Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // MOOSE includes
11 : #include "MultiApp.h"
12 :
13 : #include "AppFactory.h"
14 : #include "AuxiliarySystem.h"
15 : #include "DisplacedProblem.h"
16 : #include "Console.h"
17 : #include "Executioner.h"
18 : #include "FEProblem.h"
19 : #include "MooseMesh.h"
20 : #include "MooseUtils.h"
21 : #include "OutputWarehouse.h"
22 : #include "SetupInterface.h"
23 : #include "UserObject.h"
24 : #include "CommandLine.h"
25 : #include "Conversion.h"
26 : #include "NonlinearSystemBase.h"
27 : #include "DelimitedFileReader.h"
28 : #include "MooseAppCoordTransform.h"
29 : #include "MultiAppTransfer.h"
30 : #include "Positions.h"
31 : #include "Transient.h"
32 : #include "Backup.h"
33 : #include "Parser.h"
34 :
35 : #include "libmesh/mesh_tools.h"
36 : #include "libmesh/numeric_vector.h"
37 :
38 : // C++ includes
39 : #include <fstream>
40 : #include <iomanip>
41 : #include <iterator>
42 : #include <algorithm>
43 :
44 : // Call to "uname"
45 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
46 : #include <sys/utsname.h>
47 : #endif
48 :
49 : InputParameters
50 86501 : MultiApp::validParams()
51 : {
52 86501 : InputParameters params = MooseObject::validParams();
53 86501 : params += SetupInterface::validParams();
54 :
55 259503 : params.addParam<bool>("use_displaced_mesh",
56 173002 : false,
57 : "Whether or not this object should use the "
58 : "displaced mesh for computation. Note that "
59 : "in the case this is true but no "
60 : "displacements are provided in the Mesh block "
61 : "the undisplaced mesh will still be used.");
62 :
63 86501 : std::ostringstream app_types_strings;
64 173020 : for (const auto & name_bi_pair : AppFactory::instance().registeredObjects())
65 86519 : app_types_strings << name_bi_pair.first << " ";
66 86501 : MooseEnum app_types_options(app_types_strings.str(), "", true);
67 :
68 : // Dynamic loading
69 86501 : params.addParam<MooseEnum>("app_type",
70 : app_types_options,
71 : "The type of application to build (applications not "
72 : "registered can be loaded with dynamic libraries. Parent "
73 : "application type will be used if not provided.");
74 86501 : params.addParam<std::string>("library_path",
75 : "",
76 : "Path to search for dynamic libraries (please "
77 : "avoid committing absolute paths in addition to "
78 : "MOOSE_LIBRARY_PATH)");
79 86501 : params.addParam<std::string>(
80 : "library_name",
81 : "",
82 : "The file name of the library (*.la file) that will be dynamically loaded.");
83 259503 : params.addParam<bool>("library_load_dependencies",
84 173002 : false,
85 : "Tells MOOSE to manually load library dependencies. This should not be "
86 : "necessary and is here for debugging/troubleshooting.");
87 :
88 : // Subapp positions
89 86501 : params.addParam<std::vector<Point>>(
90 : "positions",
91 : "The positions of the App locations. Each set of 3 values will represent a "
92 : "Point. This and 'positions_file' cannot be both supplied. If this and "
93 : "'positions_file'/'_objects' are not supplied, a single position (0,0,0) will be used");
94 86501 : params.addParam<std::vector<FileName>>("positions_file",
95 : "Filename(s) that should be looked in for positions. Each"
96 : " set of 3 values in that file will represent a Point. "
97 : "This and 'positions(_objects)' cannot be both supplied");
98 86501 : params.addParam<std::vector<PositionsName>>("positions_objects",
99 : "The name of a Positions object that will contain "
100 : "the locations of the sub-apps created. This and "
101 : "'positions(_file)' cannot be both supplied");
102 259503 : params.addParam<bool>(
103 : "output_in_position",
104 173002 : false,
105 : "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
106 259503 : params.addParam<bool>(
107 : "run_in_position",
108 173002 : false,
109 : "If true this will cause the mesh from the MultiApp to be 'moved' by its position vector");
110 :
111 86501 : params.addRequiredParam<std::vector<FileName>>(
112 : "input_files",
113 : "The input file for each App. If this parameter only contains one input file "
114 : "it will be used for all of the Apps. When using 'positions_from_file' it is "
115 : "also admissable to provide one input_file per file.");
116 259503 : params.addParam<Real>("bounding_box_inflation",
117 173002 : 0.01,
118 : "Relative amount to 'inflate' the bounding box of this MultiApp.");
119 259503 : params.addParam<Point>("bounding_box_padding",
120 173002 : RealVectorValue(),
121 : "Additional padding added to the dimensions of the bounding box. The "
122 : "values are added to the x, y and z dimension respectively.");
123 :
124 86501 : params.addPrivateParam<MPI_Comm>("_mpi_comm");
125 :
126 : // Set the default execution time
127 86501 : params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
128 : // Add the POST_ADAPTIVITY execution flag.
129 : #ifdef LIBMESH_ENABLE_AMR
130 86501 : ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on");
131 86501 : exec_enum.addAvailableFlags(EXEC_POST_ADAPTIVITY);
132 86501 : params.setDocString("execute_on", exec_enum.getDocString());
133 : #endif
134 :
135 259503 : params.addParam<processor_id_type>("max_procs_per_app",
136 173002 : std::numeric_limits<processor_id_type>::max(),
137 : "Maximum number of processors to give to each App in this "
138 : "MultiApp. Useful for restricting small solves to just a few "
139 : "procs so they don't get spread out");
140 259503 : params.addParam<processor_id_type>("min_procs_per_app",
141 173002 : 1,
142 : "Minimum number of processors to give to each App in this "
143 : "MultiApp. Useful for larger, distributed mesh solves.");
144 259503 : params.addParam<bool>(
145 : "wait_for_first_app_init",
146 173002 : false,
147 : "Create the first sub-application on rank 0, then MPI_Barrier before "
148 : "creating the next N-1 apps (on all ranks). "
149 : "This is only needed if your sub-application needs to perform some setup "
150 : "actions in quiet, without other sub-applications working at the same time.");
151 :
152 259503 : params.addParam<Real>("global_time_offset",
153 173002 : 0,
154 : "The time offset relative to the parent application for the purpose of "
155 : "starting a subapp at a different time from the parent application. The "
156 : "global time will be ahead by the offset specified here.");
157 :
158 : // Resetting subapps
159 86501 : params.addParam<std::vector<Real>>(
160 : "reset_time",
161 : {},
162 : "The time(s) at which to reset Apps given by the 'reset_apps' parameter. "
163 : "Resetting an App means that it is destroyed and recreated, possibly "
164 : "modeling the insertion of 'new' material for that app.");
165 86501 : params.addParam<std::vector<unsigned int>>(
166 : "reset_apps",
167 : {},
168 : "The Apps that will be reset when 'reset_time' is hit. These are the App "
169 : "'numbers' starting with 0 corresponding to the order of the App positions. "
170 : "Resetting an App means that it is destroyed and recreated, possibly modeling "
171 : "the insertion of 'new' material for that app.");
172 :
173 : // Moving subapps
174 259503 : params.addParam<Real>(
175 : "move_time",
176 173002 : std::numeric_limits<Real>::max(),
177 : "The time at which Apps designated by move_apps are moved to move_positions.");
178 :
179 86501 : params.addParam<std::vector<unsigned int>>(
180 : "move_apps",
181 : {},
182 : "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
183 : "of the App positions, to be moved at move_time to move_positions");
184 86501 : params.addParam<std::vector<Point>>(
185 : "move_positions", {}, "The positions corresponding to each move_app.");
186 :
187 86501 : params.addParam<std::vector<CLIArgString>>(
188 : "cli_args",
189 : {},
190 : "Additional command line arguments to pass to the sub apps. If one set is provided the "
191 : "arguments are applied to all, otherwise there must be a set for each sub app.");
192 :
193 86501 : params.addParam<std::vector<FileName>>(
194 : "cli_args_files",
195 : "File names that should be looked in for additional command line arguments "
196 : "to pass to the sub apps. Each line of a file is set to each sub app. If only "
197 : "one line is provided, it will be applied to all sub apps.");
198 :
199 : // Fixed point iterations
200 259503 : params.addRangeCheckedParam<Real>("relaxation_factor",
201 173002 : 1.0,
202 : "relaxation_factor>0 & relaxation_factor<2",
203 : "Fraction of newly computed value to keep."
204 : "Set between 0 and 2.");
205 86501 : params.addDeprecatedParam<std::vector<std::string>>(
206 : "relaxed_variables",
207 : {},
208 : "Use transformed_variables.",
209 : "List of subapp variables to relax during Multiapp coupling iterations");
210 86501 : params.addParam<std::vector<std::string>>(
211 : "transformed_variables",
212 : {},
213 : "List of subapp variables to use coupling algorithm on during Multiapp coupling iterations");
214 86501 : params.addParam<std::vector<PostprocessorName>>(
215 : "transformed_postprocessors",
216 : {},
217 : "List of subapp postprocessors to use coupling "
218 : "algorithm on during Multiapp coupling iterations");
219 259503 : params.addParam<bool>("keep_solution_during_restore",
220 173002 : false,
221 : "This is useful when doing MultiApp coupling iterations. It takes the "
222 : "final solution from the previous coupling iteration"
223 : "and re-uses it as the initial guess for the next coupling iteration");
224 259503 : params.addParam<bool>("keep_aux_solution_during_restore",
225 173002 : false,
226 : "This is useful when doing MultiApp coupling iterations. It takes the "
227 : "final auxiliary solution from the previous coupling iteration"
228 : "and re-uses it as the initial guess for the next coupling iteration");
229 259503 : params.addParam<bool>(
230 : "no_backup_and_restore",
231 173002 : false,
232 : "True to turn off restore for this multiapp. This is useful when doing steady-state "
233 : "Picard iterations where we want to use the solution of previous Picard iteration as the "
234 : "initial guess of the current Picard iteration.");
235 259503 : params.addParam<unsigned int>(
236 : "max_multiapp_level",
237 173002 : 10,
238 : "Integer set by user that will stop the simulation if the multiapp level "
239 : "exceeds it. Useful for preventing infinite loops with multiapp simulations");
240 86501 : params.deprecateParam("no_backup_and_restore", "no_restore", "01/01/2025");
241 :
242 259503 : params.addDeprecatedParam<bool>("clone_master_mesh",
243 173002 : false,
244 : "True to clone parent app mesh and use it for this MultiApp.",
245 : "clone_master_mesh is deprecated, use clone_parent_mesh instead");
246 259503 : params.addParam<bool>(
247 173002 : "clone_parent_mesh", false, "True to clone parent app mesh and use it for this MultiApp.");
248 :
249 86501 : params.addPrivateParam<std::shared_ptr<CommandLine>>("_command_line");
250 86501 : params.addPrivateParam<bool>("use_positions", true);
251 86501 : params.declareControllable("enable");
252 173002 : params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
253 86501 : params.registerBase("MultiApp");
254 :
255 86501 : params.addParamNamesToGroup("use_displaced_mesh wait_for_first_app_init max_multiapp_level",
256 : "Advanced");
257 86501 : params.addParamNamesToGroup("positions positions_file positions_objects run_in_position "
258 : "output_in_position",
259 : "Positions / transformations of the MultiApp frame of reference");
260 86501 : params.addParamNamesToGroup("min_procs_per_app max_procs_per_app", "Parallelism");
261 86501 : params.addParamNamesToGroup("reset_time reset_apps", "Reset MultiApp");
262 86501 : params.addParamNamesToGroup("move_time move_apps move_positions", "Timed move of MultiApps");
263 86501 : params.addParamNamesToGroup("relaxation_factor transformed_variables transformed_postprocessors "
264 : "keep_solution_during_restore keep_aux_solution_during_restore "
265 : "no_restore",
266 : "Fixed point iteration");
267 86501 : params.addParamNamesToGroup("library_name library_path library_load_dependencies",
268 : "Dynamic loading");
269 86501 : params.addParamNamesToGroup("cli_args cli_args_files", "Passing command line argument");
270 173002 : return params;
271 173002 : }
272 :
273 7585 : MultiApp::MultiApp(const InputParameters & parameters)
274 : : MooseObject(parameters),
275 : SetupInterface(this),
276 : Restartable(this, "MultiApps"),
277 15170 : PerfGraphInterface(this, std::string("MultiApp::") + _name),
278 7585 : _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
279 10551 : _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
280 2966 : : _fe_problem.getMooseApp().type()),
281 7585 : _use_positions(getParam<bool>("use_positions")),
282 7585 : _input_files(getParam<std::vector<FileName>>("input_files")),
283 7585 : _wait_for_first_app_init(getParam<bool>("wait_for_first_app_init")),
284 7585 : _total_num_apps(0),
285 7585 : _my_num_apps(0),
286 7585 : _first_local_app(0),
287 7585 : _orig_comm(_communicator.get()),
288 7585 : _my_communicator(),
289 7585 : _my_comm(_my_communicator.get()),
290 7585 : _my_rank(0),
291 7585 : _inflation(getParam<Real>("bounding_box_inflation")),
292 7585 : _bounding_box_padding(getParam<Point>("bounding_box_padding")),
293 7585 : _max_procs_per_app(getParam<processor_id_type>("max_procs_per_app")),
294 7585 : _min_procs_per_app(getParam<processor_id_type>("min_procs_per_app")),
295 7585 : _output_in_position(getParam<bool>("output_in_position")),
296 7585 : _global_time_offset(getParam<Real>("global_time_offset")),
297 7585 : _reset_times(getParam<std::vector<Real>>("reset_time")),
298 7585 : _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
299 7585 : _reset_happened(false),
300 7585 : _move_time(getParam<Real>("move_time")),
301 7585 : _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
302 7585 : _move_positions(getParam<std::vector<Point>>("move_positions")),
303 7585 : _move_happened(false),
304 7585 : _has_an_app(true),
305 7585 : _cli_args(getParam<std::vector<CLIArgString>>("cli_args")),
306 7585 : _keep_solution_during_restore(getParam<bool>("keep_solution_during_restore")),
307 7585 : _keep_aux_solution_during_restore(getParam<bool>("keep_aux_solution_during_restore")),
308 7585 : _no_restore(getParam<bool>("no_restore")),
309 7585 : _run_in_position(getParam<bool>("run_in_position")),
310 7585 : _sub_app_backups(declareRestartableDataWithContext<SubAppBackups>("sub_app_backups", this)),
311 7585 : _solve_step_timer(registerTimedSection("solveStep", 3, "Executing MultiApps", false)),
312 7585 : _init_timer(registerTimedSection("init", 3, "Initializing MultiApp")),
313 7585 : _backup_timer(registerTimedSection("backup", 3, "Backing Up MultiApp")),
314 7585 : _restore_timer(registerTimedSection("restore", 3, "Restoring MultiApp")),
315 68265 : _reset_timer(registerTimedSection("resetApp", 3, "Resetting MultiApp"))
316 : {
317 17035 : if (parameters.isParamSetByUser("cli_args") && parameters.isParamValid("cli_args") &&
318 9450 : parameters.isParamValid("cli_args_files"))
319 4 : paramError("cli_args",
320 : "'cli_args' and 'cli_args_files' cannot be specified simultaneously in MultiApp ");
321 :
322 15162 : if (!_use_positions && (isParamValid("positions") || isParamValid("positions_file") ||
323 7581 : isParamValid("positions_objects")))
324 0 : paramError("use_positions",
325 : "This MultiApps has been set to not use positions, "
326 : "but a 'positions' parameter has been set.");
327 :
328 22674 : if ((_reset_apps.size() > 0 && _reset_times.size() == 0) ||
329 15093 : (_reset_apps.size() == 0 && _reset_times.size() > 0))
330 0 : mooseError("reset_time and reset_apps may only be specified together");
331 :
332 : // Check that the reset times are sorted by the user
333 7581 : auto sorted_times = _reset_times;
334 7581 : std::sort(sorted_times.begin(), sorted_times.end());
335 7581 : if (_reset_times.size() && _reset_times != sorted_times)
336 4 : paramError("reset_time", "List of reset times must be sorted in increasing order");
337 7577 : }
338 :
339 : void
340 7549 : MultiApp::init(unsigned int num_apps, bool batch_mode)
341 : {
342 7549 : auto config = rankConfig(
343 : processor_id(), n_processors(), num_apps, _min_procs_per_app, _max_procs_per_app, batch_mode);
344 7549 : init(num_apps, config);
345 7545 : }
346 :
347 : void
348 7549 : MultiApp::init(unsigned int num_apps, const LocalRankConfig & config)
349 : {
350 7549 : TIME_SECTION(_init_timer);
351 :
352 7549 : _total_num_apps = num_apps;
353 7549 : _rank_config = config;
354 7549 : buildComm();
355 7549 : _sub_app_backups.resize(_my_num_apps);
356 :
357 7549 : _has_bounding_box.resize(_my_num_apps, false);
358 7549 : _reset_happened.resize(_reset_times.size(), false);
359 7549 : _bounding_box.resize(_my_num_apps);
360 :
361 7549 : if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
362 4 : paramError("cli_args",
363 : "The number of items supplied must be 1 or equal to the number of sub apps.");
364 :
365 : // if cliArgs() != _cli_args, then cliArgs() was overridden and we need to check it
366 7545 : auto cla = cliArgs();
367 7545 : if (cla != std::vector<std::string>(_cli_args.begin(), _cli_args.end()))
368 : {
369 13 : if ((cla.size() > 1) && (_total_num_apps != cla.size()))
370 0 : mooseError("The number of items supplied as command line argument to subapps must be 1 or "
371 : "equal to the number of sub apps. Note: you use a multiapp that provides its own "
372 : "command line parameters so the error is not in cli_args");
373 : }
374 7545 : }
375 :
376 : void
377 7573 : MultiApp::setupPositions()
378 : {
379 7573 : if (_use_positions)
380 : {
381 7573 : fillPositions();
382 7549 : init(_positions.size());
383 7545 : createApps();
384 : }
385 7458 : }
386 :
387 : void
388 7545 : MultiApp::createApps()
389 : {
390 7545 : if (!_has_an_app)
391 54 : return;
392 :
393 7491 : TIME_SECTION("createApps", 2, "Instantiating Sub-Apps", false);
394 :
395 : // Read commandLine arguments that will be used when creating apps
396 7491 : readCommandLineArguments();
397 :
398 7471 : Moose::ScopedCommSwapper swapper(_my_comm);
399 :
400 7471 : _apps.resize(_my_num_apps);
401 :
402 : // If the user provided an unregistered app type, see if we can load it dynamically
403 7471 : if (!AppFactory::instance().isRegistered(_app_type))
404 5 : _app.dynamicAppRegistration(_app_type,
405 : getParam<std::string>("library_path"),
406 7 : getParam<std::string>("library_name"),
407 7 : getParam<bool>("library_load_dependencies"));
408 :
409 7468 : bool rank_did_quiet_init = false;
410 7468 : unsigned int local_app = libMesh::invalid_uint;
411 7468 : if (_wait_for_first_app_init)
412 : {
413 16 : if (hasLocalApp(0))
414 : {
415 8 : rank_did_quiet_init = true;
416 8 : local_app = globalAppToLocal(0);
417 8 : createLocalApp(local_app);
418 : }
419 :
420 16 : MPI_Barrier(_orig_comm);
421 : }
422 :
423 19696 : for (unsigned int i = 0; i < _my_num_apps; i++)
424 : {
425 12292 : if (rank_did_quiet_init && i == local_app)
426 8 : continue;
427 12284 : createLocalApp(i);
428 : }
429 7404 : }
430 :
431 : void
432 12292 : MultiApp::createLocalApp(const unsigned int i)
433 : {
434 12292 : createApp(i, _global_time_offset);
435 12228 : }
436 :
437 : void
438 7362 : MultiApp::initialSetup()
439 : {
440 7362 : if (!_use_positions)
441 : // if not using positions, we create the sub-apps in initialSetup instead of right after
442 : // construction of MultiApp
443 0 : createApps();
444 7362 : }
445 :
446 : void
447 7491 : MultiApp::readCommandLineArguments()
448 : {
449 7491 : if (isParamValid("cli_args_files"))
450 : {
451 59 : _cli_args_from_file.clear();
452 :
453 59 : std::vector<FileName> cli_args_files = getParam<std::vector<FileName>>("cli_args_files");
454 59 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
455 :
456 : // If we use parameter "cli_args_files", at least one file should be provided
457 59 : if (!cli_args_files.size())
458 4 : paramError("cli_args_files", "You need to provide at least one commandLine argument file ");
459 :
460 : // If we multiple input files, then we need to check if the number of input files
461 : // match with the number of argument files
462 55 : if (cli_args_files.size() != 1 && cli_args_files.size() != input_files.size())
463 8 : paramError("cli_args_files",
464 : "The number of commandLine argument files ",
465 : cli_args_files.size(),
466 : " for MultiApp ",
467 4 : name(),
468 : " must either be only one or match the number of input files ",
469 : input_files.size());
470 :
471 : // Go through all argument files
472 51 : std::vector<std::string> cli_args;
473 107 : for (unsigned int p_file_it = 0; p_file_it < cli_args_files.size(); p_file_it++)
474 : {
475 64 : std::string cli_args_file = cli_args_files[p_file_it];
476 : // Clear up
477 64 : cli_args.clear();
478 : // Read the file on the root processor then broadcast it
479 64 : if (processor_id() == 0)
480 : {
481 52 : MooseUtils::checkFileReadable(cli_args_file);
482 :
483 52 : std::ifstream is(cli_args_file.c_str());
484 : // Read by line rather than space separated like the cli_args parameter
485 52 : std::string line;
486 212 : while (std::getline(is, line))
487 160 : cli_args.push_back(line);
488 :
489 : // We do not allow empty files
490 52 : if (!cli_args.size())
491 4 : paramError("cli_args_files",
492 : "There is no commandLine argument in the commandLine argument file ",
493 : cli_args_file);
494 :
495 : // If we have position files, we need to
496 : // make sure the number of commandLine argument strings
497 : // match with the number of positions
498 48 : if (_npositions_inputfile.size())
499 : {
500 44 : auto num_positions = _npositions_inputfile[p_file_it];
501 : // Check if the number of commandLine argument strings equal to
502 : // the number of positions
503 44 : if (cli_args.size() == 1)
504 0 : for (MooseIndex(num_positions) num = 0; num < num_positions; num++)
505 0 : _cli_args_from_file.push_back(cli_args.front());
506 44 : else if (cli_args.size() == num_positions)
507 160 : for (auto && cli_arg : cli_args)
508 120 : _cli_args_from_file.push_back(cli_arg);
509 4 : else if (cli_args.size() != num_positions)
510 4 : paramError("cli_args_files",
511 : "The number of commandLine argument strings ",
512 : cli_args.size(),
513 : " in the file ",
514 : cli_args_file,
515 : " must either be only one or match the number of positions ",
516 : num_positions);
517 : }
518 : else
519 : {
520 : // If we do not have position files, we will check if the number of
521 : // commandLine argument strings match with the total number of subapps
522 24 : for (auto && cli_arg : cli_args)
523 20 : _cli_args_from_file.push_back(cli_arg);
524 : }
525 44 : }
526 56 : }
527 :
528 : // Broad cast all arguments to everyone
529 43 : _communicator.broadcast(_cli_args_from_file);
530 43 : }
531 :
532 7518 : if (_cli_args_from_file.size() && _cli_args_from_file.size() != 1 &&
533 43 : _cli_args_from_file.size() != _total_num_apps)
534 8 : mooseError(" The number of commandLine argument strings ",
535 4 : _cli_args_from_file.size(),
536 : " must either be only one or match the total "
537 : "number of sub apps ",
538 4 : _total_num_apps);
539 :
540 7471 : if (_cli_args_from_file.size() && cliArgs().size())
541 0 : mooseError("Cannot set commandLine arguments from both input_file and external files");
542 7471 : }
543 :
544 : void
545 7454 : MultiApp::fillPositions()
546 : {
547 7454 : if (_move_apps.size() != _move_positions.size())
548 0 : mooseError("The number of apps to move and the positions to move them to must be the same for "
549 : "MultiApp ",
550 0 : _name);
551 :
552 14908 : if (isParamValid("positions") + isParamValid("positions_file") +
553 7454 : isParamValid("positions_objects") >
554 : 1)
555 4 : mooseError("Only one 'positions' parameter may be specified");
556 :
557 7450 : if (isParamValid("positions"))
558 : {
559 5266 : _positions = getParam<std::vector<Point>>("positions");
560 :
561 5266 : if (_positions.size() < _input_files.size())
562 8 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
563 4 : name());
564 : }
565 2184 : else if (isParamValid("positions_file"))
566 : {
567 94 : std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
568 94 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
569 :
570 94 : if (input_files.size() != 1 && positions_files.size() != input_files.size())
571 8 : mooseError("Number of input_files for MultiApp ",
572 4 : name(),
573 : " must either be only one or match the number of positions_file files");
574 :
575 : // Clear out the _input_files because we're going to rebuild it
576 90 : if (input_files.size() != 1)
577 30 : _input_files.clear();
578 :
579 206 : for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
580 : {
581 120 : std::string positions_file = positions_files[p_file_it];
582 120 : MooseUtils::DelimitedFileReader file(positions_file, &_communicator);
583 120 : file.setFormatFlag(MooseUtils::DelimitedFileReader::FormatFlag::ROWS);
584 120 : file.read();
585 :
586 120 : const std::vector<Point> & data = file.getDataAsPoints();
587 564 : for (const auto & d : data)
588 448 : _positions.push_back(d);
589 :
590 : // Save the number of positions for this input file
591 116 : _npositions_inputfile.push_back(data.size());
592 :
593 564 : for (unsigned int i = 0; i < data.size(); ++i)
594 448 : if (input_files.size() != 1)
595 120 : _input_files.push_back(input_files[p_file_it]);
596 116 : }
597 86 : }
598 2090 : else if (isParamValid("positions_objects"))
599 : {
600 464 : const auto & positions_param_objs = getParam<std::vector<PositionsName>>("positions_objects");
601 464 : const auto & input_files = getParam<std::vector<FileName>>("input_files");
602 :
603 464 : if (input_files.size() != 1 && positions_param_objs.size() != input_files.size())
604 0 : mooseError("Number of input_files for MultiApp ",
605 0 : name(),
606 : " must either be only one or match the number of positions_objects specified");
607 :
608 : // Clear out the _input_files because we're going to rebuild it
609 464 : if (input_files.size() != 1)
610 0 : _input_files.clear();
611 :
612 : // Keeps track of where each positions object start in terms of subapp numbers
613 464 : unsigned int offset = 0;
614 :
615 967 : for (const auto p_obj_it : index_range(positions_param_objs))
616 : {
617 507 : const std::string & positions_name = positions_param_objs[p_obj_it];
618 507 : auto positions_obj = &_fe_problem.getPositionsObject(positions_name);
619 :
620 507 : const auto & data = positions_obj->getPositions(true);
621 :
622 : // Append all positions from this object
623 2200 : for (const auto & d : data)
624 1697 : _positions.push_back(d);
625 :
626 : // Save the number of positions for this input file
627 503 : _npositions_inputfile.push_back(data.size());
628 :
629 503 : if (!positions_obj)
630 0 : paramError("positions_objects",
631 0 : "'" + positions_name + "' is not of the expected type. Should be a Positions");
632 :
633 : // Keep track of which positions is tied to what subapp
634 2200 : for (unsigned int i = 0; i < data.size(); ++i)
635 : {
636 1697 : if (input_files.size() != 1)
637 0 : _input_files.push_back(input_files[p_obj_it]);
638 1697 : _positions_objs.push_back(positions_obj);
639 1697 : _positions_index_offsets.push_back(offset);
640 : }
641 503 : offset += data.size();
642 : }
643 : }
644 : else
645 : {
646 1626 : _positions = {Point()};
647 :
648 1626 : if (_positions.size() < _input_files.size())
649 8 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
650 4 : name());
651 : }
652 :
653 : mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
654 : "Number of positions and input files are not the same!");
655 7430 : }
656 :
657 : void
658 67736 : MultiApp::preTransfer(Real /*dt*/, Real target_time)
659 : {
660 : // Get a transient executioner to get a user-set tolerance
661 67736 : Real timestep_tol = 1e-13;
662 67736 : if (dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner()))
663 64930 : timestep_tol =
664 64930 : dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner())->timestepTol();
665 :
666 : // Determination on whether we need to backup the app due to changes below
667 67736 : bool backup_apps = false;
668 :
669 : // First, see if any Apps need to be reset
670 68515 : for (unsigned int i = 0; i < _reset_times.size(); i++)
671 : {
672 855 : if (!_reset_happened[i] && (target_time + timestep_tol >= _reset_times[i]))
673 : {
674 76 : _reset_happened[i] = true;
675 76 : if (_reset_apps.size() > 0)
676 152 : for (auto & app : _reset_apps)
677 76 : resetApp(app);
678 :
679 : // If we reset an application, then we delete the old objects, including the coordinate
680 : // transformation classes. Consequently we need to reset the coordinate transformation classes
681 : // in the associated transfer classes
682 109 : for (auto * const transfer : _associated_transfers)
683 33 : transfer->getAppInfo();
684 :
685 : // Similarly we need to transform the mesh again
686 76 : if (_run_in_position)
687 0 : for (const auto i : make_range(_my_num_apps))
688 : {
689 0 : auto app_ptr = _apps[i];
690 0 : if (usingPositions())
691 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
692 0 : app_ptr->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
693 : else
694 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
695 0 : app_ptr->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
696 0 : }
697 :
698 : // If the time step covers multiple reset times, set them all as having 'happened'
699 174 : for (unsigned int j = i; j < _reset_times.size(); j++)
700 98 : if (target_time + timestep_tol >= _reset_times[j])
701 87 : _reset_happened[j] = true;
702 :
703 : // Backup in case the next solve fails
704 76 : backup_apps = true;
705 :
706 76 : break;
707 : }
708 : }
709 :
710 : // Now move any apps that should be moved
711 67736 : if (_use_positions && !_move_happened && target_time + timestep_tol >= _move_time)
712 : {
713 60 : _move_happened = true;
714 116 : for (unsigned int i = 0; i < _move_apps.size(); i++)
715 60 : moveApp(_move_apps[i], _move_positions[i]);
716 :
717 : // Backup in case the next solve fails
718 56 : backup_apps = true;
719 : }
720 :
721 67732 : if (backup_apps)
722 111 : backup();
723 67732 : }
724 :
725 : Executioner *
726 0 : MultiApp::getExecutioner(unsigned int app)
727 : {
728 0 : if (!_has_an_app)
729 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
730 :
731 0 : return _apps[globalAppToLocal(app)]->getExecutioner();
732 : }
733 :
734 : void
735 4557 : MultiApp::finalize()
736 : {
737 11297 : for (const auto & app_ptr : _apps)
738 : {
739 6740 : auto * executioner = app_ptr->getExecutioner();
740 : mooseAssert(executioner, "Executioner is nullptr");
741 :
742 6740 : executioner->feProblem().execute(EXEC_FINAL);
743 6740 : executioner->feProblem().outputStep(EXEC_FINAL);
744 : }
745 4557 : }
746 :
747 : void
748 4895 : MultiApp::postExecute()
749 : {
750 12253 : for (const auto & app_ptr : _apps)
751 : {
752 7358 : auto * executioner = app_ptr->getExecutioner();
753 : mooseAssert(executioner, "Executioner is nullptr");
754 :
755 7358 : executioner->postExecute();
756 : }
757 4895 : }
758 :
759 : void
760 24985 : MultiApp::backup()
761 : {
762 24985 : TIME_SECTION(_backup_timer);
763 :
764 24985 : if (_fe_problem.verboseMultiApps())
765 1031 : _console << "Backed up MultiApp ... ";
766 :
767 59323 : for (unsigned int i = 0; i < _my_num_apps; i++)
768 34338 : _sub_app_backups[i] = _apps[i]->backup();
769 :
770 24985 : if (_fe_problem.verboseMultiApps())
771 1031 : _console << name() << std::endl;
772 24985 : }
773 :
774 : void
775 46553 : MultiApp::restore(bool force)
776 : {
777 46553 : TIME_SECTION(_restore_timer);
778 :
779 46553 : if (force || needsRestoration())
780 : {
781 : // Must be restarting / recovering from main app so hold off on restoring
782 : // Instead - the restore will happen in sub-apps' initialSetup()
783 : // Note that _backups was already populated by dataLoad() in the main app
784 10272 : if (_fe_problem.getCurrentExecuteOnFlag() == EXEC_INITIAL)
785 550 : return;
786 :
787 : // We temporarily copy and store solutions for all subapps
788 9722 : if (_keep_solution_during_restore)
789 : {
790 343 : _end_solutions.resize(_my_num_apps);
791 :
792 682 : for (unsigned int i = 0; i < _my_num_apps; i++)
793 : {
794 686 : _end_solutions[i] = _apps[i]
795 : ->getExecutioner()
796 343 : ->feProblem()
797 343 : .getNonlinearSystemBase(/*nl_sys=*/0)
798 343 : .solution()
799 686 : .clone();
800 : auto & sub_multiapps =
801 343 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects();
802 :
803 : // multiapps of each subapp should do the same things
804 : // It is implemented recursively
805 354 : for (auto & multi_app : sub_multiapps)
806 15 : multi_app->keepSolutionDuringRestore(_keep_solution_during_restore);
807 : }
808 : }
809 :
810 : // We temporarily copy and store solutions for all subapps
811 9718 : if (_keep_aux_solution_during_restore)
812 : {
813 132 : _end_aux_solutions.resize(_my_num_apps);
814 :
815 264 : for (unsigned int i = 0; i < _my_num_apps; i++)
816 132 : _end_aux_solutions[i] =
817 264 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution().clone();
818 : }
819 :
820 9718 : if (_fe_problem.verboseMultiApps())
821 516 : _console << "Restoring MultiApp ... ";
822 :
823 19621 : for (unsigned int i = 0; i < _my_num_apps; i++)
824 : {
825 9903 : _apps[i]->restore(std::move(_sub_app_backups[i]), false);
826 9903 : _sub_app_backups[i] = _apps[i]->finalizeRestore();
827 : mooseAssert(_sub_app_backups[i], "Should have a backup");
828 : }
829 :
830 9718 : if (_fe_problem.verboseMultiApps())
831 516 : _console << name() << std::endl;
832 :
833 : // Now copy the latest solutions back for each subapp
834 9718 : if (_keep_solution_during_restore)
835 : {
836 678 : for (unsigned int i = 0; i < _my_num_apps; i++)
837 : {
838 339 : _apps[i]->getExecutioner()->feProblem().getNonlinearSystemBase(/*nl_sys=*/0).solution() =
839 678 : *_end_solutions[i];
840 :
841 : // We need to synchronize solution so that local_solution has the right values
842 339 : _apps[i]->getExecutioner()->feProblem().getNonlinearSystemBase(/*nl_sys=*/0).update();
843 : }
844 :
845 339 : _end_solutions.clear();
846 : }
847 : // Now copy the latest auxiliary solutions back for each subapp
848 9718 : if (_keep_aux_solution_during_restore)
849 : {
850 264 : for (unsigned int i = 0; i < _my_num_apps; i++)
851 : {
852 132 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution() =
853 264 : *_end_aux_solutions[i];
854 :
855 : // We need to synchronize solution so that local_solution has the right values
856 132 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().update();
857 : }
858 :
859 132 : _end_aux_solutions.clear();
860 : }
861 :
862 : // Make sure the displaced mesh on the multiapp is up-to-date with displacement variables
863 19621 : for (const auto & app_ptr : _apps)
864 9903 : if (app_ptr->feProblem().getDisplacedProblem())
865 48 : app_ptr->feProblem().getDisplacedProblem()->updateMesh();
866 :
867 : // If we are restoring due to a failed solve, make sure reset the solved state in the sub-apps
868 9718 : if (!getMooseApp().getExecutioner()->lastSolveConverged())
869 1663 : for (auto & app_ptr : _apps)
870 918 : app_ptr->getExecutioner()->fixedPointSolve().clearFixedPointStatus();
871 : }
872 : else
873 : {
874 72562 : for (unsigned int i = 0; i < _my_num_apps; i++)
875 : {
876 36281 : for (auto & sub_app :
877 73033 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects())
878 471 : sub_app->restore(false);
879 : }
880 : }
881 46549 : }
882 :
883 : void
884 15 : MultiApp::keepSolutionDuringRestore(bool keep_solution_during_restore)
885 : {
886 15 : if (_pars.isParamSetByUser("keep_solution_during_restore"))
887 4 : paramError("keep_solution_during_restore",
888 : "This parameter should only be provided in parent app");
889 :
890 11 : _keep_solution_during_restore = keep_solution_during_restore;
891 11 : }
892 :
893 : void
894 422337 : MultiApp::transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform)
895 : {
896 422337 : const Real min_x = box.first(0);
897 422337 : const Real max_x = box.second(0);
898 422337 : const Real min_y = box.first(1);
899 422337 : const Real max_y = box.second(1);
900 422337 : const Real min_z = box.first(2);
901 422337 : const Real max_z = box.second(2);
902 :
903 : std::array<Point, 8> box_corners = {{Point(min_x, min_y, min_z),
904 : Point(max_x, min_y, min_z),
905 : Point(min_x, max_y, min_z),
906 : Point(max_x, max_y, min_z),
907 : Point(min_x, min_y, max_z),
908 : Point(max_x, min_y, max_z),
909 : Point(min_x, max_y, max_z),
910 422337 : Point(max_x, max_y, max_z)}};
911 :
912 : // transform each corner
913 3801033 : for (auto & corner : box_corners)
914 3378696 : corner = transform(corner);
915 :
916 : // Create new bounding box
917 422337 : Point new_box_min = box_corners[0];
918 422337 : Point new_box_max = new_box_min;
919 3378696 : for (const auto p : make_range(1, 8))
920 11825436 : for (const auto d : make_range(Moose::dim))
921 : {
922 8869077 : const Point & pt = box_corners[p];
923 8869077 : if (new_box_min(d) > pt(d))
924 2540 : new_box_min(d) = pt(d);
925 :
926 8869077 : if (new_box_max(d) < pt(d))
927 1190708 : new_box_max(d) = pt(d);
928 : }
929 422337 : box.first = new_box_min;
930 422337 : box.second = new_box_max;
931 422337 : }
932 :
933 : BoundingBox
934 354231 : MultiApp::getBoundingBox(unsigned int app,
935 : bool displaced_mesh,
936 : const MultiAppCoordTransform * const coord_transform)
937 : {
938 354231 : if (!_has_an_app)
939 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
940 :
941 354231 : unsigned int local_app = globalAppToLocal(app);
942 354231 : FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
943 9078 : MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
944 712146 : ? fe_problem_base.getDisplacedProblem()->mesh()
945 354231 : : fe_problem_base.mesh();
946 :
947 : {
948 354231 : Moose::ScopedCommSwapper swapper(_my_comm);
949 354231 : if (displaced_mesh)
950 4539 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
951 : else
952 : {
953 349692 : if (!_has_bounding_box[local_app])
954 : {
955 102 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
956 102 : _has_bounding_box[local_app] = true;
957 : }
958 : }
959 354231 : }
960 354231 : BoundingBox bbox = _bounding_box[local_app];
961 :
962 354231 : Point min = bbox.min();
963 354231 : min -= _bounding_box_padding;
964 354231 : Point max = bbox.max();
965 354231 : max += _bounding_box_padding;
966 :
967 354231 : Point inflation_amount = (max - min) * _inflation;
968 :
969 354231 : Point inflated_min = min - inflation_amount;
970 354231 : Point inflated_max = max + inflation_amount;
971 :
972 354231 : Point shifted_min = inflated_min;
973 354231 : Point shifted_max = inflated_max;
974 :
975 708462 : if ((!coord_transform || coord_transform->skipCoordinateCollapsing()) &&
976 708462 : fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
977 : {
978 : // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
979 : // FIXME: Assuming all subdomains are the same coordinate system type!
980 342175 : shifted_min(0) = -inflated_max(0);
981 342175 : shifted_min(1) = inflated_min(1);
982 342175 : shifted_min(2) = -inflated_max(0);
983 :
984 342175 : shifted_max(0) = inflated_max(0);
985 342175 : shifted_max(1) = inflated_max(1);
986 342175 : shifted_max(2) = inflated_max(0);
987 : }
988 :
989 354231 : if (coord_transform)
990 : {
991 354231 : BoundingBox transformed_bbox(shifted_min, shifted_max);
992 354231 : transformBoundingBox(transformed_bbox, *coord_transform);
993 354231 : return transformed_bbox;
994 : }
995 : else
996 : {
997 : // This is where the app is located. We need to shift by this amount.
998 0 : Point p = position(app);
999 :
1000 : // Shift them to the position they're supposed to be
1001 0 : shifted_min += p;
1002 0 : shifted_max += p;
1003 0 : return BoundingBox(shifted_min, shifted_max);
1004 : }
1005 : }
1006 :
1007 : FEProblemBase &
1008 558522 : MultiApp::appProblemBase(unsigned int app)
1009 : {
1010 558522 : if (!_has_an_app)
1011 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1012 :
1013 558522 : unsigned int local_app = globalAppToLocal(app);
1014 :
1015 558522 : return _apps[local_app]->getExecutioner()->feProblem();
1016 : }
1017 :
1018 : FEProblem &
1019 0 : MultiApp::appProblem(unsigned int app)
1020 : {
1021 0 : mooseDeprecated(
1022 : "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
1023 0 : if (!_has_an_app)
1024 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1025 :
1026 0 : unsigned int local_app = globalAppToLocal(app);
1027 :
1028 0 : return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
1029 : }
1030 :
1031 : const UserObject &
1032 186397 : MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
1033 : {
1034 186397 : if (!_has_an_app)
1035 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1036 :
1037 186397 : return appProblemBase(app).getUserObjectBase(name);
1038 : }
1039 :
1040 : Real
1041 4152 : MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
1042 : {
1043 4152 : if (!_has_an_app)
1044 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1045 :
1046 4152 : return appProblemBase(app).getPostprocessorValueByName(name);
1047 : }
1048 :
1049 : NumericVector<Number> &
1050 942 : MultiApp::appTransferVector(unsigned int app, std::string var_name)
1051 : {
1052 942 : return *(appProblemBase(app).getSystem(var_name).solution);
1053 : }
1054 :
1055 : bool
1056 150 : MultiApp::isFirstLocalRank() const
1057 : {
1058 150 : return _rank_config.is_first_local_rank;
1059 : }
1060 :
1061 : bool
1062 903715 : MultiApp::hasLocalApp(unsigned int global_app) const
1063 : {
1064 903715 : if (_has_an_app && global_app >= _first_local_app &&
1065 813830 : global_app <= _first_local_app + (_my_num_apps - 1))
1066 726080 : return true;
1067 :
1068 177635 : return false;
1069 : }
1070 :
1071 : MooseApp *
1072 11135 : MultiApp::localApp(unsigned int local_app)
1073 : {
1074 : mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
1075 11135 : return _apps[local_app].get();
1076 : }
1077 :
1078 : void
1079 76 : MultiApp::resetApp(unsigned int global_app, Real time)
1080 : {
1081 76 : TIME_SECTION(_reset_timer);
1082 :
1083 76 : Moose::ScopedCommSwapper swapper(_my_comm);
1084 :
1085 76 : if (hasLocalApp(global_app))
1086 : {
1087 76 : unsigned int local_app = globalAppToLocal(global_app);
1088 :
1089 : // Extract the file numbers from the output, so that the numbering is maintained after reset
1090 76 : std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
1091 :
1092 76 : createApp(local_app, time);
1093 :
1094 : // Reset the file numbers of the newly reset apps
1095 76 : _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
1096 76 : }
1097 76 : }
1098 :
1099 : void
1100 60 : MultiApp::moveApp(unsigned int global_app, Point p)
1101 : {
1102 60 : if (_use_positions)
1103 : {
1104 60 : _positions[global_app] = p;
1105 :
1106 60 : if (hasLocalApp(global_app))
1107 : {
1108 60 : unsigned int local_app = globalAppToLocal(global_app);
1109 :
1110 60 : if (_output_in_position)
1111 56 : _apps[local_app]->setOutputPosition(p);
1112 60 : if (_run_in_position)
1113 4 : paramError("run_in_position", "Moving apps and running apps in position is not supported");
1114 : }
1115 : }
1116 56 : }
1117 :
1118 : void
1119 21 : MultiApp::parentOutputPositionChanged()
1120 : {
1121 21 : if (_use_positions && _output_in_position)
1122 42 : for (unsigned int i = 0; i < _apps.size(); i++)
1123 21 : _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1124 21 : }
1125 :
1126 : void
1127 12368 : MultiApp::createApp(unsigned int i, Real start_time)
1128 : {
1129 : // Define the app name
1130 12368 : const std::string multiapp_name = getMultiAppName(name(), _first_local_app + i, _total_num_apps);
1131 12368 : std::string full_name;
1132 :
1133 : // Only add parent name if the parent is not the main app
1134 12368 : if (_app.multiAppLevel() > 0)
1135 525 : full_name = _app.name() + "_" + multiapp_name;
1136 : else
1137 11843 : full_name = multiapp_name;
1138 :
1139 12368 : InputParameters app_params = AppFactory::instance().getValidParams(_app_type);
1140 12368 : app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
1141 12368 : app_params.set<std::unique_ptr<Backup> *>("_initial_backup") = &_sub_app_backups[i];
1142 :
1143 : // Build the CommandLine with the relevant options for this subapp and add the
1144 : // cli args from the input file
1145 12368 : std::vector<std::string> input_cli_args;
1146 12368 : if (cliArgs().size() > 0 || _cli_args_from_file.size() > 0)
1147 3247 : input_cli_args = getCommandLineArgs(i);
1148 : // This will mark all hit CLI command line parameters that are passed to subapps
1149 : // as used within the parent app (_app)
1150 12368 : auto app_cli = _app.commandLine()->initSubAppCommandLine(name(), multiapp_name, input_cli_args);
1151 12368 : app_cli->parse();
1152 12368 : app_params.set<std::shared_ptr<CommandLine>>("_command_line") = std::move(app_cli);
1153 :
1154 12368 : if (_fe_problem.verboseMultiApps())
1155 1325 : _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
1156 1325 : << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
1157 1325 : << " on processor " << processor_id() << " with full name " << full_name
1158 1325 : << COLOR_DEFAULT << std::endl;
1159 12368 : app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
1160 12368 : app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
1161 12368 : app_params.set<const MooseMesh *>("_master_mesh") = &_fe_problem.mesh();
1162 : #ifdef MOOSE_MFEM_ENABLED
1163 16542 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1164 24813 : _app.getMFEMDevice(Moose::PassKey<MultiApp>());
1165 8271 : const auto & mfem_device_set = _app.getMFEMDevices(Moose::PassKey<MultiApp>());
1166 16542 : app_params.set<std::vector<std::string>>("_mfem_devices") =
1167 24813 : std::vector<std::string>(mfem_device_set.begin(), mfem_device_set.end());
1168 : #endif
1169 12368 : if (getParam<bool>("clone_master_mesh") || getParam<bool>("clone_parent_mesh"))
1170 : {
1171 849 : if (_fe_problem.verboseMultiApps())
1172 0 : _console << COLOR_CYAN << "Cloned parent app mesh will be used for MultiApp " << name()
1173 0 : << COLOR_DEFAULT << std::endl;
1174 849 : app_params.set<bool>("_use_master_mesh") = true;
1175 849 : auto displaced_problem = _fe_problem.getDisplacedProblem();
1176 849 : if (displaced_problem)
1177 0 : app_params.set<const MooseMesh *>("_master_displaced_mesh") = &displaced_problem->mesh();
1178 849 : }
1179 :
1180 : // If only one input file was provided, use it for all the solves
1181 12368 : const auto input_index = _input_files.size() == 1 ? 0 : _first_local_app + i;
1182 12368 : const auto & input_file = _input_files[input_index];
1183 :
1184 : // create new parser tree for the application and parse
1185 12368 : auto parser = std::make_unique<Parser>(input_file);
1186 :
1187 12368 : if (input_file.size())
1188 : {
1189 12368 : parser->parse();
1190 12368 : const auto & app_type = parser->getAppType();
1191 12368 : if (app_type.empty() && _app_type.empty())
1192 0 : mooseWarning("The application type is not specified for ",
1193 : full_name,
1194 : ". Please use [Application] block to specify the application type.");
1195 12372 : if (!app_type.empty() && app_type != _app_type &&
1196 4 : !AppFactory::instance().isRegistered(app_type))
1197 4 : mooseError("In the ",
1198 : full_name,
1199 : ", '",
1200 : app_type,
1201 : "' is not a registered application. The registered application is named: '",
1202 4 : _app_type,
1203 : "'. Please double check the [Application] block to make sure the correct "
1204 : "application is provided. \n");
1205 : }
1206 :
1207 12364 : if (parser->getAppType().empty())
1208 12351 : parser->setAppType(_app_type);
1209 :
1210 12364 : app_params.set<std::shared_ptr<Parser>>("_parser") = std::move(parser);
1211 12364 : _apps[i] = AppFactory::instance().createShared(_app_type, full_name, app_params, _my_comm);
1212 12364 : auto & app = _apps[i];
1213 :
1214 12364 : app->setGlobalTimeOffset(start_time);
1215 12364 : app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
1216 12364 : app->setRestart(_app.isRestarting());
1217 12364 : app->setRecover(_app.isRecovering());
1218 :
1219 12364 : if (_use_positions && getParam<bool>("output_in_position"))
1220 3286 : app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1221 12364 : if (_output_in_position && _run_in_position)
1222 4 : paramError("run_in_position",
1223 : "Sub-apps are already displaced, so they are already output in position");
1224 :
1225 : // Update the MultiApp level for the app that was just created
1226 12360 : app->setupOptions();
1227 : // if multiapp does not have file base in Outputs input block, output file base will
1228 : // be empty here since setupOptions() does not set the default file base with the multiapp
1229 : // input file name. Parent app will create the default file base for multiapp by taking the
1230 : // output base of the parent app problem and appending the name of the multiapp plus a number to
1231 : // it
1232 12360 : if (app->getOutputFileBase().empty())
1233 12338 : setAppOutputFileBase(i);
1234 12360 : preRunInputFile();
1235 12360 : if (_app.multiAppLevel() > getParam<unsigned int>("max_multiapp_level"))
1236 4 : mooseError("Maximum multiapp level has been reached. This is likely caused by an infinite loop "
1237 : "in your multiapp system. If additional multiapp levels are needed, "
1238 : "max_multiapp_level can be specified in the MuliApps block.");
1239 :
1240 : // Transfer coupling relaxation information to the subapps
1241 12356 : _apps[i]->fixedPointConfig().sub_relaxation_factor = getParam<Real>("relaxation_factor");
1242 12356 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1243 24712 : getParam<std::vector<std::string>>("transformed_variables");
1244 : // Handle deprecated parameter
1245 12356 : if (!parameters().isParamSetByAddParam("relaxed_variables"))
1246 0 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1247 0 : getParam<std::vector<std::string>>("relaxed_variables");
1248 12356 : _apps[i]->fixedPointConfig().sub_transformed_pps =
1249 24712 : getParam<std::vector<PostprocessorName>>("transformed_postprocessors");
1250 :
1251 12356 : app->runInputFile();
1252 12308 : auto fixed_point_solve = &(_apps[i]->getExecutioner()->fixedPointSolve());
1253 12308 : if (fixed_point_solve)
1254 12308 : fixed_point_solve->allocateStorage(false);
1255 :
1256 : // Transform the app mesh if requested
1257 12308 : if (_run_in_position)
1258 : {
1259 274 : if (usingPositions())
1260 548 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1261 274 : app->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
1262 : else
1263 0 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1264 0 : app->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
1265 : }
1266 12304 : }
1267 :
1268 : std::vector<std::string>
1269 3247 : MultiApp::getCommandLineArgs(const unsigned int local_app)
1270 : {
1271 3247 : const auto cla = cliArgs();
1272 3247 : auto cli_args_param = _cli_args_param;
1273 3247 : std::string combined_args;
1274 :
1275 : // Single set of args from cliArgs() to be provided to all apps
1276 3247 : if (cla.size() == 1)
1277 2323 : combined_args = cla[0];
1278 : // Single "cli_args_files" file to be provided to all apps
1279 924 : else if (_cli_args_from_file.size() == 1)
1280 : {
1281 0 : cli_args_param = "cli_args_files";
1282 0 : combined_args = _cli_args_from_file[0];
1283 : }
1284 : // Unique set of args from cliArgs() to be provided to each app
1285 924 : else if (cla.size())
1286 804 : combined_args = cla[local_app + _first_local_app];
1287 : // Unique set of args from "cli_args_files" to be provided to all apps
1288 : else
1289 : {
1290 120 : cli_args_param = "cli_args_files";
1291 120 : combined_args = _cli_args_from_file[local_app + _first_local_app];
1292 : }
1293 :
1294 : // Remove all of the beginning and end whitespace so we can recognize truly empty
1295 3247 : combined_args = MooseUtils::trim(combined_args);
1296 :
1297 : // MooseUtils::split will return a single empty entry if there is nothing,
1298 : // so exit early if we have nothing
1299 3247 : if (combined_args.empty())
1300 0 : return {};
1301 :
1302 : // Split the argument into a vector of arguments, and make sure
1303 : // that we don't have any empty arguments
1304 3247 : const auto args = MooseUtils::split(combined_args, ";");
1305 10687 : for (const auto & arg : args)
1306 : {
1307 7440 : if (arg.empty())
1308 : {
1309 0 : const auto error = "An empty MultiApp command line argument was provided. Your "
1310 : "combined command line string has a ';' with no argument after it.";
1311 0 : if (cli_args_param)
1312 0 : paramError(*cli_args_param, error);
1313 : else
1314 0 : mooseError(error);
1315 : }
1316 : }
1317 :
1318 3247 : return args;
1319 3247 : }
1320 :
1321 : LocalRankConfig
1322 8436 : rankConfig(processor_id_type rank,
1323 : processor_id_type nprocs,
1324 : dof_id_type napps,
1325 : processor_id_type min_app_procs,
1326 : processor_id_type max_app_procs,
1327 : bool batch_mode)
1328 : {
1329 8436 : if (min_app_procs > nprocs)
1330 0 : mooseError("minimum number of procs per app is higher than the available number of procs");
1331 8436 : else if (min_app_procs > max_app_procs)
1332 0 : mooseError("minimum number of procs per app must be lower than the max procs per app");
1333 :
1334 : mooseAssert(rank < nprocs, "rank must be smaller than the number of procs");
1335 :
1336 : // A "slot" is a group of procs/ranks that are grouped together to run a
1337 : // single (sub)app/sim in parallel.
1338 :
1339 : const processor_id_type slot_size =
1340 8436 : std::max(std::min(cast_int<processor_id_type>(nprocs / napps), max_app_procs), min_app_procs);
1341 8436 : const processor_id_type nslots = std::min(
1342 16872 : nprocs / slot_size,
1343 8436 : cast_int<processor_id_type>(std::min(
1344 8436 : static_cast<dof_id_type>(std::numeric_limits<processor_id_type>::max()), napps)));
1345 : mooseAssert(nprocs >= (nslots * slot_size),
1346 : "Ensure that leftover procs is represented by an unsigned type");
1347 8436 : const processor_id_type leftover_procs = nprocs - nslots * slot_size;
1348 8436 : const dof_id_type apps_per_slot = napps / nslots;
1349 8436 : const dof_id_type leftover_apps = napps % nslots;
1350 :
1351 8436 : std::vector<int> slot_for_rank(nprocs);
1352 8436 : processor_id_type slot = 0;
1353 8436 : processor_id_type procs_in_slot = 0;
1354 20467 : for (processor_id_type rankiter = 0; rankiter <= rank; rankiter++)
1355 : {
1356 12031 : if (slot < nslots)
1357 11883 : slot_for_rank[rankiter] = cast_int<int>(slot);
1358 : else
1359 148 : slot_for_rank[rankiter] = -1;
1360 12031 : procs_in_slot++;
1361 : // this slot keeps growing until we reach slot size plus possibly an extra
1362 : // proc if there were any leftover from the slotization of nprocs - this
1363 : // must also make sure we don't go over max app procs.
1364 12031 : if (procs_in_slot == slot_size + 1 * (slot < leftover_procs && slot_size < max_app_procs))
1365 : {
1366 8896 : procs_in_slot = 0;
1367 8896 : slot++;
1368 : }
1369 : }
1370 :
1371 8436 : if (slot_for_rank[rank] < 0)
1372 : // ranks assigned a negative slot don't have any apps running on them.
1373 72 : return {0, 0, 0, 0, false, 0};
1374 8364 : const processor_id_type slot_num = cast_int<processor_id_type>(slot_for_rank[rank]);
1375 :
1376 8364 : const bool is_first_local_rank = rank == 0 || (slot_for_rank[rank - 1] != slot_for_rank[rank]);
1377 8364 : const dof_id_type n_local_apps = apps_per_slot + 1 * (slot_num < leftover_apps);
1378 :
1379 8364 : processor_id_type my_first_rank = 0;
1380 10831 : for (processor_id_type rankiter = rank; rankiter > 0; rankiter--)
1381 2761 : if (slot_for_rank[rank] != slot_for_rank[rankiter])
1382 : {
1383 294 : my_first_rank = cast_int<processor_id_type>(slot_for_rank[rankiter + 1]);
1384 294 : break;
1385 : }
1386 :
1387 8364 : dof_id_type app_index = 0;
1388 9826 : for (processor_id_type slot = 0; slot < slot_num; slot++)
1389 : {
1390 1462 : const dof_id_type num_slot_apps = apps_per_slot + 1 * (slot < leftover_apps);
1391 1462 : app_index += num_slot_apps;
1392 : }
1393 :
1394 8364 : if (batch_mode)
1395 405 : return {n_local_apps, app_index, 1, slot_num, is_first_local_rank, my_first_rank};
1396 7959 : return {n_local_apps, app_index, n_local_apps, app_index, is_first_local_rank, my_first_rank};
1397 8436 : }
1398 :
1399 : void
1400 7549 : MultiApp::buildComm()
1401 : {
1402 : int ierr;
1403 :
1404 7549 : ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
1405 7549 : mooseCheckMPIErr(ierr);
1406 7549 : ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
1407 7549 : mooseCheckMPIErr(ierr);
1408 :
1409 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
1410 : struct utsname sysInfo;
1411 7549 : uname(&sysInfo);
1412 7549 : _node_name = sysInfo.nodename;
1413 : #else
1414 : _node_name = "Unknown";
1415 : #endif
1416 :
1417 : int rank;
1418 7549 : ierr = MPI_Comm_rank(_communicator.get(), &rank);
1419 7549 : mooseCheckMPIErr(ierr);
1420 :
1421 7549 : _my_num_apps = _rank_config.num_local_apps;
1422 7549 : _first_local_app = _rank_config.first_local_app_index;
1423 :
1424 7549 : _has_an_app = _rank_config.num_local_apps > 0;
1425 7549 : if (_rank_config.first_local_app_index >= _total_num_apps)
1426 0 : mooseError("Internal error, a processor has an undefined app.");
1427 :
1428 7549 : if (_has_an_app)
1429 : {
1430 7495 : _communicator.split(_rank_config.first_local_app_index, rank, _my_communicator);
1431 7495 : ierr = MPI_Comm_rank(_my_comm, &_my_rank);
1432 7495 : mooseCheckMPIErr(ierr);
1433 : }
1434 : else
1435 : {
1436 54 : _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
1437 54 : _my_rank = 0;
1438 : }
1439 7549 : }
1440 :
1441 : unsigned int
1442 912973 : MultiApp::globalAppToLocal(unsigned int global_app)
1443 : {
1444 912973 : if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
1445 912973 : return global_app - _first_local_app;
1446 :
1447 0 : std::stringstream ss;
1448 0 : ss << "Requesting app " << global_app << ", but processor " << processor_id() << " ";
1449 0 : if (_my_num_apps == 0)
1450 0 : ss << "does not own any apps";
1451 0 : else if (_my_num_apps == 1)
1452 0 : ss << "owns app " << _first_local_app;
1453 : else
1454 0 : ss << "owns apps " << _first_local_app << "-" << _first_local_app + (_my_num_apps - 1);
1455 0 : ss << ".";
1456 0 : mooseError("Invalid global_app!\n", ss.str());
1457 : return 0;
1458 0 : }
1459 :
1460 : void
1461 12360 : MultiApp::preRunInputFile()
1462 : {
1463 12360 : }
1464 :
1465 : void
1466 13341 : MultiApp::addAssociatedTransfer(MultiAppTransfer & transfer)
1467 : {
1468 13341 : _associated_transfers.push_back(&transfer);
1469 13341 : }
1470 :
1471 : void
1472 0 : MultiApp::setAppOutputFileBase()
1473 : {
1474 0 : for (unsigned int i = 0; i < _my_num_apps; ++i)
1475 0 : setAppOutputFileBase(i);
1476 0 : }
1477 :
1478 : std::vector<std::string>
1479 23160 : MultiApp::cliArgs() const
1480 : {
1481 : // So that we can error out with paramError("cli_args", ...);
1482 23160 : _cli_args_param = "cli_args";
1483 23160 : return std::vector<std::string>(_cli_args.begin(), _cli_args.end());
1484 : }
1485 :
1486 : void
1487 12338 : MultiApp::setAppOutputFileBase(unsigned int index)
1488 : {
1489 : const std::string multiapp_name =
1490 12338 : getMultiAppName(name(), _first_local_app + index, _total_num_apps);
1491 12338 : _apps[index]->setOutputFileBase(_app.getOutputFileBase() + "_" + multiapp_name);
1492 12338 : }
1493 :
1494 : std::string
1495 24706 : MultiApp::getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total)
1496 : {
1497 24706 : std::ostringstream multiapp_name;
1498 49412 : multiapp_name << base_name << std::setw(std::ceil(std::log10(total))) << std::setprecision(0)
1499 24706 : << std::setfill('0') << std::right << index;
1500 49412 : return multiapp_name.str();
1501 24706 : }
1502 :
1503 : const Point &
1504 234735 : MultiApp::position(unsigned int app) const
1505 : {
1506 : // If we're not using positions, it won't have changed
1507 234735 : if (_positions_objs.empty())
1508 206427 : return _positions[app];
1509 : else
1510 : // Find which Positions object is specifying it, and query a potentially updated value
1511 28308 : return _positions_objs[app]->getPosition(app - _positions_index_offsets[app], false);
1512 : }
1513 :
1514 : void
1515 2498 : dataStore(std::ostream & stream, SubAppBackups & backups, void * context)
1516 : {
1517 2498 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1518 : mooseAssert(multi_app, "Not set");
1519 :
1520 2498 : multi_app->backup();
1521 :
1522 2498 : dataStore(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1523 2498 : }
1524 :
1525 : void
1526 649 : dataLoad(std::istream & stream, SubAppBackups & backups, void * context)
1527 : {
1528 649 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1529 : mooseAssert(multi_app, "Not set");
1530 :
1531 649 : dataLoad(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1532 :
1533 649 : multi_app->restore();
1534 649 : }
|
