Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // MOOSE includes
11 : #include "MultiApp.h"
12 :
13 : #include "AppFactory.h"
14 : #include "AuxiliarySystem.h"
15 : #include "DisplacedProblem.h"
16 : #include "Console.h"
17 : #include "Executioner.h"
18 : #include "FEProblem.h"
19 : #include "MooseMesh.h"
20 : #include "MooseUtils.h"
21 : #include "OutputWarehouse.h"
22 : #include "SetupInterface.h"
23 : #include "UserObject.h"
24 : #include "CommandLine.h"
25 : #include "Conversion.h"
26 : #include "NonlinearSystemBase.h"
27 : #include "DelimitedFileReader.h"
28 : #include "MooseAppCoordTransform.h"
29 : #include "MultiAppTransfer.h"
30 : #include "Positions.h"
31 : #include "Transient.h"
32 : #include "Backup.h"
33 : #include "Parser.h"
34 :
35 : #include "libmesh/mesh_tools.h"
36 : #include "libmesh/numeric_vector.h"
37 :
38 : // C++ includes
39 : #include <fstream>
40 : #include <iomanip>
41 : #include <iterator>
42 : #include <algorithm>
43 :
44 : // Call to "uname"
45 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
46 : #include <sys/utsname.h>
47 : #endif
48 :
49 : InputParameters
50 31360 : MultiApp::validParams()
51 : {
52 31360 : InputParameters params = MooseObject::validParams();
53 31360 : params += SetupInterface::validParams();
54 :
55 62720 : params.addParam<bool>("use_displaced_mesh",
56 62720 : false,
57 : "Whether or not this object should use the "
58 : "displaced mesh for computation. Note that "
59 : "in the case this is true but no "
60 : "displacements are provided in the Mesh block "
61 : "the undisplaced mesh will still be used.");
62 :
63 31360 : std::ostringstream app_types_strings;
64 62746 : for (const auto & name_bi_pair : AppFactory::instance().registeredObjects())
65 31386 : app_types_strings << name_bi_pair.first << " ";
66 62720 : MooseEnum app_types_options(app_types_strings.str(), "", true);
67 :
68 : // Dynamic loading
69 125440 : params.addParam<MooseEnum>("app_type",
70 : app_types_options,
71 : "The type of application to build (applications not "
72 : "registered can be loaded with dynamic libraries. Parent "
73 : "application type will be used if not provided.");
74 125440 : params.addParam<std::string>("library_path",
75 : "",
76 : "Path to search for dynamic libraries (please "
77 : "avoid committing absolute paths in addition to "
78 : "MOOSE_LIBRARY_PATH)");
79 125440 : params.addParam<std::string>(
80 : "library_name",
81 : "",
82 : "The file name of the library (*.la file) that will be dynamically loaded.");
83 94080 : params.addParam<bool>("library_load_dependencies",
84 62720 : false,
85 : "Tells MOOSE to manually load library dependencies. This should not be "
86 : "necessary and is here for debugging/troubleshooting.");
87 :
88 : // Subapp positions
89 125440 : params.addParam<std::vector<Point>>(
90 : "positions",
91 : "The positions of the App locations. Each set of 3 values will represent a "
92 : "Point. This and 'positions_file' cannot be both supplied. If this and "
93 : "'positions_file'/'_objects' are not supplied, a single position (0,0,0) will be used");
94 125440 : params.addParam<std::vector<FileName>>("positions_file",
95 : "Filename(s) that should be looked in for positions. Each"
96 : " set of 3 values in that file will represent a Point. "
97 : "This and 'positions(_objects)' cannot be both supplied");
98 125440 : params.addParam<std::vector<PositionsName>>("positions_objects",
99 : "The name of a Positions object that will contain "
100 : "the locations of the sub-apps created. This and "
101 : "'positions(_file)' cannot be both supplied");
102 94080 : params.addParam<bool>(
103 : "output_in_position",
104 62720 : false,
105 : "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
106 94080 : params.addParam<bool>(
107 : "run_in_position",
108 62720 : false,
109 : "If true this will cause the mesh from the MultiApp to be 'moved' by its position vector");
110 :
111 125440 : params.addRequiredParam<std::vector<FileName>>(
112 : "input_files",
113 : "The input file for each App. If this parameter only contains one input file "
114 : "it will be used for all of the Apps. When using 'positions_from_file' it is "
115 : "also admissable to provide one input_file per file.");
116 94080 : params.addParam<Real>("bounding_box_inflation",
117 62720 : 0.01,
118 : "Relative amount to 'inflate' the bounding box of this MultiApp.");
119 94080 : params.addParam<Point>("bounding_box_padding",
120 62720 : RealVectorValue(),
121 : "Additional padding added to the dimensions of the bounding box. The "
122 : "values are added to the x, y and z dimension respectively.");
123 :
124 62720 : params.addPrivateParam<MPI_Comm>("_mpi_comm");
125 :
126 : // Set the default execution time
127 62720 : params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
128 : // Add the POST_ADAPTIVITY execution flag.
129 : #ifdef LIBMESH_ENABLE_AMR
130 31360 : ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on");
131 31360 : exec_enum.addAvailableFlags(EXEC_POST_ADAPTIVITY);
132 94080 : params.setDocString("execute_on", exec_enum.getDocString());
133 : #endif
134 :
135 94080 : params.addParam<processor_id_type>("max_procs_per_app",
136 62720 : std::numeric_limits<processor_id_type>::max(),
137 : "Maximum number of processors to give to each App in this "
138 : "MultiApp. Useful for restricting small solves to just a few "
139 : "procs so they don't get spread out");
140 94080 : params.addParam<processor_id_type>("min_procs_per_app",
141 62720 : 1,
142 : "Minimum number of processors to give to each App in this "
143 : "MultiApp. Useful for larger, distributed mesh solves.");
144 94080 : params.addParam<bool>(
145 : "wait_for_first_app_init",
146 62720 : false,
147 : "Create the first sub-application on rank 0, then MPI_Barrier before "
148 : "creating the next N-1 apps (on all ranks). "
149 : "This is only needed if your sub-application needs to perform some setup "
150 : "actions in quiet, without other sub-applications working at the same time.");
151 :
152 94080 : params.addParam<Real>("global_time_offset",
153 62720 : 0,
154 : "The time offset relative to the parent application for the purpose of "
155 : "starting a subapp at a different time from the parent application. The "
156 : "global time will be ahead by the offset specified here.");
157 :
158 : // Resetting subapps
159 125440 : params.addParam<std::vector<Real>>(
160 : "reset_time",
161 : {},
162 : "The time(s) at which to reset Apps given by the 'reset_apps' parameter. "
163 : "Resetting an App means that it is destroyed and recreated, possibly "
164 : "modeling the insertion of 'new' material for that app.");
165 125440 : params.addParam<std::vector<unsigned int>>(
166 : "reset_apps",
167 : {},
168 : "The Apps that will be reset when 'reset_time' is hit. These are the App "
169 : "'numbers' starting with 0 corresponding to the order of the App positions. "
170 : "Resetting an App means that it is destroyed and recreated, possibly modeling "
171 : "the insertion of 'new' material for that app.");
172 :
173 : // Moving subapps
174 94080 : params.addParam<Real>(
175 : "move_time",
176 62720 : std::numeric_limits<Real>::max(),
177 : "The time at which Apps designated by move_apps are moved to move_positions.");
178 :
179 125440 : params.addParam<std::vector<unsigned int>>(
180 : "move_apps",
181 : {},
182 : "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
183 : "of the App positions, to be moved at move_time to move_positions");
184 125440 : params.addParam<std::vector<Point>>(
185 : "move_positions", {}, "The positions corresponding to each move_app.");
186 :
187 125440 : params.addParam<std::vector<CLIArgString>>(
188 : "cli_args",
189 : {},
190 : "Additional command line arguments to pass to the sub apps. If one set is provided the "
191 : "arguments are applied to all, otherwise there must be a set for each sub app.");
192 :
193 125440 : params.addParam<std::vector<FileName>>(
194 : "cli_args_files",
195 : "File names that should be looked in for additional command line arguments "
196 : "to pass to the sub apps. Each line of a file is set to each sub app. If only "
197 : "one line is provided, it will be applied to all sub apps.");
198 :
199 : // Fixed point iterations
200 156800 : params.addRangeCheckedParam<Real>("relaxation_factor",
201 62720 : 1.0,
202 : "relaxation_factor>0 & relaxation_factor<2",
203 : "Fraction of newly computed value to keep."
204 : "Set between 0 and 2.");
205 188160 : params.addDeprecatedParam<std::vector<std::string>>(
206 : "relaxed_variables",
207 : {},
208 : "Use transformed_variables.",
209 : "List of subapp variables to relax during Multiapp coupling iterations");
210 125440 : params.addParam<std::vector<std::string>>(
211 : "transformed_variables",
212 : {},
213 : "List of subapp variables to use coupling algorithm on during Multiapp coupling iterations");
214 125440 : params.addParam<std::vector<PostprocessorName>>(
215 : "transformed_postprocessors",
216 : {},
217 : "List of subapp postprocessors to use coupling "
218 : "algorithm on during Multiapp coupling iterations");
219 94080 : params.addParam<bool>("keep_solution_during_restore",
220 62720 : false,
221 : "This is useful when doing MultiApp coupling iterations. It takes the "
222 : "final solution from the previous coupling iteration"
223 : "and re-uses it as the initial guess for the next coupling iteration");
224 94080 : params.addParam<bool>("keep_aux_solution_during_restore",
225 62720 : false,
226 : "This is useful when doing MultiApp coupling iterations. It takes the "
227 : "final auxiliary solution from the previous coupling iteration"
228 : "and re-uses it as the initial guess for the next coupling iteration");
229 94080 : params.addParam<bool>(
230 : "no_restore",
231 62720 : false,
232 : "True to turn off restore for this multiapp. This is useful when doing steady-state "
233 : "Picard iterations where we want to use the solution of previous Picard iteration as the "
234 : "initial guess of the current Picard iteration.");
235 94080 : params.addParam<unsigned int>(
236 : "max_multiapp_level",
237 62720 : 10,
238 : "Integer set by user that will stop the simulation if the multiapp level "
239 : "exceeds it. Useful for preventing infinite loops with multiapp simulations");
240 :
241 156800 : params.addDeprecatedParam<bool>("clone_master_mesh",
242 62720 : false,
243 : "True to clone parent app mesh and use it for this MultiApp.",
244 : "clone_master_mesh is deprecated, use clone_parent_mesh instead");
245 62720 : params.addParam<bool>(
246 62720 : "clone_parent_mesh", false, "True to clone parent app mesh and use it for this MultiApp.");
247 :
248 62720 : params.addPrivateParam<bool>("use_positions", true);
249 94080 : params.declareControllable("enable");
250 156800 : params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
251 62720 : params.registerBase("MultiApp");
252 :
253 125440 : params.addParamNamesToGroup("use_displaced_mesh wait_for_first_app_init max_multiapp_level",
254 : "Advanced");
255 125440 : params.addParamNamesToGroup("positions positions_file positions_objects run_in_position "
256 : "output_in_position",
257 : "Positions / transformations of the MultiApp frame of reference");
258 125440 : params.addParamNamesToGroup("min_procs_per_app max_procs_per_app", "Parallelism");
259 125440 : params.addParamNamesToGroup("reset_time reset_apps", "Reset MultiApp");
260 125440 : params.addParamNamesToGroup("move_time move_apps move_positions", "Timed move of MultiApps");
261 125440 : params.addParamNamesToGroup("relaxation_factor transformed_variables transformed_postprocessors "
262 : "keep_solution_during_restore keep_aux_solution_during_restore "
263 : "no_restore",
264 : "Fixed point iteration");
265 125440 : params.addParamNamesToGroup("library_name library_path library_load_dependencies",
266 : "Dynamic loading");
267 94080 : params.addParamNamesToGroup("cli_args cli_args_files", "Passing command line argument");
268 62720 : return params;
269 62720 : }
270 :
271 8025 : MultiApp::MultiApp(const InputParameters & parameters)
272 : : MooseObject(parameters),
273 : SetupInterface(this),
274 : Restartable(this, "MultiApps"),
275 16050 : PerfGraphInterface(this, std::string("MultiApp::") + _name),
276 24075 : _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
277 28436 : _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
278 3664 : : _fe_problem.getMooseApp().type()),
279 16050 : _use_positions(getParam<bool>("use_positions")),
280 16050 : _input_files(getParam<std::vector<FileName>>("input_files")),
281 16050 : _wait_for_first_app_init(getParam<bool>("wait_for_first_app_init")),
282 8025 : _total_num_apps(0),
283 8025 : _my_num_apps(0),
284 8025 : _first_local_app(0),
285 8025 : _orig_comm(_communicator.get()),
286 8025 : _my_communicator(),
287 8025 : _my_comm(_my_communicator.get()),
288 8025 : _my_rank(0),
289 16050 : _inflation(getParam<Real>("bounding_box_inflation")),
290 16050 : _bounding_box_padding(getParam<Point>("bounding_box_padding")),
291 16050 : _max_procs_per_app(getParam<processor_id_type>("max_procs_per_app")),
292 16050 : _min_procs_per_app(getParam<processor_id_type>("min_procs_per_app")),
293 16050 : _output_in_position(getParam<bool>("output_in_position")),
294 16050 : _global_time_offset(getParam<Real>("global_time_offset")),
295 16050 : _reset_times(getParam<std::vector<Real>>("reset_time")),
296 16050 : _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
297 16050 : _reset_happened(false),
298 16050 : _move_time(getParam<Real>("move_time")),
299 16050 : _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
300 16050 : _move_positions(getParam<std::vector<Point>>("move_positions")),
301 8025 : _move_happened(false),
302 8025 : _has_an_app(true),
303 16050 : _cli_args(getParam<std::vector<CLIArgString>>("cli_args")),
304 16050 : _keep_solution_during_restore(getParam<bool>("keep_solution_during_restore")),
305 16050 : _keep_aux_solution_during_restore(getParam<bool>("keep_aux_solution_during_restore")),
306 16050 : _no_restore(getParam<bool>("no_restore")),
307 16050 : _run_in_position(getParam<bool>("run_in_position")),
308 16050 : _sub_app_backups(declareRestartableDataWithContext<SubAppBackups>("sub_app_backups", this)),
309 32100 : _solve_step_timer(registerTimedSection("solveStep", 3, "Executing MultiApps", false)),
310 32100 : _init_timer(registerTimedSection("init", 3, "Initializing MultiApp")),
311 32100 : _backup_timer(registerTimedSection("backup", 3, "Backing Up MultiApp")),
312 32100 : _restore_timer(registerTimedSection("restore", 3, "Restoring MultiApp")),
313 120375 : _reset_timer(registerTimedSection("resetApp", 3, "Resetting MultiApp"))
314 : {
315 37779 : if (parameters.isParamSetByUser("cli_args") && parameters.isParamValid("cli_args") &&
316 11811 : parameters.isParamValid("cli_args_files"))
317 6 : paramError("cli_args",
318 : "'cli_args' and 'cli_args_files' cannot be specified simultaneously in MultiApp ");
319 :
320 16044 : if (!_use_positions && (isParamValid("positions") || isParamValid("positions_file") ||
321 8022 : isParamValid("positions_objects")))
322 0 : paramError("use_positions",
323 : "This MultiApps has been set to not use positions, "
324 : "but a 'positions' parameter has been set.");
325 :
326 24004 : if ((_reset_apps.size() > 0 && _reset_times.size() == 0) ||
327 15982 : (_reset_apps.size() == 0 && _reset_times.size() > 0))
328 0 : mooseError("reset_time and reset_apps may only be specified together");
329 :
330 : // Check that the reset times are sorted by the user
331 8022 : auto sorted_times = _reset_times;
332 8022 : std::sort(sorted_times.begin(), sorted_times.end());
333 8022 : if (_reset_times.size() && _reset_times != sorted_times)
334 6 : paramError("reset_time", "List of reset times must be sorted in increasing order");
335 8019 : }
336 :
337 : void
338 7998 : MultiApp::init(unsigned int num_apps, bool batch_mode)
339 : {
340 7998 : auto config = rankConfig(
341 : processor_id(), n_processors(), num_apps, _min_procs_per_app, _max_procs_per_app, batch_mode);
342 7998 : init(num_apps, config);
343 7995 : }
344 :
345 : void
346 7998 : MultiApp::init(unsigned int num_apps, const LocalRankConfig & config)
347 : {
348 7998 : TIME_SECTION(_init_timer);
349 :
350 7998 : _total_num_apps = num_apps;
351 7998 : _rank_config = config;
352 7998 : buildComm();
353 7998 : _sub_app_backups.resize(_my_num_apps);
354 :
355 7998 : _has_bounding_box.resize(_my_num_apps, false);
356 7998 : _reset_happened.resize(_reset_times.size(), false);
357 7998 : _bounding_box.resize(_my_num_apps);
358 :
359 7998 : if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
360 6 : paramError("cli_args",
361 : "The number of items supplied must be 1 or equal to the number of sub apps.");
362 :
363 : // if cliArgs() != _cli_args, then cliArgs() was overridden and we need to check it
364 7995 : auto cla = cliArgs();
365 15990 : if (cla != std::vector<std::string>(_cli_args.begin(), _cli_args.end()))
366 : {
367 12 : if ((cla.size() > 1) && (_total_num_apps != cla.size()))
368 0 : mooseError("The number of items supplied as command line argument to subapps must be 1 or "
369 : "equal to the number of sub apps. Note: you use a multiapp that provides its own "
370 : "command line parameters so the error is not in cli_args");
371 : }
372 7995 : }
373 :
374 : void
375 8016 : MultiApp::setupPositions()
376 : {
377 8016 : if (_use_positions)
378 : {
379 8016 : fillPositions();
380 7998 : init(_positions.size());
381 7995 : createApps();
382 : }
383 7929 : }
384 :
385 : void
386 7995 : MultiApp::createApps()
387 : {
388 7995 : if (!_has_an_app)
389 52 : return;
390 :
391 39715 : TIME_SECTION("createApps", 2, "Instantiating Sub-Apps", false);
392 :
393 : // Read commandLine arguments that will be used when creating apps
394 7943 : readCommandLineArguments();
395 :
396 7928 : Moose::ScopedCommSwapper swapper(_my_comm);
397 :
398 7928 : _apps.resize(_my_num_apps);
399 :
400 : // If the user provided an unregistered app type, see if we can load it dynamically
401 7928 : if (!AppFactory::instance().isRegistered(_app_type))
402 5 : _app.dynamicAppRegistration(_app_type,
403 : getParam<std::string>("library_path"),
404 7 : getParam<std::string>("library_name"),
405 14 : getParam<bool>("library_load_dependencies"));
406 :
407 7925 : bool rank_did_quiet_init = false;
408 7925 : unsigned int local_app = libMesh::invalid_uint;
409 7925 : if (_wait_for_first_app_init)
410 : {
411 14 : if (hasLocalApp(0))
412 : {
413 7 : rank_did_quiet_init = true;
414 7 : local_app = globalAppToLocal(0);
415 7 : createLocalApp(local_app);
416 : }
417 :
418 14 : MPI_Barrier(_orig_comm);
419 : }
420 :
421 19905 : for (unsigned int i = 0; i < _my_num_apps; i++)
422 : {
423 12028 : if (rank_did_quiet_init && i == local_app)
424 7 : continue;
425 12021 : createLocalApp(i);
426 : }
427 7877 : }
428 :
429 : void
430 12028 : MultiApp::createLocalApp(const unsigned int i)
431 : {
432 12028 : createApp(i, _global_time_offset);
433 11980 : }
434 :
435 : void
436 7857 : MultiApp::initialSetup()
437 : {
438 7857 : if (!_use_positions)
439 : // if not using positions, we create the sub-apps in initialSetup instead of right after
440 : // construction of MultiApp
441 0 : createApps();
442 7857 : }
443 :
444 : void
445 7943 : MultiApp::readCommandLineArguments()
446 : {
447 23829 : if (isParamValid("cli_args_files"))
448 : {
449 51 : _cli_args_from_file.clear();
450 :
451 102 : std::vector<FileName> cli_args_files = getParam<std::vector<FileName>>("cli_args_files");
452 102 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
453 :
454 : // If we use parameter "cli_args_files", at least one file should be provided
455 51 : if (!cli_args_files.size())
456 6 : paramError("cli_args_files", "You need to provide at least one commandLine argument file ");
457 :
458 : // If we multiple input files, then we need to check if the number of input files
459 : // match with the number of argument files
460 48 : if (cli_args_files.size() != 1 && cli_args_files.size() != input_files.size())
461 9 : paramError("cli_args_files",
462 : "The number of commandLine argument files ",
463 : cli_args_files.size(),
464 : " for MultiApp ",
465 3 : name(),
466 : " must either be only one or match the number of input files ",
467 : input_files.size());
468 :
469 : // Go through all argument files
470 45 : std::vector<std::string> cli_args;
471 96 : for (unsigned int p_file_it = 0; p_file_it < cli_args_files.size(); p_file_it++)
472 : {
473 57 : std::string cli_args_file = cli_args_files[p_file_it];
474 : // Clear up
475 57 : cli_args.clear();
476 : // Read the file on the root processor then broadcast it
477 57 : if (processor_id() == 0)
478 : {
479 45 : MooseUtils::checkFileReadable(cli_args_file);
480 :
481 45 : std::ifstream is(cli_args_file.c_str());
482 : // Read by line rather than space separated like the cli_args parameter
483 45 : std::string line;
484 183 : while (std::getline(is, line))
485 138 : cli_args.push_back(line);
486 :
487 : // We do not allow empty files
488 45 : if (!cli_args.size())
489 6 : paramError("cli_args_files",
490 : "There is no commandLine argument in the commandLine argument file ",
491 : cli_args_file);
492 :
493 : // If we have position files, we need to
494 : // make sure the number of commandLine argument strings
495 : // match with the number of positions
496 42 : if (_npositions_inputfile.size())
497 : {
498 39 : auto num_positions = _npositions_inputfile[p_file_it];
499 : // Check if the number of commandLine argument strings equal to
500 : // the number of positions
501 39 : if (cli_args.size() == 1)
502 0 : for (MooseIndex(num_positions) num = 0; num < num_positions; num++)
503 0 : _cli_args_from_file.push_back(cli_args.front());
504 39 : else if (cli_args.size() == num_positions)
505 144 : for (auto && cli_arg : cli_args)
506 108 : _cli_args_from_file.push_back(cli_arg);
507 3 : else if (cli_args.size() != num_positions)
508 6 : paramError("cli_args_files",
509 : "The number of commandLine argument strings ",
510 : cli_args.size(),
511 : " in the file ",
512 : cli_args_file,
513 : " must either be only one or match the number of positions ",
514 : num_positions);
515 : }
516 : else
517 : {
518 : // If we do not have position files, we will check if the number of
519 : // commandLine argument strings match with the total number of subapps
520 18 : for (auto && cli_arg : cli_args)
521 15 : _cli_args_from_file.push_back(cli_arg);
522 : }
523 39 : }
524 51 : }
525 :
526 : // Broad cast all arguments to everyone
527 39 : _communicator.broadcast(_cli_args_from_file);
528 39 : }
529 :
530 7970 : if (_cli_args_from_file.size() && _cli_args_from_file.size() != 1 &&
531 39 : _cli_args_from_file.size() != _total_num_apps)
532 6 : mooseError(" The number of commandLine argument strings ",
533 3 : _cli_args_from_file.size(),
534 : " must either be only one or match the total "
535 : "number of sub apps ",
536 3 : _total_num_apps);
537 :
538 7928 : if (_cli_args_from_file.size() && cliArgs().size())
539 0 : mooseError("Cannot set commandLine arguments from both input_file and external files");
540 7928 : }
541 :
542 : void
543 7909 : MultiApp::fillPositions()
544 : {
545 7909 : if (_move_apps.size() != _move_positions.size())
546 0 : mooseError("The number of apps to move and the positions to move them to must be the same for "
547 : "MultiApp ",
548 0 : _name);
549 :
550 39545 : if (isParamValid("positions") + isParamValid("positions_file") +
551 23727 : isParamValid("positions_objects") >
552 : 1)
553 3 : mooseError("Only one 'positions' parameter may be specified");
554 :
555 23718 : if (isParamValid("positions"))
556 : {
557 9874 : _positions = getParam<std::vector<Point>>("positions");
558 :
559 4937 : if (_positions.size() < _input_files.size())
560 3 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
561 3 : name());
562 : }
563 8907 : else if (isParamValid("positions_file"))
564 : {
565 168 : std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
566 168 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
567 :
568 84 : if (input_files.size() != 1 && positions_files.size() != input_files.size())
569 3 : mooseError("Number of input_files for MultiApp ",
570 3 : name(),
571 : " must either be only one or match the number of positions_file files");
572 :
573 : // Clear out the _input_files because we're going to rebuild it
574 81 : if (input_files.size() != 1)
575 27 : _input_files.clear();
576 :
577 186 : for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
578 : {
579 108 : std::string positions_file = positions_files[p_file_it];
580 108 : MooseUtils::DelimitedFileReader file(positions_file, &_communicator);
581 108 : file.setFormatFlag(MooseUtils::DelimitedFileReader::FormatFlag::ROWS);
582 108 : file.read();
583 :
584 108 : const std::vector<Point> & data = file.getDataAsPoints();
585 513 : for (const auto & d : data)
586 408 : _positions.push_back(d);
587 :
588 : // Save the number of positions for this input file
589 105 : _npositions_inputfile.push_back(data.size());
590 :
591 513 : for (unsigned int i = 0; i < data.size(); ++i)
592 408 : if (input_files.size() != 1)
593 108 : _input_files.push_back(input_files[p_file_it]);
594 105 : }
595 78 : }
596 8655 : else if (isParamValid("positions_objects"))
597 : {
598 928 : const auto & positions_param_objs = getParam<std::vector<PositionsName>>("positions_objects");
599 928 : const auto & input_files = getParam<std::vector<FileName>>("input_files");
600 :
601 464 : if (input_files.size() != 1 && positions_param_objs.size() != input_files.size())
602 0 : mooseError("Number of input_files for MultiApp ",
603 0 : name(),
604 : " must either be only one or match the number of positions_objects specified");
605 :
606 : // Clear out the _input_files because we're going to rebuild it
607 464 : if (input_files.size() != 1)
608 0 : _input_files.clear();
609 :
610 : // Keeps track of where each positions object start in terms of subapp numbers
611 464 : unsigned int offset = 0;
612 :
613 964 : for (const auto p_obj_it : index_range(positions_param_objs))
614 : {
615 503 : const std::string & positions_name = positions_param_objs[p_obj_it];
616 503 : auto positions_obj = &_fe_problem.getPositionsObject(positions_name);
617 :
618 503 : const auto & data = positions_obj->getPositions(true);
619 :
620 : // Append all positions from this object
621 2147 : for (const auto & d : data)
622 1647 : _positions.push_back(d);
623 :
624 : // Save the number of positions for this input file
625 500 : _npositions_inputfile.push_back(data.size());
626 :
627 500 : if (!positions_obj)
628 0 : paramError("positions_objects",
629 0 : "'" + positions_name + "' is not of the expected type. Should be a Positions");
630 :
631 : // Keep track of which positions is tied to what subapp
632 2147 : for (unsigned int i = 0; i < data.size(); ++i)
633 : {
634 1647 : if (input_files.size() != 1)
635 0 : _input_files.push_back(input_files[p_obj_it]);
636 1647 : _positions_objs.push_back(positions_obj);
637 1647 : _positions_index_offsets.push_back(offset);
638 : }
639 500 : offset += data.size();
640 : }
641 : }
642 : else
643 : {
644 2421 : _positions = {Point()};
645 :
646 2421 : if (_positions.size() < _input_files.size())
647 3 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
648 3 : name());
649 : }
650 :
651 : mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
652 : "Number of positions and input files are not the same!");
653 7891 : }
654 :
655 : void
656 64249 : MultiApp::preTransfer(Real /*dt*/, Real target_time)
657 : {
658 : // Get a transient executioner to get a user-set tolerance
659 64249 : Real timestep_tol = 1e-13;
660 64249 : if (dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner()))
661 60968 : timestep_tol =
662 60968 : dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner())->timestepTol();
663 :
664 : // Determination on whether we need to backup the app due to changes below
665 64249 : bool backup_apps = false;
666 :
667 : // First, see if any Apps need to be reset
668 64957 : for (unsigned int i = 0; i < _reset_times.size(); i++)
669 : {
670 777 : if (!_reset_happened[i] && (target_time + timestep_tol >= _reset_times[i]))
671 : {
672 69 : _reset_happened[i] = true;
673 69 : if (_reset_apps.size() > 0)
674 138 : for (auto & app : _reset_apps)
675 69 : resetApp(app);
676 :
677 : // If we reset an application, then we delete the old objects, including the coordinate
678 : // transformation classes. Consequently we need to reset the coordinate transformation classes
679 : // in the associated transfer classes
680 99 : for (auto * const transfer : _associated_transfers)
681 30 : transfer->getAppInfo();
682 :
683 : // Similarly we need to transform the mesh again
684 69 : if (_run_in_position)
685 0 : for (const auto i : make_range(_my_num_apps))
686 : {
687 0 : auto app_ptr = _apps[i];
688 0 : if (usingPositions())
689 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
690 0 : app_ptr->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
691 : else
692 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
693 0 : app_ptr->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
694 0 : }
695 :
696 : // If the time step covers multiple reset times, set them all as having 'happened'
697 158 : for (unsigned int j = i; j < _reset_times.size(); j++)
698 89 : if (target_time + timestep_tol >= _reset_times[j])
699 79 : _reset_happened[j] = true;
700 :
701 : // Backup in case the next solve fails
702 69 : backup_apps = true;
703 :
704 69 : break;
705 : }
706 : }
707 :
708 : // Now move any apps that should be moved
709 64249 : if (_use_positions && !_move_happened && target_time + timestep_tol >= _move_time)
710 : {
711 54 : _move_happened = true;
712 105 : for (unsigned int i = 0; i < _move_apps.size(); i++)
713 54 : moveApp(_move_apps[i], _move_positions[i]);
714 :
715 : // Backup in case the next solve fails
716 51 : backup_apps = true;
717 : }
718 :
719 64246 : if (backup_apps)
720 101 : backup();
721 64246 : }
722 :
723 : Executioner *
724 0 : MultiApp::getExecutioner(unsigned int app)
725 : {
726 0 : if (!_has_an_app)
727 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
728 :
729 0 : return _apps[globalAppToLocal(app)]->getExecutioner();
730 : }
731 :
732 : void
733 4572 : MultiApp::finalize()
734 : {
735 11160 : for (const auto & app_ptr : _apps)
736 : {
737 6588 : auto * executioner = app_ptr->getExecutioner();
738 : mooseAssert(executioner, "Executioner is nullptr");
739 :
740 6588 : executioner->feProblem().execute(EXEC_FINAL);
741 6588 : executioner->feProblem().outputStep(EXEC_FINAL);
742 : }
743 4572 : }
744 :
745 : void
746 4935 : MultiApp::postExecute()
747 : {
748 12186 : for (const auto & app_ptr : _apps)
749 : {
750 7251 : auto * executioner = app_ptr->getExecutioner();
751 : mooseAssert(executioner, "Executioner is nullptr");
752 :
753 7251 : executioner->postExecute();
754 : }
755 4935 : }
756 :
757 : void
758 24287 : MultiApp::backup()
759 : {
760 24287 : TIME_SECTION(_backup_timer);
761 :
762 24287 : if (_fe_problem.verboseMultiApps())
763 1098 : _console << "Backed up MultiApp ... ";
764 :
765 57066 : for (unsigned int i = 0; i < _my_num_apps; i++)
766 32779 : _sub_app_backups[i] = _apps[i]->backup();
767 :
768 24287 : if (_fe_problem.verboseMultiApps())
769 1098 : _console << name() << std::endl;
770 24287 : }
771 :
772 : void
773 43774 : MultiApp::restore(bool force)
774 : {
775 43774 : TIME_SECTION(_restore_timer);
776 :
777 43774 : if (force || needsRestoration())
778 : {
779 : // Must be restarting / recovering from main app so hold off on restoring
780 : // Instead - the restore will happen in sub-apps' initialSetup()
781 : // Note that _backups was already populated by dataLoad() in the main app
782 9330 : if (_fe_problem.getCurrentExecuteOnFlag() == EXEC_INITIAL)
783 574 : return;
784 :
785 : // We temporarily copy and store solutions for all subapps
786 8756 : if (_keep_solution_during_restore)
787 : {
788 324 : _end_solutions.resize(_my_num_apps);
789 :
790 645 : for (unsigned int i = 0; i < _my_num_apps; i++)
791 : {
792 324 : _end_solutions[i].resize(_apps[i]->getExecutioner()->feProblem().numSolverSystems());
793 658 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
794 : j++)
795 : {
796 668 : _end_solutions[i][j] = _apps[i]
797 : ->getExecutioner()
798 334 : ->feProblem()
799 334 : .getSolverSystem(/*solver_sys=*/j)
800 334 : .solution()
801 668 : .clone();
802 : }
803 : auto & sub_multiapps =
804 324 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects();
805 :
806 : // multiapps of each subapp should do the same things
807 : // It is implemented recursively
808 334 : for (auto & multi_app : sub_multiapps)
809 13 : multi_app->keepSolutionDuringRestore(_keep_solution_during_restore);
810 : }
811 : }
812 :
813 : // We temporarily copy and store solutions for all subapps
814 8753 : if (_keep_aux_solution_during_restore)
815 : {
816 120 : _end_aux_solutions.resize(_my_num_apps);
817 :
818 240 : for (unsigned int i = 0; i < _my_num_apps; i++)
819 120 : _end_aux_solutions[i] =
820 240 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution().clone();
821 : }
822 :
823 8753 : if (_fe_problem.verboseMultiApps())
824 474 : _console << "Restoring MultiApp ... ";
825 :
826 17692 : for (unsigned int i = 0; i < _my_num_apps; i++)
827 : {
828 8939 : _apps[i]->restore(std::move(_sub_app_backups[i]), false);
829 8939 : _sub_app_backups[i] = _apps[i]->finalizeRestore();
830 : mooseAssert(_sub_app_backups[i], "Should have a backup");
831 : }
832 :
833 8753 : if (_fe_problem.verboseMultiApps())
834 474 : _console << name() << std::endl;
835 :
836 : // Now copy the latest solutions back for each subapp
837 8753 : if (_keep_solution_during_restore)
838 : {
839 642 : for (unsigned int i = 0; i < _my_num_apps; i++)
840 : {
841 652 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
842 : j++)
843 : {
844 331 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).solution() =
845 662 : *_end_solutions[i][j];
846 :
847 : // We need to synchronize solution so that local_solution has the right values
848 331 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).update();
849 : }
850 : }
851 :
852 321 : _end_solutions.clear();
853 : }
854 : // Now copy the latest auxiliary solutions back for each subapp
855 8753 : if (_keep_aux_solution_during_restore)
856 : {
857 240 : for (unsigned int i = 0; i < _my_num_apps; i++)
858 : {
859 120 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution() =
860 240 : *_end_aux_solutions[i];
861 :
862 : // We need to synchronize solution so that local_solution has the right values
863 120 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().update();
864 : }
865 :
866 120 : _end_aux_solutions.clear();
867 : }
868 :
869 : // Make sure the displaced mesh on the multiapp is up-to-date with displacement variables
870 17692 : for (const auto & app_ptr : _apps)
871 8939 : if (app_ptr->feProblem().getDisplacedProblem())
872 48 : app_ptr->feProblem().getDisplacedProblem()->updateMesh();
873 :
874 : // If we are restoring due to a failed solve, make sure reset the solved state in the sub-apps
875 8753 : if (!getMooseApp().getExecutioner()->lastSolveConverged())
876 1430 : for (auto & app_ptr : _apps)
877 802 : app_ptr->getExecutioner()->fixedPointSolve().clearFixedPointStatus();
878 : }
879 : else
880 : {
881 68888 : for (unsigned int i = 0; i < _my_num_apps; i++)
882 : {
883 34444 : for (auto & sub_app :
884 69320 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects())
885 432 : sub_app->restore(false);
886 : }
887 : }
888 43771 : }
889 :
890 : void
891 13 : MultiApp::keepSolutionDuringRestore(bool keep_solution_during_restore)
892 : {
893 39 : if (_pars.isParamSetByUser("keep_solution_during_restore"))
894 6 : paramError("keep_solution_during_restore",
895 : "This parameter should only be provided in parent app");
896 :
897 10 : _keep_solution_during_restore = keep_solution_during_restore;
898 10 : }
899 :
900 : void
901 376520 : MultiApp::transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform)
902 : {
903 376520 : const Real min_x = box.first(0);
904 376520 : const Real max_x = box.second(0);
905 376520 : const Real min_y = box.first(1);
906 376520 : const Real max_y = box.second(1);
907 376520 : const Real min_z = box.first(2);
908 376520 : const Real max_z = box.second(2);
909 :
910 : std::array<Point, 8> box_corners = {{Point(min_x, min_y, min_z),
911 : Point(max_x, min_y, min_z),
912 : Point(min_x, max_y, min_z),
913 : Point(max_x, max_y, min_z),
914 : Point(min_x, min_y, max_z),
915 : Point(max_x, min_y, max_z),
916 : Point(min_x, max_y, max_z),
917 376520 : Point(max_x, max_y, max_z)}};
918 :
919 : // transform each corner
920 3388680 : for (auto & corner : box_corners)
921 3012160 : corner = transform(corner);
922 :
923 : // Create new bounding box
924 376520 : Point new_box_min = box_corners[0];
925 376520 : Point new_box_max = new_box_min;
926 3012160 : for (const auto p : make_range(1, 8))
927 10542560 : for (const auto d : make_range(Moose::dim))
928 : {
929 7906920 : const Point & pt = box_corners[p];
930 7906920 : if (new_box_min(d) > pt(d))
931 2292 : new_box_min(d) = pt(d);
932 :
933 7906920 : if (new_box_max(d) < pt(d))
934 1065095 : new_box_max(d) = pt(d);
935 : }
936 376520 : box.first = new_box_min;
937 376520 : box.second = new_box_max;
938 376520 : }
939 :
940 : BoundingBox
941 318895 : MultiApp::getBoundingBox(unsigned int app,
942 : bool displaced_mesh,
943 : const MultiAppCoordTransform * const coord_transform)
944 : {
945 318895 : if (!_has_an_app)
946 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
947 :
948 318895 : unsigned int local_app = globalAppToLocal(app);
949 318895 : FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
950 8256 : MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
951 641153 : ? fe_problem_base.getDisplacedProblem()->mesh()
952 318895 : : fe_problem_base.mesh();
953 :
954 : {
955 318895 : Moose::ScopedCommSwapper swapper(_my_comm);
956 318895 : if (displaced_mesh)
957 4128 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
958 : else
959 : {
960 314767 : if (!_has_bounding_box[local_app])
961 : {
962 92 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
963 92 : _has_bounding_box[local_app] = true;
964 : }
965 : }
966 318895 : }
967 318895 : BoundingBox bbox = _bounding_box[local_app];
968 :
969 318895 : Point min = bbox.min();
970 318895 : min -= _bounding_box_padding;
971 318895 : Point max = bbox.max();
972 318895 : max += _bounding_box_padding;
973 :
974 318895 : Point inflation_amount = (max - min) * _inflation;
975 :
976 318895 : Point inflated_min = min - inflation_amount;
977 318895 : Point inflated_max = max + inflation_amount;
978 :
979 318895 : Point shifted_min = inflated_min;
980 318895 : Point shifted_max = inflated_max;
981 :
982 637790 : if ((!coord_transform || coord_transform->skipCoordinateCollapsing()) &&
983 637790 : fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
984 : {
985 : // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
986 : // FIXME: Assuming all subdomains are the same coordinate system type!
987 308088 : shifted_min(0) = -inflated_max(0);
988 308088 : shifted_min(1) = inflated_min(1);
989 308088 : shifted_min(2) = -inflated_max(0);
990 :
991 308088 : shifted_max(0) = inflated_max(0);
992 308088 : shifted_max(1) = inflated_max(1);
993 308088 : shifted_max(2) = inflated_max(0);
994 : }
995 :
996 318895 : if (coord_transform)
997 : {
998 318895 : BoundingBox transformed_bbox(shifted_min, shifted_max);
999 318895 : transformBoundingBox(transformed_bbox, *coord_transform);
1000 318895 : return transformed_bbox;
1001 : }
1002 : else
1003 : {
1004 : // This is where the app is located. We need to shift by this amount.
1005 0 : Point p = position(app);
1006 :
1007 : // Shift them to the position they're supposed to be
1008 0 : shifted_min += p;
1009 0 : shifted_max += p;
1010 0 : return BoundingBox(shifted_min, shifted_max);
1011 : }
1012 : }
1013 :
1014 : FEProblemBase &
1015 482185 : MultiApp::appProblemBase(unsigned int app)
1016 : {
1017 482185 : if (!_has_an_app)
1018 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1019 :
1020 482185 : unsigned int local_app = globalAppToLocal(app);
1021 :
1022 482185 : return _apps[local_app]->getExecutioner()->feProblem();
1023 : }
1024 :
1025 : FEProblem &
1026 0 : MultiApp::appProblem(unsigned int app)
1027 : {
1028 0 : mooseDeprecated(
1029 : "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
1030 0 : if (!_has_an_app)
1031 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1032 :
1033 0 : unsigned int local_app = globalAppToLocal(app);
1034 :
1035 0 : return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
1036 : }
1037 :
1038 : const UserObject &
1039 167775 : MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
1040 : {
1041 167775 : if (!_has_an_app)
1042 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1043 :
1044 167775 : return appProblemBase(app).getUserObjectBase(name);
1045 : }
1046 :
1047 : Real
1048 3700 : MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
1049 : {
1050 3700 : if (!_has_an_app)
1051 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1052 :
1053 3700 : return appProblemBase(app).getPostprocessorValueByName(name);
1054 : }
1055 :
1056 : NumericVector<Number> &
1057 849 : MultiApp::appTransferVector(unsigned int app, std::string var_name)
1058 : {
1059 849 : return *(appProblemBase(app).getSystem(var_name).solution);
1060 : }
1061 :
1062 : bool
1063 136 : MultiApp::isFirstLocalRank() const
1064 : {
1065 136 : return _rank_config.is_first_local_rank;
1066 : }
1067 :
1068 : bool
1069 812272 : MultiApp::hasLocalApp(unsigned int global_app) const
1070 : {
1071 812272 : if (_has_an_app && global_app >= _first_local_app &&
1072 724831 : global_app <= _first_local_app + (_my_num_apps - 1))
1073 639528 : return true;
1074 :
1075 172744 : return false;
1076 : }
1077 :
1078 : MooseApp *
1079 11415 : MultiApp::localApp(unsigned int local_app)
1080 : {
1081 : mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
1082 11415 : return _apps[local_app].get();
1083 : }
1084 :
1085 : void
1086 69 : MultiApp::resetApp(unsigned int global_app, Real time)
1087 : {
1088 69 : TIME_SECTION(_reset_timer);
1089 :
1090 69 : Moose::ScopedCommSwapper swapper(_my_comm);
1091 :
1092 69 : if (hasLocalApp(global_app))
1093 : {
1094 69 : unsigned int local_app = globalAppToLocal(global_app);
1095 :
1096 : // Extract the file numbers from the output, so that the numbering is maintained after reset
1097 69 : std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
1098 :
1099 69 : createApp(local_app, time);
1100 :
1101 : // Reset the file numbers of the newly reset apps
1102 69 : _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
1103 69 : }
1104 69 : }
1105 :
1106 : void
1107 54 : MultiApp::moveApp(unsigned int global_app, Point p)
1108 : {
1109 54 : if (_use_positions)
1110 : {
1111 54 : _positions[global_app] = p;
1112 :
1113 54 : if (hasLocalApp(global_app))
1114 : {
1115 54 : unsigned int local_app = globalAppToLocal(global_app);
1116 :
1117 54 : if (_output_in_position)
1118 51 : _apps[local_app]->setOutputPosition(p);
1119 54 : if (_run_in_position)
1120 6 : paramError("run_in_position", "Moving apps and running apps in position is not supported");
1121 : }
1122 : }
1123 51 : }
1124 :
1125 : void
1126 19 : MultiApp::parentOutputPositionChanged()
1127 : {
1128 19 : if (_use_positions && _output_in_position)
1129 38 : for (unsigned int i = 0; i < _apps.size(); i++)
1130 19 : _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1131 19 : }
1132 :
1133 : void
1134 12097 : MultiApp::createApp(unsigned int i, Real start_time)
1135 : {
1136 : // Delete the old app if we're resetting
1137 12097 : if (_apps[i])
1138 69 : _apps[i].reset();
1139 :
1140 : // Define the app name
1141 12097 : const std::string multiapp_name = getMultiAppName(name(), _first_local_app + i, _total_num_apps);
1142 12097 : std::string full_name;
1143 :
1144 : // Only add parent name if the parent is not the main app
1145 12097 : if (_app.multiAppLevel() > 0)
1146 468 : full_name = _app.name() + "_" + multiapp_name;
1147 : else
1148 11629 : full_name = multiapp_name;
1149 :
1150 12097 : InputParameters app_params = AppFactory::instance().getValidParams(_app_type);
1151 24194 : app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
1152 24194 : app_params.set<std::unique_ptr<Backup> *>("_initial_backup") = &_sub_app_backups[i];
1153 :
1154 : // Build the CommandLine with the relevant options for this subapp and add the
1155 : // cli args from the input file
1156 12097 : std::vector<std::string> input_cli_args;
1157 12097 : if (cliArgs().size() > 0 || _cli_args_from_file.size() > 0)
1158 3156 : input_cli_args = getCommandLineArgs(i);
1159 : // This will mark all hit CLI command line parameters that are passed to subapps
1160 : // as used within the parent app (_app)
1161 12097 : auto app_cli = _app.commandLine()->initSubAppCommandLine(name(), multiapp_name, input_cli_args);
1162 12097 : app_cli->parse();
1163 :
1164 12097 : if (_fe_problem.verboseMultiApps())
1165 986 : _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
1166 986 : << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
1167 986 : << " on processor " << processor_id() << " with full name " << full_name
1168 986 : << COLOR_DEFAULT << std::endl;
1169 24194 : app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
1170 24194 : app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
1171 24194 : app_params.set<const MooseMesh *>("_master_mesh") = &_fe_problem.mesh();
1172 : #ifdef MOOSE_MFEM_ENABLED
1173 : // MFEM device must only be set once across all apps
1174 : // FIXME: this required that the base app is an MFEM app; itwill
1175 : // still fail if multiple MFEM sub-apps are launched from a libMesh base app
1176 9335 : if (i == 0)
1177 : {
1178 12244 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1179 18366 : _app.getMFEMDevice(Moose::PassKey<MultiApp>());
1180 6122 : const auto & mfem_device_set = _app.getMFEMDevices(Moose::PassKey<MultiApp>());
1181 12244 : app_params.set<std::set<std::string>>("_mfem_devices") = mfem_device_set;
1182 : }
1183 : else
1184 : {
1185 6426 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1186 9639 : _apps[0]->getMFEMDevice(Moose::PassKey<MultiApp>());
1187 3213 : const auto & mfem_device_set = _apps[0]->getMFEMDevices(Moose::PassKey<MultiApp>());
1188 6426 : app_params.set<std::set<std::string>>("_mfem_devices") = mfem_device_set;
1189 : }
1190 : #endif
1191 60485 : if (getParam<bool>("clone_master_mesh") || getParam<bool>("clone_parent_mesh"))
1192 : {
1193 781 : if (_fe_problem.verboseMultiApps())
1194 0 : _console << COLOR_CYAN << "Cloned parent app mesh will be used for MultiApp " << name()
1195 0 : << COLOR_DEFAULT << std::endl;
1196 781 : app_params.set<bool>("_use_master_mesh") = true;
1197 781 : auto displaced_problem = _fe_problem.getDisplacedProblem();
1198 781 : if (displaced_problem)
1199 0 : app_params.set<const MooseMesh *>("_master_displaced_mesh") = &displaced_problem->mesh();
1200 781 : }
1201 :
1202 : // If only one input file was provided, use it for all the solves
1203 12097 : const auto input_index = _input_files.size() == 1 ? 0 : _first_local_app + i;
1204 12097 : const auto & input_file = _input_files[input_index];
1205 :
1206 : // create new parser tree for the application and parse
1207 12097 : auto parser = std::make_unique<Parser>(input_file);
1208 12097 : parser->setCommandLineParams(app_cli->buildHitParams());
1209 12097 : parser->parse();
1210 :
1211 : // Checks on app type
1212 12097 : const auto & app_type = parser->getAppType();
1213 12097 : if (app_type.empty() && _app_type.empty())
1214 0 : mooseWarning("The application type is not specified for ",
1215 : full_name,
1216 : ". Please use [Application] block to specify the application type.");
1217 12097 : if (!app_type.empty() && app_type != _app_type && !AppFactory::instance().isRegistered(app_type))
1218 3 : mooseError("In the ",
1219 : full_name,
1220 : ", '",
1221 : app_type,
1222 : "' is not a registered application. The registered application is named: '",
1223 3 : _app_type,
1224 : "'. Please double check the [Application] block to make sure the correct "
1225 : "application is provided. \n");
1226 :
1227 12094 : if (parser->getAppType().empty())
1228 12082 : parser->setAppType(_app_type);
1229 :
1230 24188 : app_params.set<std::shared_ptr<Parser>>("_parser") = std::move(parser);
1231 24188 : app_params.set<std::shared_ptr<CommandLine>>("_command_line") = std::move(app_cli);
1232 12094 : _apps[i] = AppFactory::instance().create(_app_type, full_name, app_params, _my_comm);
1233 12094 : auto & app = _apps[i];
1234 :
1235 12094 : app->setGlobalTimeOffset(start_time);
1236 12094 : app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
1237 12094 : app->setRestart(_app.isRestarting());
1238 12094 : app->setRecover(_app.isRecovering());
1239 :
1240 36282 : if (_use_positions && getParam<bool>("output_in_position"))
1241 3359 : app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1242 12094 : if (_output_in_position && _run_in_position)
1243 6 : paramError("run_in_position",
1244 : "Sub-apps are already displaced, so they are already output in position");
1245 :
1246 : // Update the MultiApp level for the app that was just created
1247 12091 : app->setupOptions();
1248 : // if multiapp does not have file base in Outputs input block, output file base will
1249 : // be empty here since setupOptions() does not set the default file base with the multiapp
1250 : // input file name. Parent app will create the default file base for multiapp by taking the
1251 : // output base of the parent app problem and appending the name of the multiapp plus a number to
1252 : // it
1253 12091 : if (app->getOutputFileBase().empty())
1254 12055 : setAppOutputFileBase(i);
1255 12091 : preRunInputFile();
1256 36273 : if (_app.multiAppLevel() > getParam<unsigned int>("max_multiapp_level"))
1257 3 : mooseError("Maximum multiapp level has been reached. This is likely caused by an infinite loop "
1258 : "in your multiapp system. If additional multiapp levels are needed, "
1259 : "max_multiapp_level can be specified in the MuliApps block.");
1260 :
1261 : // Transfer coupling relaxation information to the subapps
1262 24176 : _apps[i]->fixedPointConfig().sub_relaxation_factor = getParam<Real>("relaxation_factor");
1263 12088 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1264 36264 : getParam<std::vector<std::string>>("transformed_variables");
1265 : // Handle deprecated parameter
1266 36264 : if (!parameters().isParamSetByAddParam("relaxed_variables"))
1267 0 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1268 0 : getParam<std::vector<std::string>>("relaxed_variables");
1269 12088 : _apps[i]->fixedPointConfig().sub_transformed_pps =
1270 36264 : getParam<std::vector<PostprocessorName>>("transformed_postprocessors");
1271 :
1272 12088 : app->runInputFile();
1273 12052 : auto fixed_point_solve = &(_apps[i]->getExecutioner()->fixedPointSolve());
1274 12052 : if (fixed_point_solve)
1275 12052 : fixed_point_solve->allocateStorage(false);
1276 :
1277 : // Transform the app mesh if requested
1278 12052 : if (_run_in_position)
1279 : {
1280 246 : if (usingPositions())
1281 492 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1282 246 : app->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
1283 : else
1284 0 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1285 0 : app->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
1286 : }
1287 12049 : }
1288 :
1289 : std::vector<std::string>
1290 3156 : MultiApp::getCommandLineArgs(const unsigned int local_app)
1291 : {
1292 3156 : const auto cla = cliArgs();
1293 3156 : auto cli_args_param = _cli_args_param;
1294 3156 : std::string combined_args;
1295 :
1296 : // Single set of args from cliArgs() to be provided to all apps
1297 3156 : if (cla.size() == 1)
1298 2362 : combined_args = cla[0];
1299 : // Single "cli_args_files" file to be provided to all apps
1300 794 : else if (_cli_args_from_file.size() == 1)
1301 : {
1302 0 : cli_args_param = "cli_args_files";
1303 0 : combined_args = _cli_args_from_file[0];
1304 : }
1305 : // Unique set of args from cliArgs() to be provided to each app
1306 794 : else if (cla.size())
1307 686 : combined_args = cla[local_app + _first_local_app];
1308 : // Unique set of args from "cli_args_files" to be provided to all apps
1309 : else
1310 : {
1311 108 : cli_args_param = "cli_args_files";
1312 108 : combined_args = _cli_args_from_file[local_app + _first_local_app];
1313 : }
1314 :
1315 : // Remove all of the beginning and end whitespace so we can recognize truly empty
1316 3156 : combined_args = MooseUtils::trim(combined_args);
1317 :
1318 : // MooseUtils::split will return a single empty entry if there is nothing,
1319 : // so exit early if we have nothing
1320 3156 : if (combined_args.empty())
1321 0 : return {};
1322 :
1323 : // Split the argument into a vector of arguments, and make sure
1324 : // that we don't have any empty arguments
1325 6312 : const auto args = MooseUtils::split(combined_args, ";");
1326 10467 : for (const auto & arg : args)
1327 : {
1328 7311 : if (arg.empty())
1329 : {
1330 0 : const auto error = "An empty MultiApp command line argument was provided. Your "
1331 : "combined command line string has a ';' with no argument after it.";
1332 0 : if (cli_args_param)
1333 0 : paramError(*cli_args_param, error);
1334 : else
1335 0 : mooseError(error);
1336 : }
1337 : }
1338 :
1339 3156 : return args;
1340 3156 : }
1341 :
1342 : LocalRankConfig
1343 8846 : rankConfig(processor_id_type rank,
1344 : processor_id_type nprocs,
1345 : dof_id_type napps,
1346 : processor_id_type min_app_procs,
1347 : processor_id_type max_app_procs,
1348 : bool batch_mode)
1349 : {
1350 8846 : if (min_app_procs > nprocs)
1351 0 : mooseError("minimum number of procs per app is higher than the available number of procs");
1352 8846 : else if (min_app_procs > max_app_procs)
1353 0 : mooseError("minimum number of procs per app must be lower than the max procs per app");
1354 :
1355 : mooseAssert(rank < nprocs, "rank must be smaller than the number of procs");
1356 :
1357 : // A "slot" is a group of procs/ranks that are grouped together to run a
1358 : // single (sub)app/sim in parallel.
1359 :
1360 : const processor_id_type slot_size =
1361 8846 : std::max(std::min(cast_int<processor_id_type>(nprocs / napps), max_app_procs), min_app_procs);
1362 8846 : const processor_id_type nslots = std::min(
1363 17692 : nprocs / slot_size,
1364 8846 : cast_int<processor_id_type>(std::min(
1365 8846 : static_cast<dof_id_type>(std::numeric_limits<processor_id_type>::max()), napps)));
1366 : mooseAssert(nprocs >= (nslots * slot_size),
1367 : "Ensure that leftover procs is represented by an unsigned type");
1368 8846 : const processor_id_type leftover_procs = nprocs - nslots * slot_size;
1369 8846 : const dof_id_type apps_per_slot = napps / nslots;
1370 8846 : const dof_id_type leftover_apps = napps % nslots;
1371 :
1372 8846 : std::vector<int> slot_for_rank(nprocs);
1373 8846 : processor_id_type slot = 0;
1374 8846 : processor_id_type procs_in_slot = 0;
1375 22249 : for (processor_id_type rankiter = 0; rankiter <= rank; rankiter++)
1376 : {
1377 13403 : if (slot < nslots)
1378 13183 : slot_for_rank[rankiter] = cast_int<int>(slot);
1379 : else
1380 220 : slot_for_rank[rankiter] = -1;
1381 13403 : procs_in_slot++;
1382 : // this slot keeps growing until we reach slot size plus possibly an extra
1383 : // proc if there were any leftover from the slotization of nprocs - this
1384 : // must also make sure we don't go over max app procs.
1385 13403 : if (procs_in_slot == slot_size + 1 * (slot < leftover_procs && slot_size < max_app_procs))
1386 : {
1387 9211 : procs_in_slot = 0;
1388 9211 : slot++;
1389 : }
1390 : }
1391 :
1392 8846 : if (slot_for_rank[rank] < 0)
1393 : // ranks assigned a negative slot don't have any apps running on them.
1394 88 : return {0, 0, 0, 0, false, 0};
1395 8758 : const processor_id_type slot_num = cast_int<processor_id_type>(slot_for_rank[rank]);
1396 :
1397 8758 : const bool is_first_local_rank = rank == 0 || (slot_for_rank[rank - 1] != slot_for_rank[rank]);
1398 8758 : const dof_id_type n_local_apps = apps_per_slot + 1 * (slot_num < leftover_apps);
1399 :
1400 8758 : processor_id_type my_first_rank = 0;
1401 11652 : for (processor_id_type rankiter = rank; rankiter > 0; rankiter--)
1402 3212 : if (slot_for_rank[rank] != slot_for_rank[rankiter])
1403 : {
1404 318 : my_first_rank = cast_int<processor_id_type>(slot_for_rank[rankiter + 1]);
1405 318 : break;
1406 : }
1407 :
1408 8758 : dof_id_type app_index = 0;
1409 10381 : for (processor_id_type slot = 0; slot < slot_num; slot++)
1410 : {
1411 1623 : const dof_id_type num_slot_apps = apps_per_slot + 1 * (slot < leftover_apps);
1412 1623 : app_index += num_slot_apps;
1413 : }
1414 :
1415 8758 : if (batch_mode)
1416 347 : return {n_local_apps, app_index, 1, slot_num, is_first_local_rank, my_first_rank};
1417 8411 : return {n_local_apps, app_index, n_local_apps, app_index, is_first_local_rank, my_first_rank};
1418 8846 : }
1419 :
1420 : void
1421 7998 : MultiApp::buildComm()
1422 : {
1423 : int ierr;
1424 :
1425 7998 : ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
1426 7998 : mooseCheckMPIErr(ierr);
1427 7998 : ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
1428 7998 : mooseCheckMPIErr(ierr);
1429 :
1430 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
1431 : struct utsname sysInfo;
1432 7998 : uname(&sysInfo);
1433 7998 : _node_name = sysInfo.nodename;
1434 : #else
1435 : _node_name = "Unknown";
1436 : #endif
1437 :
1438 : int rank;
1439 7998 : ierr = MPI_Comm_rank(_communicator.get(), &rank);
1440 7998 : mooseCheckMPIErr(ierr);
1441 :
1442 7998 : _my_num_apps = _rank_config.num_local_apps;
1443 7998 : _first_local_app = _rank_config.first_local_app_index;
1444 :
1445 7998 : _has_an_app = _rank_config.num_local_apps > 0;
1446 7998 : if (_rank_config.first_local_app_index >= _total_num_apps)
1447 0 : mooseError("Internal error, a processor has an undefined app.");
1448 :
1449 7998 : if (_has_an_app)
1450 : {
1451 7946 : _communicator.split(_rank_config.first_local_app_index, rank, _my_communicator);
1452 7946 : ierr = MPI_Comm_rank(_my_comm, &_my_rank);
1453 7946 : mooseCheckMPIErr(ierr);
1454 : }
1455 : else
1456 : {
1457 52 : _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
1458 52 : _my_rank = 0;
1459 : }
1460 7998 : }
1461 :
1462 : unsigned int
1463 801279 : MultiApp::globalAppToLocal(unsigned int global_app)
1464 : {
1465 801279 : if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
1466 801279 : return global_app - _first_local_app;
1467 :
1468 0 : std::stringstream ss;
1469 0 : ss << "Requesting app " << global_app << ", but processor " << processor_id() << " ";
1470 0 : if (_my_num_apps == 0)
1471 0 : ss << "does not own any apps";
1472 0 : else if (_my_num_apps == 1)
1473 0 : ss << "owns app " << _first_local_app;
1474 : else
1475 0 : ss << "owns apps " << _first_local_app << "-" << _first_local_app + (_my_num_apps - 1);
1476 0 : ss << ".";
1477 0 : mooseError("Invalid global_app!\n", ss.str());
1478 : return 0;
1479 0 : }
1480 :
1481 : void
1482 12091 : MultiApp::preRunInputFile()
1483 : {
1484 12091 : }
1485 :
1486 : void
1487 14183 : MultiApp::addAssociatedTransfer(MultiAppTransfer & transfer)
1488 : {
1489 14183 : _associated_transfers.push_back(&transfer);
1490 14183 : }
1491 :
1492 : void
1493 0 : MultiApp::setAppOutputFileBase()
1494 : {
1495 0 : for (unsigned int i = 0; i < _my_num_apps; ++i)
1496 0 : setAppOutputFileBase(i);
1497 0 : }
1498 :
1499 : std::vector<std::string>
1500 23248 : MultiApp::cliArgs() const
1501 : {
1502 : // So that we can error out with paramError("cli_args", ...);
1503 23248 : _cli_args_param = "cli_args";
1504 46496 : return std::vector<std::string>(_cli_args.begin(), _cli_args.end());
1505 : }
1506 :
1507 : void
1508 12055 : MultiApp::setAppOutputFileBase(unsigned int index)
1509 : {
1510 : const std::string multiapp_name =
1511 12055 : getMultiAppName(name(), _first_local_app + index, _total_num_apps);
1512 12055 : _apps[index]->setOutputFileBase(_app.getOutputFileBase() + "_" + multiapp_name);
1513 12055 : }
1514 :
1515 : std::string
1516 24152 : MultiApp::getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total)
1517 : {
1518 24152 : std::ostringstream multiapp_name;
1519 48304 : multiapp_name << base_name << std::setw(std::ceil(std::log10(total))) << std::setprecision(0)
1520 24152 : << std::setfill('0') << std::right << index;
1521 48304 : return multiapp_name.str();
1522 24152 : }
1523 :
1524 : const Point &
1525 199154 : MultiApp::position(unsigned int app) const
1526 : {
1527 : // If we're not using positions, it won't have changed
1528 199154 : if (_positions_objs.empty())
1529 183783 : return _positions[app];
1530 : else
1531 : // Find which Positions object is specifying it, and query a potentially updated value
1532 15371 : return _positions_objs[app]->getPosition(app - _positions_index_offsets[app], false);
1533 : }
1534 :
1535 : void
1536 2561 : dataStore(std::ostream & stream, SubAppBackups & backups, void * context)
1537 : {
1538 2561 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1539 : mooseAssert(multi_app, "Not set");
1540 :
1541 2561 : multi_app->backup();
1542 :
1543 2561 : dataStore(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1544 2561 : }
1545 :
1546 : void
1547 659 : dataLoad(std::istream & stream, SubAppBackups & backups, void * context)
1548 : {
1549 659 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1550 : mooseAssert(multi_app, "Not set");
1551 :
1552 659 : dataLoad(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1553 :
1554 659 : multi_app->restore();
1555 659 : }
|
