Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // StochasticTools includes
11 : #include "SamplerFullSolveMultiApp.h"
12 : #include "Sampler.h"
13 : #include "StochasticToolsTransfer.h"
14 : #include "Console.h"
15 : #include "VariadicTable.h"
16 :
17 : registerMooseObject("StochasticToolsApp", SamplerFullSolveMultiApp);
18 :
19 : InputParameters
20 3634 : SamplerFullSolveMultiApp::validParams()
21 : {
22 3634 : InputParameters params = FullSolveMultiApp::validParams();
23 3634 : params += SamplerInterface::validParams();
24 3634 : params += ReporterInterface::validParams();
25 3634 : params.addClassDescription(
26 : "Creates a full-solve type sub-application for each row of each Sampler matrix.");
27 7268 : params.addRequiredParam<SamplerName>("sampler",
28 : "The Sampler object to utilize for creating MultiApps.");
29 3634 : params.suppressParameter<std::vector<Point>>("positions");
30 3634 : params.suppressParameter<bool>("output_in_position");
31 3634 : params.suppressParameter<std::vector<FileName>>("positions_file");
32 3634 : params.suppressParameter<Real>("move_time");
33 3634 : params.suppressParameter<std::vector<Point>>("move_positions");
34 3634 : params.suppressParameter<std::vector<unsigned int>>("move_apps");
35 3634 : params.set<bool>("use_positions") = false;
36 :
37 7268 : MooseEnum modes("normal=0 batch-reset=1 batch-restore=2", "normal");
38 7268 : params.addParam<MooseEnum>(
39 : "mode",
40 : modes,
41 : "The operation mode, 'normal' creates one sub-application for each row in the Sampler and "
42 : "'batch-reset' and 'batch-restore' creates N sub-applications, where N is the minimum of "
43 : "'num_rows' in the Sampler and floor(number of processes / min_procs_per_app). To run "
44 : "the rows in the Sampler, 'batch-reset' will destroy and re-create sub-apps as needed, "
45 : "whereas the 'batch-restore' will backup and restore sub-apps to the initial state prior "
46 : "to execution, without destruction.");
47 7268 : params.addParam<ReporterName>(
48 : "should_run_reporter",
49 : "Vector reporter value determining whether a certain multiapp should be run with this "
50 : "multiapp. This only works in batch-reset or batch-restore mode.");
51 3634 : return params;
52 3634 : }
53 :
54 1809 : SamplerFullSolveMultiApp::SamplerFullSolveMultiApp(const InputParameters & parameters)
55 : : FullSolveMultiApp(parameters),
56 : SamplerInterface(this),
57 : ReporterInterface(this),
58 1809 : _sampler(getSampler("sampler")),
59 3618 : _mode(getParam<MooseEnum>("mode").getEnum<StochasticTools::MultiAppMode>()),
60 1809 : _local_batch_app_index(0),
61 3618 : _solved_once(false)
62 : {
63 3618 : if (getParam<unsigned int>("min_procs_per_app") !=
64 5427 : _sampler.getParam<unsigned int>("min_procs_per_row") ||
65 3618 : getParam<unsigned int>("max_procs_per_app") !=
66 5427 : _sampler.getParam<unsigned int>("max_procs_per_row"))
67 0 : paramError("sampler",
68 : "Sampler and multiapp communicator configuration inconsistent. Please ensure that "
69 : "'MultiApps/",
70 : name(),
71 : "/min(max)_procs_per_app' and 'Samplers/",
72 0 : _sampler.name(),
73 : "/min(max)_procs_per_row' are the same.");
74 :
75 1809 : init(_sampler.getNumberOfRows(),
76 1809 : _sampler.getRankConfig(_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
77 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE));
78 1809 : _number_of_sampler_rows = _sampler.getNumberOfRows();
79 :
80 5427 : if (isParamValid("should_run_reporter") && _mode == StochasticTools::MultiAppMode::NORMAL)
81 2 : paramError("should_run_reporter",
82 : "Conditionally run sampler multiapp only works in batch modes.");
83 1807 : }
84 :
85 : void
86 4567 : SamplerFullSolveMultiApp::backup()
87 : {
88 4567 : if (_mode != StochasticTools::MultiAppMode::BATCH_RESTORE)
89 3946 : FullSolveMultiApp::backup();
90 4567 : }
91 :
92 : void
93 4465 : SamplerFullSolveMultiApp::preTransfer(Real /*dt*/, Real /*target_time*/)
94 : {
95 : // Logic for calling initial setup again:
96 : // 1) If and only if not doing batch-reset (solveStepBatch does this at each local row)
97 : // 2) If the number of rows have changed since the communicator is re-split.
98 : // 3) If we have already solved and doing "normal" execution, effectively resetting the apps
99 : bool initial_setup_required = false;
100 :
101 : // Reinitialize MultiApp size
102 4465 : const auto num_rows = _sampler.getNumberOfRows();
103 4465 : if (num_rows != _number_of_sampler_rows)
104 : {
105 177 : init(num_rows,
106 177 : _sampler.getRankConfig(_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
107 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE));
108 177 : _number_of_sampler_rows = num_rows;
109 177 : _row_data.clear();
110 177 : initial_setup_required = _mode != StochasticTools::MultiAppMode::BATCH_RESET;
111 : }
112 4288 : else if (_solved_once)
113 2525 : initial_setup_required = _mode == StochasticTools::MultiAppMode::NORMAL;
114 :
115 : // Call initial setup based on the logic above
116 2702 : if (initial_setup_required)
117 : {
118 1537 : initialSetup();
119 1537 : _solved_once = false;
120 : }
121 : // Otherwise we need to restore for batch-restore
122 2928 : else if (_solved_once && _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
123 26 : restore();
124 :
125 8930 : if (isParamValid("should_run_reporter"))
126 1097 : _should_run = &getReporterValue<std::vector<bool>>("should_run_reporter");
127 4465 : }
128 :
129 : bool
130 4430 : SamplerFullSolveMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
131 : {
132 8860 : TIME_SECTION("solveStep", 3, "Solving SamplerFullSolveMultiApp");
133 :
134 : mooseAssert(_my_num_apps, _sampler.getNumberOfLocalRows());
135 :
136 : bool last_solve_converged = true;
137 :
138 4430 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
139 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
140 2204 : last_solve_converged = solveStepBatch(dt, target_time, auto_advance);
141 : else
142 2226 : last_solve_converged = FullSolveMultiApp::solveStep(dt, target_time, auto_advance);
143 :
144 4428 : _solved_once = true;
145 :
146 4428 : return last_solve_converged;
147 4428 : }
148 :
149 : bool
150 2204 : SamplerFullSolveMultiApp::solveStepBatch(Real dt, Real target_time, bool auto_advance)
151 : {
152 4408 : TIME_SECTION("solveStepBatch", 3, "Solving Step Batch For SamplerFullSolveMultiApp");
153 :
154 3301 : if (_should_run && _should_run->size() < _sampler.getNumberOfLocalRows())
155 0 : paramError("should_run_reporter",
156 : "Reporter deteriming multiapp run must be of size greater than or equal to the "
157 : "number of local rows in the sampler, ",
158 0 : _should_run->size(),
159 : " < ",
160 0 : _sampler.getNumberOfLocalRows(),
161 : ".");
162 :
163 : // Value to return
164 : bool last_solve_converged = true;
165 :
166 : // List of active relevant Transfer objects
167 : std::vector<std::shared_ptr<StochasticToolsTransfer>> to_transfers =
168 2204 : getActiveStochasticToolsTransfers(MultiAppTransfer::TO_MULTIAPP);
169 : std::vector<std::shared_ptr<StochasticToolsTransfer>> from_transfers =
170 2204 : getActiveStochasticToolsTransfers(MultiAppTransfer::FROM_MULTIAPP);
171 :
172 : // Initialize to/from transfers
173 4058 : for (auto transfer : to_transfers)
174 : {
175 1854 : transfer->setGlobalMultiAppIndex(_rank_config.first_local_app_index);
176 1854 : transfer->initializeToMultiapp();
177 : }
178 4499 : for (auto transfer : from_transfers)
179 : {
180 2295 : transfer->setGlobalMultiAppIndex(_rank_config.first_local_app_index);
181 2295 : transfer->initializeFromMultiapp();
182 : }
183 :
184 2204 : if (!_solved_once && _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
185 550 : FullSolveMultiApp::backup();
186 :
187 : // Perform batch MultiApp solves
188 2204 : _local_batch_app_index = 0;
189 13940 : for (dof_id_type i = _rank_config.first_local_sim_index;
190 13940 : i < _rank_config.first_local_sim_index + _rank_config.num_local_sims;
191 : ++i)
192 : {
193 11736 : updateRowData(_local_batch_app_index);
194 :
195 11736 : bool run = true;
196 11736 : if (_should_run)
197 : {
198 1249 : if (isRootProcessor())
199 1075 : run = (*_should_run)[_local_batch_app_index];
200 1249 : _my_communicator.broadcast(run, 0);
201 : }
202 11736 : if (!run)
203 : {
204 687 : _local_batch_app_index++;
205 687 : continue;
206 : }
207 :
208 : // Given that we don't initialize in preTransfer for batch-reset mode, we need
209 : // a different logic for resetting the apps for every sample:
210 : // - batch-restore: after (re-)initializing the problem, we only need to restore
211 : // starting from the second sample
212 11049 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
213 : {
214 5650 : if (i != _rank_config.first_local_sim_index)
215 5074 : restore();
216 : }
217 : // - batch-reset: we don't need to initialize for the first sample in the first
218 : // solve. After that, we initialize every time. This is mainly to avoid unnecessary
219 : // initializations for cases when the multiapp does not need to be executed (conditional runs)
220 : else
221 : {
222 5399 : if (i != _rank_config.first_local_sim_index || _solved_once)
223 4910 : initialSetup();
224 : }
225 :
226 11049 : execBatchTransfers(to_transfers,
227 : i,
228 11049 : _row_data,
229 : MultiAppTransfer::TO_MULTIAPP,
230 11049 : _fe_problem.verboseMultiApps(),
231 11049 : _console);
232 :
233 : // Set the file base based on the current row
234 22098 : for (unsigned int ai = 0; ai < _my_num_apps; ++ai)
235 : {
236 11049 : const std::string mname = getMultiAppName(name(), i, _number_of_sampler_rows);
237 33147 : _apps[ai]->setOutputFileBase(_app.getOutputFileBase() + "_" + mname);
238 : }
239 :
240 : const bool curr_last_solve_converged =
241 11049 : FullSolveMultiApp::solveStep(dt, target_time, auto_advance);
242 11049 : last_solve_converged = last_solve_converged && curr_last_solve_converged;
243 :
244 11049 : execBatchTransfers(from_transfers,
245 : i,
246 : _row_data,
247 : MultiAppTransfer::FROM_MULTIAPP,
248 11049 : _fe_problem.verboseMultiApps(),
249 : _console);
250 :
251 11049 : _local_batch_app_index++;
252 : }
253 2204 : _local_batch_app_index = 0;
254 :
255 : // Finalize to/from transfers
256 4058 : for (auto transfer : to_transfers)
257 1854 : transfer->finalizeToMultiapp();
258 4499 : for (auto transfer : from_transfers)
259 2295 : transfer->finalizeFromMultiapp();
260 :
261 2204 : return last_solve_converged;
262 2204 : }
263 :
264 : void
265 23798 : SamplerFullSolveMultiApp::execBatchTransfers(
266 : const std::vector<std::shared_ptr<StochasticToolsTransfer>> & transfers,
267 : dof_id_type global_row_index,
268 : const std::vector<Real> & row_data,
269 : Transfer::DIRECTION direction,
270 : bool verbose,
271 : const ConsoleStream & console)
272 : {
273 23798 : if (verbose && transfers.size())
274 : {
275 0 : console << COLOR_CYAN << "\nBatch transfers for row " << global_row_index;
276 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
277 0 : console << " To ";
278 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
279 0 : console << " From ";
280 0 : console << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
281 :
282 0 : console << "Sampler row " << global_row_index << " data: [" << Moose::stringify(row_data) << "]"
283 0 : << std::endl;
284 :
285 : // Build Table of Transfer Info
286 : VariadicTable<std::string, std::string, std::string, std::string> table(
287 0 : {"Name", "Type", "From", "To"});
288 0 : for (const auto & transfer : transfers)
289 0 : table.addRow(
290 0 : transfer->name(), transfer->type(), transfer->getFromName(), transfer->getToName());
291 0 : table.print(console);
292 0 : }
293 :
294 43811 : for (auto & transfer : transfers)
295 : {
296 20013 : transfer->setGlobalRowIndex(global_row_index);
297 : transfer->setCurrentRow(row_data);
298 20013 : if (direction == MultiAppTransfer::TO_MULTIAPP)
299 7879 : transfer->executeToMultiapp();
300 12134 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
301 12134 : transfer->executeFromMultiapp();
302 : }
303 :
304 23798 : if (verbose && transfers.size())
305 0 : console << COLOR_CYAN << "Batch transfers for row " << global_row_index << " Are Finished\n"
306 0 : << COLOR_DEFAULT << std::endl;
307 23798 : }
308 :
309 : void
310 18580 : SamplerFullSolveMultiApp::showStatusMessage(unsigned int i) const
311 : {
312 : // Local row is the app index if in normal mode, otherwise it's _local_batch_app_index
313 : const dof_id_type local_row =
314 18580 : _mode == StochasticTools::MultiAppMode::NORMAL ? (dof_id_type)i : _local_batch_app_index;
315 : // If the local row is less than the number of local sims, we aren't finished yet
316 18580 : if (local_row < _rank_config.num_local_sims - 1)
317 : return;
318 :
319 : // Loop through processors to communicate completeness
320 12452 : for (const auto & pid : make_range(n_processors()))
321 : {
322 : // This is what is being sent to trigger completeness
323 17382 : dof_id_type last_row = _rank_config.is_first_local_rank
324 8691 : ? _rank_config.first_local_sim_index + _rank_config.num_local_sims
325 : : 0;
326 : // Cannot send/receive to the same processor, so avoid if root
327 8691 : if (pid > 0)
328 : {
329 : // Send data to root
330 4930 : if (pid == processor_id())
331 1620 : _communicator.send(0, last_row);
332 : // Receive data from source
333 3310 : else if (processor_id() == 0)
334 1620 : _communicator.receive(pid, last_row);
335 : }
336 :
337 : // Output the samples that are complete if it's the main processor for the batch
338 8691 : if (last_row)
339 7845 : _console << COLOR_CYAN << type() << " [" << name() << "] " << last_row << "/"
340 7387 : << _number_of_sampler_rows << " samples complete!" << std::endl;
341 : }
342 : }
343 :
344 : std::vector<std::shared_ptr<StochasticToolsTransfer>>
345 4408 : SamplerFullSolveMultiApp::getActiveStochasticToolsTransfers(Transfer::DIRECTION direction)
346 : {
347 : std::vector<std::shared_ptr<StochasticToolsTransfer>> output;
348 : const ExecuteMooseObjectWarehouse<Transfer> & warehouse =
349 4408 : _fe_problem.getMultiAppTransferWarehouse(direction);
350 9085 : for (std::shared_ptr<Transfer> transfer : warehouse.getActiveObjects())
351 : {
352 4677 : auto ptr = std::dynamic_pointer_cast<StochasticToolsTransfer>(transfer);
353 9354 : if (ptr && ptr->getMultiApp().get() == this)
354 4149 : output.push_back(ptr);
355 : }
356 4408 : return output;
357 0 : }
358 :
359 : std::vector<std::string>
360 10262 : SamplerFullSolveMultiApp::getCommandLineArgs(const unsigned int local_app)
361 : {
362 10262 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
363 202 : return FullSolveMultiApp::getCommandLineArgs(local_app);
364 :
365 : std::vector<std::string> args;
366 :
367 : // With multiple processors per app, there are no local rows for non-root processors
368 10060 : if (isRootProcessor())
369 : {
370 : // Since we only store param_names in cli_args, we need to find the values for each param from
371 : // sampler data and combine them to get full command line option strings.
372 9907 : updateRowData(_mode == StochasticTools::MultiAppMode::NORMAL ? local_app
373 : : _local_batch_app_index);
374 :
375 19812 : args = sampledCommandLineArgs(_row_data, FullSolveMultiApp::getCommandLineArgs(local_app));
376 : }
377 :
378 10058 : _my_communicator.broadcast(args);
379 : return args;
380 10058 : }
381 :
382 : void
383 21643 : SamplerFullSolveMultiApp::updateRowData(dof_id_type local_index)
384 : {
385 21643 : if (!isRootProcessor())
386 : return;
387 :
388 : mooseAssert(local_index < _sampler.getNumberOfLocalRows(),
389 : "Local index must be less than number of local rows.");
390 :
391 21117 : if (_row_data.empty() ||
392 19593 : (_local_row_index == _sampler.getNumberOfLocalRows() - 1 && local_index == 0))
393 : {
394 : mooseAssert(local_index == 0,
395 : "The first time calling updateRowData must have a local index of 0.");
396 3170 : _local_row_index = 0;
397 3170 : _row_data = _sampler.getNextLocalRow();
398 : }
399 17947 : else if (local_index - _local_row_index == 1)
400 : {
401 13916 : _local_row_index++;
402 13916 : _row_data = _sampler.getNextLocalRow();
403 : }
404 :
405 : mooseAssert(local_index == _local_row_index,
406 : "Local index must be equal or one greater than the index previously called.");
407 : }
408 :
409 : std::vector<std::string>
410 20720 : SamplerFullSolveMultiApp::sampledCommandLineArgs(const std::vector<Real> & row,
411 : const std::vector<std::string> & full_args_name)
412 : {
413 : std::vector<std::string> args;
414 :
415 : // Find parameters that are meant to be assigned by sampler values
416 : std::vector<std::string> cli_args_name;
417 76374 : for (const auto & fan : full_args_name)
418 : {
419 : // If it has an '=', then it is not meant to be modified
420 55654 : if (fan.find("=") == std::string::npos)
421 55100 : cli_args_name.push_back(fan);
422 : else
423 554 : args.push_back(fan);
424 : }
425 :
426 : // Make sure the parameters either all have brackets, or none of them do
427 : bool has_brackets = false;
428 20720 : if (cli_args_name.size())
429 : {
430 20578 : has_brackets = cli_args_name[0].find("[") != std::string::npos;
431 55100 : for (unsigned int i = 1; i < cli_args_name.size(); ++i)
432 34522 : if (has_brackets != (cli_args_name[i].find("[") != std::string::npos))
433 0 : ::mooseError("If the bracket is used, it must be provided to every parameter.");
434 : }
435 20720 : if (!has_brackets && cli_args_name.size() && cli_args_name.size() != row.size())
436 4 : ::mooseError("Number of command line arguments does not match number of sampler columns.");
437 :
438 75806 : for (unsigned int i = 0; i < cli_args_name.size(); ++i)
439 : {
440 : // Assign bracketed parameters
441 55092 : if (has_brackets)
442 : {
443 : // Split param name and vector assignment: "param[0,(3.14),1]" -> {"param", "0,(3.14),1]"}
444 2346 : const std::vector<std::string> & vector_param = MooseUtils::split(cli_args_name[i], "[");
445 : // Get indices of vector: "0,(3.14),1]" -> {"0", "(3.14)", "1"}
446 : const std::vector<std::string> & index_string =
447 4692 : MooseUtils::split(vector_param[1].substr(0, vector_param[1].find("]")), ",");
448 :
449 : // Loop through indices and assign parameter: param='row[0] 3.14 row[1]'
450 : std::vector<std::string> values;
451 12033 : for (const auto & istr : index_string)
452 : {
453 : Real value;
454 :
455 : // If the value is enclosed in parentheses, then it isn't an index, it's a value
456 9689 : if (istr.find("(") != std::string::npos)
457 34 : value = std::stod(istr.substr(istr.find("(") + 1));
458 : // Assign the value from row if it is an index
459 : else
460 : {
461 9672 : unsigned int index = MooseUtils::stringToInteger(istr);
462 9672 : if (index >= row.size())
463 2 : ::mooseError("The provided global column index (",
464 : index,
465 : ") for ",
466 : vector_param[0],
467 : " is out of bound.");
468 9670 : value = row[index];
469 : }
470 :
471 19374 : values.push_back(Moose::stringifyExact(value));
472 : }
473 :
474 : // If there is only one value, do not put quotes around it
475 2344 : if (values.size() == 1)
476 794 : args.push_back(vector_param[0] + "=" + values[0]);
477 : else
478 5841 : args.push_back(vector_param[0] + "='" + MooseUtils::stringJoin(values) + "'");
479 2344 : }
480 : // Assign scalar parameters
481 : else
482 105492 : args.push_back(cli_args_name[i] + "=" + Moose::stringifyExact(row[i]));
483 : }
484 :
485 20714 : return args;
486 20714 : }
|
