Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // StochasticTools includes
11 : #include "SamplerFullSolveMultiApp.h"
12 : #include "Sampler.h"
13 : #include "StochasticToolsTransfer.h"
14 : #include "Console.h"
15 : #include "VariadicTable.h"
16 :
17 : registerMooseObject("StochasticToolsApp", SamplerFullSolveMultiApp);
18 :
19 : InputParameters
20 8198 : SamplerFullSolveMultiApp::validParams()
21 : {
22 8198 : InputParameters params = FullSolveMultiApp::validParams();
23 8198 : params += SamplerInterface::validParams();
24 8198 : params += ReporterInterface::validParams();
25 8198 : params.addClassDescription(
26 : "Creates a full-solve type sub-application for each row of each Sampler matrix.");
27 16396 : params.addRequiredParam<SamplerName>("sampler",
28 : "The Sampler object to utilize for creating MultiApps.");
29 8198 : params.suppressParameter<std::vector<Point>>("positions");
30 8198 : params.suppressParameter<bool>("output_in_position");
31 8198 : params.suppressParameter<std::vector<FileName>>("positions_file");
32 8198 : params.suppressParameter<Real>("move_time");
33 8198 : params.suppressParameter<std::vector<Point>>("move_positions");
34 8198 : params.suppressParameter<std::vector<unsigned int>>("move_apps");
35 8198 : params.set<bool>("use_positions") = false;
36 :
37 16396 : MooseEnum modes("normal=0 batch-reset=1 batch-restore=2", "normal");
38 16396 : params.addParam<MooseEnum>(
39 : "mode",
40 : modes,
41 : "The operation mode, 'normal' creates one sub-application for each row in the Sampler and "
42 : "'batch-reset' and 'batch-restore' creates N sub-applications, where N is the minimum of "
43 : "'num_rows' in the Sampler and floor(number of processes / min_procs_per_app). To run "
44 : "the rows in the Sampler, 'batch-reset' will destroy and re-create sub-apps as needed, "
45 : "whereas the 'batch-restore' will backup and restore sub-apps to the initial state prior "
46 : "to execution, without destruction.");
47 16396 : params.addParam<ReporterName>(
48 : "should_run_reporter",
49 : "Vector reporter value determining whether a certain multiapp should be run with this "
50 : "multiapp. This only works in batch-reset or batch-restore mode.");
51 8198 : return params;
52 8198 : }
53 :
54 4083 : SamplerFullSolveMultiApp::SamplerFullSolveMultiApp(const InputParameters & parameters)
55 : : FullSolveMultiApp(parameters),
56 : SamplerInterface(this),
57 : ReporterInterface(this),
58 4083 : _sampler(getSampler("sampler")),
59 8166 : _mode(getParam<MooseEnum>("mode").getEnum<StochasticTools::MultiAppMode>()),
60 4083 : _local_batch_app_index(0),
61 8166 : _solved_once(false)
62 : {
63 8166 : if (getParam<unsigned int>("min_procs_per_app") !=
64 12249 : _sampler.getParam<unsigned int>("min_procs_per_row") ||
65 8166 : getParam<unsigned int>("max_procs_per_app") !=
66 12249 : _sampler.getParam<unsigned int>("max_procs_per_row"))
67 0 : paramError("sampler",
68 : "Sampler and multiapp communicator configuration inconsistent. Please ensure that "
69 : "'MultiApps/",
70 : name(),
71 : "/min(max)_procs_per_app' and 'Samplers/",
72 0 : _sampler.name(),
73 : "/min(max)_procs_per_row' are the same.");
74 :
75 4083 : init(_sampler.getNumberOfRows(),
76 4083 : _sampler.getRankConfig(_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
77 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE));
78 4083 : _number_of_sampler_rows = _sampler.getNumberOfRows();
79 :
80 12249 : if (isParamValid("should_run_reporter") && _mode == StochasticTools::MultiAppMode::NORMAL)
81 4 : paramError("should_run_reporter",
82 : "Conditionally run sampler multiapp only works in batch modes.");
83 4079 : }
84 :
85 : void
86 10046 : SamplerFullSolveMultiApp::backup()
87 : {
88 10046 : if (_mode != StochasticTools::MultiAppMode::BATCH_RESTORE)
89 8660 : FullSolveMultiApp::backup();
90 10046 : }
91 :
92 : void
93 10131 : SamplerFullSolveMultiApp::preTransfer(Real /*dt*/, Real /*target_time*/)
94 : {
95 : // Logic for calling initial setup again:
96 : // 1) If and only if not doing batch-reset (solveStepBatch does this at each local row)
97 : // 2) If the number of rows have changed since the communicator is re-split.
98 : // 3) If we have already solved and doing "normal" execution, effectively resetting the apps
99 : bool initial_setup_required = false;
100 :
101 : // Reinitialize MultiApp size
102 10131 : const auto num_rows = _sampler.getNumberOfRows();
103 10131 : if (num_rows != _number_of_sampler_rows)
104 : {
105 448 : init(num_rows,
106 448 : _sampler.getRankConfig(_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
107 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE));
108 448 : _number_of_sampler_rows = num_rows;
109 448 : _row_data.clear();
110 448 : initial_setup_required = _mode != StochasticTools::MultiAppMode::BATCH_RESET;
111 : }
112 9683 : else if (_solved_once)
113 5698 : initial_setup_required = _mode == StochasticTools::MultiAppMode::NORMAL;
114 :
115 : // Call initial setup based on the logic above
116 6146 : if (initial_setup_required)
117 : {
118 3384 : initialSetup();
119 3384 : _solved_once = false;
120 : }
121 : // Otherwise we need to restore for batch-restore
122 6747 : else if (_solved_once && _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
123 47 : restore();
124 :
125 20262 : if (isParamValid("should_run_reporter"))
126 2625 : _should_run = &getReporterValue<std::vector<bool>>("should_run_reporter");
127 10131 : }
128 :
129 : bool
130 10046 : SamplerFullSolveMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
131 : {
132 20092 : TIME_SECTION("solveStep", 3, "Solving SamplerFullSolveMultiApp");
133 :
134 : mooseAssert(_my_num_apps, _sampler.getNumberOfLocalRows());
135 :
136 : bool last_solve_converged = true;
137 :
138 10046 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
139 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
140 5195 : last_solve_converged = solveStepBatch(dt, target_time, auto_advance);
141 : else
142 4851 : last_solve_converged = FullSolveMultiApp::solveStep(dt, target_time, auto_advance);
143 :
144 10042 : _solved_once = true;
145 :
146 10042 : return last_solve_converged;
147 : }
148 :
149 : bool
150 5195 : SamplerFullSolveMultiApp::solveStepBatch(Real dt, Real target_time, bool auto_advance)
151 : {
152 10390 : TIME_SECTION("solveStepBatch", 3, "Solving Step Batch For SamplerFullSolveMultiApp");
153 :
154 7820 : if (_should_run && _should_run->size() < _sampler.getNumberOfLocalRows())
155 0 : paramError("should_run_reporter",
156 : "Reporter deteriming multiapp run must be of size greater than or equal to the "
157 : "number of local rows in the sampler, ",
158 0 : _should_run->size(),
159 : " < ",
160 0 : _sampler.getNumberOfLocalRows(),
161 : ".");
162 :
163 : // Value to return
164 : bool last_solve_converged = true;
165 :
166 : // List of active relevant Transfer objects
167 : std::vector<std::shared_ptr<StochasticToolsTransfer>> to_transfers =
168 5195 : getActiveStochasticToolsTransfers(MultiAppTransfer::TO_MULTIAPP);
169 : std::vector<std::shared_ptr<StochasticToolsTransfer>> from_transfers =
170 5195 : getActiveStochasticToolsTransfers(MultiAppTransfer::FROM_MULTIAPP);
171 :
172 : // Initialize to/from transfers
173 9634 : for (auto transfer : to_transfers)
174 : {
175 4439 : transfer->setGlobalMultiAppIndex(_rank_config.first_local_app_index);
176 4439 : transfer->initializeToMultiapp();
177 : }
178 10629 : for (auto transfer : from_transfers)
179 : {
180 5434 : transfer->setGlobalMultiAppIndex(_rank_config.first_local_app_index);
181 5434 : transfer->initializeFromMultiapp();
182 : }
183 :
184 5195 : if (!_solved_once && _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
185 1339 : FullSolveMultiApp::backup();
186 :
187 : // Perform batch MultiApp solves
188 5195 : _local_batch_app_index = 0;
189 31391 : for (dof_id_type i = _rank_config.first_local_sim_index;
190 31391 : i < _rank_config.first_local_sim_index + _rank_config.num_local_sims;
191 : ++i)
192 : {
193 26196 : updateRowData(_local_batch_app_index);
194 :
195 26196 : bool run = true;
196 26196 : if (_should_run)
197 : {
198 2953 : if (isRootProcessor())
199 2431 : run = (*_should_run)[_local_batch_app_index];
200 2953 : _my_communicator.broadcast(run, 0);
201 : }
202 26196 : if (!run)
203 : {
204 1651 : _local_batch_app_index++;
205 1651 : continue;
206 : }
207 :
208 : // Given that we don't initialize in preTransfer for batch-reset mode, we need
209 : // a different logic for resetting the apps for every sample:
210 : // - batch-restore: after (re-)initializing the problem, we only need to restore
211 : // starting from the second sample
212 24545 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
213 : {
214 12683 : if (i != _rank_config.first_local_sim_index)
215 11297 : restore();
216 : }
217 : // - batch-reset: we don't need to initialize for the first sample in the first
218 : // solve. After that, we initialize every time. This is mainly to avoid unnecessary
219 : // initializations for cases when the multiapp does not need to be executed (conditional runs)
220 : else
221 : {
222 11862 : if (i != _rank_config.first_local_sim_index || _solved_once)
223 10768 : initialSetup();
224 : }
225 :
226 24545 : execBatchTransfers(to_transfers,
227 : i,
228 24545 : _row_data,
229 : MultiAppTransfer::TO_MULTIAPP,
230 24545 : _fe_problem.verboseMultiApps(),
231 24545 : _console);
232 :
233 : // Set the file base based on the current row
234 49090 : for (unsigned int ai = 0; ai < _my_num_apps; ++ai)
235 : {
236 24545 : const std::string mname = getMultiAppName(name(), i, _number_of_sampler_rows);
237 73635 : _apps[ai]->setOutputFileBase(_app.getOutputFileBase() + "_" + mname);
238 : }
239 :
240 : const bool curr_last_solve_converged =
241 24545 : FullSolveMultiApp::solveStep(dt, target_time, auto_advance);
242 24545 : last_solve_converged = last_solve_converged && curr_last_solve_converged;
243 :
244 24545 : execBatchTransfers(from_transfers,
245 : i,
246 : _row_data,
247 : MultiAppTransfer::FROM_MULTIAPP,
248 24545 : _fe_problem.verboseMultiApps(),
249 : _console);
250 :
251 24545 : _local_batch_app_index++;
252 : }
253 5195 : _local_batch_app_index = 0;
254 :
255 : // Finalize to/from transfers
256 9634 : for (auto transfer : to_transfers)
257 4439 : transfer->finalizeToMultiapp();
258 10629 : for (auto transfer : from_transfers)
259 5434 : transfer->finalizeFromMultiapp();
260 :
261 5195 : return last_solve_converged;
262 5195 : }
263 :
264 : void
265 52830 : SamplerFullSolveMultiApp::execBatchTransfers(
266 : const std::vector<std::shared_ptr<StochasticToolsTransfer>> & transfers,
267 : dof_id_type global_row_index,
268 : const std::vector<Real> & row_data,
269 : Transfer::DIRECTION direction,
270 : bool verbose,
271 : const ConsoleStream & console)
272 : {
273 52830 : if (verbose && transfers.size())
274 : {
275 0 : console << COLOR_CYAN << "\nBatch transfers for row " << global_row_index;
276 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
277 0 : console << " To ";
278 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
279 0 : console << " From ";
280 0 : console << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
281 :
282 0 : console << "Sampler row " << global_row_index << " data: [" << Moose::stringify(row_data) << "]"
283 0 : << std::endl;
284 :
285 : // Build Table of Transfer Info
286 : VariadicTable<std::string, std::string, std::string, std::string> table(
287 0 : {"Name", "Type", "From", "To"});
288 0 : for (const auto & transfer : transfers)
289 0 : table.addRow(
290 0 : transfer->name(), transfer->type(), transfer->getFromName(), transfer->getToName());
291 0 : table.print(console);
292 0 : }
293 :
294 97608 : for (auto & transfer : transfers)
295 : {
296 44778 : transfer->setGlobalRowIndex(global_row_index);
297 : transfer->setCurrentRow(row_data);
298 44778 : if (direction == MultiAppTransfer::TO_MULTIAPP)
299 17736 : transfer->executeToMultiapp();
300 27042 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
301 27042 : transfer->executeFromMultiapp();
302 : }
303 :
304 52830 : if (verbose && transfers.size())
305 0 : console << COLOR_CYAN << "Batch transfers for row " << global_row_index << " Are Finished\n"
306 0 : << COLOR_DEFAULT << std::endl;
307 52830 : }
308 :
309 : void
310 39813 : SamplerFullSolveMultiApp::showStatusMessage(unsigned int i) const
311 : {
312 : // Local row is the app index if in normal mode, otherwise it's _local_batch_app_index
313 : const dof_id_type local_row =
314 39813 : _mode == StochasticTools::MultiAppMode::NORMAL ? (dof_id_type)i : _local_batch_app_index;
315 : // If the local row is less than the number of local sims, we aren't finished yet
316 39813 : if (local_row < _rank_config.num_local_sims - 1)
317 : return;
318 :
319 : // Loop through processors to communicate completeness
320 28326 : for (const auto & pid : make_range(n_processors()))
321 : {
322 : // This is what is being sent to trigger completeness
323 39782 : dof_id_type last_row = _rank_config.is_first_local_rank
324 19891 : ? _rank_config.first_local_sim_index + _rank_config.num_local_sims
325 : : 0;
326 : // Cannot send/receive to the same processor, so avoid if root
327 19891 : if (pid > 0)
328 : {
329 : // Send data to root
330 11456 : if (pid == processor_id())
331 3935 : _communicator.send(0, last_row);
332 : // Receive data from source
333 7521 : else if (processor_id() == 0)
334 3935 : _communicator.receive(pid, last_row);
335 : }
336 :
337 : // Output the samples that are complete if it's the main processor for the batch
338 19891 : if (last_row)
339 23241 : _console << COLOR_CYAN << type() << " [" << name() << "] " << last_row << "/"
340 16583 : << _number_of_sampler_rows << " samples complete!" << std::endl;
341 : }
342 : }
343 :
344 : std::vector<std::shared_ptr<StochasticToolsTransfer>>
345 10390 : SamplerFullSolveMultiApp::getActiveStochasticToolsTransfers(Transfer::DIRECTION direction)
346 : {
347 : std::vector<std::shared_ptr<StochasticToolsTransfer>> output;
348 : const ExecuteMooseObjectWarehouse<Transfer> & warehouse =
349 10390 : _fe_problem.getMultiAppTransferWarehouse(direction);
350 21563 : for (std::shared_ptr<Transfer> transfer : warehouse.getActiveObjects())
351 : {
352 11173 : auto ptr = std::dynamic_pointer_cast<StochasticToolsTransfer>(transfer);
353 22346 : if (ptr && ptr->getMultiApp().get() == this)
354 9873 : output.push_back(ptr);
355 : }
356 10390 : return output;
357 0 : }
358 :
359 : std::vector<std::string>
360 20943 : SamplerFullSolveMultiApp::getCommandLineArgs(const unsigned int local_app)
361 : {
362 : std::vector<std::string> args;
363 :
364 : // With multiple processors per app, there are no local rows for non-root processors
365 20943 : if (isRootProcessor())
366 : {
367 : // Since we only store param_names in cli_args, we need to find the values for each param from
368 : // sampler data and combine them to get full command line option strings.
369 20567 : updateRowData(_mode == StochasticTools::MultiAppMode::NORMAL ? local_app
370 : : _local_batch_app_index);
371 :
372 41130 : args = sampledCommandLineArgs(_row_data, FullSolveMultiApp::getCommandLineArgs(local_app));
373 : }
374 :
375 20939 : _my_communicator.broadcast(args);
376 20939 : return args;
377 0 : }
378 :
379 : void
380 46763 : SamplerFullSolveMultiApp::updateRowData(dof_id_type local_index)
381 : {
382 46763 : if (!isRootProcessor())
383 : return;
384 :
385 : mooseAssert(local_index < _sampler.getNumberOfLocalRows(),
386 : "Local index must be less than number of local rows.");
387 :
388 45444 : if (_row_data.empty() ||
389 41967 : (_local_row_index == _sampler.getNumberOfLocalRows() - 1 && local_index == 0))
390 : {
391 : mooseAssert(local_index == 0,
392 : "The first time calling updateRowData must have a local index of 0.");
393 6974 : _local_row_index = 0;
394 6974 : _row_data = _sampler.getNextLocalRow();
395 : }
396 38470 : else if (local_index - _local_row_index == 1)
397 : {
398 29350 : _local_row_index++;
399 29350 : _row_data = _sampler.getNextLocalRow();
400 : }
401 :
402 : mooseAssert(local_index == _local_row_index,
403 : "Local index must be equal or one greater than the index previously called.");
404 : }
405 :
406 : std::vector<std::string>
407 44853 : SamplerFullSolveMultiApp::sampledCommandLineArgs(const std::vector<Real> & row,
408 : const std::vector<std::string> & full_args_name)
409 : {
410 : std::vector<std::string> args;
411 :
412 : // Find parameters that are meant to be assigned by sampler values
413 : std::vector<std::string> cli_args_name;
414 164725 : for (const auto & fan : full_args_name)
415 : {
416 : // If it has an '=', then it is not meant to be modified
417 119872 : if (fan.find("=") == std::string::npos)
418 118130 : cli_args_name.push_back(fan);
419 : else
420 1742 : args.push_back(fan);
421 : }
422 :
423 : // Make sure the parameters either all have brackets, or none of them do
424 : bool has_brackets = false;
425 44853 : if (cli_args_name.size())
426 : {
427 44036 : has_brackets = cli_args_name[0].find("[") != std::string::npos;
428 118130 : for (unsigned int i = 1; i < cli_args_name.size(); ++i)
429 74094 : if (has_brackets != (cli_args_name[i].find("[") != std::string::npos))
430 0 : ::mooseError("If the bracket is used, it must be provided to every parameter.");
431 : }
432 44853 : if (!has_brackets && cli_args_name.size() && cli_args_name.size() != row.size())
433 8 : ::mooseError("Number of command line arguments does not match number of sampler columns.");
434 :
435 162955 : for (unsigned int i = 0; i < cli_args_name.size(); ++i)
436 : {
437 : // Assign bracketed parameters
438 118114 : if (has_brackets)
439 : {
440 : // Split param name and vector assignment: "param[0,(3.14),1]" -> {"param", "0,(3.14),1]"}
441 5245 : const std::vector<std::string> & vector_param = MooseUtils::split(cli_args_name[i], "[");
442 : // Get indices of vector: "0,(3.14),1]" -> {"0", "(3.14)", "1"}
443 : const std::vector<std::string> & index_string =
444 10490 : MooseUtils::split(vector_param[1].substr(0, vector_param[1].find("]")), ",");
445 :
446 : // Loop through indices and assign parameter: param='row[0] 3.14 row[1]'
447 : std::vector<std::string> values;
448 26903 : for (const auto & istr : index_string)
449 : {
450 : Real value;
451 :
452 : // If the value is enclosed in parentheses, then it isn't an index, it's a value
453 21662 : if (istr.find("(") != std::string::npos)
454 74 : value = std::stod(istr.substr(istr.find("(") + 1));
455 : // Assign the value from row if it is an index
456 : else
457 : {
458 21625 : unsigned int index = MooseUtils::stringToInteger(istr);
459 21625 : if (index >= row.size())
460 4 : ::mooseError("The provided global column index (",
461 : index,
462 : ") for ",
463 : vector_param[0],
464 : " is out of bound.");
465 21621 : value = row[index];
466 : }
467 :
468 43316 : values.push_back(Moose::stringifyExact(value));
469 : }
470 :
471 10482 : args.push_back(vector_param[0] + "='" + MooseUtils::stringJoin(values) + "'");
472 5241 : }
473 : // Assign scalar parameters
474 : else
475 225738 : args.push_back(cli_args_name[i] + "=" + Moose::stringifyExact(row[i]));
476 : }
477 :
478 44841 : return args;
479 44841 : }
|
