Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // StochasticTools includes
11 : #include "SamplerFullSolveMultiApp.h"
12 : #include "Sampler.h"
13 : #include "StochasticToolsTransfer.h"
14 : #include "Console.h"
15 : #include "VariadicTable.h"
16 :
17 : registerMooseObject("StochasticToolsApp", SamplerFullSolveMultiApp);
18 :
19 : InputParameters
20 7508 : SamplerFullSolveMultiApp::validParams()
21 : {
22 7508 : InputParameters params = FullSolveMultiApp::validParams();
23 7508 : params += SamplerInterface::validParams();
24 7508 : params += ReporterInterface::validParams();
25 7508 : params.addClassDescription(
26 : "Creates a full-solve type sub-application for each row of each Sampler matrix.");
27 15016 : params.addRequiredParam<SamplerName>("sampler",
28 : "The Sampler object to utilize for creating MultiApps.");
29 7508 : params.suppressParameter<std::vector<Point>>("positions");
30 7508 : params.suppressParameter<bool>("output_in_position");
31 7508 : params.suppressParameter<std::vector<FileName>>("positions_file");
32 7508 : params.suppressParameter<Real>("move_time");
33 7508 : params.suppressParameter<std::vector<Point>>("move_positions");
34 7508 : params.suppressParameter<std::vector<unsigned int>>("move_apps");
35 7508 : params.set<bool>("use_positions") = false;
36 :
37 15016 : MooseEnum modes("normal=0 batch-reset=1 batch-restore=2", "normal");
38 15016 : params.addParam<MooseEnum>(
39 : "mode",
40 : modes,
41 : "The operation mode, 'normal' creates one sub-application for each row in the Sampler and "
42 : "'batch-reset' and 'batch-restore' creates N sub-applications, where N is the minimum of "
43 : "'num_rows' in the Sampler and floor(number of processes / min_procs_per_app). To run "
44 : "the rows in the Sampler, 'batch-reset' will destroy and re-create sub-apps as needed, "
45 : "whereas the 'batch-restore' will backup and restore sub-apps to the initial state prior "
46 : "to execution, without destruction.");
47 15016 : params.addParam<ReporterName>(
48 : "should_run_reporter",
49 : "Vector reporter value determining whether a certain multiapp should be run with this "
50 : "multiapp. This only works in batch-reset or batch-restore mode.");
51 7508 : return params;
52 7508 : }
53 :
54 3738 : SamplerFullSolveMultiApp::SamplerFullSolveMultiApp(const InputParameters & parameters)
55 : : FullSolveMultiApp(parameters),
56 : SamplerInterface(this),
57 : ReporterInterface(this),
58 3738 : _sampler(getSampler("sampler")),
59 7476 : _mode(getParam<MooseEnum>("mode").getEnum<StochasticTools::MultiAppMode>()),
60 3738 : _local_batch_app_index(0),
61 7476 : _solved_once(false)
62 : {
63 7476 : if (getParam<unsigned int>("min_procs_per_app") !=
64 11214 : _sampler.getParam<unsigned int>("min_procs_per_row") ||
65 7476 : getParam<unsigned int>("max_procs_per_app") !=
66 11214 : _sampler.getParam<unsigned int>("max_procs_per_row"))
67 0 : paramError("sampler",
68 : "Sampler and multiapp communicator configuration inconsistent. Please ensure that "
69 : "'MultiApps/",
70 : name(),
71 : "/min(max)_procs_per_app' and 'Samplers/",
72 0 : _sampler.name(),
73 : "/min(max)_procs_per_row' are the same.");
74 :
75 3738 : init(_sampler.getNumberOfRows(),
76 3738 : _sampler.getRankConfig(_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
77 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE));
78 3738 : _number_of_sampler_rows = _sampler.getNumberOfRows();
79 :
80 11214 : if (isParamValid("should_run_reporter") && _mode == StochasticTools::MultiAppMode::NORMAL)
81 4 : paramError("should_run_reporter",
82 : "Conditionally run sampler multiapp only works in batch modes.");
83 3734 : }
84 :
85 : void
86 8974 : SamplerFullSolveMultiApp::preTransfer(Real /*dt*/, Real /*target_time*/)
87 : {
88 : // Reinitialize MultiApp size
89 8974 : const auto num_rows = _sampler.getNumberOfRows();
90 8974 : if (num_rows != _number_of_sampler_rows)
91 : {
92 96 : init(num_rows,
93 96 : _sampler.getRankConfig(_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
94 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE));
95 96 : _number_of_sampler_rows = num_rows;
96 : _row_data.clear();
97 : }
98 :
99 : // Reinitialize app to original state prior to solve, if a solve has occured.
100 : // Since the app is reinitialized in the solve step either way, we skip this
101 : // for batch-reset mode.
102 8974 : if (_solved_once && _mode != StochasticTools::MultiAppMode::BATCH_RESET)
103 2962 : initialSetup();
104 :
105 17948 : if (isParamValid("should_run_reporter"))
106 2436 : _should_run = &getReporterValue<std::vector<bool>>("should_run_reporter");
107 8974 : }
108 :
109 : bool
110 8894 : SamplerFullSolveMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
111 : {
112 17788 : TIME_SECTION("solveStep", 3, "Solving SamplerFullSolveMultiApp");
113 :
114 : mooseAssert(_my_num_apps, _sampler.getNumberOfLocalRows());
115 :
116 : bool last_solve_converged = true;
117 :
118 8894 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESET ||
119 : _mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
120 4488 : last_solve_converged = solveStepBatch(dt, target_time, auto_advance);
121 : else
122 4406 : last_solve_converged = FullSolveMultiApp::solveStep(dt, target_time, auto_advance);
123 :
124 8890 : _solved_once = true;
125 :
126 8890 : return last_solve_converged;
127 : }
128 :
129 : bool
130 4488 : SamplerFullSolveMultiApp::solveStepBatch(Real dt, Real target_time, bool auto_advance)
131 : {
132 8976 : TIME_SECTION("solveStepBatch", 3, "Solving Step Batch For SamplerFullSolveMultiApp");
133 :
134 4488 : if (_should_run && _should_run->size() < _sampler.getNumberOfLocalRows())
135 0 : paramError("should_run_reporter",
136 : "Reporter deteriming multiapp run must be of size greater than or equal to the "
137 : "number of local rows in the sampler, ",
138 0 : _should_run->size(),
139 : " < ",
140 0 : _sampler.getNumberOfLocalRows(),
141 : ".");
142 :
143 : // Value to return
144 : bool last_solve_converged = true;
145 :
146 : // List of active relevant Transfer objects
147 : std::vector<std::shared_ptr<StochasticToolsTransfer>> to_transfers =
148 4488 : getActiveStochasticToolsTransfers(MultiAppTransfer::TO_MULTIAPP);
149 : std::vector<std::shared_ptr<StochasticToolsTransfer>> from_transfers =
150 4488 : getActiveStochasticToolsTransfers(MultiAppTransfer::FROM_MULTIAPP);
151 :
152 : // Initialize to/from transfers
153 8446 : for (auto transfer : to_transfers)
154 : {
155 3958 : transfer->setGlobalMultiAppIndex(_rank_config.first_local_app_index);
156 3958 : transfer->initializeToMultiapp();
157 : }
158 9196 : for (auto transfer : from_transfers)
159 : {
160 4708 : transfer->setGlobalMultiAppIndex(_rank_config.first_local_app_index);
161 4708 : transfer->initializeFromMultiapp();
162 : }
163 :
164 4488 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
165 1128 : backup();
166 :
167 : // Perform batch MultiApp solves
168 4488 : _local_batch_app_index = 0;
169 27500 : for (dof_id_type i = _rank_config.first_local_sim_index;
170 27500 : i < _rank_config.first_local_sim_index + _rank_config.num_local_sims;
171 : ++i)
172 : {
173 23012 : updateRowData(_local_batch_app_index);
174 :
175 23012 : bool run = true;
176 23012 : if (_should_run)
177 : {
178 2732 : if (isRootProcessor())
179 2210 : run = (*_should_run)[_local_batch_app_index];
180 2732 : _my_communicator.broadcast(run, 0);
181 : }
182 23012 : if (!run)
183 : {
184 1532 : _local_batch_app_index++;
185 1532 : continue;
186 : }
187 :
188 : // Given that we don't initialize in preTransfer for batch-reset mode, we need
189 : // a different logic for resetting the apps for every sample:
190 : // - batch-restore: after (re-)initializing the problem, we only need to restore
191 : // starting from the second sample
192 21480 : if (_mode == StochasticTools::MultiAppMode::BATCH_RESTORE)
193 : {
194 11170 : if (i != _rank_config.first_local_sim_index)
195 10042 : restore();
196 : }
197 : // - batch-reset: we don't need to initialize for the first sample in the first
198 : // solve. After that, we initialize every time. This is mainly to avoid unnecessary
199 : // initializations for cases when the multiapp does not need to be executed (conditional runs)
200 : else
201 : {
202 10310 : if (i != _rank_config.first_local_sim_index || _solved_once)
203 9320 : initialSetup();
204 : }
205 :
206 21480 : execBatchTransfers(to_transfers,
207 : i,
208 21480 : _row_data,
209 : MultiAppTransfer::TO_MULTIAPP,
210 21480 : _fe_problem.verboseMultiApps(),
211 21480 : _console);
212 :
213 : // Set the file base based on the current row
214 42960 : for (unsigned int ai = 0; ai < _my_num_apps; ++ai)
215 : {
216 21480 : const std::string mname = getMultiAppName(name(), i, _number_of_sampler_rows);
217 64440 : _apps[ai]->setOutputFileBase(_app.getOutputFileBase() + "_" + mname);
218 : }
219 :
220 : const bool curr_last_solve_converged =
221 21480 : FullSolveMultiApp::solveStep(dt, target_time, auto_advance);
222 21480 : last_solve_converged = last_solve_converged && curr_last_solve_converged;
223 :
224 21480 : execBatchTransfers(from_transfers,
225 : i,
226 : _row_data,
227 : MultiAppTransfer::FROM_MULTIAPP,
228 21480 : _fe_problem.verboseMultiApps(),
229 : _console);
230 :
231 21480 : _local_batch_app_index++;
232 : }
233 4488 : _local_batch_app_index = 0;
234 :
235 : // Finalize to/from transfers
236 8446 : for (auto transfer : to_transfers)
237 3958 : transfer->finalizeToMultiapp();
238 9196 : for (auto transfer : from_transfers)
239 4708 : transfer->finalizeFromMultiapp();
240 :
241 4488 : return last_solve_converged;
242 4488 : }
243 :
244 : void
245 46360 : SamplerFullSolveMultiApp::execBatchTransfers(
246 : const std::vector<std::shared_ptr<StochasticToolsTransfer>> & transfers,
247 : dof_id_type global_row_index,
248 : const std::vector<Real> & row_data,
249 : Transfer::DIRECTION direction,
250 : bool verbose,
251 : const ConsoleStream & console)
252 : {
253 46360 : if (verbose && transfers.size())
254 : {
255 0 : console << COLOR_CYAN << "\nBatch transfers for row " << global_row_index;
256 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
257 0 : console << " To ";
258 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
259 0 : console << " From ";
260 0 : console << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
261 :
262 0 : console << "Sampler row " << global_row_index << " data: [" << Moose::stringify(row_data) << "]"
263 0 : << std::endl;
264 :
265 : // Build Table of Transfer Info
266 : VariadicTable<std::string, std::string, std::string, std::string> table(
267 0 : {"Name", "Type", "From", "To"});
268 0 : for (const auto & transfer : transfers)
269 0 : table.addRow(
270 0 : transfer->name(), transfer->type(), transfer->getFromName(), transfer->getToName());
271 0 : table.print(console);
272 0 : }
273 :
274 85890 : for (auto & transfer : transfers)
275 : {
276 39530 : transfer->setGlobalRowIndex(global_row_index);
277 : transfer->setCurrentRow(row_data);
278 39530 : if (direction == MultiAppTransfer::TO_MULTIAPP)
279 15780 : transfer->executeToMultiapp();
280 23750 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
281 23750 : transfer->executeFromMultiapp();
282 : }
283 :
284 46360 : if (verbose && transfers.size())
285 0 : console << COLOR_CYAN << "Batch transfers for row " << global_row_index << " Are Finished\n"
286 0 : << COLOR_DEFAULT << std::endl;
287 46360 : }
288 :
289 : void
290 35218 : SamplerFullSolveMultiApp::showStatusMessage(unsigned int i) const
291 : {
292 : // Local row is the app index if in normal mode, otherwise it's _local_batch_app_index
293 : const dof_id_type local_row =
294 35218 : _mode == StochasticTools::MultiAppMode::NORMAL ? (dof_id_type)i : _local_batch_app_index;
295 : // If the local row is less than the number of local sims, we aren't finished yet
296 35218 : if (local_row < _rank_config.num_local_sims - 1)
297 : return;
298 :
299 : // Loop through processors to communicate completeness
300 25164 : for (const auto & pid : make_range(n_processors()))
301 : {
302 : // This is what is being sent to trigger completeness
303 35532 : dof_id_type last_row = _rank_config.is_first_local_rank
304 17766 : ? _rank_config.first_local_sim_index + _rank_config.num_local_sims
305 : : 0;
306 : // Cannot send/receive to the same processor, so avoid if root
307 17766 : if (pid > 0)
308 : {
309 : // Send data to root
310 10368 : if (pid == processor_id())
311 3554 : _communicator.send(0, last_row);
312 : // Receive data from source
313 6814 : else if (processor_id() == 0)
314 3554 : _communicator.receive(pid, last_row);
315 : }
316 :
317 : // Output the samples that are complete if it's the main processor for the batch
318 17766 : if (last_row)
319 15168 : _console << COLOR_CYAN << type() << " [" << name() << "] " << last_row << "/"
320 14720 : << _number_of_sampler_rows << " samples complete!" << std::endl;
321 : }
322 : }
323 :
324 : std::vector<std::shared_ptr<StochasticToolsTransfer>>
325 8976 : SamplerFullSolveMultiApp::getActiveStochasticToolsTransfers(Transfer::DIRECTION direction)
326 : {
327 : std::vector<std::shared_ptr<StochasticToolsTransfer>> output;
328 : const ExecuteMooseObjectWarehouse<Transfer> & warehouse =
329 8976 : _fe_problem.getMultiAppTransferWarehouse(direction);
330 18826 : for (std::shared_ptr<Transfer> transfer : warehouse.getActiveObjects())
331 : {
332 9850 : auto ptr = std::dynamic_pointer_cast<StochasticToolsTransfer>(transfer);
333 19700 : if (ptr && ptr->getMultiApp().get() == this)
334 8666 : output.push_back(ptr);
335 : }
336 8976 : return output;
337 0 : }
338 :
339 : std::vector<std::string>
340 18242 : SamplerFullSolveMultiApp::getCommandLineArgs(const unsigned int local_app)
341 : {
342 : std::vector<std::string> args;
343 :
344 : // With multiple processors per app, there are no local rows for non-root processors
345 18242 : if (isRootProcessor())
346 : {
347 : // Since we only store param_names in cli_args, we need to find the values for each param from
348 : // sampler data and combine them to get full command line option strings.
349 17932 : updateRowData(_mode == StochasticTools::MultiAppMode::NORMAL ? local_app
350 : : _local_batch_app_index);
351 :
352 35860 : args = sampledCommandLineArgs(_row_data, FullSolveMultiApp::getCommandLineArgs(local_app));
353 : }
354 :
355 18238 : _my_communicator.broadcast(args);
356 18238 : return args;
357 0 : }
358 :
359 : void
360 40944 : SamplerFullSolveMultiApp::updateRowData(dof_id_type local_index)
361 : {
362 40944 : if (!isRootProcessor())
363 : return;
364 :
365 : mooseAssert(local_index < _sampler.getNumberOfLocalRows(),
366 : "Local index must be less than number of local rows.");
367 :
368 39722 : if (_row_data.empty() ||
369 36834 : (_local_row_index == _sampler.getNumberOfLocalRows() - 1 && local_index == 0))
370 : {
371 : mooseAssert(local_index == 0,
372 : "The first time calling updateRowData must have a local index of 0.");
373 5970 : _local_row_index = 0;
374 11940 : _row_data = _sampler.getNextLocalRow();
375 : }
376 33752 : else if (local_index - _local_row_index == 1)
377 : {
378 25990 : _local_row_index++;
379 51980 : _row_data = _sampler.getNextLocalRow();
380 : }
381 :
382 : mooseAssert(local_index == _local_row_index,
383 : "Local index must be equal or one greater than the index previously called.");
384 : }
385 :
386 : std::vector<std::string>
387 39672 : SamplerFullSolveMultiApp::sampledCommandLineArgs(const std::vector<Real> & row,
388 : const std::vector<std::string> & full_args_name)
389 : {
390 : std::vector<std::string> args;
391 :
392 : // Find parameters that are meant to be assigned by sampler values
393 : std::vector<std::string> cli_args_name;
394 147334 : for (const auto & fan : full_args_name)
395 : {
396 : // If it has an '=', then it is not meant to be modified
397 107662 : if (fan.find("=") == std::string::npos)
398 106198 : cli_args_name.push_back(fan);
399 : else
400 1464 : args.push_back(fan);
401 : }
402 :
403 : // Make sure the parameters either all have brackets, or none of them do
404 : bool has_brackets = false;
405 39672 : if (cli_args_name.size())
406 : {
407 39050 : has_brackets = cli_args_name[0].find("[") != std::string::npos;
408 106198 : for (unsigned int i = 1; i < cli_args_name.size(); ++i)
409 67148 : if (has_brackets != (cli_args_name[i].find("[") != std::string::npos))
410 0 : ::mooseError("If the bracket is used, it must be provided to every parameter.");
411 : }
412 39672 : if (!has_brackets && cli_args_name.size() && cli_args_name.size() != row.size())
413 8 : ::mooseError("Number of command line arguments does not match number of sampler columns.");
414 :
415 145842 : for (unsigned int i = 0; i < cli_args_name.size(); ++i)
416 : {
417 : // Assign bracketed parameters
418 106182 : if (has_brackets)
419 : {
420 : // Split param name and vector assignment: "param[0,(3.14),1]" -> {"param", "0,(3.14),1]"}
421 3102 : const std::vector<std::string> & vector_param = MooseUtils::split(cli_args_name[i], "[");
422 : // Get indices of vector: "0,(3.14),1]" -> {"0", "(3.14)", "1"}
423 : const std::vector<std::string> & index_string =
424 6204 : MooseUtils::split(vector_param[1].substr(0, vector_param[1].find("]")), ",");
425 :
426 : // Loop through indices and assign parameter: param='row[0] 3.14 row[1]'
427 : std::vector<std::string> values;
428 14622 : for (const auto & istr : index_string)
429 : {
430 : Real value;
431 :
432 : // If the value is enclosed in parentheses, then it isn't an index, it's a value
433 11524 : if (istr.find("(") != std::string::npos)
434 68 : value = std::stod(istr.substr(istr.find("(") + 1));
435 : // Assign the value from row if it is an index
436 : else
437 : {
438 11490 : unsigned int index = MooseUtils::stringToInteger(istr);
439 11490 : if (index >= row.size())
440 4 : ::mooseError("The provided global column index (",
441 : index,
442 : ") for ",
443 : vector_param[0],
444 : " is out of bound.");
445 11486 : value = row[index];
446 : }
447 :
448 23040 : values.push_back(Moose::stringifyExact(value));
449 : }
450 :
451 6196 : args.push_back(vector_param[0] + "='" + MooseUtils::stringJoin(values) + "'");
452 3098 : }
453 : // Assign scalar parameters
454 : else
455 206160 : args.push_back(cli_args_name[i] + "=" + Moose::stringifyExact(row[i]));
456 : }
457 :
458 39660 : return args;
459 39660 : }
|
