Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : // StochasticTools includes
      11             : #include "SerializedSolutionTransfer.h"
      12             : #include "NonlinearSystemBase.h"
      13             : #include "Sampler.h"
      14             : 
      15             : registerMooseObject("StochasticToolsApp", SerializedSolutionTransfer);
      16             : 
      17             : InputParameters
      18         858 : SerializedSolutionTransfer::validParams()
      19             : {
      20         858 :   InputParameters params = StochasticToolsTransfer::validParams();
      21         858 :   params.addClassDescription(
      22             :       "Serializes and transfers solution vectors for given variables from sub-applications.");
      23        1716 :   params.addRequiredParam<std::string>("parallel_storage",
      24             :                                        "The name of the parallel storage reporter.");
      25        1716 :   params.addRequiredParam<std::string>("solution_container",
      26             :                                        "The name of the solution container on the subapp.");
      27        1716 :   params.addRequiredParam<std::vector<VariableName>>(
      28             :       "variables",
      29             :       "The names of the variables which should be serialized and transferred to this application.");
      30        1716 :   params.addParam<bool>("serialize_on_root",
      31        1716 :                         false,
      32             :                         "If we want to gather the solution fields only on the root processors of "
      33             :                         "the subapps before transfering to the main app.");
      34         858 :   return params;
      35           0 : }
      36             : 
      37         429 : SerializedSolutionTransfer::SerializedSolutionTransfer(const InputParameters & parameters)
      38             :   : StochasticToolsTransfer(parameters),
      39         858 :     _variable_names(getParam<std::vector<VariableName>>("variables")),
      40        1287 :     _serialize_on_root(getParam<bool>("serialize_on_root"))
      41             : {
      42         429 :   if (hasToMultiApp())
      43           0 :     paramError("to_multi_app", "To and between multiapp directions are not implemented");
      44         429 : }
      45             : 
      46             : void
      47         429 : SerializedSolutionTransfer::initialSetup()
      48             : {
      49             :   // Check if we have the storage space to receive the serialized solution fields
      50         429 :   _parallel_storage = &_fe_problem.getUserObject<ParallelSolutionStorage>(
      51         429 :       getParam<std::string>("parallel_storage"));
      52         429 : }
      53             : 
      54             : void
      55          88 : SerializedSolutionTransfer::initializeInNormalMode()
      56             : {
      57          88 :   _solution_container.clear();
      58          88 :   const auto n = getFromMultiApp()->numGlobalApps();
      59          88 :   const auto & serialized_solution_reporter = getParam<std::string>("solution_container");
      60             : 
      61         440 :   for (MooseIndex(n) i = 0; i < n; i++)
      62         704 :     if (getFromMultiApp()->hasLocalApp(i))
      63             :     {
      64         176 :       FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(i);
      65             :       _solution_container.push_back(
      66         176 :           &app_problem.getUserObject<SolutionContainer>(serialized_solution_reporter));
      67             :     }
      68          88 : }
      69             : 
      70             : void
      71        1276 : SerializedSolutionTransfer::initializeInBatchMode()
      72             : {
      73             :   // First we fetch the solution containers from the subapps. This function is used
      74             :   // in batch mode only so we will have one solution container on each rank
      75        1276 :   _solution_container.clear();
      76             : 
      77        1276 :   FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(_app_index);
      78             : 
      79             :   _solution_container.push_back(
      80        2552 :       &app_problem.getUserObject<SolutionContainer>(getParam<std::string>("solution_container")));
      81        1276 : }
      82             : 
      83             : void
      84          88 : SerializedSolutionTransfer::execute()
      85             : {
      86          88 :   initializeInNormalMode();
      87             : 
      88          88 :   const auto n = getFromMultiApp()->numGlobalApps();
      89             : 
      90         440 :   for (MooseIndex(n) i = 0; i < n; i++)
      91             :   {
      92         704 :     if (getFromMultiApp()->hasLocalApp(i))
      93             :     {
      94         176 :       FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(i);
      95             : 
      96             :       // Converting the local indexing to global sample indices
      97         176 :       const unsigned int local_i = i - _sampler_ptr->getLocalRowBegin();
      98             : 
      99             :       // Here we have to branch out based on if only the root processors
     100             :       // need to participate in the transfer or if we would like to distribute the
     101             :       // data among every processor of the subapplication
     102         176 :       if (_serialize_on_root)
     103          88 :         transferToSubAppRoot(app_problem, *_solution_container[local_i], i);
     104             :       else
     105          88 :         transferInParallel(app_problem, *_solution_container[local_i], i);
     106             :     }
     107             :   }
     108          88 : }
     109             : 
     110             : void
     111        1276 : SerializedSolutionTransfer::executeFromMultiapp()
     112             : {
     113        1276 :   initializeInBatchMode();
     114             : 
     115        2552 :   if (getFromMultiApp()->hasLocalApp(_app_index))
     116             :   {
     117        1276 :     FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(_app_index);
     118             : 
     119             :     // Here we have to branch out based on if only the root processors
     120             :     // need to participate in the transfer or if we would like to distribute the
     121             :     // data among every processor of the subapplication
     122        1276 :     if (_serialize_on_root)
     123         308 :       transferToSubAppRoot(app_problem, *_solution_container[0], _global_index);
     124             :     else
     125         968 :       transferInParallel(app_problem, *_solution_container[0], _global_index);
     126             :   }
     127        1276 : }
     128             : 
     129             : void
     130        1056 : SerializedSolutionTransfer::transferInParallel(FEProblemBase & app_problem,
     131             :                                                SolutionContainer & solution_container,
     132             :                                                const dof_id_type global_i)
     133             : {
     134        1056 :   unsigned int local_app_index = global_i - _sampler_ptr->getLocalRowBegin();
     135             : 
     136             :   // Looping over the variables to extract the corresponding solution values
     137             :   // and copy them into the container.
     138        2376 :   for (unsigned int var_i = 0; var_i < _variable_names.size(); ++var_i)
     139             :   {
     140        1320 :     SystemBase & system = getSystem(app_problem, _variable_names[var_i]);
     141             : 
     142             :     // We need to go through this communicator because the multiapp's
     143             :     // communicator is not necessarily the communicator of the underlying MooseObject.
     144             :     const auto & comm = system.comm();
     145        1320 :     dof_id_type num_entries = _sampler_ptr->getNumberOfLocalRows();
     146        1320 :     comm.sum(num_entries);
     147             : 
     148             :     // We shall distribute the samples on the given application between its processors.
     149             :     // Only using a linear partitioning here for the sake of simplicity.
     150             :     dof_id_type new_local_entries_begin;
     151             :     dof_id_type new_local_entries_end;
     152             :     dof_id_type num_new_local_entries;
     153             : 
     154        1320 :     MooseUtils::linearPartitionItems(num_entries,
     155             :                                      comm.size(),
     156             :                                      comm.rank(),
     157             :                                      num_new_local_entries,
     158             :                                      new_local_entries_begin,
     159             :                                      new_local_entries_end);
     160             : 
     161             :     // Getting the corresponding DoF indices for the variable.
     162        1320 :     system.setVariableGlobalDoFs(_variable_names[var_i]);
     163             : 
     164        2640 :     for (const auto & snapshot : solution_container.getSnapshots())
     165             :     {
     166        1320 :       DenseVector<Real> serialized_solution;
     167             : 
     168             :       // Localize the solution and add it to the local container on the rank
     169             :       // which is supposed to own it
     170        1320 :       snapshot.localize(
     171             :           serialized_solution.get_values(),
     172         990 :           (local_app_index >= new_local_entries_begin && local_app_index < new_local_entries_end)
     173        3300 :               ? system.getVariableGlobalDoFs()
     174             :               : std::vector<dof_id_type>());
     175             : 
     176        1320 :       if (local_app_index >= new_local_entries_begin && local_app_index < new_local_entries_end)
     177         660 :         _parallel_storage->addEntry(_variable_names[var_i], global_i, serialized_solution);
     178             :     }
     179             :   }
     180        1056 : }
     181             : 
     182             : void
     183         396 : SerializedSolutionTransfer::transferToSubAppRoot(FEProblemBase & app_problem,
     184             :                                                  SolutionContainer & solution_container,
     185             :                                                  const dof_id_type global_i)
     186             : {
     187             :   // Looping over the variables to extract the corresponding solution values
     188        1056 :   for (unsigned int var_i = 0; var_i < _variable_names.size(); ++var_i)
     189             :   {
     190         660 :     SystemBase & system = getSystem(app_problem, _variable_names[var_i]);
     191             : 
     192             :     // Getting the corresponding DoF indices for the variable.
     193         660 :     system.setVariableGlobalDoFs(_variable_names[var_i]);
     194             : 
     195        1320 :     for (const auto & snapshot : solution_container.getSnapshots())
     196             :     {
     197         660 :       DenseVector<Real> serialized_solution;
     198             : 
     199             :       // In this case we always serialize on the root processor of the application.
     200         660 :       snapshot.localize(serialized_solution.get_values(),
     201        1320 :                         getFromMultiApp()->isRootProcessor() ? system.getVariableGlobalDoFs()
     202             :                                                              : std::vector<dof_id_type>());
     203             : 
     204        1320 :       if (getFromMultiApp()->isRootProcessor())
     205         352 :         _parallel_storage->addEntry(_variable_names[var_i], global_i, serialized_solution);
     206             :     }
     207             :   }
     208         396 : }
     209             : 
     210             : SystemBase &
     211        1980 : SerializedSolutionTransfer::getSystem(FEProblemBase & app_problem, const VariableName & vname)
     212             : {
     213        1980 :   auto & variable = app_problem.getVariable(_tid, vname);
     214        1980 :   return variable.sys();
     215             : }