Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : // StochasticTools includes
      11             : #include "SerializedSolutionTransfer.h"
      12             : #include "NonlinearSystemBase.h"
      13             : #include "Sampler.h"
      14             : 
      15             : registerMooseObject("StochasticToolsApp", SerializedSolutionTransfer);
      16             : 
      17             : InputParameters
      18         780 : SerializedSolutionTransfer::validParams()
      19             : {
      20         780 :   InputParameters params = StochasticToolsTransfer::validParams();
      21         780 :   params.addClassDescription(
      22             :       "Serializes and transfers solution vectors for given variables from sub-applications.");
      23        1560 :   params.addRequiredParam<std::string>("parallel_storage",
      24             :                                        "The name of the parallel storage reporter.");
      25        1560 :   params.addRequiredParam<std::string>("solution_container",
      26             :                                        "The name of the solution container on the subapp.");
      27        1560 :   params.addRequiredParam<std::vector<VariableName>>(
      28             :       "variables",
      29             :       "The names of the variables which should be serialized and transferred to this application.");
      30        1560 :   params.addParam<bool>("serialize_on_root",
      31        1560 :                         false,
      32             :                         "If we want to gather the solution fields only on the root processors of "
      33             :                         "the subapps before transfering to the main app.");
      34         780 :   return params;
      35           0 : }
      36             : 
      37         390 : SerializedSolutionTransfer::SerializedSolutionTransfer(const InputParameters & parameters)
      38             :   : StochasticToolsTransfer(parameters),
      39         780 :     _variable_names(getParam<std::vector<VariableName>>("variables")),
      40        1170 :     _serialize_on_root(getParam<bool>("serialize_on_root"))
      41             : {
      42         390 :   if (hasToMultiApp())
      43           0 :     paramError("to_multi_app", "To and between multiapp directions are not implemented");
      44         390 : }
      45             : 
      46             : void
      47         390 : SerializedSolutionTransfer::initialSetup()
      48             : {
      49             :   // Check if we have the storage space to receive the serialized solution fields
      50         390 :   _parallel_storage = &_fe_problem.getUserObject<ParallelSolutionStorage>(
      51         390 :       getParam<std::string>("parallel_storage"));
      52         390 : }
      53             : 
      54             : void
      55          80 : SerializedSolutionTransfer::initializeInNormalMode()
      56             : {
      57          80 :   _solution_container.clear();
      58          80 :   const auto n = getFromMultiApp()->numGlobalApps();
      59          80 :   const auto & serialized_solution_reporter = getParam<std::string>("solution_container");
      60             : 
      61         400 :   for (MooseIndex(n) i = 0; i < n; i++)
      62         640 :     if (getFromMultiApp()->hasLocalApp(i))
      63             :     {
      64         160 :       FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(i);
      65             :       _solution_container.push_back(
      66         160 :           &app_problem.getUserObject<SolutionContainer>(serialized_solution_reporter));
      67             :     }
      68          80 : }
      69             : 
      70             : void
      71        1160 : SerializedSolutionTransfer::initializeInBatchMode()
      72             : {
      73             :   // First we fetch the solution containers from the subapps. This function is used
      74             :   // in batch mode only so we will have one solution container on each rank
      75        1160 :   _solution_container.clear();
      76             : 
      77        1160 :   FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(_app_index);
      78             : 
      79             :   _solution_container.push_back(
      80        2320 :       &app_problem.getUserObject<SolutionContainer>(getParam<std::string>("solution_container")));
      81        1160 : }
      82             : 
      83             : void
      84          80 : SerializedSolutionTransfer::execute()
      85             : {
      86          80 :   initializeInNormalMode();
      87             : 
      88          80 :   const auto n = getFromMultiApp()->numGlobalApps();
      89             : 
      90         400 :   for (MooseIndex(n) i = 0; i < n; i++)
      91             :   {
      92         640 :     if (getFromMultiApp()->hasLocalApp(i))
      93             :     {
      94         160 :       FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(i);
      95             : 
      96             :       // Converting the local indexing to global sample indices
      97         160 :       const unsigned int local_i = i - _sampler_ptr->getLocalRowBegin();
      98             : 
      99             :       // Here we have to branch out based on if only the root processors
     100             :       // need to participate in the transfer or if we would like to distribute the
     101             :       // data among every processor of the subapplication
     102         160 :       if (_serialize_on_root)
     103          80 :         transferToSubAppRoot(app_problem, *_solution_container[local_i], i);
     104             :       else
     105          80 :         transferInParallel(app_problem, *_solution_container[local_i], i);
     106             :     }
     107             :   }
     108          80 : }
     109             : 
     110             : void
     111        1160 : SerializedSolutionTransfer::executeFromMultiapp()
     112             : {
     113        1160 :   initializeInBatchMode();
     114             : 
     115        2320 :   if (getFromMultiApp()->hasLocalApp(_app_index))
     116             :   {
     117        1160 :     FEProblemBase & app_problem = getFromMultiApp()->appProblemBase(_app_index);
     118             : 
     119             :     // Here we have to branch out based on if only the root processors
     120             :     // need to participate in the transfer or if we would like to distribute the
     121             :     // data among every processor of the subapplication
     122        1160 :     if (_serialize_on_root)
     123         280 :       transferToSubAppRoot(app_problem, *_solution_container[0], _global_index);
     124             :     else
     125         880 :       transferInParallel(app_problem, *_solution_container[0], _global_index);
     126             :   }
     127        1160 : }
     128             : 
     129             : void
     130         960 : SerializedSolutionTransfer::transferInParallel(FEProblemBase & app_problem,
     131             :                                                SolutionContainer & solution_container,
     132             :                                                const dof_id_type global_i)
     133             : {
     134         960 :   unsigned int local_app_index = global_i - _sampler_ptr->getLocalRowBegin();
     135             : 
     136             :   // Looping over the variables to extract the corresponding solution values
     137             :   // and copy them into the container.
     138        2160 :   for (unsigned int var_i = 0; var_i < _variable_names.size(); ++var_i)
     139             :   {
     140        1200 :     SystemBase & system = getSystem(app_problem, _variable_names[var_i]);
     141             : 
     142             :     // We need to go through this communicator because the multiapp's
     143             :     // communicator is not necessarily the communicator of the underlying MooseObject.
     144             :     const auto & comm = system.comm();
     145        1200 :     dof_id_type num_entries = _sampler_ptr->getNumberOfLocalRows();
     146        1200 :     comm.sum(num_entries);
     147             : 
     148             :     // We shall distribute the samples on the given application between its processors.
     149             :     // Only using a linear partitioning here for the sake of simplicity.
     150             :     dof_id_type new_local_entries_begin;
     151             :     dof_id_type new_local_entries_end;
     152             :     dof_id_type num_new_local_entries;
     153             : 
     154        1200 :     MooseUtils::linearPartitionItems(num_entries,
     155             :                                      comm.size(),
     156             :                                      comm.rank(),
     157             :                                      num_new_local_entries,
     158             :                                      new_local_entries_begin,
     159             :                                      new_local_entries_end);
     160             : 
     161             :     // Getting the corresponding DoF indices for the variable.
     162        1200 :     system.setVariableGlobalDoFs(_variable_names[var_i]);
     163             : 
     164        2400 :     for (const auto & snapshot : solution_container.getSnapshots())
     165             :     {
     166        1200 :       DenseVector<Real> serialized_solution;
     167             : 
     168             :       // Localize the solution and add it to the local container on the rank
     169             :       // which is supposed to own it
     170        1200 :       snapshot.localize(
     171             :           serialized_solution.get_values(),
     172         900 :           (local_app_index >= new_local_entries_begin && local_app_index < new_local_entries_end)
     173        1800 :               ? system.getVariableGlobalDoFs()
     174             :               : std::vector<dof_id_type>());
     175             : 
     176        1200 :       if (local_app_index >= new_local_entries_begin && local_app_index < new_local_entries_end)
     177         600 :         _parallel_storage->addEntry(_variable_names[var_i], global_i, serialized_solution);
     178             :     }
     179             :   }
     180         960 : }
     181             : 
     182             : void
     183         360 : SerializedSolutionTransfer::transferToSubAppRoot(FEProblemBase & app_problem,
     184             :                                                  SolutionContainer & solution_container,
     185             :                                                  const dof_id_type global_i)
     186             : {
     187             :   // Looping over the variables to extract the corresponding solution values
     188         960 :   for (unsigned int var_i = 0; var_i < _variable_names.size(); ++var_i)
     189             :   {
     190         600 :     SystemBase & system = getSystem(app_problem, _variable_names[var_i]);
     191             : 
     192             :     // Getting the corresponding DoF indices for the variable.
     193         600 :     system.setVariableGlobalDoFs(_variable_names[var_i]);
     194             : 
     195        1200 :     for (const auto & snapshot : solution_container.getSnapshots())
     196             :     {
     197         600 :       DenseVector<Real> serialized_solution;
     198             : 
     199             :       // In this case we always serialize on the root processor of the application.
     200         600 :       snapshot.localize(serialized_solution.get_values(),
     201         600 :                         getFromMultiApp()->isRootProcessor() ? system.getVariableGlobalDoFs()
     202             :                                                              : std::vector<dof_id_type>());
     203             : 
     204        1200 :       if (getFromMultiApp()->isRootProcessor())
     205         320 :         _parallel_storage->addEntry(_variable_names[var_i], global_i, serialized_solution);
     206             :     }
     207             :   }
     208         360 : }
     209             : 
     210             : SystemBase &
     211        1800 : SerializedSolutionTransfer::getSystem(FEProblemBase & app_problem, const VariableName & vname)
     212             : {
     213        1800 :   auto & variable = app_problem.getVariable(_tid, vname);
     214        1800 :   return variable.sys();
     215             : }