Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : #include "FullSolveMultiApp.h"
      11             : #include "LayeredSideDiffusiveFluxAverage.h"
      12             : #include "Executioner.h"
      13             : #include "TransientBase.h"
      14             : #include "FEProblemBase.h"
      15             : #include "MaterialPropertyStorage.h"
      16             : #include "Console.h"
      17             : 
      18             : // libMesh
      19             : #include "libmesh/mesh_tools.h"
      20             : 
      21             : registerMooseObject("MooseApp", FullSolveMultiApp);
      22             : 
      23             : InputParameters
      24       11378 : FullSolveMultiApp::validParams()
      25             : {
      26       11378 :   InputParameters params = MultiApp::validParams();
      27       22756 :   params.addClassDescription("Performs a complete simulation during each execution.");
      28       34134 :   params.addParam<bool>(
      29             :       "keep_full_output_history",
      30       22756 :       false,
      31             :       "Whether or not to keep the full output history when this multiapp has multiple entries");
      32       34134 :   params.addParam<bool>("ignore_solve_not_converge",
      33       22756 :                         false,
      34             :                         "True to continue main app even if a sub app's solve does not converge.");
      35       22756 :   params.addParam<bool>("update_old_solution_when_keeping_solution_during_restore",
      36       22756 :                         true,
      37             :                         "Whether to update the old solution vector (to the previous fixed point "
      38             :                         "iteration solution) when keeping the solution during restore.");
      39       11378 :   return params;
      40           0 : }
      41             : 
      42        2627 : FullSolveMultiApp::FullSolveMultiApp(const InputParameters & parameters)
      43             :   : MultiApp(parameters),
      44        2627 :     _ignore_diverge(getParam<bool>("ignore_solve_not_converge")),
      45        2627 :     _update_old_state_when_keeping_solution_during_restore(
      46       10508 :         getParam<bool>("update_old_solution_when_keeping_solution_during_restore"))
      47             : {
      48             :   // You could end up with some dirty hidden behavior if you do this. We could remove this check,
      49             :   // but I don't think that it's sane to do so.
      50        2627 :   if (_no_restore && (_app.isRecovering() || _app.isRestarting()))
      51           0 :     paramError("no_restore",
      52             :                "The parent app is restarting or recovering, restoration cannot be disabled");
      53             :   // Force the user to make a decision on updating or not the old state of variables
      54        2675 :   if (_keep_solution_during_restore &&
      55        2771 :       !isParamSetByUser("update_old_solution_when_keeping_solution_during_restore"))
      56           0 :     paramError("update_old_solution_when_keeping_solution_during_restore",
      57             :                "Due to 'keep_solution_during_restore' parameter being true, which is an "
      58             :                "optimization for fixed point iterations, the "
      59             :                "unrestored solution will be kept as the starting solution for the next solve "
      60             :                "of the MultiApp. You must set this parameter to decide if this solution should "
      61             :                "be copied as the old solution at the beginning of the next time step, "
      62             :                "or not. If the MultiApp is running a transient, you likely want to set this to "
      63             :                "true. If the MultiApp is a quasi-static simulation, you likely want to set "
      64             :                "this to false. If you don't know what this error message means, please set "
      65             :                "'keep_solution_during_restore' to false and no need to set "
      66             :                "'update_old_solution_when_keeping_solution_during_restore'.");
      67        7929 :   if (isParamSetByUser("update_old_solution_when_keeping_solution_during_restore") &&
      68          48 :       !_keep_solution_during_restore)
      69           0 :     paramError("update_old_solution_when_keeping_solution_during_restore",
      70             :                "Should not be set if not keeping the solution during restore "
      71             :                "(keep_solution_during_restore=false)");
      72        2627 : }
      73             : 
      74             : void
      75        1306 : FullSolveMultiApp::restore(bool force)
      76             : {
      77        1306 :   if (!_no_restore)
      78         901 :     MultiApp::restore(force);
      79        1303 : }
      80             : 
      81             : void
      82        2543 : FullSolveMultiApp::initialSetup()
      83             : {
      84        2543 :   MultiApp::initialSetup();
      85             : 
      86        2543 :   if (_has_an_app)
      87             :   {
      88        2543 :     Moose::ScopedCommSwapper swapper(_my_comm);
      89             : 
      90        2543 :     _executioners.resize(_my_num_apps);
      91             : 
      92             :     // Grab Executioner from each app
      93        6480 :     for (unsigned int i = 0; i < _my_num_apps; i++)
      94             :     {
      95        3940 :       auto & app = _apps[i];
      96        3940 :       Executioner * ex = app->getExecutioner();
      97             : 
      98        3940 :       if (!ex)
      99           0 :         mooseError("Executioner does not exist!");
     100             : 
     101        3940 :       if (_ignore_diverge)
     102             :       {
     103          15 :         TransientBase * tex = dynamic_cast<TransientBase *>(ex);
     104          15 :         if (tex && tex->parameters().get<bool>("error_on_dtmin"))
     105           3 :           mooseError("Requesting to ignore failed solutions, but 'Executioner/error_on_dtmin' is "
     106             :                      "true in sub-application. Set this parameter to false in sub-application to "
     107             :                      "avoid an error if Transient solve fails.");
     108             :       }
     109             : 
     110        3937 :       ex->init();
     111             : 
     112        3950 :       if (_keep_solution_during_restore && _update_old_state_when_keeping_solution_during_restore &&
     113          13 :           (appProblemBase(_first_local_app + i).getMaterialPropertyStorage().hasStatefulProperties()
     114             : #ifdef KOKKOS_ENABLED
     115             :            || appProblemBase(_first_local_app + i)
     116             :                   .getKokkosMaterialPropertyStorage()
     117             :                   .hasStatefulProperties()
     118             : #endif
     119             :                ))
     120           0 :         paramError(
     121             :             "update_old_solution_when_keeping_solution_during_restore",
     122             :             "While we are updating old solutions using the solution from the previous fixed "
     123             :             "point iteration, we are not updating the old stateful material properties as "
     124             :             "well. This is not consistent. We recommend you consider using the 'no_restore' "
     125             :             "parameter instead of 'keep_solution_during_restore', or stop using the latter.");
     126        3985 :       if (_keep_solution_during_restore &&
     127          48 :           appProblemBase(_first_local_app + i)
     128          48 :               .hasSolutionState(2, Moose::SolutionIterationType::Time))
     129           0 :         mooseDoOnce(paramWarning(
     130             :             "keep_solution_during_restore",
     131             :             "This FullSolveMultiApp simulation(s) uses older time step variable states (notably "
     132             :             "from two time steps prior in transients). Due to 'keep_solution_during_restore' "
     133             :             "parameter being true, which is an optimization for fixed point iterations, the "
     134             :             "unrestored solution will be kept as the starting solution. It would normally be "
     135             :             "copied onto the 'old' state at the beginning of the first time step. This copy can be "
     136             :             "skipped using 'update_old_solution_when_keeping_solution_during_restore', while the "
     137             :             "copy of the old state onto the 'older' state and the stateful material properties "
     138             :             "state updates do not have such an option at this time. This warning relates to this "
     139             :             "inconsistency. If you suspect this is a problem, please set "
     140             :             "'keep_solution_during_restore' to false"));
     141             : 
     142        3937 :       _executioners[i] = ex;
     143             :     }
     144        2540 :   }
     145        2540 : }
     146             : 
     147             : bool
     148        3691 : FullSolveMultiApp::solveStep(Real /*dt*/, Real /*target_time*/, bool auto_advance)
     149             : {
     150        3691 :   if (!auto_advance)
     151           0 :     mooseError("FullSolveMultiApp is not compatible with auto_advance=false");
     152             : 
     153        3691 :   if (!_has_an_app)
     154           0 :     return true;
     155             : 
     156        3691 :   TIME_SECTION(_solve_step_timer);
     157             : 
     158        3691 :   Moose::ScopedCommSwapper swapper(_my_comm);
     159             : 
     160             :   int rank;
     161             :   int ierr;
     162        3691 :   ierr = MPI_Comm_rank(_communicator.get(), &rank);
     163        3691 :   mooseCheckMPIErr(ierr);
     164             : 
     165        3691 :   bool last_solve_converged = true;
     166        8569 :   for (unsigned int i = 0; i < _my_num_apps; i++)
     167             :   {
     168             :     // reset output system if desired
     169       14652 :     if (!getParam<bool>("keep_full_output_history"))
     170        4602 :       _apps[i]->getOutputWarehouse().reset();
     171             :     // Prevent the copy of the post-FP iteration solution state onto the old vector
     172        4884 :     if (!_update_old_state_when_keeping_solution_during_restore)
     173         152 :       appProblemBase(_first_local_app + i).skipNextForwardSolutionCopyToOld();
     174             : 
     175        4884 :     Executioner * ex = _executioners[i];
     176        4884 :     ex->execute();
     177             : 
     178        4878 :     last_solve_converged = last_solve_converged && ex->lastSolveConverged();
     179             : 
     180        4878 :     showStatusMessage(i);
     181             :   }
     182             : 
     183        3685 :   return last_solve_converged || _ignore_diverge;
     184        3685 : }
     185             : 
     186             : void
     187        4878 : FullSolveMultiApp::showStatusMessage(unsigned int i) const
     188             : {
     189        9566 :   if (!_fe_problem.verboseMultiApps() &&
     190        9566 :       _apps[i]->getOutputWarehouse().getOutputs<Console>().size() > 0)
     191        4688 :     return;
     192         190 :   else if (!_executioners[i]->lastSolveConverged())
     193           0 :     _console << COLOR_RED << "Subapp " << _apps[i]->name() << " solve Did NOT Converge!"
     194           0 :              << COLOR_DEFAULT << std::endl;
     195             :   else
     196         190 :     _console << COLOR_GREEN << "Subapp " << _apps[i]->name() << " solve converged!" << COLOR_DEFAULT
     197         190 :              << std::endl;
     198             : }