Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : // C POSIX includes
      11             : #include <sstream>
      12             : #include <sys/stat.h>
      13             : 
      14             : #include <system_error>
      15             : 
      16             : // Moose includes
      17             : #include "Checkpoint.h"
      18             : #include "FEProblem.h"
      19             : #include "MooseApp.h"
      20             : #include "MaterialPropertyStorage.h"
      21             : #include "MooseMesh.h"
      22             : #include "MeshMetaDataInterface.h"
      23             : #include "RestartableDataWriter.h"
      24             : 
      25             : #include "libmesh/checkpoint_io.h"
      26             : #include "libmesh/enum_xdr_mode.h"
      27             : #include "libmesh/utility.h"
      28             : 
      29             : using namespace libMesh;
      30             : 
      31             : registerMooseObject("MooseApp", Checkpoint);
      32             : 
      33             : InputParameters
      34      102506 : Checkpoint::validParams()
      35             : {
      36             :   // Get the parameters from the base classes
      37      102506 :   InputParameters params = FileOutput::validParams();
      38             : 
      39      205012 :   params.addClassDescription("Output for MOOSE recovery checkpoint files.");
      40             : 
      41             :   // Typical checkpoint options
      42      410024 :   params.addParam<unsigned int>("num_files", 2, "Number of the restart files to save");
      43      410024 :   params.addParam<std::string>(
      44             :       "suffix",
      45             :       "cp",
      46             :       "This will be appended to the file_base to create the directory name for checkpoint files.");
      47             :   // For checkpoints, set the wall time output interval to defualt of 1 hour (3600 s)
      48      307518 :   params.addParam<Real>(
      49      205012 :       "wall_time_interval", 3600, "The target wall time interval (in seconds) at which to output");
      50             : 
      51             :   // Since it makes the most sense to write checkpoints at the end of time steps,
      52             :   // change the default value of execute_on to TIMESTEP_END
      53      102506 :   ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on", true);
      54      205012 :   exec_enum = {EXEC_TIMESTEP_END};
      55             : 
      56      102506 :   return params;
      57      102506 : }
      58             : 
      59       49708 : Checkpoint::Checkpoint(const InputParameters & parameters)
      60             :   : FileOutput(parameters),
      61       49708 :     _num_files(getParam<unsigned int>("num_files")),
      62      198832 :     _suffix(getParam<std::string>("suffix"))
      63             : {
      64             :   // Prevent the checkpoint from executing at any time other than INITIAL,
      65             :   // TIMESTEP_END, and FINAL
      66       49708 :   validateExecuteOn();
      67             : 
      68             :   // The following updates the value of _wall_time_interval if the
      69             :   // '--output-wall-time-interval' command line parameter is used.
      70             :   // If it is not used, _wall_time_interval keeps its current value.
      71             :   // 'The --output-wall-time-interval parameter is necessary for testing
      72             :   // and should only be used in the test suite.
      73       49705 :   Output::setWallTimeIntervalFromCommandLineParam();
      74       49705 : }
      75             : 
      76             : std::string
      77       62040 : Checkpoint::filename()
      78             : {
      79             :   // Get the time step with correct zero padding
      80       62040 :   std::ostringstream output;
      81      124080 :   output << directory() << "/" << std::setw(_padding) << std::setprecision(0) << std::setfill('0')
      82       62040 :          << std::right << timeStep();
      83             : 
      84      124080 :   return output.str();
      85       62040 : }
      86             : 
      87             : std::string
      88       75151 : Checkpoint::directory() const
      89             : {
      90      150302 :   return _file_base + "_" + _suffix;
      91             : }
      92             : 
      93             : bool
      94      307622 : Checkpoint::shouldOutput()
      95             : {
      96             :   // should_output_parent ensures that we output only when _execute_on contains
      97             :   // _current_execute_flag (see Output::shouldOutput), ensuring that we wait
      98             :   // until the end of the timestep to write, preventing the output of an
      99             :   // unconverged solution.
     100      307622 :   const bool should_output_parent = FileOutput::shouldOutput();
     101      307622 :   if (!should_output_parent)
     102      294511 :     return false; // No point in continuing
     103             : 
     104             :   // Check for signal
     105             :   // Reading checkpoint on time step 0 is not supported
     106       13111 :   const bool should_output_signal = (Moose::interrupt_signal_number != 0) && (timeStep() > 0);
     107       13111 :   if (should_output_signal)
     108             :   {
     109          22 :     _console << "Unix signal SIGUSR1 detected. Outputting checkpoint file.\n";
     110             :     // Reset signal number since we output
     111          22 :     Moose::interrupt_signal_number = 0;
     112          22 :     return true;
     113             :   }
     114             : 
     115             :   // Check if enough wall time has elapsed to output
     116       13089 :   const bool should_output_wall_time = _wall_time_since_last_output >= _wall_time_interval;
     117       13089 :   if (should_output_wall_time)
     118         275 :     return true;
     119             : 
     120             :   // Check if the checkpoint should "normally" output, i.e. if it was created
     121             :   // through the input file
     122       12814 :   const bool should_output = (onInterval() || _current_execute_flag == EXEC_FINAL);
     123             : 
     124       12814 :   return should_output;
     125             : }
     126             : 
     127             : void
     128       13111 : Checkpoint::output()
     129             : {
     130             :   // Create the output directory
     131       13111 :   const auto cp_dir = directory();
     132       13111 :   Utility::mkdir(cp_dir.c_str());
     133             : 
     134             :   // Create the output filename
     135       13111 :   const auto current_file = filename();
     136             : 
     137             :   // Create checkpoint file structure
     138       13111 :   CheckpointFileNames curr_file_struct;
     139             : 
     140       13111 :   curr_file_struct.checkpoint = current_file + _app.checkpointSuffix();
     141             : 
     142       13111 :   const auto mesh_paths = _problem_ptr->mesh().writeRecoveryFiles(curr_file_struct.checkpoint);
     143       26222 :   curr_file_struct.restart.insert(
     144       13111 :       curr_file_struct.restart.end(), mesh_paths.begin(), mesh_paths.end());
     145             : 
     146             :   // Write out meta data if there is any (only on processor zero)
     147       13111 :   if (processor_id() == 0)
     148             :   {
     149       11466 :     const auto paths = _app.writeRestartableMetaData(curr_file_struct.checkpoint);
     150       11466 :     curr_file_struct.restart.insert(curr_file_struct.restart.begin(), paths.begin(), paths.end());
     151       11466 :   }
     152             : 
     153             :   // Write out the backup
     154       13111 :   const auto paths = _app.backup(_app.restartFolderBase(current_file));
     155       13108 :   curr_file_struct.restart.insert(curr_file_struct.restart.begin(), paths.begin(), paths.end());
     156             : 
     157             :   // Remove old checkpoint files
     158       13108 :   updateCheckpointFiles(curr_file_struct);
     159       13108 : }
     160             : 
     161             : void
     162       13108 : Checkpoint::updateCheckpointFiles(CheckpointFileNames file_struct)
     163             : {
     164             :   // It is possible to have already written a checkpoint with the same file
     165             :   // names contained in file_struct. If this is the case, file_struct will
     166             :   // already be stored in _file_names. When this happens, the current state of
     167             :   // the simulation is likely different than the state when the duplicately
     168             :   // named checkpoint was last written. Because of this, we want to go ahead and
     169             :   // rewrite the duplicately named checkpoint, overwritting the files
     170             :   // representing the old state. For accurate bookkeeping, we will delete the
     171             :   // existing instance of file_struct from _file_names and re-append it to the
     172             :   // end of _file_names (to keep the order in which checkpoints are written
     173             :   // accurate).
     174             : 
     175       13108 :   const auto it = std::find(_file_names.begin(), _file_names.end(), file_struct);
     176             :   // file_struct was found in _file_names.
     177             :   // Delete it so it can be re-added as the last element.
     178       13108 :   if (it != _file_names.end())
     179         121 :     _file_names.erase(it);
     180             : 
     181       13108 :   _file_names.push_back(file_struct);
     182             : 
     183             :   // Remove the file and the corresponding directory if it's empty
     184       23492 :   const auto remove_file = [this](const std::filesystem::path & path)
     185             :   {
     186       23492 :     std::error_code err;
     187             : 
     188       23492 :     if (!std::filesystem::remove(path, err))
     189           0 :       mooseWarning("Error during the deletion of checkpoint file\n",
     190           0 :                    std::filesystem::absolute(path),
     191             :                    "\n\n",
     192           0 :                    err.message());
     193             : 
     194       23492 :     const auto dir = path.parent_path();
     195       23492 :     if (std::filesystem::is_empty(dir))
     196       11709 :       if (!std::filesystem::remove(dir, err))
     197           0 :         mooseError("Error during the deletion of checkpoint directory\n",
     198           0 :                    std::filesystem::absolute(dir),
     199             :                    "\n\n",
     200           0 :                    err.message());
     201       23492 :   };
     202             : 
     203             :   // Remove un-wanted files
     204       13108 :   if (_file_names.size() > _num_files)
     205             :   {
     206             :     // Extract the filenames to be removed
     207        6128 :     CheckpointFileNames delete_files = _file_names.front();
     208             : 
     209             :     // Remove these filenames from the list
     210        6128 :     _file_names.pop_front();
     211             : 
     212             :     // Delete restartable data
     213       29620 :     for (const auto & path : delete_files.restart)
     214       23492 :       remove_file(path);
     215             : 
     216             :     // Delete checkpoint files
     217             :     // This file may not exist so don't worry about checking for success
     218        6128 :     if (processor_id() == 0)
     219        5581 :       CheckpointIO::cleanup(delete_files.checkpoint,
     220        5581 :                             _problem_ptr->mesh().isDistributedMesh() ? comm().size() : 1);
     221        6128 :   }
     222       13108 : }
     223             : 
     224             : void
     225       49708 : Checkpoint::validateExecuteOn() const
     226             : {
     227       99416 :   const auto & execute_on = getParam<ExecFlagEnum>("execute_on");
     228      248540 :   const std::set<ExecFlagType> allowed = {EXEC_INITIAL, EXEC_TIMESTEP_END, EXEC_FINAL};
     229      100035 :   for (const auto & value : execute_on)
     230       50330 :     if (!allowed.count(value))
     231           6 :       paramError("execute_on",
     232             :                  "The exec flag ",
     233             :                  value,
     234             :                  " is not allowed. Allowed flags are INITIAL, TIMESTEP_END, and FINAL.");
     235       99413 : }
     236             : 
     237             : std::stringstream
     238       75125 : Checkpoint::checkpointInfo() const
     239             : {
     240             :   static const unsigned int console_field_width = 27;
     241       75125 :   std::stringstream checkpoint_info;
     242             : 
     243       75125 :   std::stringstream interval_info_ss;
     244       75125 :   interval_info_ss << "Every " << std::defaultfloat << _wall_time_interval << " s";
     245       75125 :   const std::string interval_info = interval_info_ss.str();
     246             : 
     247       75125 :   checkpoint_info << std::left << std::setw(console_field_width)
     248       75125 :                   << "  Wall Time Interval:" << interval_info << "\n";
     249             : 
     250       75125 :   const std::string user_info = "Outputs/" + name();
     251             : 
     252       75125 :   checkpoint_info << std::left << std::setw(console_field_width) << "  Checkpoint:" << user_info
     253       75125 :                   << "\n";
     254             : 
     255       75125 :   checkpoint_info << std::left << std::setw(console_field_width)
     256       75125 :                   << "  # Checkpoints Kept:" << std::to_string(_num_files) << "\n";
     257       75125 :   std::string exec_on_values = "";
     258      151279 :   for (const auto & item : _execute_on)
     259       76154 :     exec_on_values += item.name() + " ";
     260       75125 :   checkpoint_info << std::left << std::setw(console_field_width)
     261       75125 :                   << "  Execute On:" << exec_on_values << "\n";
     262             : 
     263      150250 :   return checkpoint_info;
     264       75125 : }