Checkpoint.C

Go to the documentation of this file.

//* This file is part of the MOOSE framework
//* https://mooseframework.inl.gov
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html

// C POSIX includes
#include <sstream>
#include <sys/stat.h>

#include <system_error>

// Moose includes
#include "Checkpoint.h"
#include "FEProblem.h"
#include "MooseApp.h"
#include "MaterialPropertyStorage.h"
#include "MooseMesh.h"
#include "MeshMetaDataInterface.h"
#include "RestartableDataWriter.h"

#include "libmesh/checkpoint_io.h"
#include "libmesh/enum_xdr_mode.h"
#include "libmesh/utility.h"

using namespace libMesh;

registerMooseObject("MooseApp", Checkpoint);

InputParameters
Checkpoint::validParams()
{
  // Get the parameters from the base classes
  InputParameters params = FileOutput::validParams();

  params.addClassDescription("Output for MOOSE recovery checkpoint files.");

  // Typical checkpoint options
  params.addParam<unsigned int>("num_files", 2, "Number of the restart files to save");
  params.addParam<std::string>(
      "suffix",
      "cp",
      "This will be appended to the file_base to create the directory name for checkpoint files.");
  // For checkpoints, set the wall time output interval to defualt of 1 hour (3600 s)
  params.addParam<Real>(
      "wall_time_interval", 3600, "The target wall time interval (in seconds) at which to output");

  // Since it makes the most sense to write checkpoints at the end of time steps,
  // change the default value of execute_on to TIMESTEP_END
  ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on", true);
  exec_enum = {EXEC_TIMESTEP_END};

  return params;
}

Checkpoint::Checkpoint(const InputParameters & parameters)
  : FileOutput(parameters),
    _num_files(getParam<unsigned int>("num_files")),
    _suffix(getParam<std::string>("suffix"))
{
  // Prevent the checkpoint from executing at any time other than INITIAL,
  // TIMESTEP_END, and FINAL
  validateExecuteOn();

  // The following updates the value of _wall_time_interval if the
  // '--output-wall-time-interval' command line parameter is used.
  // If it is not used, _wall_time_interval keeps its current value.
  // 'The --output-wall-time-interval parameter is necessary for testing
  // and should only be used in the test suite.
  Output::setWallTimeIntervalFromCommandLineParam();
}

std::string
Checkpoint::filename()
{
  // Get the time step with correct zero padding
  std::ostringstream output;
  output << directory() << "/" << std::setw(_padding) << std::setprecision(0) << std::setfill('0')
         << std::right << timeStep();

  return output.str();
}

std::string
Checkpoint::directory() const
{
  return _file_base + "_" + _suffix;
}

bool
Checkpoint::shouldOutput()
{
  // should_output_parent ensures that we output only when _execute_on contains
  // _current_execute_flag (see Output::shouldOutput), ensuring that we wait
  // until the end of the timestep to write, preventing the output of an
  // unconverged solution.
  const bool should_output_parent = FileOutput::shouldOutput();
  if (!should_output_parent)
    return false; // No point in continuing

  // Check for signal
  // Reading checkpoint on time step 0 is not supported
  const bool should_output_signal = (Moose::interrupt_signal_number != 0) && (timeStep() > 0);
  if (should_output_signal)
  {
    _console << "Unix signal SIGUSR1 detected. Outputting checkpoint file.\n";
    // Reset signal number since we output
    Moose::interrupt_signal_number = 0;
    return true;
  }

  // Check if enough wall time has elapsed to output
  const bool should_output_wall_time = _wall_time_since_last_output >= _wall_time_interval;
  if (should_output_wall_time)
    return true;

  // Check if the checkpoint should "normally" output, i.e. if it was created
  // through the input file
  const bool should_output = (onInterval() || _current_execute_flag == EXEC_FINAL);

  return should_output;
}

void
Checkpoint::output()
{
  // Create the output directory
  const auto cp_dir = directory();
  Utility::mkdir(cp_dir.c_str());

  // Create the output filename
  const auto current_file = filename();

  // Create checkpoint file structure
  CheckpointFileNames curr_file_struct;

  curr_file_struct.checkpoint = current_file + _app.checkpointSuffix();

  const auto mesh_paths = _problem_ptr->mesh().writeRecoveryFiles(curr_file_struct.checkpoint);
  curr_file_struct.restart.insert(
      curr_file_struct.restart.end(), mesh_paths.begin(), mesh_paths.end());

  // Write out meta data if there is any (only on processor zero)
  if (processor_id() == 0)
  {
    const auto paths = _app.writeRestartableMetaData(curr_file_struct.checkpoint);
    curr_file_struct.restart.insert(curr_file_struct.restart.begin(), paths.begin(), paths.end());
  }

  // Write out the backup
  const auto paths = _app.backup(_app.restartFolderBase(current_file));
  curr_file_struct.restart.insert(curr_file_struct.restart.begin(), paths.begin(), paths.end());

  // Remove old checkpoint files
  updateCheckpointFiles(curr_file_struct);
}

void
Checkpoint::updateCheckpointFiles(CheckpointFileNames file_struct)
{
  // It is possible to have already written a checkpoint with the same file
  // names contained in file_struct. If this is the case, file_struct will
  // already be stored in _file_names. When this happens, the current state of
  // the simulation is likely different than the state when the duplicately
  // named checkpoint was last written. Because of this, we want to go ahead and
  // rewrite the duplicately named checkpoint, overwritting the files
  // representing the old state. For accurate bookkeeping, we will delete the
  // existing instance of file_struct from _file_names and re-append it to the
  // end of _file_names (to keep the order in which checkpoints are written
  // accurate).

  const auto it = std::find(_file_names.begin(), _file_names.end(), file_struct);
  // file_struct was found in _file_names.
  // Delete it so it can be re-added as the last element.
  if (it != _file_names.end())
    _file_names.erase(it);

  _file_names.push_back(file_struct);

  // Remove the file and the corresponding directory if it's empty
  const auto remove_file = [this](const std::filesystem::path & path)
  {
    std::error_code err;

    if (!std::filesystem::remove(path, err))
      mooseWarning("Error during the deletion of checkpoint file\n",
                   std::filesystem::absolute(path),
                   "\n\n",
                   err.message());

    const auto dir = path.parent_path();
    if (std::filesystem::is_empty(dir))
      if (!std::filesystem::remove(dir, err))
        mooseError("Error during the deletion of checkpoint directory\n",
                   std::filesystem::absolute(dir),
                   "\n\n",
                   err.message());
  };

  // Remove un-wanted files
  if (_file_names.size() > _num_files)
  {
    // Extract the filenames to be removed
    CheckpointFileNames delete_files = _file_names.front();

    // Remove these filenames from the list
    _file_names.pop_front();

    // Delete restartable data
    for (const auto & path : delete_files.restart)
      remove_file(path);

    // Delete checkpoint files
    // This file may not exist so don't worry about checking for success
    if (processor_id() == 0)
      CheckpointIO::cleanup(delete_files.checkpoint,
                            _problem_ptr->mesh().isDistributedMesh() ? comm().size() : 1);
  }
}

void
Checkpoint::validateExecuteOn() const
{
  const auto & execute_on = getParam<ExecFlagEnum>("execute_on");
  const std::set<ExecFlagType> allowed = {EXEC_INITIAL, EXEC_TIMESTEP_END, EXEC_FINAL};
  for (const auto & value : execute_on)
    if (!allowed.count(value))
      paramError("execute_on",
                 "The exec flag ",
                 value,
                 " is not allowed. Allowed flags are INITIAL, TIMESTEP_END, and FINAL.");
}

std::stringstream
Checkpoint::checkpointInfo() const
{
  static const unsigned int console_field_width = 27;
  std::stringstream checkpoint_info;

  std::stringstream interval_info_ss;
  interval_info_ss << "Every " << std::defaultfloat << _wall_time_interval << " s";
  const std::string interval_info = interval_info_ss.str();

  checkpoint_info << std::left << std::setw(console_field_width)
                  << "  Wall Time Interval:" << interval_info << "\n";

  const std::string user_info = "Outputs/" + name();

  checkpoint_info << std::left << std::setw(console_field_width) << "  Checkpoint:" << user_info
                  << "\n";

  checkpoint_info << std::left << std::setw(console_field_width)
                  << "  # Checkpoints Kept:" << std::to_string(_num_files) << "\n";
  std::string exec_on_values = "";
  for (const auto & item : _execute_on)
    exec_on_values += item.name() + " ";
  checkpoint_info << std::left << std::setw(console_field_width)
                  << "  Execute On:" << exec_on_values << "\n";

  return checkpoint_info;
}