Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : #ifdef MOOSE_MFEM_ENABLED
      11             : 
      12             : #include "MFEMMesh.h"
      13             : #include "MooseApp.h"
      14             : #include "libmesh/mesh_generation.h"
      15             : 
      16             : #include <fstream>
      17             : 
      18             : registerMooseObject("MooseApp", MFEMMesh);
      19             : 
      20             : InputParameters
      21        6254 : MFEMMesh::validParams()
      22             : {
      23        6254 :   InputParameters params = FileMesh::validParams();
      24       18762 :   params.addParam<unsigned int>(
      25             :       "serial_refine",
      26       12508 :       0,
      27             :       "Number of serial refinements to perform on the mesh. Equivalent to uniform_refine.");
      28       18762 :   params.addParam<unsigned int>(
      29             :       "uniform_refine",
      30       12508 :       0,
      31             :       "Number of serial refinements to perform on the mesh. Equivalent to serial_refine");
      32       18762 :   params.addParam<unsigned int>(
      33       12508 :       "parallel_refine", 0, "Number of parallel refinements to perform on the mesh.");
      34       25016 :   params.addParam<std::string>("displacement", "Optional variable to use for mesh displacement.");
      35       18762 :   params.addParam<bool>("nonconforming",
      36       12508 :                         false,
      37             :                         "Ensures the mesh is non-conforming: necessary for refining quad/hex "
      38             :                         "meshes and load (re)balancing.");
      39       18762 :   params.addParam<bool>("reorder_mesh",
      40       12508 :                         false,
      41             :                         "Determines whether we reorder the mesh to improve dynamic partitioning. "
      42             :                         "Only Hilbert sorting is supported at present.");
      43             : 
      44        6254 :   params.addClassDescription("Class to read in and store an mfem::ParMesh from file.");
      45             : 
      46        6254 :   return params;
      47           0 : }
      48             : 
      49        2058 : MFEMMesh::MFEMMesh(const InputParameters & parameters) : FileMesh(parameters) {}
      50             : 
      51        2390 : MFEMMesh::~MFEMMesh() {}
      52             : 
      53             : void
      54        1702 : MFEMMesh::init()
      55             : {
      56             :   // MooseMesh::init() handles the libMesh dummy mesh:
      57             :   //   - recovery:    reads the libMesh mesh back from its checkpoint file
      58             :   //   - normal run:  calls buildMesh(), which for MFEMMesh builds both the
      59             :   //                  dummy libMesh mesh and the MFEM ParMesh
      60        1702 :   MooseMesh::init();
      61             : 
      62        1702 :   if (_app.isRecovering() && allowRecovery() && _app.isUltimateMaster())
      63             :   {
      64             :     // MooseMesh::init() already restored the libMesh dummy mesh from its checkpoint.
      65             :     // Now restore the MFEM parallel mesh from its own checkpoint file.
      66          22 :     const auto checkpoint_file = _app.getRestartRecoverFileBase() + _app.checkpointSuffix() +
      67          33 :                                  ".mfem.mesh." + std::to_string(this->processor_id());
      68          11 :     std::ifstream input(checkpoint_file);
      69          11 :     if (!input)
      70           0 :       mooseError("Unable to open MFEM recovery mesh file '", checkpoint_file, "'.");
      71             : 
      72          11 :     _mfem_par_mesh = std::make_shared<mfem::ParMesh>(this->comm().get(), input);
      73             : 
      74          33 :     if (isParamSetByUser("displacement"))
      75           3 :       _mesh_displacement_variable.emplace(getParam<std::string>("displacement"));
      76          11 :   }
      77        1702 : }
      78             : 
      79             : void
      80        2047 : MFEMMesh::buildMesh()
      81             : {
      82       10235 :   TIME_SECTION("buildMesh", 2, "Reading Mesh");
      83             : 
      84             :   // Build the MFEM ParMesh from a serial MFEM mesh
      85        2047 :   mfem::Mesh mfem_ser_mesh(getFileName());
      86             : 
      87        6197 :   if (isParamSetByUser("serial_refine") && isParamSetByUser("uniform_refine"))
      88           0 :     paramError(
      89             :         "Cannot define serial_refine and uniform_refine to be nonzero at the same time (they "
      90             :         "are the same variable). Please choose one.\n");
      91             : 
      92        2047 :   uniformRefinement(mfem_ser_mesh,
      93        6197 :                     isParamSetByUser("serial_refine") ? getParam<unsigned int>("serial_refine")
      94        8104 :                                                       : getParam<unsigned int>("uniform_refine"));
      95             : 
      96             :   // MFEM supports load balancing of parallel non-conforming meshes
      97             :   // with a space-filling curve partitioning, and we can improve it
      98             :   // by re-ordering the mesh. For now, we only support the Hilbert
      99             :   // ordering, although there is one other option.
     100        6141 :   if (getParam<bool>("reorder_mesh"))
     101             :   {
     102           0 :     mfem::Array<int> ordering;
     103           0 :     mfem_ser_mesh.GetHilbertElementOrdering(ordering);
     104           0 :     mfem_ser_mesh.ReorderElements(ordering);
     105           0 :   }
     106             : 
     107             :   // Make sure mesh is in non-conforming mode to enable local refinement of
     108             :   // quadrilaterals/hexahedra (c.f. MFEM example 6p). The argument (true/false)
     109             :   // determines whether a simplex mesh is considered to be non-conforming.
     110        6141 :   if (getParam<bool>("nonconforming"))
     111          26 :     mfem_ser_mesh.EnsureNCMesh(true);
     112             : 
     113             :   // multi app should take the mpi comm from moose so is split correctly??
     114        2047 :   auto comm = this->comm().get();
     115        2047 :   _mfem_par_mesh = std::make_shared<mfem::ParMesh>(comm, mfem_ser_mesh);
     116             : 
     117             :   // Perform parallel refinements
     118        4094 :   uniformRefinement(*_mfem_par_mesh, getParam<unsigned int>("parallel_refine"));
     119             : 
     120        6141 :   if (isParamSetByUser("displacement"))
     121          93 :     _mesh_displacement_variable.emplace(getParam<std::string>("displacement"));
     122             : 
     123             :   // Build a dummy MOOSE mesh to enable this class to work with other MOOSE classes.
     124        2047 :   buildDummyMooseMesh();
     125        2047 : }
     126             : 
     127             : std::vector<std::filesystem::path>
     128         118 : MFEMMesh::writeRecoveryFiles(const std::filesystem::path & file_base)
     129             : {
     130         118 :   MooseMesh::writeRecoveryFiles(file_base);
     131             : 
     132             :   mooseAssert(_mfem_par_mesh, "MFEM parallel mesh is not initialized");
     133             : 
     134             :   const auto checkpoint_file =
     135         118 :       file_base.string() + ".mfem.mesh." + std::to_string(this->processor_id());
     136         118 :   std::ofstream output(checkpoint_file);
     137         118 :   if (!output)
     138           0 :     mooseError("Unable to open MFEM recovery mesh file '", checkpoint_file, "' for writing.");
     139             : 
     140         236 :   _mfem_par_mesh->ParPrint(output);
     141         472 :   return {checkpoint_file};
     142         236 : }
     143             : 
     144             : void
     145          38 : MFEMMesh::displace(mfem::GridFunction const & displacement)
     146             : {
     147          38 :   _mfem_par_mesh->EnsureNodes();
     148          38 :   mfem::GridFunction * nodes = _mfem_par_mesh->GetNodes();
     149             : 
     150          38 :   *nodes += displacement;
     151          38 : }
     152             : 
     153             : void
     154        2047 : MFEMMesh::buildDummyMooseMesh()
     155             : {
     156        2047 :   auto & dummy = static_cast<UnstructuredMesh &>(getMesh());
     157        2047 :   MeshTools::Generation::build_point(dummy);
     158        2047 :   if (dimension() >= 2)
     159        1969 :     MeshTools::Generation::build_square(dummy, 1, 1, 0., 1., 0., 1., ElemType::QUAD9);
     160        2047 : }
     161             : 
     162             : void
     163        4094 : MFEMMesh::uniformRefinement(mfem::Mesh & mesh, const unsigned int nref) const
     164             : {
     165        4288 :   for (unsigned int i = 0; i < nref; ++i)
     166         194 :     mesh.UniformRefinement();
     167        4094 : }
     168             : 
     169             : std::unique_ptr<MooseMesh>
     170           8 : MFEMMesh::safeClone() const
     171             : {
     172           8 :   return _app.getFactory().copyConstruct(*this);
     173             : }
     174             : 
     175             : #endif