Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : #include "GridPartitioner.h"
      11             : 
      12             : #include "GeneratedMesh.h"
      13             : #include "MooseApp.h"
      14             : 
      15             : #include "libmesh/mesh_tools.h"
      16             : #include "libmesh/elem.h"
      17             : 
      18             : registerMooseObject("MooseApp", GridPartitioner);
      19             : 
      20             : #include <memory>
      21             : 
      22             : InputParameters
      23       16689 : GridPartitioner::validParams()
      24             : {
      25       16689 :   InputParameters params = MoosePartitioner::validParams();
      26             : 
      27       16689 :   MooseEnum method("manual automatic", "manual");
      28       16689 :   params.addParam<MooseEnum>(
      29             :       "grid_computation",
      30             :       method,
      31             :       "Whether to determine the grid manually (using nx, ny and nz) or automatically. When using "
      32             :       "the automatic mode, the user can impose a certain value for nx, ny or nz, and the automatic "
      33             :       "factorization will adjust the number of processors in the other directions.");
      34             : 
      35             :   // Users specify how many processors they need along each direction
      36       16689 :   params.addParam<unsigned int>(
      37             :       "nx", "Number of processors in the X direction. Defaults to 1 in manual mode");
      38       16689 :   params.addParam<unsigned int>(
      39             :       "ny", "Number of processors in the Y direction. Defaults to 1 in manual mode");
      40       16689 :   params.addParam<unsigned int>(
      41             :       "nz", "Number of processors in the Z direction. Defaults to 1 in manual mode");
      42             : 
      43       16689 :   params.addClassDescription("Create a uniform grid that overlays the mesh to be partitioned.  "
      44             :                              "Assign all elements within each cell of the grid to the same "
      45             :                              "processor.");
      46             : 
      47       33378 :   return params;
      48       16689 : }
      49             : 
      50        1826 : GridPartitioner::GridPartitioner(const InputParameters & params)
      51        1826 :   : MoosePartitioner(params), _mesh(*getCheckedPointerParam<MooseMesh *>("mesh"))
      52             : {
      53        1826 : }
      54             : 
      55        3652 : GridPartitioner::~GridPartitioner() {}
      56             : 
      57             : std::unique_ptr<Partitioner>
      58        1228 : GridPartitioner::clone() const
      59             : {
      60        1228 :   return _app.getFactory().clone(*this);
      61             : }
      62             : 
      63             : void
      64         706 : GridPartitioner::_do_partition(MeshBase & mesh, const unsigned int /*n*/)
      65             : {
      66             :   // By default, there are one processor along each direction
      67             :   // nx: the number of processors along x direction
      68             :   // ny: the number of processors along y direction
      69             :   // nz: the number of processors along z direction
      70         706 :   unsigned int nx = 1, ny = 1, nz = 1;
      71             : 
      72             :   // Figure out the physical bounds of the given mesh
      73         706 :   auto bounding_box = MeshTools::create_bounding_box(mesh);
      74         706 :   const auto & min = bounding_box.min();
      75         706 :   const auto & max = bounding_box.max();
      76             : 
      77         706 :   auto dim = mesh.mesh_dimension();
      78             : 
      79             :   // Gather user parameters
      80         706 :   nx = isParamValid("nx") ? getParam<unsigned int>("nx") : nx;
      81         706 :   if (dim >= 2)
      82         706 :     ny = isParamValid("ny") ? getParam<unsigned int>("ny") : ny;
      83         706 :   if (dim == 3)
      84         286 :     nz = isParamValid("nz") ? getParam<unsigned int>("nz") : nz;
      85             : 
      86             :   // simple info message unused parameters as this can be normal: we could be partitioning
      87             :   // a 2D mesh before extruding it to 3D. The nz parameter is needed for the 3D mesh
      88         706 :   if (dim < 2 && isParamValid("ny") && getParam<unsigned int>("ny") > 1)
      89           0 :     paramInfo("ny", "Parameter ignored as mesh is currently of dimension less than 2.");
      90         706 :   if (dim < 3 && isParamValid("nz") && getParam<unsigned int>("nz") > 1)
      91          32 :     paramInfo("nz", "Parameter ignored as mesh is currently of dimension less than 3.");
      92             : 
      93             :   // User parameters, which should match the number of partitions needed
      94         706 :   if (getParam<MooseEnum>("grid_computation") == "manual")
      95             :   {
      96             :     //  Need to make sure the number of grid cells matches the number of procs to partition for
      97         504 :     if ((nx * ny * nz) != mesh.n_partitions())
      98           0 :       mooseError("Number of grid cells (" + std::to_string(nx * ny * nz) +
      99           0 :                  ") does not match the number of MPI processes (" +
     100           0 :                  std::to_string(mesh.n_partitions()) + ")");
     101             :   }
     102             : 
     103         202 :   else if (getParam<MooseEnum>("grid_computation") == "automatic")
     104             :   {
     105             :     // remove over-constraint and tell user
     106         202 :     if (nx * ny * nz > mesh.n_partitions())
     107             :     {
     108          28 :       nx = 0;
     109          28 :       ny = 0;
     110          28 :       nz = 0;
     111          28 :       paramWarning("grid_computation",
     112             :                    "User specified (nx,ny,nz) grid exceeded number of partitions, these parameters "
     113             :                    "will be ignored.");
     114             :     }
     115             :     // 0 means no restriction on which number to choose
     116         404 :     int dims[] = {isParamValid("nx") ? int(nx) : 0,
     117         404 :                   isParamValid("ny") ? int(ny) : 0,
     118         404 :                   isParamValid("nz") ? int(nz) : 0};
     119             : 
     120         124 :     if ((dims[0] > 0 && dim == 1 && dims[0] != int(mesh.n_partitions())) ||
     121         496 :         (dims[0] > 0 && dims[1] > 0 && dim == 2 && dims[0] * dims[1] != int(mesh.n_partitions())) ||
     122         170 :         (dims[0] > 0 && dims[1] > 0 && dims[2] > 0 && dim == 3 &&
     123          64 :          dims[0] * dims[1] * dims[2] != int(mesh.n_partitions())))
     124             :     {
     125          32 :       dims[0] = 0;
     126          32 :       dims[1] = 0;
     127          32 :       dims[2] = 0;
     128          32 :       paramWarning("grid_computation",
     129          64 :                    "User specified grid for the current dimension of the mesh (" +
     130         128 :                        std::to_string(dim) + ") does not fit the number of partitions (" +
     131         128 :                        std::to_string(mesh.n_partitions()) +
     132             :                        ") and constrain the grid partitioner in every direction, these parameters "
     133             :                        "will be ignored.");
     134             :     }
     135             : 
     136             :     // This will error if the factorization is not possible
     137         202 :     MPI_Dims_create(mesh.n_partitions(), dim, dims);
     138             : 
     139         202 :     nx = dims[0];
     140         202 :     if (dim >= 2)
     141         202 :       ny = dims[1];
     142         202 :     if (dim == 3)
     143          70 :       nz = dims[2];
     144             :   }
     145             : 
     146             :   // hx: grid interval along x direction
     147             :   // hy: grid interval along y direction
     148             :   // hz: grid interval along z direction
     149             :   // Lx: domain length along x direction
     150             :   // Ly: domain length along y direction
     151             :   // Lz: domain length along z direction
     152         706 :   Real hx = 1., hy = 1., hz = 1., Lx = 1., Ly = 1., Lz = 1.;
     153         706 :   Lx = max(0) - min(0);
     154         706 :   hx = Lx / nx;
     155         706 :   if (dim >= 2)
     156             :   {
     157         706 :     Ly = max(1) - min(1);
     158         706 :     hy = Ly / ny;
     159             :   }
     160             : 
     161         706 :   if (dim == 3)
     162             :   {
     163         286 :     Lz = max(2) - min(2);
     164         286 :     hz = Lz / nz;
     165             :   }
     166             : 
     167             :   // Processor coordinates along x, y, z directions
     168         706 :   unsigned int k = 0, j = 0, i = 0;
     169             :   // Coordinates for current element centroid
     170         706 :   Real coordx = 0, coordy = 0, coordz = 0;
     171             : 
     172             :   // Loop over all of the elements in the given mesh
     173      251138 :   for (auto & elem_ptr : mesh.active_element_ptr_range())
     174             :   {
     175             :     // Find the element it lands in in the GeneratedMesh
     176      250432 :     auto centroid = elem_ptr->vertex_average();
     177             : 
     178      250432 :     coordx = centroid(0);
     179             :     mooseAssert(coordx >= min(0) && coordy <= max(0),
     180             :                 "element is outside of bounding box along x direction");
     181      250432 :     if (dim >= 2)
     182             :     {
     183      250432 :       coordy = centroid(1);
     184             :       mooseAssert(coordy >= min(1) && coordy <= max(1),
     185             :                   "element is outside of bounding box along y direction");
     186             :     }
     187      250432 :     if (dim == 3)
     188             :     {
     189      212912 :       coordz = centroid(2);
     190             :       mooseAssert(coordz >= min(2) && coordz <= max(2),
     191             :                   "element is outside of bounding box along z direction");
     192             :     }
     193             : 
     194             :     // Compute processor coordinates
     195      250432 :     j = k = 0;
     196      250432 :     i = (coordx - min(0)) / hx;
     197      250432 :     if (dim >= 2)
     198      250432 :       j = (coordy - min(1)) / hy;
     199      250432 :     if (dim == 3)
     200      212912 :       k = (coordz - min(2)) / hz;
     201             : 
     202             :     mooseAssert(i < nx, "Index calculation is wrong along x direction");
     203             :     mooseAssert(j < ny || ny == 0, "Index calculation is wrong along y direction");
     204             :     mooseAssert(k < nz || nz == 0, "Index calculation is wrong along z direction");
     205             :     // Assign processor ID to current element
     206      250432 :     elem_ptr->processor_id() = k * nx * ny + j * nx + i;
     207         706 :   }
     208         706 : }