Loading [MathJax]/extensions/tex2jax.js
https://mooseframework.inl.gov
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
HierarchicalGridPartitioner.C
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://mooseframework.inl.gov
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
11 
12 #include "GeneratedMesh.h"
13 #include "MooseApp.h"
14 
15 #include "libmesh/elem.h"
16 #include "libmesh/mesh_tools.h"
17 #include "libmesh/replicated_mesh.h"
18 #include "libmesh/mesh_generation.h"
19 #include "libmesh/linear_partitioner.h"
20 
22 
23 #include <memory>
24 
27 {
28  auto params = MoosePartitioner::validParams();
29 
30  // Options for automatic grid computations
31  MooseEnum method("manual automatic", "manual");
32  params.addParam<MooseEnum>(
33  "nodes_grid_computation",
34  method,
35  "Whether to determine the compute node grid manually (using nx_nodes, ny_nodes and nz_nodes) "
36  "or automatically. When using the automatic mode, the user can impose a certain value for "
37  "nx, ny or nz, and the automatic factorization will adjust the number of processors in the "
38  "other directions.");
39  params.addParam<unsigned int>(
40  "number_nodes", "Number of nodes. Used for determining the node grid automatically");
41  params.addParam<MooseEnum>(
42  "processors_grid_computation",
43  method,
44  "Whether to determine the processors grid on each node manually (using nx_procs, ny_procs "
45  "and nz_procs) or automatically. When using the automatic mode, the user can impose a "
46  "certain value for nx, ny or nz, and the automatic factorization will adjust the number of "
47  "processors in the other directions.");
48  params.addParam<unsigned int>("number_procs_per_node",
49  "Number of processors per node. Used for determining the processor "
50  "grid on each node automatically");
51 
52  // Node grid
53  params.addRangeCheckedParam<unsigned int>(
54  "nx_nodes", "nx_nodes >= 1", "Number of compute nodes in the X direction");
55  params.addRangeCheckedParam<unsigned int>(
56  "ny_nodes", "ny_nodes >= 1", "Number of compute nodes in the Y direction");
57  params.addRangeCheckedParam<unsigned int>(
58  "nz_nodes", "nz_nodes >= 1", "Number of compute nodes in the Z direction");
59 
60  // Processor grid on each node
61  params.addRangeCheckedParam<unsigned int>(
62  "nx_procs", "nx_procs >= 1", "Number of processors in the X direction within each node");
63  params.addRangeCheckedParam<unsigned int>(
64  "ny_procs", "ny_procs >= 1", "Number of processors in the Y direction within each node");
65  params.addRangeCheckedParam<unsigned int>(
66  "nz_procs", "nz_procs >= 1", "Number of processors in the Z direction within each node");
67 
68  params.addClassDescription("Partitions a mesh into sub-partitions for each computational node"
69  " then into partitions within that node. All partitions are made"
70  " using a regular grid.");
71 
72  return params;
73 }
74 
76  : MoosePartitioner(params), _mesh(*getCheckedPointerParam<MooseMesh *>("mesh"))
77 {
78 }
79 
81 
82 std::unique_ptr<Partitioner>
84 {
85  return _app.getFactory().clone(*this);
86 }
87 
88 void
89 HierarchicalGridPartitioner::_do_partition(MeshBase & mesh, const unsigned int /*n*/)
90 {
91  const auto dim = mesh.spatial_dimension();
92 
93  // Process user parameters
94  _nx_nodes = isParamValid("nx_nodes") ? getParam<unsigned int>("nx_nodes") : 0;
95  _ny_nodes = isParamValid("ny_nodes") ? getParam<unsigned int>("ny_nodes") : 0;
96  _nz_nodes = isParamValid("nz_nodes") ? getParam<unsigned int>("nz_nodes") : 0;
97  _nx_procs = isParamValid("nx_procs") ? getParam<unsigned int>("nx_procs") : 0;
98  _ny_procs = isParamValid("ny_procs") ? getParam<unsigned int>("ny_procs") : 0;
99  _nz_procs = isParamValid("nz_procs") ? getParam<unsigned int>("nz_procs") : 0;
100 
101  if (getParam<MooseEnum>("nodes_grid_computation") == "manual")
102  {
103  if (_nx_nodes == 0)
104  paramError("nx_nodes", "Required for manual nodes grid specification");
105  if (_ny_nodes == 0 && dim > 1)
106  paramError("ny_nodes", "Required for ", dim, "D meshes");
107  if (_nz_nodes == 0 && dim == 3)
108  paramError("nz_nodes", "Required for 3D meshes");
109  }
110  else
111  {
112  if (!isParamValid("number_nodes"))
113  paramError("number_nodes", "Required for automatic nodes grid computation");
114  // 0 means no restriction on which number to choose
115  int dims[] = {int(_nx_nodes), int(_ny_nodes), int(_nz_nodes)};
116  // This will error if the factorization is not possible
117  MPI_Dims_create(getParam<unsigned int>("number_nodes"), dim, dims);
118 
119  _nx_nodes = dims[0];
120  _ny_nodes = (dim >= 2) ? dims[1] : 0;
121  _nz_nodes = (dim == 3) ? dims[2] : 0;
122  }
123 
124  if (getParam<MooseEnum>("processors_grid_computation") == "manual")
125  {
126  if (_nx_procs == 0)
127  paramError("nx_procs", "Required for manual processors grid specification");
128  if (_ny_procs == 0 && dim > 1)
129  paramError("ny_procs", "Required for ", dim, "D meshes");
130  if (_nz_procs == 0 && dim == 3)
131  paramError("nz_procs", "Required for 3D meshes");
132  }
133  else
134  {
135  if (!isParamValid("number_procs_per_node"))
136  paramError("number_procs_per_node", "Required for automatic processors grid computation");
137  // 0 means no restriction on which number to choose
138  int dims[] = {int(_nx_procs), int(_ny_procs), int(_nz_procs)};
139  // This will error if the factorization is not possible
140  MPI_Dims_create(getParam<unsigned int>("number_procs_per_node"), dim, dims);
141 
142  _nx_procs = dims[0];
143  _ny_procs = (dim >= 2) ? dims[1] : 0;
144  _nz_procs = (dim == 3) ? dims[2] : 0;
145  }
146 
147  // Sanity checks on both grids
148  auto total_nodes = _nx_nodes;
149  if (mesh.spatial_dimension() >= 2)
150  total_nodes *= _ny_nodes;
151  if (mesh.spatial_dimension() == 3)
152  total_nodes *= _nz_nodes;
153  if (isParamValid("number_nodes") && total_nodes != getParam<unsigned int>("number_nodes") &&
154  processor_id() == 0)
155  paramError("number_nodes",
156  "Computed number of nodes (" + std::to_string(total_nodes) + ") does not match");
157 
158  auto procs_per_node = _nx_procs;
159  if (mesh.spatial_dimension() >= 2)
160  procs_per_node *= _ny_procs;
161  if (mesh.spatial_dimension() == 3)
162  procs_per_node *= _nz_procs;
163  if (isParamValid("number_procs_per_node") &&
164  procs_per_node != getParam<unsigned int>("number_procs_per_node") && processor_id() == 0)
165  paramError("number_procs_per_node",
166  "Computed number of processors per node (" + std::to_string(procs_per_node) +
167  ") does not match");
168 
169  if (procs_per_node * total_nodes != mesh.n_partitions() && processor_id() == 0)
170  mooseError("Partitioning creates ",
171  procs_per_node * total_nodes,
172  " partitions, which does not add up to the total number of processors: ",
173  mesh.n_partitions());
174 
175  // Figure out the physical bounds of the given mesh
176  auto nodes_bounding_box = MeshTools::create_bounding_box(mesh);
177  const auto & nodes_min = nodes_bounding_box.min();
178  const auto & nodes_max = nodes_bounding_box.max();
179 
180  // Bound the coarse mesh (n_nodes * n_nodes)
181  auto nodes_mesh = std::make_unique<ReplicatedMesh>(this->_communicator);
182  nodes_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
183 
184  if (mesh.spatial_dimension() == 2)
185  MeshTools::Generation::build_cube(*nodes_mesh,
186  _nx_nodes,
187  _ny_nodes,
188  _nz_nodes,
189  nodes_min(0),
190  nodes_max(0),
191  nodes_min(1),
192  nodes_max(1),
193  nodes_min(2),
194  nodes_max(2),
195  QUAD4);
196  else
197  MeshTools::Generation::build_cube(*nodes_mesh,
198  _nx_nodes,
199  _ny_nodes,
200  _nz_nodes,
201  nodes_min(0),
202  nodes_max(0),
203  nodes_min(1),
204  nodes_max(1),
205  nodes_min(2),
206  nodes_max(2),
207  HEX8);
208 
209  // Now build the procs meshes
210  std::vector<std::unique_ptr<ReplicatedMesh>> procs_meshes(nodes_mesh->n_elem());
211 
212  for (const auto & elem_ptr : nodes_mesh->active_element_ptr_range())
213  {
214  // Need to find the bounds of the elem
221 
222  // Loop over all the nodes
223  for (const auto & node : elem_ptr->node_ref_range())
224  {
225  min(0) = std::min(min(0), node(0));
226  min(1) = std::min(min(1), node(1));
227  min(2) = std::min(min(2), node(2));
228 
229  max(0) = std::max(max(0), node(0));
230  max(1) = std::max(max(1), node(1));
231  max(2) = std::max(max(2), node(2));
232  }
233 
234  auto procs_mesh = std::make_unique<ReplicatedMesh>(this->_communicator);
235  procs_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
236 
237  if (mesh.spatial_dimension() == 2)
238  MeshTools::Generation::build_cube(*procs_mesh,
239  _nx_procs,
240  _ny_procs,
241  _nz_procs,
242  min(0),
243  max(0),
244  min(1),
245  max(1),
246  min(2),
247  max(2),
248  QUAD4);
249  else
250  MeshTools::Generation::build_cube(*procs_mesh,
251  _nx_procs,
252  _ny_procs,
253  _nz_procs,
254  min(0),
255  max(0),
256  min(1),
257  max(1),
258  min(2),
259  max(2),
260  HEX8);
261 
262  procs_meshes[elem_ptr->id()] = std::move(procs_mesh);
263  }
264 
265  auto nodes_point_locator_ptr = nodes_mesh->sub_point_locator();
266 
267  std::vector<std::unique_ptr<libMesh::PointLocatorBase>> procs_point_locators(procs_meshes.size());
268 
269  for (unsigned int i = 0; i < procs_meshes.size(); i++)
270  procs_point_locators[i] = procs_meshes[i]->sub_point_locator();
271 
272  // Loop over all of the elements in the given mesh
273  for (auto & elem_ptr : mesh.active_element_ptr_range())
274  {
275  auto elem_centroid = elem_ptr->vertex_average();
276 
277  // Find the element it lands in in the Nodes mesh
278  auto nodes_elem_ptr = (*nodes_point_locator_ptr)(elem_centroid);
279 
280  auto nodes_elem_id = nodes_elem_ptr->id();
281 
282  // True if we found something
283  if (nodes_elem_ptr)
284  {
285  // Now see where it lands within the procs mesh of that node
286  auto procs_elem_ptr = (*procs_point_locators[nodes_elem_id])(elem_centroid);
287 
288  // Assign the _id_ of the cell to the processor_id (plus an offset for which node we're in)
289  elem_ptr->processor_id() = procs_elem_ptr->id() + (nodes_elem_id * procs_per_node);
290  }
291  else // Should never happen (seriously - we create bounding boxes that should disallow this!)
292  mooseError("HierarchicalGridPartitioner unable to locate element within the grid!");
293  }
294 }
virtual void _do_partition(MeshBase &mesh, const unsigned int n) override
HEX8
static InputParameters validParams()
unsigned int _ny_nodes
Number of nodes in the Y direction.
registerMooseObject("MooseApp", HierarchicalGridPartitioner)
Partitions a mesh into sub-partitions for each computational node then into partitions within that no...
MeshBase & mesh
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
unsigned int _nz_nodes
Number of nodes in the Z direction.
static constexpr std::size_t dim
This is the dimension of all vector and tensor datastructures used in MOOSE.
Definition: Moose.h:152
const Parallel::Communicator & _communicator
Factory & getFactory()
Retrieve a writable reference to the Factory associated with this App.
Definition: MooseApp.h:434
auto max(const L &left, const R &right)
QUAD4
bool isParamValid(const std::string &name) const
Test if the supplied parameter is valid.
unsigned int _ny_procs
Number of processors on each node in the Y direction.
MooseMesh wraps a libMesh::Mesh object and enhances its capabilities by caching additional data and s...
Definition: MooseMesh.h:88
This is a "smart" enum class intended to replace many of the shortcomings in the C++ enum type It sho...
Definition: MooseEnum.h:33
static InputParameters validParams()
unsigned int _nz_procs
Number of processors on each node in the Z direction.
MooseApp & _app
The MOOSE application this is associated with.
Definition: MooseBase.h:84
void paramError(const std::string &param, Args... args) const
Emits an error prefixed with the file and line number of the given param (from the input file) along ...
virtual std::unique_ptr< Partitioner > clone() const override
Base class for MOOSE partitioner.
unsigned int _nx_procs
Number of processors on each node in the X direction.
void mooseError(Args &&... args) const
Emits an error prefixed with object name and type.
std::unique_ptr< T > clone(const T &object)
Clones the object object.
Definition: Factory.h:275
unsigned int _nx_nodes
Number of nodes in the X direction.
processor_id_type processor_id() const
auto min(const L &left, const R &right)
void ErrorVector unsigned int
HierarchicalGridPartitioner(const InputParameters &params)