15 #include "libmesh/elem.h" 16 #include "libmesh/mesh_tools.h" 17 #include "libmesh/replicated_mesh.h" 18 #include "libmesh/mesh_generation.h" 19 #include "libmesh/linear_partitioner.h" 31 MooseEnum method(
"manual automatic",
"manual");
33 "nodes_grid_computation",
35 "Whether to determine the compute node grid manually (using nx_nodes, ny_nodes and nz_nodes) " 36 "or automatically. When using the automatic mode, the user can impose a certain value for " 37 "nx, ny or nz, and the automatic factorization will adjust the number of processors in the " 39 params.addParam<
unsigned int>(
40 "number_nodes",
"Number of nodes. Used for determining the node grid automatically");
42 "processors_grid_computation",
44 "Whether to determine the processors grid on each node manually (using nx_procs, ny_procs " 45 "and nz_procs) or automatically. When using the automatic mode, the user can impose a " 46 "certain value for nx, ny or nz, and the automatic factorization will adjust the number of " 47 "processors in the other directions.");
48 params.addParam<
unsigned int>(
"number_procs_per_node",
49 "Number of processors per node. Used for determining the processor " 50 "grid on each node automatically");
53 params.addRangeCheckedParam<
unsigned int>(
54 "nx_nodes",
"nx_nodes >= 1",
"Number of compute nodes in the X direction");
55 params.addRangeCheckedParam<
unsigned int>(
56 "ny_nodes",
"ny_nodes >= 1",
"Number of compute nodes in the Y direction");
57 params.addRangeCheckedParam<
unsigned int>(
58 "nz_nodes",
"nz_nodes >= 1",
"Number of compute nodes in the Z direction");
61 params.addRangeCheckedParam<
unsigned int>(
62 "nx_procs",
"nx_procs >= 1",
"Number of processors in the X direction within each node");
63 params.addRangeCheckedParam<
unsigned int>(
64 "ny_procs",
"ny_procs >= 1",
"Number of processors in the Y direction within each node");
65 params.addRangeCheckedParam<
unsigned int>(
66 "nz_procs",
"nz_procs >= 1",
"Number of processors in the Z direction within each node");
68 params.addClassDescription(
"Partitions a mesh into sub-partitions for each computational node" 69 " then into partitions within that node. All partitions are made" 70 " using a regular grid.");
82 std::unique_ptr<Partitioner>
91 const auto dim =
mesh.spatial_dimension();
101 if (getParam<MooseEnum>(
"nodes_grid_computation") ==
"manual")
104 paramError(
"nx_nodes",
"Required for manual nodes grid specification");
108 paramError(
"nz_nodes",
"Required for 3D meshes");
113 paramError(
"number_nodes",
"Required for automatic nodes grid computation");
117 MPI_Dims_create(getParam<unsigned int>(
"number_nodes"),
dim, dims);
124 if (getParam<MooseEnum>(
"processors_grid_computation") ==
"manual")
127 paramError(
"nx_procs",
"Required for manual processors grid specification");
131 paramError(
"nz_procs",
"Required for 3D meshes");
136 paramError(
"number_procs_per_node",
"Required for automatic processors grid computation");
140 MPI_Dims_create(getParam<unsigned int>(
"number_procs_per_node"),
dim, dims);
149 if (
mesh.spatial_dimension() >= 2)
151 if (
mesh.spatial_dimension() == 3)
153 if (
isParamValid(
"number_nodes") && total_nodes != getParam<unsigned int>(
"number_nodes") &&
156 "Computed number of nodes (" + std::to_string(total_nodes) +
") does not match");
159 if (
mesh.spatial_dimension() >= 2)
161 if (
mesh.spatial_dimension() == 3)
164 procs_per_node != getParam<unsigned int>(
"number_procs_per_node") &&
processor_id() == 0)
166 "Computed number of processors per node (" + std::to_string(procs_per_node) +
169 if (procs_per_node * total_nodes !=
mesh.n_partitions() &&
processor_id() == 0)
171 procs_per_node * total_nodes,
172 " partitions, which does not add up to the total number of processors: ",
173 mesh.n_partitions());
176 auto nodes_bounding_box = MeshTools::create_bounding_box(
mesh);
177 const auto & nodes_min = nodes_bounding_box.min();
178 const auto & nodes_max = nodes_bounding_box.max();
181 auto nodes_mesh = std::make_unique<ReplicatedMesh>(this->
_communicator);
182 nodes_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
184 if (
mesh.spatial_dimension() == 2)
185 MeshTools::Generation::build_cube(*nodes_mesh,
197 MeshTools::Generation::build_cube(*nodes_mesh,
210 std::vector<std::unique_ptr<ReplicatedMesh>> procs_meshes(nodes_mesh->n_elem());
212 for (
const auto & elem_ptr : nodes_mesh->active_element_ptr_range())
223 for (
const auto & node : elem_ptr->node_ref_range())
234 auto procs_mesh = std::make_unique<ReplicatedMesh>(this->
_communicator);
235 procs_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
237 if (
mesh.spatial_dimension() == 2)
238 MeshTools::Generation::build_cube(*procs_mesh,
250 MeshTools::Generation::build_cube(*procs_mesh,
262 procs_meshes[elem_ptr->id()] = std::move(procs_mesh);
265 auto nodes_point_locator_ptr = nodes_mesh->sub_point_locator();
267 std::vector<std::unique_ptr<libMesh::PointLocatorBase>> procs_point_locators(procs_meshes.size());
269 for (
unsigned int i = 0; i < procs_meshes.size(); i++)
270 procs_point_locators[i] = procs_meshes[i]->sub_point_locator();
273 for (
auto & elem_ptr :
mesh.active_element_ptr_range())
275 auto elem_centroid = elem_ptr->vertex_average();
278 auto nodes_elem_ptr = (*nodes_point_locator_ptr)(elem_centroid);
280 auto nodes_elem_id = nodes_elem_ptr->id();
286 auto procs_elem_ptr = (*procs_point_locators[nodes_elem_id])(elem_centroid);
289 elem_ptr->processor_id() = procs_elem_ptr->id() + (nodes_elem_id * procs_per_node);
292 mooseError(
"HierarchicalGridPartitioner unable to locate element within the grid!");
virtual ~HierarchicalGridPartitioner()
virtual void _do_partition(MeshBase &mesh, const unsigned int n) override
static InputParameters validParams()
unsigned int _ny_nodes
Number of nodes in the Y direction.
registerMooseObject("MooseApp", HierarchicalGridPartitioner)
Partitions a mesh into sub-partitions for each computational node then into partitions within that no...
unsigned int _nz_nodes
Number of nodes in the Z direction.
static constexpr std::size_t dim
This is the dimension of all vector and tensor datastructures used in MOOSE.
const Parallel::Communicator & _communicator
Factory & getFactory()
Retrieve a writable reference to the Factory associated with this App.
auto max(const L &left, const R &right)
bool isParamValid(const std::string &name) const
Test if the supplied parameter is valid.
unsigned int _ny_procs
Number of processors on each node in the Y direction.
MooseMesh wraps a libMesh::Mesh object and enhances its capabilities by caching additional data and s...
This is a "smart" enum class intended to replace many of the shortcomings in the C++ enum type It sho...
static InputParameters validParams()
unsigned int _nz_procs
Number of processors on each node in the Z direction.
MooseApp & _app
The MOOSE application this is associated with.
void paramError(const std::string ¶m, Args... args) const
Emits an error prefixed with the file and line number of the given param (from the input file) along ...
virtual std::unique_ptr< Partitioner > clone() const override
Base class for MOOSE partitioner.
unsigned int _nx_procs
Number of processors on each node in the X direction.
void mooseError(Args &&... args) const
Emits an error prefixed with object name and type.
std::unique_ptr< T > clone(const T &object)
Clones the object object.
unsigned int _nx_nodes
Number of nodes in the X direction.
processor_id_type processor_id() const
auto min(const L &left, const R &right)
void ErrorVector unsigned int
HierarchicalGridPartitioner(const InputParameters ¶ms)