Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "HierarchicalGridPartitioner.h"
11 :
12 : #include "GeneratedMesh.h"
13 : #include "MooseApp.h"
14 :
15 : #include "libmesh/elem.h"
16 : #include "libmesh/mesh_tools.h"
17 : #include "libmesh/replicated_mesh.h"
18 : #include "libmesh/mesh_generation.h"
19 : #include "libmesh/linear_partitioner.h"
20 :
21 : registerMooseObject("MooseApp", HierarchicalGridPartitioner);
22 :
23 : #include <memory>
24 :
25 : InputParameters
26 16189 : HierarchicalGridPartitioner::validParams()
27 : {
28 16189 : auto params = MoosePartitioner::validParams();
29 :
30 : // Options for automatic grid computations
31 16189 : MooseEnum method("manual automatic", "manual");
32 16189 : params.addParam<MooseEnum>(
33 : "nodes_grid_computation",
34 : method,
35 : "Whether to determine the compute node grid manually (using nx_nodes, ny_nodes and nz_nodes) "
36 : "or automatically. When using the automatic mode, the user can impose a certain value for "
37 : "nx, ny or nz, and the automatic factorization will adjust the number of processors in the "
38 : "other directions.");
39 16189 : params.addParam<unsigned int>(
40 : "number_nodes", "Number of nodes. Used for determining the node grid automatically");
41 16189 : params.addParam<MooseEnum>(
42 : "processors_grid_computation",
43 : method,
44 : "Whether to determine the processors grid on each node manually (using nx_procs, ny_procs "
45 : "and nz_procs) or automatically. When using the automatic mode, the user can impose a "
46 : "certain value for nx, ny or nz, and the automatic factorization will adjust the number of "
47 : "processors in the other directions.");
48 16189 : params.addParam<unsigned int>("number_procs_per_node",
49 : "Number of processors per node. Used for determining the processor "
50 : "grid on each node automatically");
51 :
52 : // Node grid
53 16189 : params.addRangeCheckedParam<unsigned int>(
54 : "nx_nodes", "nx_nodes >= 1", "Number of compute nodes in the X direction");
55 16189 : params.addRangeCheckedParam<unsigned int>(
56 : "ny_nodes", "ny_nodes >= 1", "Number of compute nodes in the Y direction");
57 16189 : params.addRangeCheckedParam<unsigned int>(
58 : "nz_nodes", "nz_nodes >= 1", "Number of compute nodes in the Z direction");
59 :
60 : // Processor grid on each node
61 16189 : params.addRangeCheckedParam<unsigned int>(
62 : "nx_procs", "nx_procs >= 1", "Number of processors in the X direction within each node");
63 16189 : params.addRangeCheckedParam<unsigned int>(
64 : "ny_procs", "ny_procs >= 1", "Number of processors in the Y direction within each node");
65 16189 : params.addRangeCheckedParam<unsigned int>(
66 : "nz_procs", "nz_procs >= 1", "Number of processors in the Z direction within each node");
67 :
68 16189 : params.addClassDescription("Partitions a mesh into sub-partitions for each computational node"
69 : " then into partitions within that node. All partitions are made"
70 : " using a regular grid.");
71 :
72 32378 : return params;
73 16189 : }
74 :
75 1452 : HierarchicalGridPartitioner::HierarchicalGridPartitioner(const InputParameters & params)
76 1452 : : MoosePartitioner(params), _mesh(*getCheckedPointerParam<MooseMesh *>("mesh"))
77 : {
78 1452 : }
79 :
80 2904 : HierarchicalGridPartitioner::~HierarchicalGridPartitioner() {}
81 :
82 : std::unique_ptr<Partitioner>
83 980 : HierarchicalGridPartitioner::clone() const
84 : {
85 980 : return _app.getFactory().clone(*this);
86 : }
87 :
88 : void
89 456 : HierarchicalGridPartitioner::_do_partition(MeshBase & mesh, const unsigned int /*n*/)
90 : {
91 456 : const auto dim = mesh.spatial_dimension();
92 :
93 : // Process user parameters
94 456 : _nx_nodes = isParamValid("nx_nodes") ? getParam<unsigned int>("nx_nodes") : 0;
95 456 : _ny_nodes = isParamValid("ny_nodes") ? getParam<unsigned int>("ny_nodes") : 0;
96 456 : _nz_nodes = isParamValid("nz_nodes") ? getParam<unsigned int>("nz_nodes") : 0;
97 456 : _nx_procs = isParamValid("nx_procs") ? getParam<unsigned int>("nx_procs") : 0;
98 456 : _ny_procs = isParamValid("ny_procs") ? getParam<unsigned int>("ny_procs") : 0;
99 456 : _nz_procs = isParamValid("nz_procs") ? getParam<unsigned int>("nz_procs") : 0;
100 :
101 456 : if (getParam<MooseEnum>("nodes_grid_computation") == "manual")
102 : {
103 328 : if (_nx_nodes == 0)
104 0 : paramError("nx_nodes", "Required for manual nodes grid specification");
105 328 : if (_ny_nodes == 0 && dim > 1)
106 0 : paramError("ny_nodes", "Required for ", dim, "D meshes");
107 328 : if (_nz_nodes == 0 && dim == 3)
108 0 : paramError("nz_nodes", "Required for 3D meshes");
109 : }
110 : else
111 : {
112 128 : if (!isParamValid("number_nodes"))
113 0 : paramError("number_nodes", "Required for automatic nodes grid computation");
114 : // 0 means no restriction on which number to choose
115 128 : int dims[] = {int(_nx_nodes), int(_ny_nodes), int(_nz_nodes)};
116 : // This will error if the factorization is not possible
117 128 : MPI_Dims_create(getParam<unsigned int>("number_nodes"), dim, dims);
118 :
119 128 : _nx_nodes = dims[0];
120 128 : _ny_nodes = (dim >= 2) ? dims[1] : 0;
121 128 : _nz_nodes = (dim == 3) ? dims[2] : 0;
122 : }
123 :
124 456 : if (getParam<MooseEnum>("processors_grid_computation") == "manual")
125 : {
126 328 : if (_nx_procs == 0)
127 0 : paramError("nx_procs", "Required for manual processors grid specification");
128 328 : if (_ny_procs == 0 && dim > 1)
129 0 : paramError("ny_procs", "Required for ", dim, "D meshes");
130 328 : if (_nz_procs == 0 && dim == 3)
131 0 : paramError("nz_procs", "Required for 3D meshes");
132 : }
133 : else
134 : {
135 128 : if (!isParamValid("number_procs_per_node"))
136 0 : paramError("number_procs_per_node", "Required for automatic processors grid computation");
137 : // 0 means no restriction on which number to choose
138 128 : int dims[] = {int(_nx_procs), int(_ny_procs), int(_nz_procs)};
139 : // This will error if the factorization is not possible
140 128 : MPI_Dims_create(getParam<unsigned int>("number_procs_per_node"), dim, dims);
141 :
142 128 : _nx_procs = dims[0];
143 128 : _ny_procs = (dim >= 2) ? dims[1] : 0;
144 128 : _nz_procs = (dim == 3) ? dims[2] : 0;
145 : }
146 :
147 : // Sanity checks on both grids
148 456 : auto total_nodes = _nx_nodes;
149 456 : if (mesh.spatial_dimension() >= 2)
150 456 : total_nodes *= _ny_nodes;
151 456 : if (mesh.spatial_dimension() == 3)
152 72 : total_nodes *= _nz_nodes;
153 456 : if (isParamValid("number_nodes") && total_nodes != getParam<unsigned int>("number_nodes") &&
154 0 : processor_id() == 0)
155 0 : paramError("number_nodes",
156 0 : "Computed number of nodes (" + std::to_string(total_nodes) + ") does not match");
157 :
158 456 : auto procs_per_node = _nx_procs;
159 456 : if (mesh.spatial_dimension() >= 2)
160 456 : procs_per_node *= _ny_procs;
161 456 : if (mesh.spatial_dimension() == 3)
162 72 : procs_per_node *= _nz_procs;
163 912 : if (isParamValid("number_procs_per_node") &&
164 912 : procs_per_node != getParam<unsigned int>("number_procs_per_node") && processor_id() == 0)
165 0 : paramError("number_procs_per_node",
166 0 : "Computed number of processors per node (" + std::to_string(procs_per_node) +
167 : ") does not match");
168 :
169 456 : if (procs_per_node * total_nodes != mesh.n_partitions() && processor_id() == 0)
170 0 : mooseError("Partitioning creates ",
171 0 : procs_per_node * total_nodes,
172 : " partitions, which does not add up to the total number of processors: ",
173 0 : mesh.n_partitions());
174 :
175 : // Figure out the physical bounds of the given mesh
176 456 : auto nodes_bounding_box = MeshTools::create_bounding_box(mesh);
177 456 : const auto & nodes_min = nodes_bounding_box.min();
178 456 : const auto & nodes_max = nodes_bounding_box.max();
179 :
180 : // Bound the coarse mesh (n_nodes * n_nodes)
181 456 : auto nodes_mesh = std::make_unique<ReplicatedMesh>(this->_communicator);
182 456 : nodes_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
183 :
184 456 : if (mesh.spatial_dimension() == 2)
185 384 : MeshTools::Generation::build_cube(*nodes_mesh,
186 : _nx_nodes,
187 : _ny_nodes,
188 : _nz_nodes,
189 384 : nodes_min(0),
190 384 : nodes_max(0),
191 384 : nodes_min(1),
192 384 : nodes_max(1),
193 384 : nodes_min(2),
194 384 : nodes_max(2),
195 : QUAD4);
196 : else
197 72 : MeshTools::Generation::build_cube(*nodes_mesh,
198 : _nx_nodes,
199 : _ny_nodes,
200 : _nz_nodes,
201 72 : nodes_min(0),
202 72 : nodes_max(0),
203 72 : nodes_min(1),
204 72 : nodes_max(1),
205 72 : nodes_min(2),
206 72 : nodes_max(2),
207 : HEX8);
208 :
209 : // Now build the procs meshes
210 456 : std::vector<std::unique_ptr<ReplicatedMesh>> procs_meshes(nodes_mesh->n_elem());
211 :
212 3816 : for (const auto & elem_ptr : nodes_mesh->active_element_ptr_range())
213 : {
214 : // Need to find the bounds of the elem
215 : Point min(std::numeric_limits<Real>::max(),
216 : std::numeric_limits<Real>::max(),
217 1680 : std::numeric_limits<Real>::max());
218 1680 : Point max(-std::numeric_limits<Real>::max(),
219 1680 : -std::numeric_limits<Real>::max(),
220 1680 : -std::numeric_limits<Real>::max());
221 :
222 : // Loop over all the nodes
223 8976 : for (const auto & node : elem_ptr->node_ref_range())
224 : {
225 7296 : min(0) = std::min(min(0), node(0));
226 7296 : min(1) = std::min(min(1), node(1));
227 7296 : min(2) = std::min(min(2), node(2));
228 :
229 7296 : max(0) = std::max(max(0), node(0));
230 7296 : max(1) = std::max(max(1), node(1));
231 7296 : max(2) = std::max(max(2), node(2));
232 : }
233 :
234 1680 : auto procs_mesh = std::make_unique<ReplicatedMesh>(this->_communicator);
235 1680 : procs_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
236 :
237 1680 : if (mesh.spatial_dimension() == 2)
238 1536 : MeshTools::Generation::build_cube(*procs_mesh,
239 : _nx_procs,
240 : _ny_procs,
241 : _nz_procs,
242 1536 : min(0),
243 1536 : max(0),
244 1536 : min(1),
245 1536 : max(1),
246 1536 : min(2),
247 1536 : max(2),
248 : QUAD4);
249 : else
250 144 : MeshTools::Generation::build_cube(*procs_mesh,
251 : _nx_procs,
252 : _ny_procs,
253 : _nz_procs,
254 144 : min(0),
255 144 : max(0),
256 144 : min(1),
257 144 : max(1),
258 144 : min(2),
259 144 : max(2),
260 : HEX8);
261 :
262 1680 : procs_meshes[elem_ptr->id()] = std::move(procs_mesh);
263 2136 : }
264 :
265 456 : auto nodes_point_locator_ptr = nodes_mesh->sub_point_locator();
266 :
267 456 : std::vector<std::unique_ptr<libMesh::PointLocatorBase>> procs_point_locators(procs_meshes.size());
268 :
269 2136 : for (unsigned int i = 0; i < procs_meshes.size(); i++)
270 1680 : procs_point_locators[i] = procs_meshes[i]->sub_point_locator();
271 :
272 : // Loop over all of the elements in the given mesh
273 121608 : for (auto & elem_ptr : mesh.active_element_ptr_range())
274 : {
275 60576 : auto elem_centroid = elem_ptr->vertex_average();
276 :
277 : // Find the element it lands in in the Nodes mesh
278 60576 : auto nodes_elem_ptr = (*nodes_point_locator_ptr)(elem_centroid);
279 :
280 60576 : auto nodes_elem_id = nodes_elem_ptr->id();
281 :
282 : // True if we found something
283 60576 : if (nodes_elem_ptr)
284 : {
285 : // Now see where it lands within the procs mesh of that node
286 60576 : auto procs_elem_ptr = (*procs_point_locators[nodes_elem_id])(elem_centroid);
287 :
288 : // Assign the _id_ of the cell to the processor_id (plus an offset for which node we're in)
289 60576 : elem_ptr->processor_id() = procs_elem_ptr->id() + (nodes_elem_id * procs_per_node);
290 : }
291 : else // Should never happen (seriously - we create bounding boxes that should disallow this!)
292 0 : mooseError("HierarchicalGridPartitioner unable to locate element within the grid!");
293 456 : }
294 456 : }
|
