Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "HierarchicalGridPartitioner.h"
11 :
12 : #include "GeneratedMesh.h"
13 : #include "MooseApp.h"
14 :
15 : #include "libmesh/elem.h"
16 : #include "libmesh/mesh_tools.h"
17 : #include "libmesh/replicated_mesh.h"
18 : #include "libmesh/mesh_generation.h"
19 : #include "libmesh/linear_partitioner.h"
20 :
21 : registerMooseObject("MooseApp", HierarchicalGridPartitioner);
22 :
23 : #include <memory>
24 :
25 : InputParameters
26 15977 : HierarchicalGridPartitioner::validParams()
27 : {
28 15977 : auto params = MoosePartitioner::validParams();
29 :
30 : // Options for automatic grid computations
31 15977 : MooseEnum method("manual automatic", "manual");
32 15977 : params.addParam<MooseEnum>(
33 : "nodes_grid_computation",
34 : method,
35 : "Whether to determine the compute node grid manually (using nx_nodes, ny_nodes and nz_nodes) "
36 : "or automatically. When using the automatic mode, the user can impose a certain value for "
37 : "nx, ny or nz, and the automatic factorization will adjust the number of processors in the "
38 : "other directions.");
39 15977 : params.addParam<unsigned int>(
40 : "number_nodes", "Number of nodes. Used for determining the node grid automatically");
41 15977 : params.addParam<MooseEnum>(
42 : "processors_grid_computation",
43 : method,
44 : "Whether to determine the processors grid on each node manually (using nx_procs, ny_procs "
45 : "and nz_procs) or automatically. When using the automatic mode, the user can impose a "
46 : "certain value for nx, ny or nz, and the automatic factorization will adjust the number of "
47 : "processors in the other directions.");
48 15977 : params.addParam<unsigned int>("number_procs_per_node",
49 : "Number of processors per node. Used for determining the processor "
50 : "grid on each node automatically");
51 :
52 : // Node grid
53 15977 : params.addRangeCheckedParam<unsigned int>(
54 : "nx_nodes", "nx_nodes >= 1", "Number of compute nodes in the X direction");
55 15977 : params.addRangeCheckedParam<unsigned int>(
56 : "ny_nodes", "ny_nodes >= 1", "Number of compute nodes in the Y direction");
57 15977 : params.addRangeCheckedParam<unsigned int>(
58 : "nz_nodes", "nz_nodes >= 1", "Number of compute nodes in the Z direction");
59 :
60 : // Processor grid on each node
61 15977 : params.addRangeCheckedParam<unsigned int>(
62 : "nx_procs", "nx_procs >= 1", "Number of processors in the X direction within each node");
63 15977 : params.addRangeCheckedParam<unsigned int>(
64 : "ny_procs", "ny_procs >= 1", "Number of processors in the Y direction within each node");
65 15977 : params.addRangeCheckedParam<unsigned int>(
66 : "nz_procs", "nz_procs >= 1", "Number of processors in the Z direction within each node");
67 :
68 15977 : params.addClassDescription("Partitions a mesh into sub-partitions for each computational node"
69 : " then into partitions within that node. All partitions are made"
70 : " using a regular grid.");
71 :
72 31954 : return params;
73 15977 : }
74 :
75 1292 : HierarchicalGridPartitioner::HierarchicalGridPartitioner(const InputParameters & params)
76 1292 : : MoosePartitioner(params), _mesh(*getCheckedPointerParam<MooseMesh *>("mesh"))
77 : {
78 1292 : }
79 :
80 2584 : HierarchicalGridPartitioner::~HierarchicalGridPartitioner() {}
81 :
82 : std::unique_ptr<Partitioner>
83 872 : HierarchicalGridPartitioner::clone() const
84 : {
85 872 : return _app.getFactory().clone(*this);
86 : }
87 :
88 : void
89 400 : HierarchicalGridPartitioner::_do_partition(MeshBase & mesh, const unsigned int /*n*/)
90 : {
91 400 : const auto dim = mesh.spatial_dimension();
92 :
93 : // Process user parameters
94 400 : _nx_nodes = isParamValid("nx_nodes") ? getParam<unsigned int>("nx_nodes") : 0;
95 400 : _ny_nodes = isParamValid("ny_nodes") ? getParam<unsigned int>("ny_nodes") : 0;
96 400 : _nz_nodes = isParamValid("nz_nodes") ? getParam<unsigned int>("nz_nodes") : 0;
97 400 : _nx_procs = isParamValid("nx_procs") ? getParam<unsigned int>("nx_procs") : 0;
98 400 : _ny_procs = isParamValid("ny_procs") ? getParam<unsigned int>("ny_procs") : 0;
99 400 : _nz_procs = isParamValid("nz_procs") ? getParam<unsigned int>("nz_procs") : 0;
100 :
101 400 : if (getParam<MooseEnum>("nodes_grid_computation") == "manual")
102 : {
103 288 : if (_nx_nodes == 0)
104 0 : paramError("nx_nodes", "Required for manual nodes grid specification");
105 288 : if (_ny_nodes == 0 && dim > 1)
106 0 : paramError("ny_nodes", "Required for ", dim, "D meshes");
107 288 : if (_nz_nodes == 0 && dim == 3)
108 0 : paramError("nz_nodes", "Required for 3D meshes");
109 : }
110 : else
111 : {
112 112 : if (!isParamValid("number_nodes"))
113 0 : paramError("number_nodes", "Required for automatic nodes grid computation");
114 : // 0 means no restriction on which number to choose
115 112 : int dims[] = {int(_nx_nodes), int(_ny_nodes), int(_nz_nodes)};
116 : // This will error if the factorization is not possible
117 112 : MPI_Dims_create(getParam<unsigned int>("number_nodes"), dim, dims);
118 :
119 112 : _nx_nodes = dims[0];
120 112 : _ny_nodes = (dim >= 2) ? dims[1] : 0;
121 112 : _nz_nodes = (dim == 3) ? dims[2] : 0;
122 : }
123 :
124 400 : if (getParam<MooseEnum>("processors_grid_computation") == "manual")
125 : {
126 288 : if (_nx_procs == 0)
127 0 : paramError("nx_procs", "Required for manual processors grid specification");
128 288 : if (_ny_procs == 0 && dim > 1)
129 0 : paramError("ny_procs", "Required for ", dim, "D meshes");
130 288 : if (_nz_procs == 0 && dim == 3)
131 0 : paramError("nz_procs", "Required for 3D meshes");
132 : }
133 : else
134 : {
135 112 : if (!isParamValid("number_procs_per_node"))
136 0 : paramError("number_procs_per_node", "Required for automatic processors grid computation");
137 : // 0 means no restriction on which number to choose
138 112 : int dims[] = {int(_nx_procs), int(_ny_procs), int(_nz_procs)};
139 : // This will error if the factorization is not possible
140 112 : MPI_Dims_create(getParam<unsigned int>("number_procs_per_node"), dim, dims);
141 :
142 112 : _nx_procs = dims[0];
143 112 : _ny_procs = (dim >= 2) ? dims[1] : 0;
144 112 : _nz_procs = (dim == 3) ? dims[2] : 0;
145 : }
146 :
147 : // Sanity checks on both grids
148 400 : auto total_nodes = _nx_nodes;
149 400 : if (mesh.spatial_dimension() >= 2)
150 400 : total_nodes *= _ny_nodes;
151 400 : if (mesh.spatial_dimension() == 3)
152 64 : total_nodes *= _nz_nodes;
153 400 : if (isParamValid("number_nodes") && total_nodes != getParam<unsigned int>("number_nodes") &&
154 0 : processor_id() == 0)
155 0 : paramError("number_nodes",
156 0 : "Computed number of nodes (" + std::to_string(total_nodes) + ") does not match");
157 :
158 400 : auto procs_per_node = _nx_procs;
159 400 : if (mesh.spatial_dimension() >= 2)
160 400 : procs_per_node *= _ny_procs;
161 400 : if (mesh.spatial_dimension() == 3)
162 64 : procs_per_node *= _nz_procs;
163 800 : if (isParamValid("number_procs_per_node") &&
164 800 : procs_per_node != getParam<unsigned int>("number_procs_per_node") && processor_id() == 0)
165 0 : paramError("number_procs_per_node",
166 0 : "Computed number of processors per node (" + std::to_string(procs_per_node) +
167 : ") does not match");
168 :
169 400 : if (procs_per_node * total_nodes != mesh.n_partitions() && processor_id() == 0)
170 0 : mooseError("Partitioning creates ",
171 0 : procs_per_node * total_nodes,
172 : " partitions, which does not add up to the total number of processors: ",
173 0 : mesh.n_partitions());
174 :
175 : // Figure out the physical bounds of the given mesh
176 400 : auto nodes_bounding_box = MeshTools::create_bounding_box(mesh);
177 400 : const auto & nodes_min = nodes_bounding_box.min();
178 400 : const auto & nodes_max = nodes_bounding_box.max();
179 :
180 : // Bound the coarse mesh (n_nodes * n_nodes)
181 400 : auto nodes_mesh = std::make_unique<ReplicatedMesh>(this->_communicator);
182 400 : nodes_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
183 :
184 400 : if (mesh.spatial_dimension() == 2)
185 336 : MeshTools::Generation::build_cube(*nodes_mesh,
186 : _nx_nodes,
187 : _ny_nodes,
188 : _nz_nodes,
189 336 : nodes_min(0),
190 336 : nodes_max(0),
191 336 : nodes_min(1),
192 336 : nodes_max(1),
193 336 : nodes_min(2),
194 336 : nodes_max(2),
195 : QUAD4);
196 : else
197 64 : MeshTools::Generation::build_cube(*nodes_mesh,
198 : _nx_nodes,
199 : _ny_nodes,
200 : _nz_nodes,
201 64 : nodes_min(0),
202 64 : nodes_max(0),
203 64 : nodes_min(1),
204 64 : nodes_max(1),
205 64 : nodes_min(2),
206 64 : nodes_max(2),
207 : HEX8);
208 :
209 : // Now build the procs meshes
210 400 : std::vector<std::unique_ptr<ReplicatedMesh>> procs_meshes(nodes_mesh->n_elem());
211 :
212 3344 : for (const auto & elem_ptr : nodes_mesh->active_element_ptr_range())
213 : {
214 : // Need to find the bounds of the elem
215 : Point min(std::numeric_limits<Real>::max(),
216 : std::numeric_limits<Real>::max(),
217 1472 : std::numeric_limits<Real>::max());
218 1472 : Point max(-std::numeric_limits<Real>::max(),
219 1472 : -std::numeric_limits<Real>::max(),
220 1472 : -std::numeric_limits<Real>::max());
221 :
222 : // Loop over all the nodes
223 7872 : for (const auto & node : elem_ptr->node_ref_range())
224 : {
225 6400 : min(0) = std::min(min(0), node(0));
226 6400 : min(1) = std::min(min(1), node(1));
227 6400 : min(2) = std::min(min(2), node(2));
228 :
229 6400 : max(0) = std::max(max(0), node(0));
230 6400 : max(1) = std::max(max(1), node(1));
231 6400 : max(2) = std::max(max(2), node(2));
232 : }
233 :
234 1472 : auto procs_mesh = std::make_unique<ReplicatedMesh>(this->_communicator);
235 1472 : procs_mesh->partitioner() = std::make_unique<libMesh::LinearPartitioner>();
236 :
237 1472 : if (mesh.spatial_dimension() == 2)
238 1344 : MeshTools::Generation::build_cube(*procs_mesh,
239 : _nx_procs,
240 : _ny_procs,
241 : _nz_procs,
242 1344 : min(0),
243 1344 : max(0),
244 1344 : min(1),
245 1344 : max(1),
246 1344 : min(2),
247 1344 : max(2),
248 : QUAD4);
249 : else
250 128 : MeshTools::Generation::build_cube(*procs_mesh,
251 : _nx_procs,
252 : _ny_procs,
253 : _nz_procs,
254 128 : min(0),
255 128 : max(0),
256 128 : min(1),
257 128 : max(1),
258 128 : min(2),
259 128 : max(2),
260 : HEX8);
261 :
262 1472 : procs_meshes[elem_ptr->id()] = std::move(procs_mesh);
263 1872 : }
264 :
265 400 : auto nodes_point_locator_ptr = nodes_mesh->sub_point_locator();
266 :
267 400 : std::vector<std::unique_ptr<libMesh::PointLocatorBase>> procs_point_locators(procs_meshes.size());
268 :
269 1872 : for (unsigned int i = 0; i < procs_meshes.size(); i++)
270 1472 : procs_point_locators[i] = procs_meshes[i]->sub_point_locator();
271 :
272 : // Loop over all of the elements in the given mesh
273 107408 : for (auto & elem_ptr : mesh.active_element_ptr_range())
274 : {
275 53504 : auto elem_centroid = elem_ptr->vertex_average();
276 :
277 : // Find the element it lands in in the Nodes mesh
278 53504 : auto nodes_elem_ptr = (*nodes_point_locator_ptr)(elem_centroid);
279 :
280 53504 : auto nodes_elem_id = nodes_elem_ptr->id();
281 :
282 : // True if we found something
283 53504 : if (nodes_elem_ptr)
284 : {
285 : // Now see where it lands within the procs mesh of that node
286 53504 : auto procs_elem_ptr = (*procs_point_locators[nodes_elem_id])(elem_centroid);
287 :
288 : // Assign the _id_ of the cell to the processor_id (plus an offset for which node we're in)
289 53504 : elem_ptr->processor_id() = procs_elem_ptr->id() + (nodes_elem_id * procs_per_node);
290 : }
291 : else // Should never happen (seriously - we create bounding boxes that should disallow this!)
292 0 : mooseError("HierarchicalGridPartitioner unable to locate element within the grid!");
293 400 : }
294 400 : }
|
