www.mooseframework.org
PetscExternalPartitioner.C
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://www.mooseframework.org
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
11 
12 #include "GeneratedMesh.h"
13 #include "MooseApp.h"
14 
15 #include "libmesh/mesh_tools.h"
16 #include "libmesh/linear_partitioner.h"
17 #include "libmesh/elem.h"
18 #include "libmesh/mesh_base.h"
19 
21 
22 #include <memory>
23 
24 template <>
27 {
29 
30  MooseEnum partPackage("parmetis ptscotch chaco party hierarch", "parmetis", false);
31 
32  params.addParam<MooseEnum>("part_package",
33  partPackage,
34  "The external package is used for partitioning the mesh via PETSc");
35 
36  params.addParam<bool>("apply_element_weight",
37  false,
38  "Indicate if we are going to apply element weights to partitioners");
39 
40  params.addParam<bool>(
41  "apply_side_weight", false, "Indicate if we are going to apply side weights to partitioners");
42 
43  params.addClassDescription(
44  "Partition mesh using external packages via PETSc MatPartitioning interface");
45 
46  return params;
47 }
48 
50  : MoosePartitioner(params),
51  _part_package(params.get<MooseEnum>("part_package")),
52  _apply_element_weight(params.get<bool>("apply_element_weight")),
53  _apply_side_weight(params.get<bool>("apply_side_weight"))
54 {
56  (_part_package == "chaco" || _part_package == "party"))
57  mooseError(_part_package, " does not support weighted graph");
58 }
59 
60 std::unique_ptr<Partitioner>
62 {
63  return libmesh_make_unique<PetscExternalPartitioner>(_pars);
64 }
65 
66 void
68 {
69  // Temporarily cache the old partition method
70  auto old_partitioner = std::move(mesh.partitioner());
71  // Create a linear partitioner
72  mesh.partitioner().reset(new LinearPartitioner);
73  // Partition mesh
74  mesh.partition(n_processors());
75  // Restore the old partition
76  mesh.partitioner() = std::move(old_partitioner);
77 }
78 
79 void
80 PetscExternalPartitioner::partition(MeshBase & mesh, const unsigned int n_parts)
81 {
82  // We want to use a parallel partitioner that requires a distributed graph
83  // Simply calling a linear partitioner provides us the distributed graph
84  // We shold not do anything when using a distributed mesh since the mesh itself
85  // is already distributed
86  // When n_parts=1, we do not need to run any partitioner, instead, let libmesh
87  // handle this
88  if (mesh.is_replicated() && n_parts > 1)
89  preLinearPartition(mesh);
90 
91  Partitioner::partition(mesh, n_parts);
92 }
93 
94 void
95 PetscExternalPartitioner::_do_partition(MeshBase & mesh, const unsigned int n_parts)
96 {
97 #ifdef LIBMESH_HAVE_PETSC
98  // construct a dual graph
99  Mat dual;
100  PetscInt *i, *j, *values, *elem_weights, nrows, nj, ncols, local_elem_id;
101  const PetscInt * parts;
102  unsigned int side;
103  MatPartitioning part;
104  IS is;
105  // Let us check PETSc return code
106  PetscErrorCode ierr;
107 
108  i = 0;
109  j = 0;
110  values = 0;
111  elem_weights = 0;
112 
113  build_graph(mesh);
114  nrows = _dual_graph.size();
115  ierr = PetscCalloc1(nrows + 1, &i);
116  CHKERRABORT(mesh.comm().get(), ierr);
118  {
119  ierr = PetscCalloc1(nrows + 1, &elem_weights);
120  CHKERRABORT(mesh.comm().get(), ierr);
121  }
122 
123  // Set graph offsets and
124  // compute element weight
125  for (PetscInt k = 0; k < nrows; k++)
126  {
127  i[k + 1] = i[k] + _dual_graph[k].size();
128 
130  {
131  // Get the original element
132  mooseAssert(k < static_cast<PetscInt>(_local_id_to_elem.size()),
133  "Local element id " << k << " is not smaller than " << _local_id_to_elem.size());
134  auto elem = _local_id_to_elem[k];
135 
136  elem_weights[k] = computeElementWeight(*elem);
137  }
138  }
139 
140  // Graph adjacency
141  ierr = PetscCalloc1(i[nrows], &j);
142  CHKERRABORT(mesh.comm().get(), ierr);
143 
144  // Edge weights represent the communication
145  if (_apply_side_weight)
146  {
147  ierr = PetscCalloc1(i[nrows], &values);
148  CHKERRABORT(mesh.comm().get(), ierr);
149  }
150 
151  local_elem_id = 0;
152  nj = 0;
153  for (auto & row : _dual_graph)
154  {
155  mooseAssert(local_elem_id < static_cast<PetscInt>(_local_id_to_elem.size()),
156  "Local element id " << local_elem_id << " is not smaller than "
157  << _local_id_to_elem.size());
158  auto elem = _local_id_to_elem[local_elem_id];
159  unsigned int n_neighbors = 0;
160 
161  side = 0;
162  for (auto neighbor : elem->neighbor_ptr_range())
163  {
164  // Skip boundary sides since they do not connect to
165  // anything.
166  if (neighbor != nullptr && neighbor->active())
167  {
168  j[nj] = row[n_neighbors++];
169 
170  if (_apply_side_weight)
171  values[nj] = computeSideWeight(*elem, side);
172 
173  nj++;
174  }
175 
176  side++;
177  }
178  if (n_neighbors != row.size())
179  mooseError(
180  "Cannot construct dual graph correctly since the number of neighbors is inconsistent");
181 
182  local_elem_id++;
183  }
184 
185  ncols = 0;
186 
187  for (processor_id_type pid = 0; pid < mesh.n_processors(); pid++)
188  {
189  mooseAssert(pid < _n_active_elem_on_proc.size(),
190  "Processor id " << pid << " is not smaller than " << _n_active_elem_on_proc.size());
191  ncols += _n_active_elem_on_proc[pid];
192  }
193 
194  ierr = MatCreateMPIAdj(mesh.comm().get(), nrows, ncols, i, j, values, &dual);
195  CHKERRABORT(mesh.comm().get(), ierr);
196  ierr = MatPartitioningCreate(mesh.comm().get(), &part);
197  CHKERRABORT(mesh.comm().get(), ierr);
198  ierr = MatPartitioningSetAdjacency(part, dual);
199  CHKERRABORT(mesh.comm().get(), ierr);
200 
202  {
203  ierr = MatPartitioningSetVertexWeights(part, elem_weights);
204  CHKERRABORT(mesh.comm().get(), ierr);
205  }
206 
207  ierr = MatPartitioningSetNParts(part, n_parts);
208  CHKERRABORT(mesh.comm().get(), ierr);
209 #if PETSC_VERSION_LESS_THAN(3, 9, 2)
210  if (_part_package == "party")
211  mooseError("PETSc-3.9.3 or higher is required for using party");
212 #endif
213 #if PETSC_VERSION_LESS_THAN(3, 9, 0)
214  if (_part_package == "chaco")
215  mooseError("PETSc-3.9.0 or higher is required for using chaco");
216 #endif
217  ierr = MatPartitioningSetType(part, _part_package.c_str());
218  CHKERRABORT(mesh.comm().get(), ierr);
219  ierr = MatPartitioningSetFromOptions(part);
220  CHKERRABORT(mesh.comm().get(), ierr);
221  ierr = MatPartitioningApply(part, &is);
222  CHKERRABORT(mesh.comm().get(), ierr);
223 
224  ierr = ISGetIndices(is, &parts);
225  CHKERRABORT(mesh.comm().get(), ierr);
226 
227  std::vector<dof_id_type> libmesh_parts;
228  std::copy(parts, parts + nrows, std::back_inserter(libmesh_parts));
229 
230  ierr = ISRestoreIndices(is, &parts);
231  CHKERRABORT(mesh.comm().get(), ierr);
232 
233  assign_partitioning(mesh, libmesh_parts);
234 
235  ierr = ISRestoreIndices(is, &parts);
236  CHKERRABORT(mesh.comm().get(), ierr);
237 
238  ierr = MatPartitioningDestroy(&part);
239  CHKERRABORT(mesh.comm().get(), ierr);
240  ierr = ISDestroy(&is);
241  CHKERRABORT(mesh.comm().get(), ierr);
242  ierr = MatDestroy(&dual);
243  CHKERRABORT(mesh.comm().get(), ierr);
244 #else
245  mooseError("Petsc is required for this partitioner");
246 #endif
247 }
248 
249 dof_id_type
251 {
252  return 1;
253 }
254 
255 dof_id_type
256 PetscExternalPartitioner::computeSideWeight(Elem & /*elem*/, unsigned int /*side*/)
257 {
258  return 1;
259 }
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
void mooseError(Args &&... args) const
Definition: MooseObject.h:147
virtual void partition(MeshBase &mesh, const unsigned int n) override
virtual dof_id_type computeElementWeight(Elem &elm)
virtual dof_id_type computeSideWeight(Elem &elem, unsigned int side)
registerMooseObject("MooseApp", PetscExternalPartitioner)
This is a "smart" enum class intended to replace many of the shortcomings in the C++ enum type It sho...
Definition: MooseEnum.h:31
void preLinearPartition(MeshBase &mesh)
const InputParameters & _pars
Parameters of this object, references the InputParameters stored in the InputParametersWarehouse.
Definition: MooseObject.h:174
InputParameters validParams< PetscExternalPartitioner >()
Base class for MOOSE partitioner.
void addClassDescription(const std::string &doc_string)
This method adds a description of the class that will be displayed in the input file syntax dump...
PetscExternalPartitioner(const InputParameters &params)
void addParam(const std::string &name, const S &value, const std::string &doc_string)
These methods add an option parameter and a documentation string to the InputParameters object...
ierr
Partitions a mesh using a regular grid.
InputParameters validParams< MoosePartitioner >()
virtual void _do_partition(MeshBase &mesh, const unsigned int n) override
virtual std::unique_ptr< Partitioner > clone() const override