Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 278256 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 278256 : unsigned char d) :
44 277760 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 277760 : _is_serial_on_proc_0(true),
46 277760 : _deleted_coarse_elements(false),
47 277760 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 278504 : _next_free_local_node_id(this->processor_id()),
49 278256 : _next_free_local_elem_id(this->processor_id()),
50 278256 : _next_free_unpartitioned_node_id(this->n_processors()),
51 278256 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 278256 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 278256 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 278504 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 556512 : default_partitioner);
64 : _partitioner = Partitioner::build
65 556264 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 278256 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 320366 : DistributedMesh::~DistributedMesh ()
143 : {
144 298475 : this->DistributedMesh::clear(); // Free nodes and elements
145 320366 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17734 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 17734 : DistributedMesh(static_cast<const MeshBase &>(other_mesh))
153 : {
154 17734 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
155 :
156 : // Need to copy extra_ghost_elems
157 17734 : for (auto & elem : other_mesh._extra_ghost_elems)
158 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
159 17734 : }
160 :
161 :
162 :
163 20219 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh) :
164 20219 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
165 20309 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
166 20039 : _deleted_coarse_elements(true), // better safe than sorry...
167 20039 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
168 20309 : _next_free_local_node_id(this->processor_id()),
169 20219 : _next_free_local_elem_id(this->processor_id()),
170 20219 : _next_free_unpartitioned_node_id(this->n_processors()),
171 40348 : _next_free_unpartitioned_elem_id(this->n_processors())
172 : {
173 : // Just copy, skipping preparation
174 20219 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
175 :
176 180 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
177 180 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
178 180 : this->allow_renumbering(other_mesh.allow_renumbering());
179 180 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
180 180 : this->skip_partitioning(other_mesh.skip_partitioning());
181 :
182 20219 : this->copy_constraint_rows(other_mesh);
183 :
184 90 : auto & this_boundary_info = this->get_boundary_info();
185 90 : const auto & other_boundary_info = other_mesh.get_boundary_info();
186 :
187 20219 : this_boundary_info = other_boundary_info;
188 :
189 90 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
190 :
191 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
192 : // nodes/elements they consider locally owned (or unpartitioned), so after
193 : // copying only the objects visible to this processor, those counts
194 : // reflect an incomplete local view rather than other_mesh's totals,
195 : // and need to be copied afresh.
196 20219 : _n_nodes = other_mesh.n_nodes();
197 20219 : _n_elem = other_mesh.n_elem();
198 20219 : _max_node_id = other_mesh.max_node_id();
199 20219 : _max_elem_id = other_mesh.max_elem_id();
200 :
201 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
202 : // counters directly, mirroring other_mesh's own bookkeeping exactly
203 20219 : if (const DistributedMesh * other_dist_mesh = dynamic_cast<const DistributedMesh *>(&other_mesh))
204 : {
205 19782 : _next_free_local_node_id = other_dist_mesh->_next_free_local_node_id;
206 19782 : _next_free_local_elem_id = other_dist_mesh->_next_free_local_elem_id;
207 19782 : _next_free_unpartitioned_node_id = other_dist_mesh->_next_free_unpartitioned_node_id;
208 19782 : _next_free_unpartitioned_elem_id = other_dist_mesh->_next_free_unpartitioned_elem_id;
209 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
210 19782 : _next_unique_id = other_dist_mesh->_next_unique_id;
211 19782 : _next_unpartitioned_unique_id = other_dist_mesh->_next_unpartitioned_unique_id;
212 : #endif
213 : }
214 : else
215 : {
216 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
217 : // raw counterpart fields to copy, so derive the free-id-range
218 : // counters from _max_elem_id/_max_node_id the same way
219 : // update_parallel_id_counts() does via the set_next_ids() method. For
220 : // the unique id counters we copy other_mesh's own local next_unique_id()
221 : // and derive the next available unique ids
222 : // (partitioned and unpartitioned) via set_next_unique_ids().
223 437 : this->set_next_ids();
224 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
225 70 : libmesh_assert(other_mesh.comm().verify(other_mesh.next_unique_id()));
226 437 : this->set_next_unique_ids(other_mesh.next_unique_id());
227 : #endif
228 : }
229 :
230 : // Copy other_mesh's actual preparation state, including whatever it
231 : // reports for has_synched_id_counts: the counts above are a faithful copy
232 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
233 : // verified computation, so our synced-ness should match other_mesh's.
234 20219 : this->_preparation = other_mesh.preparation();
235 20219 : }
236 :
237 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
238 : {
239 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
240 :
241 4 : this->_nodes = std::move(other_mesh._nodes);
242 270 : this->_n_nodes = other_mesh.n_nodes();
243 :
244 4 : this->_elements = std::move(other_mesh._elements);
245 270 : this->_n_elem = other_mesh.n_elem();
246 :
247 270 : _is_serial = other_mesh.is_serial();
248 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
249 270 : _deleted_coarse_elements = true; // Better safe than sorry
250 :
251 270 : _max_node_id = other_mesh.max_node_id();
252 270 : _max_elem_id = other_mesh.max_elem_id();
253 :
254 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
255 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
256 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
257 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
258 :
259 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
260 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
261 : #endif
262 270 : }
263 :
264 1916919 : void DistributedMesh::set_next_ids()
265 : {
266 1916919 : _next_free_unpartitioned_elem_id =
267 1918683 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
268 1916919 : (this->n_processors() + 1) + this->n_processors();
269 1916919 : _next_free_local_elem_id =
270 1916919 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
271 1916919 : (this->n_processors() + 1) + this->processor_id();
272 :
273 1916919 : _next_free_unpartitioned_node_id =
274 1918683 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
275 1916919 : (this->n_processors() + 1) + this->n_processors();
276 1916919 : _next_free_local_node_id =
277 1916919 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
278 1916919 : (this->n_processors() + 1) + this->processor_id();
279 1916919 : }
280 :
281 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
282 1916919 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
283 : {
284 : // Unique ids are laid out in repeating groups of (n_processors()+1)
285 : // consecutive integers ("alignment" here means a value's position, i.e.
286 : // its residue mod (n_processors()+1), within one of these groups). In
287 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
288 : // are each permanently earmarked for one processor's locally-owned
289 : // objects, and the last slot (residue n_processors()) is permanently
290 : // earmarked for unpartitioned objects.
291 : //
292 : // parallel_max_unique_id carries no guarantee of landing on any
293 : // particular residue, so rather than simply offsetting it by
294 : // processor_id()/n_processors(), we have to explicitly round up to the
295 : // start of the next full, entirely-unused group before adding this
296 : // processor's (or the unpartitioned pool's) fixed offset into that group.
297 : //
298 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
299 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
300 : // the number of complete groups needed to reach at least past the
301 : // current max id. Multiplying back by (n_processors()+1) gives the first
302 : // id of the next group that is guaranteed to be entirely unused; adding
303 : // n_processors(), the unpartitioned pool's fixed offset within a group,
304 : // lands on that pool's reserved slot in that group.
305 1916919 : _next_unpartitioned_unique_id =
306 1918683 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
307 1916919 : (this->n_processors() + 1) +
308 1916919 : this->n_processors();
309 :
310 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
311 : // the rounding accounts for processor_id() possibly being less than the
312 : // residue the raw max id already occupies within its group; this can
313 : // land this processor's slot in the same group as the unpartitioned
314 : // pool's above, or push it into the following group, depending on where
315 : // the max id and this processor's offset happen to fall.
316 1916919 : _next_unique_id =
317 1916919 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
318 1916919 : (this->n_processors() + 1) +
319 1916919 : this->processor_id();
320 1916919 : }
321 : #endif
322 :
323 : // We use cached values for these so they can be called
324 : // from one processor without bothering the rest, but
325 : // we may need to update those caches before doing a full
326 : // renumbering
327 1916482 : void DistributedMesh::update_parallel_id_counts()
328 : {
329 : // This function must be run on all processors at once
330 1694 : parallel_object_only();
331 :
332 1916482 : _n_elem = this->parallel_n_elem();
333 1916482 : _n_nodes = this->parallel_n_nodes();
334 1916482 : _max_node_id = this->parallel_max_node_id();
335 1916482 : _max_elem_id = this->parallel_max_elem_id();
336 :
337 1916482 : this->set_next_ids();
338 :
339 : // set_next_unique_ids() needs the true parallel max unique id (not just
340 : // this processor's possibly-stale cached _next_unique_id) since the whole
341 : // point of this function is to recompute counters that may have drifted
342 : // out of sync.
343 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
344 1916482 : this->set_next_unique_ids(this->parallel_max_unique_id());
345 : #endif
346 :
347 1916482 : this->_preparation.has_synched_id_counts = true;
348 1916482 : }
349 :
350 :
351 : // Or in debug mode we may want to test the uncached values without
352 : // changing the cache
353 1924207 : dof_id_type DistributedMesh::parallel_n_elem() const
354 : {
355 : // This function must be run on all processors at once
356 2542 : parallel_object_only();
357 :
358 1924207 : dof_id_type n_local = this->n_local_elem();
359 1924207 : this->comm().sum(n_local);
360 1924207 : n_local += this->n_unpartitioned_elem();
361 1924207 : return n_local;
362 : }
363 :
364 :
365 :
366 1923478 : dof_id_type DistributedMesh::parallel_max_elem_id() const
367 : {
368 : // This function must be run on all processors at once
369 8690 : parallel_object_only();
370 :
371 1923478 : dof_id_type max_local = 0;
372 :
373 : dofobject_container<Elem>::const_reverse_veclike_iterator
374 8690 : rit = _elements.rbegin();
375 :
376 : const dofobject_container<Elem>::const_reverse_veclike_iterator
377 8690 : rend = _elements.rend();
378 :
379 : // Look for the maximum element id. Search backwards through
380 : // elements so we can break out early. Beware of nullptr entries that
381 : // haven't yet been cleared from _elements.
382 10556256 : for (; rit != rend; ++rit)
383 : {
384 10035125 : const DofObject *d = *rit;
385 19001 : if (d)
386 : {
387 8475 : libmesh_assert(_elements[d->id()] == d);
388 1402347 : max_local = d->id() + 1;
389 1402347 : break;
390 : }
391 : }
392 :
393 1923478 : this->comm().max(max_local);
394 1923478 : return max_local;
395 : }
396 :
397 :
398 :
399 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
400 1999910 : unique_id_type DistributedMesh::parallel_max_unique_id() const
401 : {
402 : // This function must be run on all processors at once
403 1796 : parallel_object_only();
404 :
405 3999820 : unique_id_type max_local = std::max(_next_unique_id,
406 2290298 : _next_unpartitioned_unique_id);
407 1999910 : this->comm().max(max_local);
408 1999910 : return max_local;
409 : }
410 :
411 :
412 :
413 78327 : void DistributedMesh::set_next_unique_id(unique_id_type id)
414 : {
415 144 : _next_unique_id = id;
416 78327 : _next_unpartitioned_unique_id =
417 78471 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
418 78327 : (this->n_processors() + 1) + this->n_processors();
419 78327 : _next_unique_id =
420 78327 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
421 78327 : (this->n_processors() + 1) + this->processor_id();
422 78327 : }
423 : #endif
424 :
425 :
426 :
427 1924655 : dof_id_type DistributedMesh::parallel_n_nodes() const
428 : {
429 : // This function must be run on all processors at once
430 2542 : parallel_object_only();
431 :
432 1924655 : dof_id_type n_local = this->n_local_nodes();
433 1924655 : this->comm().sum(n_local);
434 1924655 : n_local += this->n_unpartitioned_nodes();
435 1924655 : return n_local;
436 : }
437 :
438 :
439 :
440 1921062 : dof_id_type DistributedMesh::parallel_max_node_id() const
441 : {
442 : // This function must be run on all processors at once
443 6274 : parallel_object_only();
444 :
445 1921062 : dof_id_type max_local = 0;
446 :
447 : dofobject_container<Node>::const_reverse_veclike_iterator
448 6274 : rit = _nodes.rbegin();
449 :
450 : const dofobject_container<Node>::const_reverse_veclike_iterator
451 6274 : rend = _nodes.rend();
452 :
453 : // Look for the maximum node id. Search backwards through
454 : // nodes so we can break out early. Beware of nullptr entries that
455 : // haven't yet been cleared from _nodes
456 38134626 : for (; rit != rend; ++rit)
457 : {
458 37613565 : const DofObject *d = *rit;
459 63385 : if (d)
460 : {
461 6129 : libmesh_assert(_nodes[d->id()] == d);
462 1400001 : max_local = d->id() + 1;
463 1400001 : break;
464 : }
465 : }
466 :
467 1921062 : this->comm().max(max_local);
468 1921062 : return max_local;
469 : }
470 :
471 :
472 :
473 42416396 : const Point & DistributedMesh::point (const dof_id_type i) const
474 : {
475 42416396 : return this->node_ref(i);
476 : }
477 :
478 :
479 :
480 43521785 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
481 : {
482 136653 : libmesh_assert(_nodes[i]);
483 136653 : libmesh_assert_equal_to (_nodes[i]->id(), i);
484 :
485 43521785 : return _nodes[i];
486 : }
487 :
488 :
489 :
490 :
491 643652387 : Node * DistributedMesh::node_ptr (const dof_id_type i)
492 : {
493 489296 : libmesh_assert(_nodes[i]);
494 489296 : libmesh_assert_equal_to (_nodes[i]->id(), i);
495 :
496 643652387 : return _nodes[i];
497 : }
498 :
499 :
500 :
501 :
502 3109134 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
503 : {
504 3109134 : if (const auto it = _nodes.find(i);
505 657411 : it != _nodes.end())
506 : {
507 2820622 : const Node * n = *it;
508 368899 : libmesh_assert (!n || n->id() == i);
509 368899 : return n;
510 : }
511 :
512 288512 : return nullptr;
513 : }
514 :
515 :
516 :
517 :
518 367387736 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
519 : {
520 367387736 : if (auto it = _nodes.find(i);
521 159367 : it != _nodes.end())
522 : {
523 284598818 : Node * n = *it;
524 111970 : libmesh_assert (!n || n->id() == i);
525 111970 : return n;
526 : }
527 :
528 47397 : return nullptr;
529 : }
530 :
531 :
532 :
533 :
534 3050216 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
535 : {
536 10087 : libmesh_assert(_elements[i]);
537 10087 : libmesh_assert_equal_to (_elements[i]->id(), i);
538 :
539 3050216 : return _elements[i];
540 : }
541 :
542 :
543 :
544 :
545 286911794 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
546 : {
547 167221 : libmesh_assert(_elements[i]);
548 167221 : libmesh_assert_equal_to (_elements[i]->id(), i);
549 :
550 286911794 : return _elements[i];
551 : }
552 :
553 :
554 :
555 :
556 1457373 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
557 : {
558 1457373 : if (const auto it = _elements.find(i);
559 618403 : it != _elements.end())
560 : {
561 1099794 : const Elem * e = *it;
562 269739 : libmesh_assert (!e || e->id() == i);
563 269739 : return e;
564 : }
565 :
566 348664 : return nullptr;
567 : }
568 :
569 :
570 :
571 :
572 603663700 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
573 : {
574 603663700 : if (auto it = _elements.find(i);
575 194370 : it != _elements.end())
576 : {
577 518160090 : Elem * e = *it;
578 190690 : libmesh_assert (!e || e->id() == i);
579 190690 : return e;
580 : }
581 :
582 3680 : return nullptr;
583 : }
584 :
585 :
586 :
587 :
588 44050320 : Elem * DistributedMesh::add_elem (Elem * e)
589 : {
590 : // Don't try to add nullptrs!
591 18945 : libmesh_assert(e);
592 :
593 : // Trying to add an existing element is a no-op
594 44050320 : if (e->valid_id() && _elements[e->id()] == e)
595 0 : return e;
596 :
597 44050320 : const processor_id_type elem_procid = e->processor_id();
598 :
599 44050320 : if (!e->valid_id())
600 : {
601 : // We should only be creating new ids past the end of the range
602 : // of existing ids
603 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
604 : _max_elem_id);
605 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
606 :
607 : // Use the unpartitioned ids for unpartitioned elems, and
608 : // temporarily for ghost elems
609 3860185 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
610 3863753 : if (elem_procid == this->processor_id())
611 1133684 : next_id = &_next_free_local_elem_id;
612 3860185 : e->set_id (*next_id);
613 : }
614 :
615 : {
616 : // Advance next_ids up high enough that each is pointing to an
617 : // unused id and any subsequent increments will still point us
618 : // to unused ids
619 88100640 : _max_elem_id = std::max(_max_elem_id,
620 44050320 : static_cast<dof_id_type>(e->id()+1));
621 :
622 44050320 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
623 5167248 : _next_free_unpartitioned_elem_id =
624 5167248 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
625 5167248 : (this->n_processors() + 1) + this->n_processors();
626 44050320 : if (_next_free_local_elem_id < _max_elem_id)
627 3463752 : _next_free_local_elem_id =
628 3470005 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
629 3463752 : (this->n_processors() + 1) + this->processor_id();
630 :
631 : #ifndef NDEBUG
632 : // We need a const dofobject_container so we don't inadvertently create
633 : // nullptr entries when testing for non-nullptr ones
634 18945 : const dofobject_container<Elem> & const_elements = _elements;
635 : #endif
636 18945 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
637 18945 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
638 : }
639 :
640 : // Don't try to overwrite existing elems
641 18945 : libmesh_assert (!_elements[e->id()]);
642 :
643 44050320 : _elements[e->id()] = e;
644 :
645 : // We actually added a new element. Some of our caches might still
646 : // be valid, but we should clear the ones which definitely are not.
647 44050320 : this->clear_point_locator();
648 44050320 : this->clear_stored_ranges();
649 :
650 : // Try to make the cached elem data more accurate
651 44069265 : if (elem_procid == this->processor_id() ||
652 : elem_procid == DofObject::invalid_processor_id)
653 12914092 : _n_elem++;
654 :
655 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
656 44050320 : if (!e->valid_unique_id())
657 : {
658 13510369 : if (processor_id() == e->processor_id())
659 : {
660 1133684 : e->set_unique_id(_next_unique_id);
661 1133684 : _next_unique_id += this->n_processors() + 1;
662 : }
663 : else
664 : {
665 12376685 : e->set_unique_id(_next_unpartitioned_unique_id);
666 12376685 : _next_unpartitioned_unique_id += this->n_processors() + 1;
667 : }
668 : }
669 : else
670 : {
671 30564328 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
672 30539951 : _next_unique_id =
673 30539951 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
674 30539951 : (this->n_processors() + 1) + this->processor_id();
675 : }
676 : #endif
677 :
678 : // Unpartitioned elems should be added on every processor
679 : // And shouldn't be added in the same batch as ghost elems
680 : // But we might be just adding on processor 0 to
681 : // broadcast later
682 : // #ifdef DEBUG
683 : // if (elem_procid == DofObject::invalid_processor_id)
684 : // {
685 : // dof_id_type elem_id = e->id();
686 : // this->comm().max(elem_id);
687 : // libmesh_assert_equal_to (elem_id, e->id());
688 : // }
689 : // #endif
690 :
691 : // Make sure any new element is given space for any extra integers
692 : // we've requested
693 44050320 : e->add_extra_integers(_elem_integer_names.size(),
694 44050320 : _elem_integer_default_values);
695 :
696 : // And set mapping type and data on any new element
697 37890 : e->set_mapping_type(this->default_mapping_type());
698 37890 : e->set_mapping_data(this->default_mapping_data());
699 :
700 44050320 : return e;
701 : }
702 :
703 :
704 :
705 14746574 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
706 : {
707 : // The mesh now takes ownership of the Elem. Eventually the guts of
708 : // add_elem() will get moved to a private helper function, and
709 : // calling add_elem() directly will be deprecated.
710 14746574 : return add_elem(e.release());
711 : }
712 :
713 :
714 :
715 3351724 : Elem * DistributedMesh::insert_elem (Elem * e)
716 : {
717 3351724 : if (_elements[e->id()])
718 3351724 : this->delete_elem(_elements[e->id()]);
719 :
720 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
721 3351724 : if (!e->valid_unique_id())
722 : {
723 0 : if (processor_id() == e->processor_id())
724 : {
725 0 : e->set_unique_id(_next_unique_id);
726 0 : _next_unique_id += this->n_processors() + 1;
727 : }
728 : else
729 : {
730 0 : e->set_unique_id(_next_unpartitioned_unique_id);
731 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
732 : }
733 : }
734 : else
735 : {
736 3351724 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
737 3351724 : _next_unique_id =
738 3355420 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
739 3351724 : (this->n_processors() + 1) + this->processor_id();
740 : }
741 : #endif
742 :
743 : // Try to make the cached elem data more accurate
744 3351724 : processor_id_type elem_procid = e->processor_id();
745 3355420 : if (elem_procid == this->processor_id() ||
746 : elem_procid == DofObject::invalid_processor_id)
747 3220856 : _n_elem++;
748 :
749 3351724 : _elements[e->id()] = e;
750 :
751 : // We actually added a new element. Some of our caches might still
752 : // be valid, but we should clear the ones which definitely are not.
753 3351724 : this->clear_point_locator();
754 3351724 : this->clear_stored_ranges();
755 :
756 : // Make sure any new element is given space for any extra integers
757 : // we've requested
758 3351724 : e->add_extra_integers(_elem_integer_names.size(),
759 3351724 : _elem_integer_default_values);
760 :
761 : // And set mapping type and data on any new element
762 7392 : e->set_mapping_type(this->default_mapping_type());
763 7392 : e->set_mapping_data(this->default_mapping_data());
764 :
765 3351724 : return e;
766 : }
767 :
768 3351724 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
769 : {
770 : // The mesh now takes ownership of the Elem. Eventually the guts of
771 : // insert_elem(Elem*) will get moved to a private helper function, and
772 : // calling insert_elem(Elem*) directly will be deprecated.
773 3351724 : return insert_elem(e.release());
774 : }
775 :
776 :
777 40097977 : void DistributedMesh::delete_elem(Elem * e)
778 : {
779 7610 : libmesh_assert (e);
780 :
781 : // Try to make the cached elem data more accurate
782 40097977 : _n_elem--;
783 :
784 : // Was this a coarse element, not just a coarsening where we still
785 : // have some ancestor structure? Was it a *local* element, that we
786 : // might have been depending on as an owner of local nodes? We'll
787 : // have to be more careful with our nodes in contract() later; no
788 : // telling if we just locally orphaned a node that should be
789 : // globally retained.
790 40107407 : if (e->processor_id() == this->processor_id() &&
791 3640 : !e->parent())
792 80793 : _deleted_coarse_elements = true;
793 :
794 : // Delete the element from the BoundaryInfo object
795 40097977 : this->get_boundary_info().remove(e);
796 :
797 : // But not yet from the container; we might invalidate
798 : // an iterator that way!
799 :
800 : //_elements.erase(e->id());
801 :
802 : // Instead, we set it to nullptr for now
803 :
804 40097977 : _elements[e->id()] = nullptr;
805 :
806 : // delete the element
807 40097977 : delete e;
808 :
809 : // Some of our caches might still be valid, but we should clear the
810 : // ones which definitely are not.
811 40097977 : this->clear_point_locator();
812 40097977 : this->clear_stored_ranges();
813 40097977 : }
814 :
815 :
816 :
817 3208100 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
818 : const dof_id_type new_id)
819 : {
820 : // This could be a no-op
821 3208100 : if (old_id == new_id)
822 0 : return;
823 :
824 3208100 : Elem * el = _elements[old_id];
825 1160 : libmesh_assert (el);
826 1160 : libmesh_assert_equal_to (el->id(), old_id);
827 :
828 3208100 : el->set_id(new_id);
829 1160 : libmesh_assert (!_elements[new_id]);
830 3208100 : _elements[new_id] = el;
831 3208100 : _elements.erase(old_id);
832 :
833 : // Should we delete any caches here? Our point locator indexes by
834 : // element pointer and should be fine with an id change. Our stored
835 : // ranges are no longer sorted, which is *probably* fine, but let's
836 : // just be safe.
837 3208100 : this->clear_stored_ranges();
838 : }
839 :
840 :
841 :
842 40955093 : Node * DistributedMesh::add_point (const Point & p,
843 : const dof_id_type id,
844 : const processor_id_type proc_id)
845 : {
846 40955093 : Node * old_n = this->query_node_ptr(id);
847 :
848 40955093 : if (old_n)
849 : {
850 0 : *old_n = p;
851 0 : old_n->processor_id() = proc_id;
852 :
853 0 : return old_n;
854 : }
855 :
856 40915194 : Node * n = Node::build(p, id).release();
857 40955093 : n->processor_id() = proc_id;
858 :
859 40955093 : return DistributedMesh::add_node(n);
860 : }
861 :
862 :
863 4130 : void DistributedMesh::own_node (Node & n)
864 : {
865 : // This had better be a node in our mesh
866 0 : libmesh_assert(_nodes[n.id()] == &n);
867 :
868 4130 : _nodes[n.id()] = nullptr;
869 4130 : _n_nodes--;
870 :
871 0 : n.set_id(DofObject::invalid_id);
872 4130 : n.processor_id() = this->processor_id();
873 :
874 4130 : this->add_node(&n);
875 4130 : }
876 :
877 :
878 71733001 : Node * DistributedMesh::add_node (Node * n)
879 : {
880 : // Don't try to add nullptrs!
881 41611 : libmesh_assert(n);
882 :
883 : // Trying to add an existing node is a no-op
884 71733001 : if (n->valid_id() && _nodes[n->id()] == n)
885 0 : return n;
886 :
887 71733001 : const processor_id_type node_procid = n->processor_id();
888 :
889 71733001 : if (!n->valid_id())
890 : {
891 : // We should only be creating new ids past the end of the range
892 : // of existing ids
893 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
894 : _max_node_id);
895 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
896 :
897 : // Use the unpartitioned ids for unpartitioned nodes,
898 : // and temporarily for ghost nodes
899 17191263 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
900 17209092 : if (node_procid == this->processor_id())
901 2128730 : next_id = &_next_free_local_node_id;
902 17191263 : n->set_id (*next_id);
903 : }
904 :
905 : {
906 : // Advance next_ids up high enough that each is pointing to an
907 : // unused id and any subsequent increments will still point us
908 : // to unused ids
909 143466002 : _max_node_id = std::max(_max_node_id,
910 71733001 : static_cast<dof_id_type>(n->id()+1));
911 :
912 71733001 : if (_next_free_unpartitioned_node_id < _max_node_id)
913 20115915 : _next_free_unpartitioned_node_id =
914 20115915 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
915 20115915 : (this->n_processors() + 1) + this->n_processors();
916 71733001 : if (_next_free_local_node_id < _max_node_id)
917 11727360 : _next_free_local_node_id =
918 11745528 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
919 11727360 : (this->n_processors() + 1) + this->processor_id();
920 :
921 : #ifndef NDEBUG
922 : // We need a const dofobject_container so we don't inadvertently create
923 : // nullptr entries when testing for non-nullptr ones
924 41611 : const dofobject_container<Node> & const_nodes = _nodes;
925 : #endif
926 41611 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
927 41611 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
928 : }
929 :
930 : // Don't try to overwrite existing nodes
931 41611 : libmesh_assert (!_nodes[n->id()]);
932 :
933 71733001 : _nodes[n->id()] = n;
934 :
935 : // Try to make the cached node data more accurate
936 71774612 : if (node_procid == this->processor_id() ||
937 : node_procid == DofObject::invalid_processor_id)
938 38540731 : _n_nodes++;
939 :
940 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
941 71733001 : if (!n->valid_unique_id())
942 : {
943 40956973 : if (processor_id() == n->processor_id())
944 : {
945 2545412 : n->set_unique_id(_next_unique_id);
946 2545412 : _next_unique_id += this->n_processors() + 1;
947 : }
948 : else
949 : {
950 38411561 : n->set_unique_id(_next_unpartitioned_unique_id);
951 38411561 : _next_unpartitioned_unique_id += this->n_processors() + 1;
952 : }
953 : }
954 : else
955 : {
956 30906548 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
957 30776028 : _next_unique_id =
958 30776028 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
959 30776028 : (this->n_processors() + 1) + this->processor_id();
960 : }
961 : #endif
962 :
963 71733001 : n->add_extra_integers(_node_integer_names.size(),
964 71733001 : _node_integer_default_values);
965 :
966 : // Unpartitioned nodes should be added on every processor
967 : // And shouldn't be added in the same batch as ghost nodes
968 : // But we might be just adding on processor 0 to
969 : // broadcast later
970 : // #ifdef DEBUG
971 : // if (node_procid == DofObject::invalid_processor_id)
972 : // {
973 : // dof_id_type node_id = n->id();
974 : // this->comm().max(node_id);
975 : // libmesh_assert_equal_to (node_id, n->id());
976 : // }
977 : // #endif
978 :
979 71733001 : return n;
980 : }
981 :
982 30773778 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
983 : {
984 : // The mesh now takes ownership of the Node. Eventually the guts of
985 : // add_node() will get moved to a private helper function, and
986 : // calling add_node() directly will be deprecated.
987 30773778 : return add_node(n.release());
988 : }
989 :
990 50495302 : void DistributedMesh::delete_node(Node * n)
991 : {
992 4497 : libmesh_assert(n);
993 4497 : libmesh_assert(_nodes[n->id()]);
994 :
995 : // Try to make the cached elem data more accurate
996 50495302 : _n_nodes--;
997 :
998 : // Delete the node from the BoundaryInfo object
999 50495302 : this->get_boundary_info().remove(n);
1000 8994 : _constraint_rows.erase(n);
1001 :
1002 : // But not yet from the container; we might invalidate
1003 : // an iterator that way!
1004 :
1005 : //_nodes.erase(n->id());
1006 :
1007 : // Instead, we set it to nullptr for now
1008 :
1009 50495302 : _nodes[n->id()] = nullptr;
1010 :
1011 : // delete the node
1012 50495302 : delete n;
1013 50495302 : }
1014 :
1015 :
1016 :
1017 6899073 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1018 : const dof_id_type new_id)
1019 : {
1020 : // This could be a no-op
1021 6899073 : if (old_id == new_id)
1022 0 : return;
1023 :
1024 6898512 : Node * nd = _nodes[old_id];
1025 3234 : libmesh_assert (nd);
1026 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1027 :
1028 : // If we have nodes shipped to this processor for NodeConstraints
1029 : // use, then those nodes will exist in _nodes, but may not be
1030 : // locatable via a TopologyMap due to the insufficiency of elements
1031 : // connecting to them. If local refinement then wants to create a
1032 : // *new* node in the same location, it will initially get a temporary
1033 : // id, and then make_node_ids_parallel_consistent() will try to move
1034 : // it to the canonical id. We need to account for this case to
1035 : // avoid false positives and memory leaks.
1036 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1037 6468 : if (_nodes[new_id])
1038 : {
1039 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1040 : *(Point *)_nodes[old_id]);
1041 0 : _nodes.erase(new_id);
1042 : }
1043 : #else
1044 : // If we aren't shipping nodes for NodeConstraints, there should be
1045 : // no reason for renumbering one node onto another.
1046 : libmesh_assert (!_nodes[new_id]);
1047 : #endif
1048 6898512 : _nodes[new_id] = nd;
1049 6898512 : nd->set_id(new_id);
1050 :
1051 6898512 : _nodes.erase(old_id);
1052 : }
1053 :
1054 :
1055 :
1056 556952 : void DistributedMesh::clear ()
1057 : {
1058 : // Call parent clear function
1059 556952 : MeshBase::clear();
1060 :
1061 : // Clear our elements and nodes
1062 : // There is no need to remove them from
1063 : // the BoundaryInfo data structure since we
1064 : // already cleared it.
1065 556952 : this->DistributedMesh::clear_elems();
1066 :
1067 22594334 : for (auto & node : _nodes)
1068 40907060 : delete node;
1069 :
1070 578 : _nodes.clear();
1071 :
1072 : // We're no longer distributed if we were before
1073 556952 : _is_serial = true;
1074 556952 : _is_serial_on_proc_0 = true;
1075 :
1076 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1077 : // all is copacetic.
1078 556952 : _deleted_coarse_elements = false;
1079 :
1080 : // Correct our caches
1081 556952 : _n_nodes = 0;
1082 556952 : _max_node_id = 0;
1083 557530 : _next_free_local_node_id = this->processor_id();
1084 556952 : _next_free_unpartitioned_node_id = this->n_processors();
1085 556952 : }
1086 :
1087 :
1088 :
1089 563352 : void DistributedMesh::clear_elems ()
1090 : {
1091 9213447 : for (auto & elem : _elements)
1092 8650095 : delete elem;
1093 :
1094 578 : _elements.clear();
1095 :
1096 : // Correct our caches
1097 563352 : _n_elem = 0;
1098 563352 : _max_elem_id = 0;
1099 563930 : _next_free_local_elem_id = this->processor_id();
1100 563352 : _next_free_unpartitioned_elem_id = this->n_processors();
1101 563352 : }
1102 :
1103 :
1104 :
1105 560012 : void DistributedMesh::redistribute ()
1106 : {
1107 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1108 560012 : if (!this->is_serial())
1109 : {
1110 : // Construct a MeshCommunication object to actually redistribute the nodes
1111 : // and elements according to the partitioner, and then to re-gather the neighbors.
1112 : MeshCommunication mc;
1113 68322 : mc.redistribute(*this);
1114 :
1115 68322 : this->update_parallel_id_counts();
1116 :
1117 : // We ought to still have valid neighbor links; we communicate
1118 : // them for newly-redistributed elements
1119 : // this->find_neighbors();
1120 :
1121 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1122 : // anyway... but users can always call partition directly, in which case we do need
1123 : // to call delete_remote_elements()...
1124 : //
1125 : // Regardless of whether it's necessary, it isn't safe. We
1126 : // haven't communicated new node processor_ids yet, and we can't
1127 : // delete nodes until we do.
1128 : // this->delete_remote_elements();
1129 : }
1130 : else
1131 : // The base class can handle non-distributed things, like
1132 : // notifying any GhostingFunctors of changes
1133 491690 : MeshBase::redistribute();
1134 560012 : }
1135 :
1136 :
1137 :
1138 481565 : void DistributedMesh::update_post_partitioning ()
1139 : {
1140 : // this->recalculate_n_partitions();
1141 :
1142 : // Let's do the base class cache clearing first, just in case our
1143 : // later computations are ever changed to make use of a local
1144 : // elements range cache
1145 481565 : this->MeshBase::update_post_partitioning();
1146 :
1147 : // Partitioning changes our numbers of unpartitioned objects
1148 481565 : this->update_parallel_id_counts();
1149 481565 : }
1150 :
1151 :
1152 :
1153 : template <typename T>
1154 3356 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1155 : {
1156 : // This function must be run on all processors at once
1157 3356 : parallel_object_only();
1158 :
1159 3356 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1160 3356 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1161 3356 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1162 :
1163 894268 : for (dof_id_type i=0; i != pmax_id; ++i)
1164 : {
1165 890912 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1166 :
1167 : // Local lookups by id should return the requested object
1168 890912 : libmesh_assert(!obj || obj->id() == i);
1169 :
1170 : // All processors with an object should agree on id
1171 : #ifndef NDEBUG
1172 890912 : const dof_id_type dofid = obj && obj->valid_id() ?
1173 : obj->id() : DofObject::invalid_id;
1174 890912 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1175 : #endif
1176 :
1177 : // All processors with an object should agree on processor id
1178 890912 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1179 : obj->processor_id() : DofObject::invalid_processor_id;
1180 890912 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1181 :
1182 890912 : dof_id_type min_procid = procid;
1183 890912 : this->comm().min(min_procid);
1184 :
1185 : // Either:
1186 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1187 : // or
1188 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1189 :
1190 : // Original assert logic
1191 : // libmesh_assert (min_procid != this->processor_id() || obj);
1192 :
1193 : // More human-understandable logic...
1194 890912 : libmesh_assert (
1195 : ((min_procid == this->processor_id()) && obj)
1196 : ||
1197 : (min_procid != this->processor_id())
1198 : );
1199 :
1200 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1201 : // All processors with an object should agree on unique id
1202 890912 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1203 890912 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1204 : #endif
1205 : }
1206 3356 : }
1207 :
1208 :
1209 :
1210 1678 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1211 : {
1212 1678 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1213 1678 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1214 1678 : }
1215 :
1216 :
1217 :
1218 402 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1219 : {
1220 : #ifndef NDEBUG
1221 : // This function must be run on all processors at once
1222 402 : parallel_object_only();
1223 :
1224 402 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1225 :
1226 21958 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1227 : {
1228 21556 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1229 :
1230 21556 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1231 :
1232 : // All processors with an active element should agree on p level
1233 21556 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1234 : }
1235 : #endif
1236 402 : }
1237 :
1238 :
1239 :
1240 :
1241 1626 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1242 : {
1243 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1244 : // This function must be run on all processors at once
1245 1626 : parallel_object_only();
1246 :
1247 1626 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1248 :
1249 144668 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1250 : {
1251 143042 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1252 :
1253 143042 : unsigned int refinement_flag = el ?
1254 65709 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1255 143042 : unsigned int p_refinement_flag = el ?
1256 65709 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1257 :
1258 143042 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1259 :
1260 : // p refinement flags aren't always kept correct on inactive
1261 : // ghost elements
1262 143042 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1263 : }
1264 : #endif // LIBMESH_ENABLE_AMR
1265 1626 : }
1266 :
1267 :
1268 :
1269 : template <typename T>
1270 : dof_id_type
1271 1717566 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1272 : {
1273 : // This function must be run on all processors at once
1274 800 : parallel_object_only();
1275 :
1276 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1277 :
1278 : // In parallel we may not know what objects other processors have.
1279 : // Start by figuring out how many
1280 800 : dof_id_type unpartitioned_objects = 0;
1281 :
1282 : std::unordered_map<processor_id_type, dof_id_type>
1283 1600 : ghost_objects_from_proc;
1284 :
1285 1717566 : object_iterator it = objects.begin();
1286 800 : object_iterator end = objects.end();
1287 :
1288 109777364 : while (it != end)
1289 : {
1290 108059798 : T * obj = *it;
1291 :
1292 : // Remove any nullptr container entries while we're here.
1293 108059798 : if (!obj)
1294 1417935 : it = objects.erase(it);
1295 : else
1296 : {
1297 106641863 : processor_id_type obj_procid = obj->processor_id();
1298 106641863 : if (obj_procid == DofObject::invalid_processor_id)
1299 42414931 : unpartitioned_objects++;
1300 : else
1301 64226932 : ghost_objects_from_proc[obj_procid]++;
1302 :
1303 : // Finally, increment the iterator
1304 56414 : ++it;
1305 : }
1306 : }
1307 :
1308 1719166 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1309 1718366 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1310 800 : this->comm().allgather
1311 1718057 : ((this_it == ghost_objects_from_proc.end()) ?
1312 491 : dof_id_type(0) : this_it->second, objects_on_proc);
1313 :
1314 : #ifndef NDEBUG
1315 800 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1316 2400 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1317 1600 : if (ghost_objects_from_proc.count(p))
1318 951 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1319 : else
1320 649 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1321 : #endif
1322 :
1323 : // We'll renumber objects in blocks by processor id
1324 1719166 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1325 18225342 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1326 16510976 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1327 800 : objects_on_proc[i-1];
1328 1718366 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1329 1717566 : dof_id_type first_free_id =
1330 1719166 : first_object_on_proc[this->n_processors()-1] +
1331 800 : objects_on_proc[this->n_processors()-1] +
1332 : unpartitioned_objects;
1333 :
1334 : // First set new local object ids and build request sets
1335 : // for non-local object ids
1336 :
1337 : // Request sets to send to each processor
1338 : std::map<processor_id_type, std::vector<dof_id_type>>
1339 800 : requested_ids;
1340 :
1341 : // We know how many objects live on each processor, so reserve() space for
1342 : // each.
1343 800 : auto ghost_end = ghost_objects_from_proc.end();
1344 19942908 : for (auto p : make_range(this->n_processors()))
1345 18226942 : if (p != this->processor_id())
1346 : {
1347 16508576 : if (const auto p_it = ghost_objects_from_proc.find(p);
1348 800 : p_it != ghost_end)
1349 3322991 : requested_ids[p].reserve(p_it->second);
1350 : }
1351 :
1352 800 : end = objects.end();
1353 108359429 : for (it = objects.begin(); it != end; ++it)
1354 : {
1355 106641863 : T * obj = *it;
1356 106641863 : if (!obj)
1357 0 : continue;
1358 106698277 : if (obj->processor_id() == this->processor_id())
1359 19773492 : obj->set_id(next_id++);
1360 86868371 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1361 44453440 : requested_ids[obj->processor_id()].push_back(obj->id());
1362 : }
1363 :
1364 : // Next set ghost object ids from other processors
1365 :
1366 1734737 : auto gather_functor =
1367 47743009 : [
1368 : #ifndef NDEBUG
1369 : this,
1370 : &first_object_on_proc,
1371 : &objects_on_proc,
1372 : #endif
1373 : &objects]
1374 : (processor_id_type, const std::vector<dof_id_type> & ids,
1375 920 : std::vector<dof_id_type> & new_ids)
1376 : {
1377 920 : std::size_t ids_size = ids.size();
1378 3322991 : new_ids.resize(ids_size);
1379 :
1380 47776431 : for (std::size_t i=0; i != ids_size; ++i)
1381 : {
1382 44453440 : T * obj = objects[ids[i]];
1383 16251 : libmesh_assert(obj);
1384 16251 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1385 44469691 : new_ids[i] = obj->id();
1386 :
1387 16251 : libmesh_assert_greater_equal (new_ids[i],
1388 : first_object_on_proc[this->processor_id()]);
1389 16251 : libmesh_assert_less (new_ids[i],
1390 : first_object_on_proc[this->processor_id()] +
1391 : objects_on_proc[this->processor_id()]);
1392 : }
1393 : };
1394 :
1395 1734737 : auto action_functor =
1396 47743009 : [
1397 : #ifndef NDEBUG
1398 : &first_object_on_proc,
1399 : &objects_on_proc,
1400 : #endif
1401 : &objects]
1402 : (processor_id_type libmesh_dbg_var(pid),
1403 : const std::vector<dof_id_type> & ids,
1404 920 : const std::vector<dof_id_type> & data)
1405 : {
1406 : // Copy the id changes we've now been informed of
1407 47776431 : for (auto i : index_range(ids))
1408 : {
1409 44453440 : T * obj = objects[ids[i]];
1410 16251 : libmesh_assert (obj);
1411 16251 : libmesh_assert_equal_to (obj->processor_id(), pid);
1412 16251 : libmesh_assert_greater_equal (data[i],
1413 : first_object_on_proc[pid]);
1414 16251 : libmesh_assert_less (data[i],
1415 : first_object_on_proc[pid] +
1416 : objects_on_proc[pid]);
1417 44469691 : obj->set_id(data[i]);
1418 : }
1419 : };
1420 :
1421 800 : const dof_id_type * ex = nullptr;
1422 : Parallel::pull_parallel_vector_data
1423 1717566 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1424 :
1425 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1426 1734737 : auto unique_gather_functor =
1427 47743009 : [
1428 : #ifndef NDEBUG
1429 : this,
1430 : #endif
1431 : &objects]
1432 : (processor_id_type, const std::vector<dof_id_type> & ids,
1433 920 : std::vector<unique_id_type> & data)
1434 : {
1435 920 : std::size_t ids_size = ids.size();
1436 3322991 : data.resize(ids_size);
1437 :
1438 47776431 : for (std::size_t i=0; i != ids_size; ++i)
1439 : {
1440 44453440 : T * obj = objects[ids[i]];
1441 16251 : libmesh_assert(obj);
1442 16251 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1443 88906880 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1444 : }
1445 : };
1446 :
1447 1734737 : auto unique_action_functor =
1448 47743009 : [&objects]
1449 : (processor_id_type libmesh_dbg_var(pid),
1450 : const std::vector<dof_id_type> & ids,
1451 920 : const std::vector<unique_id_type> & data)
1452 : {
1453 47776431 : for (auto i : index_range(ids))
1454 : {
1455 44453440 : T * obj = objects[ids[i]];
1456 16251 : libmesh_assert (obj);
1457 16251 : libmesh_assert_equal_to (obj->processor_id(), pid);
1458 44453440 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1459 0 : obj->set_unique_id(data[i]);
1460 : }
1461 : };
1462 :
1463 800 : const unique_id_type * unique_ex = nullptr;
1464 : Parallel::pull_parallel_vector_data
1465 1717566 : (this->comm(), requested_ids, unique_gather_functor,
1466 : unique_action_functor, unique_ex);
1467 : #endif
1468 :
1469 : // Next set unpartitioned object ids
1470 800 : next_id = 0;
1471 19942908 : for (auto i : make_range(this->n_processors()))
1472 18226942 : next_id += objects_on_proc[i];
1473 108359429 : for (it = objects.begin(); it != end; ++it)
1474 : {
1475 106641863 : T * obj = *it;
1476 106641863 : if (!obj)
1477 0 : continue;
1478 106641863 : if (obj->processor_id() == DofObject::invalid_processor_id)
1479 42414931 : obj->set_id(next_id++);
1480 : }
1481 :
1482 : // Finally shuffle around objects so that container indices
1483 : // match ids
1484 1717566 : it = objects.begin();
1485 800 : end = objects.end();
1486 114830935 : while (it != end)
1487 : {
1488 113113369 : T * obj = *it;
1489 113113369 : if (obj) // don't try shuffling already-nullptr entries
1490 : {
1491 113113369 : T * next = objects[obj->id()];
1492 : // If we have to move this object
1493 113113369 : if (next != obj)
1494 : {
1495 : // nullptr out its original position for now
1496 : // (our shuffling may put another object there shortly)
1497 39560477 : *it = nullptr;
1498 :
1499 : // There may already be another object with this id that
1500 : // needs to be moved itself
1501 51028603 : while (next)
1502 : {
1503 : // We shouldn't be trying to give two objects the
1504 : // same id
1505 9329 : libmesh_assert_not_equal_to (next->id(), obj->id());
1506 11468126 : objects[obj->id()] = obj;
1507 9329 : obj = next;
1508 11468126 : next = objects[obj->id()];
1509 : }
1510 39560477 : objects[obj->id()] = obj;
1511 : }
1512 : }
1513 :
1514 : // Remove any container entries that were left as nullptr.
1515 115296 : if (!obj)
1516 0 : it = objects.erase(it);
1517 : else
1518 57648 : ++it;
1519 : }
1520 :
1521 1718366 : return first_free_id;
1522 : }
1523 :
1524 :
1525 860453 : void DistributedMesh::renumber_nodes_and_elements ()
1526 : {
1527 402 : parallel_object_only();
1528 :
1529 : #ifdef DEBUG
1530 : // Make sure our ids and flags are consistent
1531 402 : this->libmesh_assert_valid_parallel_ids();
1532 402 : this->libmesh_assert_valid_parallel_flags();
1533 402 : this->libmesh_assert_valid_parallel_p_levels();
1534 : #endif
1535 :
1536 402 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1537 :
1538 : // Nodes not connected to any elements, and nullptr node entries
1539 : // in our container, should be deleted. But wait! If we've deleted coarse
1540 : // local elements on some processor, other processors might have ghosted
1541 : // nodes from it that are now no longer connected to any elements on it, but
1542 : // that are connected to their own semilocal elements. We'll have to
1543 : // communicate to ascertain if that's the case.
1544 860453 : this->comm().max(_deleted_coarse_elements);
1545 :
1546 : // What used nodes do we see on our proc?
1547 402 : std::set<dof_id_type> used_nodes;
1548 :
1549 : // What used node info should we send from our proc? Could we take ownership
1550 : // of each node if we needed to?
1551 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1552 402 : used_nodes_on_proc;
1553 :
1554 : // flag the nodes we need
1555 56897540 : for (auto & elem : this->element_ptr_range())
1556 296435061 : for (const Node & node : elem->node_ref_range())
1557 : {
1558 268817129 : const dof_id_type n = node.id();
1559 268672759 : used_nodes.insert(n);
1560 268817129 : if (_deleted_coarse_elements)
1561 : {
1562 3145701 : const processor_id_type p = node.processor_id();
1563 3145985 : if (p != this->processor_id())
1564 : {
1565 2257624 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1566 2257766 : if (elem->processor_id() == this->processor_id())
1567 216210 : used_nodes_on_p[n] = true;
1568 : else
1569 105 : if (!used_nodes_on_p.count(n))
1570 525626 : used_nodes_on_p[n] = false;
1571 : }
1572 : }
1573 859649 : }
1574 :
1575 860453 : if (_deleted_coarse_elements)
1576 : {
1577 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1578 : typedef unsigned char boolish;
1579 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1580 28 : used_nodes_on_proc_vecs;
1581 73209 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1582 56827 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1583 :
1584 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1585 : std::map<processor_id_type, std::set<dof_id_type>>
1586 28 : repartitioned_node_sets_to_push;
1587 :
1588 : auto ids_action_functor =
1589 56801 : [&used_nodes, &repartitioned_node_pids,
1590 : &repartitioned_node_sets_to_push]
1591 : (processor_id_type pid,
1592 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1593 : {
1594 592596 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1595 : {
1596 : // If we don't see a use for our own node, but someone
1597 : // else does, better figure out who should own it next.
1598 97 : if (!used_nodes.count(n))
1599 : {
1600 2 : if (auto it = repartitioned_node_pids.find(n);
1601 2 : sender_could_become_owner)
1602 : {
1603 1 : if (it != repartitioned_node_pids.end() &&
1604 1 : pid < it->second)
1605 1 : it->second = pid;
1606 : else
1607 0 : repartitioned_node_pids[n] = pid;
1608 : }
1609 : else
1610 1 : if (it == repartitioned_node_pids.end())
1611 1 : repartitioned_node_pids[n] =
1612 : DofObject::invalid_processor_id;
1613 :
1614 2 : repartitioned_node_sets_to_push[pid].insert(n);
1615 : }
1616 : }
1617 16394 : };
1618 :
1619 : // We need two pushes instead of a pull here because we need to
1620 : // know *all* the queries for a particular node before we can
1621 : // respond to *any* of them.
1622 : Parallel::push_parallel_vector_data
1623 16382 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1624 :
1625 : // Repartition (what used to be) our own nodes first
1626 16383 : for (auto & [n, p] : repartitioned_node_pids)
1627 : {
1628 1 : Node & node = this->node_ref(n);
1629 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1630 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1631 1 : node.processor_id() = p;
1632 : }
1633 :
1634 : // Then push to repartition others' ghosted copies.
1635 :
1636 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1637 28 : repartitioned_node_vecs;
1638 :
1639 16384 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1640 : {
1641 2 : auto & rn_vec = repartitioned_node_vecs[p];
1642 4 : for (auto n : nodeset)
1643 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1644 : }
1645 :
1646 : auto repartition_node_functor =
1647 2 : [this]
1648 : (processor_id_type libmesh_dbg_var(pid),
1649 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1650 : {
1651 4 : for (auto [n, p] : ids_and_pids)
1652 : {
1653 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1654 2 : Node & node = this->node_ref(n);
1655 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1656 2 : node.processor_id() = p;
1657 : }
1658 16354 : };
1659 :
1660 : Parallel::push_parallel_vector_data
1661 16382 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1662 : }
1663 :
1664 860453 : _deleted_coarse_elements = false;
1665 :
1666 : // Nodes not connected to any local elements, and nullptr node entries
1667 : // in our container, are deleted
1668 : {
1669 860453 : node_iterator_imp it = _nodes.begin();
1670 402 : node_iterator_imp end = _nodes.end();
1671 :
1672 86718050 : while (it != end)
1673 : {
1674 85857597 : Node * nd = *it;
1675 85857597 : if (!nd)
1676 3471824 : it = _nodes.erase(it);
1677 82385773 : else if (!used_nodes.count(nd->id()))
1678 : {
1679 : // remove any boundary information associated with
1680 : // this node
1681 2328191 : this->get_boundary_info().remove (nd);
1682 2828 : _constraint_rows.erase(nd);
1683 :
1684 : // delete the node
1685 2328191 : delete nd;
1686 :
1687 2328191 : it = _nodes.erase(it);
1688 : }
1689 : else
1690 45269 : ++it;
1691 : }
1692 : }
1693 :
1694 860453 : this->_preparation.has_removed_orphaned_nodes = true;
1695 :
1696 860453 : if (_skip_renumber_nodes_and_elements)
1697 : {
1698 1670 : this->update_parallel_id_counts();
1699 2 : return;
1700 : }
1701 :
1702 : // Finally renumber all the elements
1703 858783 : _n_elem = this->renumber_dof_objects (this->_elements);
1704 :
1705 : // and all the remaining nodes
1706 858783 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1707 :
1708 : // And figure out what IDs we should use when adding new nodes and
1709 : // new elements
1710 858783 : this->update_parallel_id_counts();
1711 :
1712 : // Make sure our caches are up to date and our
1713 : // DofObjects are well packed
1714 : #ifdef DEBUG
1715 400 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1716 400 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1717 400 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1718 400 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1719 400 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1720 400 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1721 400 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1722 400 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1723 :
1724 : // Make sure our ids and flags are consistent
1725 400 : this->libmesh_assert_valid_parallel_ids();
1726 400 : this->libmesh_assert_valid_parallel_flags();
1727 :
1728 : // And make sure we've made our numbering monotonic
1729 400 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1730 : #endif
1731 : }
1732 :
1733 :
1734 :
1735 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1736 : {
1737 : // We can't use range-for here because we need access to the special
1738 : // iterators' methods, not just to their dereferenced values.
1739 :
1740 : // Nodes first
1741 0 : for (auto pr = this->_nodes.begin(),
1742 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1743 : {
1744 5427360 : Node * n = *pr;
1745 5427360 : if (n != nullptr)
1746 : {
1747 0 : const dof_id_type id = pr.index();
1748 5204550 : n->set_id() = id;
1749 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1750 : }
1751 : }
1752 :
1753 : // Elements next
1754 0 : for (auto pr = this->_elements.begin(),
1755 5671231 : end = this->_elements.end(); pr != end; ++pr)
1756 : {
1757 5653821 : Elem * e = *pr;
1758 5653821 : if (e != nullptr)
1759 : {
1760 0 : const dof_id_type id = pr.index();
1761 5590229 : e->set_id() = id;
1762 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1763 : }
1764 : }
1765 17410 : }
1766 :
1767 :
1768 :
1769 638691 : dof_id_type DistributedMesh::n_active_elem () const
1770 : {
1771 550 : parallel_object_only();
1772 :
1773 : // Get local active elements first
1774 : dof_id_type active_elements =
1775 1276832 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1776 1914973 : this->active_local_elements_end()));
1777 638691 : this->comm().sum(active_elements);
1778 :
1779 : // Then add unpartitioned active elements, which should exist on
1780 : // every processor
1781 638691 : active_elements +=
1782 638691 : static_cast<dof_id_type>(std::distance
1783 1276832 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1784 639241 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1785 638691 : return active_elements;
1786 : }
1787 :
1788 :
1789 :
1790 404641 : void DistributedMesh::delete_remote_elements()
1791 : {
1792 : #ifdef DEBUG
1793 : // Make sure our neighbor links are all fine
1794 388 : MeshTools::libmesh_assert_valid_neighbors(*this);
1795 :
1796 : // And our child/parent links, and our flags
1797 388 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1798 :
1799 : // Make sure our ids and flags are consistent
1800 388 : this->libmesh_assert_valid_parallel_ids();
1801 388 : this->libmesh_assert_valid_parallel_flags();
1802 :
1803 388 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1804 388 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1805 388 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1806 388 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1807 388 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1808 388 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1809 : #endif
1810 :
1811 404641 : _is_serial = false;
1812 404641 : _is_serial_on_proc_0 = false;
1813 :
1814 404641 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1815 :
1816 388 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1817 :
1818 : // Now make sure the containers actually shrink - strip
1819 : // any newly-created nullptr voids out of the element array
1820 404641 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1821 388 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1822 53430172 : while (e_it != e_end)
1823 53025531 : if (!*e_it)
1824 39493540 : e_it = _elements.erase(e_it);
1825 : else
1826 17389 : ++e_it;
1827 :
1828 404641 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1829 388 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1830 99941974 : while (n_it != n_end)
1831 99537333 : if (!*n_it)
1832 66380838 : n_it = _nodes.erase(n_it);
1833 : else
1834 48597 : ++n_it;
1835 :
1836 : // We may have deleted no-longer-connected nodes or coarsened-away
1837 : // elements; let's update our caches.
1838 404641 : this->update_parallel_id_counts();
1839 :
1840 : // We may have deleted nodes or elements that were the only local
1841 : // representatives of some particular boundary id(s); let's update
1842 : // those caches.
1843 404641 : this->get_boundary_info().regenerate_id_sets();
1844 :
1845 : #ifdef DEBUG
1846 : // We might not have well-packed objects if the user didn't allow us
1847 : // to renumber
1848 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1849 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1850 :
1851 : // Make sure our neighbor links are all fine
1852 388 : MeshTools::libmesh_assert_valid_neighbors(*this);
1853 :
1854 : // And our child/parent links, and our flags
1855 388 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1856 :
1857 : // Make sure our ids and flags are consistent
1858 388 : this->libmesh_assert_valid_parallel_ids();
1859 388 : this->libmesh_assert_valid_parallel_flags();
1860 : #endif
1861 :
1862 404641 : this->_preparation.has_removed_remote_elements = true;
1863 404641 : }
1864 :
1865 :
1866 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1867 : {
1868 : // First add the elem like normal
1869 0 : add_elem(e);
1870 :
1871 : // Now add it to the set that won't be deleted when we call
1872 : // delete_remote_elements()
1873 0 : _extra_ghost_elems.insert(e);
1874 0 : }
1875 :
1876 : void
1877 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1878 : {
1879 0 : std::set<Elem *> tmp;
1880 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1881 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1882 0 : std::inserter(tmp, tmp.begin()));
1883 0 : _extra_ghost_elems = tmp;
1884 0 : }
1885 :
1886 97139 : void DistributedMesh::allgather()
1887 : {
1888 97139 : if (_is_serial)
1889 0 : return;
1890 97137 : MeshCommunication().allgather(*this);
1891 97137 : _is_serial = true;
1892 97137 : _is_serial_on_proc_0 = true;
1893 :
1894 : // Make sure our caches are up to date and our
1895 : // DofObjects are well packed
1896 : #ifdef DEBUG
1897 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1898 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1899 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1900 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1901 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1902 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1903 :
1904 : // If we've disabled renumbering we can't be sure we're contiguous
1905 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1906 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1907 :
1908 : // Make sure our neighbor links are all fine
1909 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1910 :
1911 : // Make sure our ids and flags are consistent
1912 48 : this->libmesh_assert_valid_parallel_ids();
1913 48 : this->libmesh_assert_valid_parallel_flags();
1914 : #endif
1915 : }
1916 :
1917 18352 : void DistributedMesh::gather_to_zero()
1918 : {
1919 18352 : if (_is_serial_on_proc_0)
1920 4 : return;
1921 :
1922 14114 : _is_serial_on_proc_0 = true;
1923 14114 : MeshCommunication().gather(0, *this);
1924 : }
1925 :
1926 :
1927 : } // namespace libMesh
|
