Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 298736 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 298736 : unsigned char d) :
44 298240 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 298240 : _is_serial_on_proc_0(true),
46 298240 : _deleted_coarse_elements(false),
47 298240 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 298984 : _next_free_local_node_id(this->processor_id()),
49 298736 : _next_free_local_elem_id(this->processor_id()),
50 298736 : _next_free_unpartitioned_node_id(this->n_processors()),
51 298736 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 298736 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 298736 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 298984 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 597472 : default_partitioner);
64 : _partitioner = Partitioner::build
65 597224 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 298736 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 340846 : DistributedMesh::~DistributedMesh ()
143 : {
144 318955 : this->DistributedMesh::clear(); // Free nodes and elements
145 340846 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17734 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 : DistributedMesh(static_cast<const MeshBase &>(other_mesh),
153 17734 : /*bool other_is_distributed_type=*/true)
154 : {
155 17734 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
156 17734 : _next_free_local_node_id =
157 17734 : other_mesh._next_free_local_node_id;
158 17734 : _next_free_local_elem_id =
159 17734 : other_mesh._next_free_local_elem_id;
160 17734 : _next_free_unpartitioned_node_id =
161 17734 : other_mesh._next_free_unpartitioned_node_id;
162 17734 : _next_free_unpartitioned_elem_id =
163 17734 : other_mesh._next_free_unpartitioned_elem_id;
164 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
165 17734 : _next_unique_id =
166 17734 : other_mesh._next_unique_id;
167 17734 : _next_unpartitioned_unique_id =
168 17734 : other_mesh._next_unpartitioned_unique_id;
169 : #endif
170 :
171 : // Need to copy extra_ghost_elems
172 17734 : for (auto & elem : other_mesh._extra_ghost_elems)
173 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
174 17734 : }
175 :
176 :
177 :
178 20219 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
179 20219 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
180 20309 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
181 20039 : _deleted_coarse_elements(true), // better safe than sorry...
182 20039 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
183 20309 : _next_free_local_node_id(this->processor_id()),
184 20219 : _next_free_local_elem_id(this->processor_id()),
185 20219 : _next_free_unpartitioned_node_id(this->n_processors()),
186 40348 : _next_free_unpartitioned_elem_id(this->n_processors())
187 : {
188 : // Just copy, skipping preparation
189 20219 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
190 :
191 180 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
192 180 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
193 180 : this->allow_renumbering(other_mesh.allow_renumbering());
194 180 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
195 180 : this->skip_partitioning(other_mesh.skip_partitioning());
196 :
197 20219 : this->copy_constraint_rows(other_mesh);
198 :
199 90 : auto & this_boundary_info = this->get_boundary_info();
200 90 : const auto & other_boundary_info = other_mesh.get_boundary_info();
201 :
202 20219 : this_boundary_info = other_boundary_info;
203 :
204 90 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
205 :
206 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
207 : // nodes/elements they consider locally owned (or unpartitioned), so after
208 : // copying only the objects visible to this processor, those counts
209 : // reflect an incomplete local view rather than other_mesh's totals,
210 : // and need to be copied afresh.
211 20219 : _n_nodes = other_mesh.n_nodes();
212 20219 : _n_elem = other_mesh.n_elem();
213 20219 : _max_node_id = other_mesh.max_node_id();
214 20219 : _max_elem_id = other_mesh.max_elem_id();
215 :
216 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
217 : // counters directly, mirroring other_mesh's own bookkeeping exactly. This
218 : // is done in the DistributedMesh copy constructor.
219 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
220 : // raw counterpart fields to copy, so derive the free-id-range
221 : // counters from _max_elem_id/_max_node_id the same way
222 : // update_parallel_id_counts() does via the set_next_ids() method. For
223 : // the unique id counters we copy other_mesh's own local next_unique_id()
224 : // and derive the next available unique ids
225 : // (partitioned and unpartitioned) via set_next_unique_ids().
226 20219 : if (!other_is_distributed_type)
227 : {
228 2485 : this->set_next_ids();
229 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
230 70 : libmesh_assert(other_mesh.comm().verify(other_mesh.next_unique_id()));
231 2485 : this->set_next_unique_ids(other_mesh.next_unique_id());
232 : #endif
233 : }
234 :
235 : // Copy other_mesh's actual preparation state, including whatever it
236 : // reports for has_synched_id_counts: the counts above are a faithful copy
237 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
238 : // verified computation, so our synced-ness should match other_mesh's.
239 20219 : this->_preparation = other_mesh.preparation();
240 20219 : }
241 :
242 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
243 : {
244 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
245 :
246 4 : this->_nodes = std::move(other_mesh._nodes);
247 270 : this->_n_nodes = other_mesh.n_nodes();
248 :
249 4 : this->_elements = std::move(other_mesh._elements);
250 270 : this->_n_elem = other_mesh.n_elem();
251 :
252 270 : _is_serial = other_mesh.is_serial();
253 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
254 270 : _deleted_coarse_elements = true; // Better safe than sorry
255 :
256 270 : _max_node_id = other_mesh.max_node_id();
257 270 : _max_elem_id = other_mesh.max_elem_id();
258 :
259 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
260 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
261 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
262 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
263 :
264 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
265 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
266 : #endif
267 270 : }
268 :
269 2034394 : void DistributedMesh::set_next_ids()
270 : {
271 2034394 : _next_free_unpartitioned_elem_id =
272 2036158 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
273 2034394 : (this->n_processors() + 1) + this->n_processors();
274 2034394 : _next_free_local_elem_id =
275 2034394 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
276 2034394 : (this->n_processors() + 1) + this->processor_id();
277 :
278 2034394 : _next_free_unpartitioned_node_id =
279 2036158 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
280 2034394 : (this->n_processors() + 1) + this->n_processors();
281 2034394 : _next_free_local_node_id =
282 2034394 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
283 2034394 : (this->n_processors() + 1) + this->processor_id();
284 2034394 : }
285 :
286 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
287 2034394 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
288 : {
289 : // Unique ids are laid out in repeating groups of (n_processors()+1)
290 : // consecutive integers ("alignment" here means a value's position, i.e.
291 : // its residue mod (n_processors()+1), within one of these groups). In
292 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
293 : // are each permanently earmarked for one processor's locally-owned
294 : // objects, and the last slot (residue n_processors()) is permanently
295 : // earmarked for unpartitioned objects.
296 : //
297 : // parallel_max_unique_id carries no guarantee of landing on any
298 : // particular residue, so rather than simply offsetting it by
299 : // processor_id()/n_processors(), we have to explicitly round up to the
300 : // start of the next full, entirely-unused group before adding this
301 : // processor's (or the unpartitioned pool's) fixed offset into that group.
302 : //
303 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
304 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
305 : // the number of complete groups needed to reach at least past the
306 : // current max id. Multiplying back by (n_processors()+1) gives the first
307 : // id of the next group that is guaranteed to be entirely unused; adding
308 : // n_processors(), the unpartitioned pool's fixed offset within a group,
309 : // lands on that pool's reserved slot in that group.
310 2034394 : _next_unpartitioned_unique_id =
311 2036158 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
312 2034394 : (this->n_processors() + 1) +
313 2034394 : this->n_processors();
314 :
315 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
316 : // the rounding accounts for processor_id() possibly being less than the
317 : // residue the raw max id already occupies within its group; this can
318 : // land this processor's slot in the same group as the unpartitioned
319 : // pool's above, or push it into the following group, depending on where
320 : // the max id and this processor's offset happen to fall.
321 2034394 : _next_unique_id =
322 2034394 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
323 2034394 : (this->n_processors() + 1) +
324 2034394 : this->processor_id();
325 2034394 : }
326 : #endif
327 :
328 : // We use cached values for these so they can be called
329 : // from one processor without bothering the rest, but
330 : // we may need to update those caches before doing a full
331 : // renumbering
332 2031909 : void DistributedMesh::update_parallel_id_counts()
333 : {
334 : // This function must be run on all processors at once
335 1694 : parallel_object_only();
336 :
337 2031909 : _n_elem = this->parallel_n_elem();
338 2031909 : _n_nodes = this->parallel_n_nodes();
339 2031909 : _max_node_id = this->parallel_max_node_id();
340 2031909 : _max_elem_id = this->parallel_max_elem_id();
341 :
342 2031909 : this->set_next_ids();
343 :
344 : // set_next_unique_ids() needs the true parallel max unique id (not just
345 : // this processor's possibly-stale cached _next_unique_id) since the whole
346 : // point of this function is to recompute counters that may have drifted
347 : // out of sync.
348 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
349 2031909 : this->set_next_unique_ids(this->parallel_max_unique_id());
350 : #endif
351 :
352 2031909 : this->_preparation.has_synched_id_counts = true;
353 2031909 : }
354 :
355 :
356 : // Or in debug mode we may want to test the uncached values without
357 : // changing the cache
358 2039634 : dof_id_type DistributedMesh::parallel_n_elem() const
359 : {
360 : // This function must be run on all processors at once
361 2542 : parallel_object_only();
362 :
363 2039634 : dof_id_type n_local = this->n_local_elem();
364 2039634 : this->comm().sum(n_local);
365 2039634 : n_local += this->n_unpartitioned_elem();
366 2039634 : return n_local;
367 : }
368 :
369 :
370 :
371 2038905 : dof_id_type DistributedMesh::parallel_max_elem_id() const
372 : {
373 : // This function must be run on all processors at once
374 8690 : parallel_object_only();
375 :
376 2038905 : dof_id_type max_local = 0;
377 :
378 : dofobject_container<Elem>::const_reverse_veclike_iterator
379 8690 : rit = _elements.rbegin();
380 :
381 : const dofobject_container<Elem>::const_reverse_veclike_iterator
382 8690 : rend = _elements.rend();
383 :
384 : // Look for the maximum element id. Search backwards through
385 : // elements so we can break out early. Beware of nullptr entries that
386 : // haven't yet been cleared from _elements.
387 10672652 : for (; rit != rend; ++rit)
388 : {
389 10109652 : const DofObject *d = *rit;
390 19001 : if (d)
391 : {
392 8475 : libmesh_assert(_elements[d->id()] == d);
393 1475905 : max_local = d->id() + 1;
394 1475905 : break;
395 : }
396 : }
397 :
398 2038905 : this->comm().max(max_local);
399 2038905 : return max_local;
400 : }
401 :
402 :
403 :
404 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
405 2117385 : unique_id_type DistributedMesh::parallel_max_unique_id() const
406 : {
407 : // This function must be run on all processors at once
408 1796 : parallel_object_only();
409 :
410 4234770 : unique_id_type max_local = std::max(_next_unique_id,
411 2429328 : _next_unpartitioned_unique_id);
412 2117385 : this->comm().max(max_local);
413 2117385 : return max_local;
414 : }
415 :
416 :
417 :
418 80375 : void DistributedMesh::set_next_unique_id(unique_id_type id)
419 : {
420 144 : _next_unique_id = id;
421 80375 : _next_unpartitioned_unique_id =
422 80519 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
423 80375 : (this->n_processors() + 1) + this->n_processors();
424 80375 : _next_unique_id =
425 80375 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
426 80375 : (this->n_processors() + 1) + this->processor_id();
427 80375 : }
428 : #endif
429 :
430 :
431 :
432 2040082 : dof_id_type DistributedMesh::parallel_n_nodes() const
433 : {
434 : // This function must be run on all processors at once
435 2542 : parallel_object_only();
436 :
437 2040082 : dof_id_type n_local = this->n_local_nodes();
438 2040082 : this->comm().sum(n_local);
439 2040082 : n_local += this->n_unpartitioned_nodes();
440 2040082 : return n_local;
441 : }
442 :
443 :
444 :
445 2036489 : dof_id_type DistributedMesh::parallel_max_node_id() const
446 : {
447 : // This function must be run on all processors at once
448 6274 : parallel_object_only();
449 :
450 2036489 : dof_id_type max_local = 0;
451 :
452 : dofobject_container<Node>::const_reverse_veclike_iterator
453 6274 : rit = _nodes.rbegin();
454 :
455 : const dofobject_container<Node>::const_reverse_veclike_iterator
456 6274 : rend = _nodes.rend();
457 :
458 : // Look for the maximum node id. Search backwards through
459 : // nodes so we can break out early. Beware of nullptr entries that
460 : // haven't yet been cleared from _nodes
461 38856316 : for (; rit != rend; ++rit)
462 : {
463 38293386 : const DofObject *d = *rit;
464 63385 : if (d)
465 : {
466 6129 : libmesh_assert(_nodes[d->id()] == d);
467 1473559 : max_local = d->id() + 1;
468 1473559 : break;
469 : }
470 : }
471 :
472 2036489 : this->comm().max(max_local);
473 2036489 : return max_local;
474 : }
475 :
476 :
477 :
478 42988812 : const Point & DistributedMesh::point (const dof_id_type i) const
479 : {
480 42988812 : return this->node_ref(i);
481 : }
482 :
483 :
484 :
485 44094201 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
486 : {
487 136653 : libmesh_assert(_nodes[i]);
488 136653 : libmesh_assert_equal_to (_nodes[i]->id(), i);
489 :
490 44094201 : return _nodes[i];
491 : }
492 :
493 :
494 :
495 :
496 648003882 : Node * DistributedMesh::node_ptr (const dof_id_type i)
497 : {
498 489296 : libmesh_assert(_nodes[i]);
499 489296 : libmesh_assert_equal_to (_nodes[i]->id(), i);
500 :
501 648003882 : return _nodes[i];
502 : }
503 :
504 :
505 :
506 :
507 3109134 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
508 : {
509 3109134 : if (const auto it = _nodes.find(i);
510 657411 : it != _nodes.end())
511 : {
512 2820622 : const Node * n = *it;
513 368899 : libmesh_assert (!n || n->id() == i);
514 368899 : return n;
515 : }
516 :
517 288512 : return nullptr;
518 : }
519 :
520 :
521 :
522 :
523 371658053 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
524 : {
525 371658053 : if (auto it = _nodes.find(i);
526 159369 : it != _nodes.end())
527 : {
528 288212213 : Node * n = *it;
529 111972 : libmesh_assert (!n || n->id() == i);
530 111972 : return n;
531 : }
532 :
533 47397 : return nullptr;
534 : }
535 :
536 :
537 :
538 :
539 3058368 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
540 : {
541 10087 : libmesh_assert(_elements[i]);
542 10087 : libmesh_assert_equal_to (_elements[i]->id(), i);
543 :
544 3058368 : return _elements[i];
545 : }
546 :
547 :
548 :
549 :
550 288492737 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
551 : {
552 167221 : libmesh_assert(_elements[i]);
553 167221 : libmesh_assert_equal_to (_elements[i]->id(), i);
554 :
555 288492737 : return _elements[i];
556 : }
557 :
558 :
559 :
560 :
561 1457373 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
562 : {
563 1457373 : if (const auto it = _elements.find(i);
564 618403 : it != _elements.end())
565 : {
566 1099794 : const Elem * e = *it;
567 269739 : libmesh_assert (!e || e->id() == i);
568 269739 : return e;
569 : }
570 :
571 348664 : return nullptr;
572 : }
573 :
574 :
575 :
576 :
577 608695730 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
578 : {
579 608695730 : if (auto it = _elements.find(i);
580 194370 : it != _elements.end())
581 : {
582 523112317 : Elem * e = *it;
583 190690 : libmesh_assert (!e || e->id() == i);
584 190690 : return e;
585 : }
586 :
587 3680 : return nullptr;
588 : }
589 :
590 :
591 :
592 :
593 44197646 : Elem * DistributedMesh::add_elem (Elem * e)
594 : {
595 : // Don't try to add nullptrs!
596 18945 : libmesh_assert(e);
597 :
598 : // Trying to add an existing element is a no-op
599 44197646 : if (e->valid_id() && _elements[e->id()] == e)
600 0 : return e;
601 :
602 44197646 : const processor_id_type elem_procid = e->processor_id();
603 :
604 44197646 : if (!e->valid_id())
605 : {
606 : // We should only be creating new ids past the end of the range
607 : // of existing ids
608 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
609 : _max_elem_id);
610 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
611 :
612 : // Use the unpartitioned ids for unpartitioned elems, and
613 : // temporarily for ghost elems
614 3860753 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
615 3864321 : if (elem_procid == this->processor_id())
616 1133684 : next_id = &_next_free_local_elem_id;
617 3860753 : e->set_id (*next_id);
618 : }
619 :
620 : {
621 : // Advance next_ids up high enough that each is pointing to an
622 : // unused id and any subsequent increments will still point us
623 : // to unused ids
624 88395292 : _max_elem_id = std::max(_max_elem_id,
625 44197646 : static_cast<dof_id_type>(e->id()+1));
626 :
627 44197646 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
628 5178364 : _next_free_unpartitioned_elem_id =
629 5178364 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
630 5178364 : (this->n_processors() + 1) + this->n_processors();
631 44197646 : if (_next_free_local_elem_id < _max_elem_id)
632 3475074 : _next_free_local_elem_id =
633 3481327 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
634 3475074 : (this->n_processors() + 1) + this->processor_id();
635 :
636 : #ifndef NDEBUG
637 : // We need a const dofobject_container so we don't inadvertently create
638 : // nullptr entries when testing for non-nullptr ones
639 18945 : const dofobject_container<Elem> & const_elements = _elements;
640 : #endif
641 18945 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
642 18945 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
643 : }
644 :
645 : // Don't try to overwrite existing elems
646 18945 : libmesh_assert (!_elements[e->id()]);
647 :
648 44197646 : _elements[e->id()] = e;
649 :
650 : // We actually added a new element. Some of our caches might still
651 : // be valid, but we should clear the ones which definitely are not.
652 44197646 : this->clear_point_locator();
653 44197646 : this->clear_stored_ranges();
654 :
655 : // Try to make the cached elem data more accurate
656 44216591 : if (elem_procid == this->processor_id() ||
657 : elem_procid == DofObject::invalid_processor_id)
658 13057438 : _n_elem++;
659 :
660 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
661 44197646 : if (!e->valid_unique_id())
662 : {
663 13653657 : if (processor_id() == e->processor_id())
664 : {
665 1133684 : e->set_unique_id(_next_unique_id);
666 1133684 : _next_unique_id += this->n_processors() + 1;
667 : }
668 : else
669 : {
670 12519973 : e->set_unique_id(_next_unpartitioned_unique_id);
671 12519973 : _next_unpartitioned_unique_id += this->n_processors() + 1;
672 : }
673 : }
674 : else
675 : {
676 30568366 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
677 30543989 : _next_unique_id =
678 30543989 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
679 30543989 : (this->n_processors() + 1) + this->processor_id();
680 : }
681 : #endif
682 :
683 : // Unpartitioned elems should be added on every processor
684 : // And shouldn't be added in the same batch as ghost elems
685 : // But we might be just adding on processor 0 to
686 : // broadcast later
687 : // #ifdef DEBUG
688 : // if (elem_procid == DofObject::invalid_processor_id)
689 : // {
690 : // dof_id_type elem_id = e->id();
691 : // this->comm().max(elem_id);
692 : // libmesh_assert_equal_to (elem_id, e->id());
693 : // }
694 : // #endif
695 :
696 : // Make sure any new element is given space for any extra integers
697 : // we've requested
698 44197646 : e->add_extra_integers(_elem_integer_names.size(),
699 44197646 : _elem_integer_default_values);
700 :
701 : // And set mapping type and data on any new element
702 37890 : e->set_mapping_type(this->default_mapping_type());
703 37890 : e->set_mapping_data(this->default_mapping_data());
704 :
705 44197646 : return e;
706 : }
707 :
708 :
709 :
710 14889862 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
711 : {
712 : // The mesh now takes ownership of the Elem. Eventually the guts of
713 : // add_elem() will get moved to a private helper function, and
714 : // calling add_elem() directly will be deprecated.
715 14889862 : return add_elem(e.release());
716 : }
717 :
718 :
719 :
720 3423532 : Elem * DistributedMesh::insert_elem (Elem * e)
721 : {
722 3423532 : if (_elements[e->id()])
723 3423532 : this->delete_elem(_elements[e->id()]);
724 :
725 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
726 3423532 : if (!e->valid_unique_id())
727 : {
728 0 : if (processor_id() == e->processor_id())
729 : {
730 0 : e->set_unique_id(_next_unique_id);
731 0 : _next_unique_id += this->n_processors() + 1;
732 : }
733 : else
734 : {
735 0 : e->set_unique_id(_next_unpartitioned_unique_id);
736 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
737 : }
738 : }
739 : else
740 : {
741 3423532 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
742 3423532 : _next_unique_id =
743 3427228 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
744 3423532 : (this->n_processors() + 1) + this->processor_id();
745 : }
746 : #endif
747 :
748 : // Try to make the cached elem data more accurate
749 3423532 : processor_id_type elem_procid = e->processor_id();
750 3427228 : if (elem_procid == this->processor_id() ||
751 : elem_procid == DofObject::invalid_processor_id)
752 3292664 : _n_elem++;
753 :
754 3423532 : _elements[e->id()] = e;
755 :
756 : // We actually added a new element. Some of our caches might still
757 : // be valid, but we should clear the ones which definitely are not.
758 3423532 : this->clear_point_locator();
759 3423532 : this->clear_stored_ranges();
760 :
761 : // Make sure any new element is given space for any extra integers
762 : // we've requested
763 3423532 : e->add_extra_integers(_elem_integer_names.size(),
764 3423532 : _elem_integer_default_values);
765 :
766 : // And set mapping type and data on any new element
767 7392 : e->set_mapping_type(this->default_mapping_type());
768 7392 : e->set_mapping_data(this->default_mapping_data());
769 :
770 3423532 : return e;
771 : }
772 :
773 3423532 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
774 : {
775 : // The mesh now takes ownership of the Elem. Eventually the guts of
776 : // insert_elem(Elem*) will get moved to a private helper function, and
777 : // calling insert_elem(Elem*) directly will be deprecated.
778 3423532 : return insert_elem(e.release());
779 : }
780 :
781 :
782 40223458 : void DistributedMesh::delete_elem(Elem * e)
783 : {
784 7610 : libmesh_assert (e);
785 :
786 : // Try to make the cached elem data more accurate
787 40223458 : _n_elem--;
788 :
789 : // Was this a coarse element, not just a coarsening where we still
790 : // have some ancestor structure? Was it a *local* element, that we
791 : // might have been depending on as an owner of local nodes? We'll
792 : // have to be more careful with our nodes in contract() later; no
793 : // telling if we just locally orphaned a node that should be
794 : // globally retained.
795 40232888 : if (e->processor_id() == this->processor_id() &&
796 3640 : !e->parent())
797 80793 : _deleted_coarse_elements = true;
798 :
799 : // Delete the element from the BoundaryInfo object
800 40223458 : this->get_boundary_info().remove(e);
801 :
802 : // But not yet from the container; we might invalidate
803 : // an iterator that way!
804 :
805 : //_elements.erase(e->id());
806 :
807 : // Instead, we set it to nullptr for now
808 :
809 40223458 : _elements[e->id()] = nullptr;
810 :
811 : // delete the element
812 40223458 : delete e;
813 :
814 : // Some of our caches might still be valid, but we should clear the
815 : // ones which definitely are not.
816 40223458 : this->clear_point_locator();
817 40223458 : this->clear_stored_ranges();
818 40223458 : }
819 :
820 :
821 :
822 3208092 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
823 : const dof_id_type new_id)
824 : {
825 : // This could be a no-op
826 3208092 : if (old_id == new_id)
827 0 : return;
828 :
829 3208092 : Elem * el = _elements[old_id];
830 1160 : libmesh_assert (el);
831 1160 : libmesh_assert_equal_to (el->id(), old_id);
832 :
833 3208092 : el->set_id(new_id);
834 1160 : libmesh_assert (!_elements[new_id]);
835 3208092 : _elements[new_id] = el;
836 3208092 : _elements.erase(old_id);
837 :
838 : // Should we delete any caches here? Our point locator indexes by
839 : // element pointer and should be fine with an id change. Our stored
840 : // ranges are no longer sorted, which is *probably* fine, but let's
841 : // just be safe.
842 3208092 : this->clear_stored_ranges();
843 : }
844 :
845 :
846 :
847 41599161 : Node * DistributedMesh::add_point (const Point & p,
848 : const dof_id_type id,
849 : const processor_id_type proc_id)
850 : {
851 41599161 : Node * old_n = this->query_node_ptr(id);
852 :
853 41599161 : if (old_n)
854 : {
855 0 : *old_n = p;
856 0 : old_n->processor_id() = proc_id;
857 :
858 0 : return old_n;
859 : }
860 :
861 41559262 : Node * n = Node::build(p, id).release();
862 41599161 : n->processor_id() = proc_id;
863 :
864 41599161 : return DistributedMesh::add_node(n);
865 : }
866 :
867 :
868 4130 : void DistributedMesh::own_node (Node & n)
869 : {
870 : // This had better be a node in our mesh
871 0 : libmesh_assert(_nodes[n.id()] == &n);
872 :
873 4130 : _nodes[n.id()] = nullptr;
874 4130 : _n_nodes--;
875 :
876 0 : n.set_id(DofObject::invalid_id);
877 4130 : n.processor_id() = this->processor_id();
878 :
879 4130 : this->add_node(&n);
880 4130 : }
881 :
882 :
883 72390461 : Node * DistributedMesh::add_node (Node * n)
884 : {
885 : // Don't try to add nullptrs!
886 41611 : libmesh_assert(n);
887 :
888 : // Trying to add an existing node is a no-op
889 72390461 : if (n->valid_id() && _nodes[n->id()] == n)
890 0 : return n;
891 :
892 72390461 : const processor_id_type node_procid = n->processor_id();
893 :
894 72390461 : if (!n->valid_id())
895 : {
896 : // We should only be creating new ids past the end of the range
897 : // of existing ids
898 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
899 : _max_node_id);
900 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
901 :
902 : // Use the unpartitioned ids for unpartitioned nodes,
903 : // and temporarily for ghost nodes
904 17428099 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
905 17445928 : if (node_procid == this->processor_id())
906 2128728 : next_id = &_next_free_local_node_id;
907 17428099 : n->set_id (*next_id);
908 : }
909 :
910 : {
911 : // Advance next_ids up high enough that each is pointing to an
912 : // unused id and any subsequent increments will still point us
913 : // to unused ids
914 144780922 : _max_node_id = std::max(_max_node_id,
915 72390461 : static_cast<dof_id_type>(n->id()+1));
916 :
917 72390461 : if (_next_free_unpartitioned_node_id < _max_node_id)
918 20384413 : _next_free_unpartitioned_node_id =
919 20384413 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
920 20384413 : (this->n_processors() + 1) + this->n_processors();
921 72390461 : if (_next_free_local_node_id < _max_node_id)
922 11876735 : _next_free_local_node_id =
923 11894902 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
924 11876735 : (this->n_processors() + 1) + this->processor_id();
925 :
926 : #ifndef NDEBUG
927 : // We need a const dofobject_container so we don't inadvertently create
928 : // nullptr entries when testing for non-nullptr ones
929 41611 : const dofobject_container<Node> & const_nodes = _nodes;
930 : #endif
931 41611 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
932 41611 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
933 : }
934 :
935 : // Don't try to overwrite existing nodes
936 41611 : libmesh_assert (!_nodes[n->id()]);
937 :
938 72390461 : _nodes[n->id()] = n;
939 :
940 : // Try to make the cached node data more accurate
941 72432072 : if (node_procid == this->processor_id() ||
942 : node_procid == DofObject::invalid_processor_id)
943 39185017 : _n_nodes++;
944 :
945 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
946 72390461 : if (!n->valid_unique_id())
947 : {
948 41601233 : if (processor_id() == n->processor_id())
949 : {
950 2545410 : n->set_unique_id(_next_unique_id);
951 2545410 : _next_unique_id += this->n_processors() + 1;
952 : }
953 : else
954 : {
955 39055823 : n->set_unique_id(_next_unpartitioned_unique_id);
956 39055823 : _next_unpartitioned_unique_id += this->n_processors() + 1;
957 : }
958 : }
959 : else
960 : {
961 30919717 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
962 30789228 : _next_unique_id =
963 30789228 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
964 30789228 : (this->n_processors() + 1) + this->processor_id();
965 : }
966 : #endif
967 :
968 72390461 : n->add_extra_integers(_node_integer_names.size(),
969 72390461 : _node_integer_default_values);
970 :
971 : // Unpartitioned nodes should be added on every processor
972 : // And shouldn't be added in the same batch as ghost nodes
973 : // But we might be just adding on processor 0 to
974 : // broadcast later
975 : // #ifdef DEBUG
976 : // if (node_procid == DofObject::invalid_processor_id)
977 : // {
978 : // dof_id_type node_id = n->id();
979 : // this->comm().max(node_id);
980 : // libmesh_assert_equal_to (node_id, n->id());
981 : // }
982 : // #endif
983 :
984 72390461 : return n;
985 : }
986 :
987 30787170 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
988 : {
989 : // The mesh now takes ownership of the Node. Eventually the guts of
990 : // add_node() will get moved to a private helper function, and
991 : // calling add_node() directly will be deprecated.
992 30787170 : return add_node(n.release());
993 : }
994 :
995 50735984 : void DistributedMesh::delete_node(Node * n)
996 : {
997 4497 : libmesh_assert(n);
998 4497 : libmesh_assert(_nodes[n->id()]);
999 :
1000 : // Try to make the cached elem data more accurate
1001 50735984 : _n_nodes--;
1002 :
1003 : // Delete the node from the BoundaryInfo object
1004 50735984 : this->get_boundary_info().remove(n);
1005 8994 : _constraint_rows.erase(n);
1006 :
1007 : // But not yet from the container; we might invalidate
1008 : // an iterator that way!
1009 :
1010 : //_nodes.erase(n->id());
1011 :
1012 : // Instead, we set it to nullptr for now
1013 :
1014 50735984 : _nodes[n->id()] = nullptr;
1015 :
1016 : // delete the node
1017 50735984 : delete n;
1018 50735984 : }
1019 :
1020 :
1021 :
1022 6898984 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1023 : const dof_id_type new_id)
1024 : {
1025 : // This could be a no-op
1026 6898984 : if (old_id == new_id)
1027 0 : return;
1028 :
1029 6898395 : Node * nd = _nodes[old_id];
1030 3234 : libmesh_assert (nd);
1031 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1032 :
1033 : // If we have nodes shipped to this processor for NodeConstraints
1034 : // use, then those nodes will exist in _nodes, but may not be
1035 : // locatable via a TopologyMap due to the insufficiency of elements
1036 : // connecting to them. If local refinement then wants to create a
1037 : // *new* node in the same location, it will initially get a temporary
1038 : // id, and then make_node_ids_parallel_consistent() will try to move
1039 : // it to the canonical id. We need to account for this case to
1040 : // avoid false positives and memory leaks.
1041 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1042 6468 : if (_nodes[new_id])
1043 : {
1044 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1045 : *(Point *)_nodes[old_id]);
1046 0 : _nodes.erase(new_id);
1047 : }
1048 : #else
1049 : // If we aren't shipping nodes for NodeConstraints, there should be
1050 : // no reason for renumbering one node onto another.
1051 : libmesh_assert (!_nodes[new_id]);
1052 : #endif
1053 6898395 : _nodes[new_id] = nd;
1054 6898395 : nd->set_id(new_id);
1055 :
1056 6898395 : _nodes.erase(old_id);
1057 : }
1058 :
1059 :
1060 :
1061 597080 : void DistributedMesh::clear ()
1062 : {
1063 : // Call parent clear function
1064 597080 : MeshBase::clear();
1065 :
1066 : // Clear our elements and nodes
1067 : // There is no need to remove them from
1068 : // the BoundaryInfo data structure since we
1069 : // already cleared it.
1070 597080 : this->DistributedMesh::clear_elems();
1071 :
1072 22986106 : for (auto & node : _nodes)
1073 41598490 : delete node;
1074 :
1075 578 : _nodes.clear();
1076 :
1077 : // We're no longer distributed if we were before
1078 597080 : _is_serial = true;
1079 597080 : _is_serial_on_proc_0 = true;
1080 :
1081 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1082 : // all is copacetic.
1083 597080 : _deleted_coarse_elements = false;
1084 :
1085 : // Correct our caches
1086 597080 : _n_nodes = 0;
1087 597080 : _max_node_id = 0;
1088 597658 : _next_free_local_node_id = this->processor_id();
1089 597080 : _next_free_unpartitioned_node_id = this->n_processors();
1090 597080 : }
1091 :
1092 :
1093 :
1094 603480 : void DistributedMesh::clear_elems ()
1095 : {
1096 9353555 : for (auto & elem : _elements)
1097 8750075 : delete elem;
1098 :
1099 578 : _elements.clear();
1100 :
1101 : // Correct our caches
1102 603480 : _n_elem = 0;
1103 603480 : _max_elem_id = 0;
1104 604058 : _next_free_local_elem_id = this->processor_id();
1105 603480 : _next_free_unpartitioned_elem_id = this->n_processors();
1106 603480 : }
1107 :
1108 :
1109 :
1110 595014 : void DistributedMesh::redistribute ()
1111 : {
1112 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1113 595014 : if (!this->is_serial())
1114 : {
1115 : // Construct a MeshCommunication object to actually redistribute the nodes
1116 : // and elements according to the partitioner, and then to re-gather the neighbors.
1117 : MeshCommunication mc;
1118 70338 : mc.redistribute(*this);
1119 :
1120 70338 : this->update_parallel_id_counts();
1121 :
1122 : // We communicate valid neighbor links for newly-redistributed
1123 : // elements, but we may still have remote_elem links that should
1124 : // be corrected but whose correction wasn't communicated.
1125 70610 : this->find_neighbors(/*reset_remote_elements*/ false,
1126 : /*reset_current_list*/ false /*non-default*/,
1127 : /*assert_valid*/ false,
1128 272 : /*check_non_remote*/ false /*non-default!*/);
1129 :
1130 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1131 : // anyway... but users can always call partition directly, in which case we do need
1132 : // to call delete_remote_elements()...
1133 : //
1134 : // Regardless of whether it's necessary, it isn't safe. We
1135 : // haven't communicated new node processor_ids yet, and we can't
1136 : // delete nodes until we do.
1137 : // this->delete_remote_elements();
1138 : }
1139 : else
1140 : // The base class can handle non-distributed things, like
1141 : // notifying any GhostingFunctors of changes
1142 524676 : MeshBase::redistribute();
1143 595014 : }
1144 :
1145 :
1146 :
1147 507616 : void DistributedMesh::update_post_partitioning ()
1148 : {
1149 : // this->recalculate_n_partitions();
1150 :
1151 : // Let's do the base class cache clearing first, just in case our
1152 : // later computations are ever changed to make use of a local
1153 : // elements range cache
1154 507616 : this->MeshBase::update_post_partitioning();
1155 :
1156 : // Partitioning changes our numbers of unpartitioned objects
1157 507616 : this->update_parallel_id_counts();
1158 507616 : }
1159 :
1160 :
1161 :
1162 : template <typename T>
1163 3356 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1164 : {
1165 : // This function must be run on all processors at once
1166 3356 : parallel_object_only();
1167 :
1168 3356 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1169 3356 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1170 3356 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1171 :
1172 894268 : for (dof_id_type i=0; i != pmax_id; ++i)
1173 : {
1174 890912 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1175 :
1176 : // Local lookups by id should return the requested object
1177 890912 : libmesh_assert(!obj || obj->id() == i);
1178 :
1179 : // All processors with an object should agree on id
1180 : #ifndef NDEBUG
1181 890912 : const dof_id_type dofid = obj && obj->valid_id() ?
1182 : obj->id() : DofObject::invalid_id;
1183 890912 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1184 : #endif
1185 :
1186 : // All processors with an object should agree on processor id
1187 890912 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1188 : obj->processor_id() : DofObject::invalid_processor_id;
1189 890912 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1190 :
1191 890912 : dof_id_type min_procid = procid;
1192 890912 : this->comm().min(min_procid);
1193 :
1194 : // Either:
1195 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1196 : // or
1197 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1198 :
1199 : // Original assert logic
1200 : // libmesh_assert (min_procid != this->processor_id() || obj);
1201 :
1202 : // More human-understandable logic...
1203 890912 : libmesh_assert (
1204 : ((min_procid == this->processor_id()) && obj)
1205 : ||
1206 : (min_procid != this->processor_id())
1207 : );
1208 :
1209 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1210 : // All processors with an object should agree on unique id
1211 890912 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1212 890912 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1213 : #endif
1214 : }
1215 3356 : }
1216 :
1217 :
1218 :
1219 1678 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1220 : {
1221 1678 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1222 1678 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1223 1678 : }
1224 :
1225 :
1226 :
1227 402 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1228 : {
1229 : #ifndef NDEBUG
1230 : // This function must be run on all processors at once
1231 402 : parallel_object_only();
1232 :
1233 402 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1234 :
1235 21958 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1236 : {
1237 21556 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1238 :
1239 21556 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1240 :
1241 : // All processors with an active element should agree on p level
1242 21556 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1243 : }
1244 : #endif
1245 402 : }
1246 :
1247 :
1248 :
1249 :
1250 1626 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1251 : {
1252 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1253 : // This function must be run on all processors at once
1254 1626 : parallel_object_only();
1255 :
1256 1626 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1257 :
1258 144668 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1259 : {
1260 143042 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1261 :
1262 143042 : unsigned int refinement_flag = el ?
1263 65709 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1264 143042 : unsigned int p_refinement_flag = el ?
1265 65709 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1266 :
1267 143042 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1268 :
1269 : // p refinement flags aren't always kept correct on inactive
1270 : // ghost elements
1271 143042 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1272 : }
1273 : #endif // LIBMESH_ENABLE_AMR
1274 1626 : }
1275 :
1276 :
1277 :
1278 : template <typename T>
1279 : dof_id_type
1280 1843134 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1281 : {
1282 : // This function must be run on all processors at once
1283 800 : parallel_object_only();
1284 :
1285 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1286 :
1287 : // In parallel we may not know what objects other processors have.
1288 : // Start by figuring out how many
1289 800 : dof_id_type unpartitioned_objects = 0;
1290 :
1291 : std::unordered_map<processor_id_type, dof_id_type>
1292 1600 : ghost_objects_from_proc;
1293 :
1294 1843134 : object_iterator it = objects.begin();
1295 800 : object_iterator end = objects.end();
1296 :
1297 111874288 : while (it != end)
1298 : {
1299 110031154 : T * obj = *it;
1300 :
1301 : // Remove any nullptr container entries while we're here.
1302 110031154 : if (!obj)
1303 1424830 : it = objects.erase(it);
1304 : else
1305 : {
1306 108606324 : processor_id_type obj_procid = obj->processor_id();
1307 108606324 : if (obj_procid == DofObject::invalid_processor_id)
1308 43119507 : unpartitioned_objects++;
1309 : else
1310 65486817 : ghost_objects_from_proc[obj_procid]++;
1311 :
1312 : // Finally, increment the iterator
1313 56414 : ++it;
1314 : }
1315 : }
1316 :
1317 1844734 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1318 1843934 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1319 800 : this->comm().allgather
1320 1843625 : ((this_it == ghost_objects_from_proc.end()) ?
1321 491 : dof_id_type(0) : this_it->second, objects_on_proc);
1322 :
1323 : #ifndef NDEBUG
1324 800 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1325 2400 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1326 1600 : if (ghost_objects_from_proc.count(p))
1327 951 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1328 : else
1329 649 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1330 : #endif
1331 :
1332 : // We'll renumber objects in blocks by processor id
1333 1844734 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1334 19559502 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1335 17719568 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1336 800 : objects_on_proc[i-1];
1337 1843934 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1338 1843134 : dof_id_type first_free_id =
1339 1844734 : first_object_on_proc[this->n_processors()-1] +
1340 800 : objects_on_proc[this->n_processors()-1] +
1341 : unpartitioned_objects;
1342 :
1343 : // First set new local object ids and build request sets
1344 : // for non-local object ids
1345 :
1346 : // Request sets to send to each processor
1347 : std::map<processor_id_type, std::vector<dof_id_type>>
1348 800 : requested_ids;
1349 :
1350 : // We know how many objects live on each processor, so reserve() space for
1351 : // each.
1352 800 : auto ghost_end = ghost_objects_from_proc.end();
1353 21402636 : for (auto p : make_range(this->n_processors()))
1354 19561102 : if (p != this->processor_id())
1355 : {
1356 17717168 : if (const auto p_it = ghost_objects_from_proc.find(p);
1357 800 : p_it != ghost_end)
1358 3481134 : requested_ids[p].reserve(p_it->second);
1359 : }
1360 :
1361 800 : end = objects.end();
1362 110449458 : for (it = objects.begin(); it != end; ++it)
1363 : {
1364 108606324 : T * obj = *it;
1365 108606324 : if (!obj)
1366 0 : continue;
1367 108662738 : if (obj->processor_id() == this->processor_id())
1368 20002566 : obj->set_id(next_id++);
1369 88603758 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1370 45484251 : requested_ids[obj->processor_id()].push_back(obj->id());
1371 : }
1372 :
1373 : // Next set ghost object ids from other processors
1374 :
1375 1860305 : auto gather_functor =
1376 48931963 : [
1377 : #ifndef NDEBUG
1378 : this,
1379 : &first_object_on_proc,
1380 : &objects_on_proc,
1381 : #endif
1382 : &objects]
1383 : (processor_id_type, const std::vector<dof_id_type> & ids,
1384 920 : std::vector<dof_id_type> & new_ids)
1385 : {
1386 920 : std::size_t ids_size = ids.size();
1387 3481134 : new_ids.resize(ids_size);
1388 :
1389 48965385 : for (std::size_t i=0; i != ids_size; ++i)
1390 : {
1391 45484251 : T * obj = objects[ids[i]];
1392 16251 : libmesh_assert(obj);
1393 16251 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1394 45500502 : new_ids[i] = obj->id();
1395 :
1396 16251 : libmesh_assert_greater_equal (new_ids[i],
1397 : first_object_on_proc[this->processor_id()]);
1398 16251 : libmesh_assert_less (new_ids[i],
1399 : first_object_on_proc[this->processor_id()] +
1400 : objects_on_proc[this->processor_id()]);
1401 : }
1402 : };
1403 :
1404 1860305 : auto action_functor =
1405 48931963 : [
1406 : #ifndef NDEBUG
1407 : &first_object_on_proc,
1408 : &objects_on_proc,
1409 : #endif
1410 : &objects]
1411 : (processor_id_type libmesh_dbg_var(pid),
1412 : const std::vector<dof_id_type> & ids,
1413 920 : const std::vector<dof_id_type> & data)
1414 : {
1415 : // Copy the id changes we've now been informed of
1416 48965385 : for (auto i : index_range(ids))
1417 : {
1418 45484251 : T * obj = objects[ids[i]];
1419 16251 : libmesh_assert (obj);
1420 16251 : libmesh_assert_equal_to (obj->processor_id(), pid);
1421 16251 : libmesh_assert_greater_equal (data[i],
1422 : first_object_on_proc[pid]);
1423 16251 : libmesh_assert_less (data[i],
1424 : first_object_on_proc[pid] +
1425 : objects_on_proc[pid]);
1426 45500502 : obj->set_id(data[i]);
1427 : }
1428 : };
1429 :
1430 800 : const dof_id_type * ex = nullptr;
1431 : Parallel::pull_parallel_vector_data
1432 1843134 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1433 :
1434 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1435 1860305 : auto unique_gather_functor =
1436 48931963 : [
1437 : #ifndef NDEBUG
1438 : this,
1439 : #endif
1440 : &objects]
1441 : (processor_id_type, const std::vector<dof_id_type> & ids,
1442 920 : std::vector<unique_id_type> & data)
1443 : {
1444 920 : std::size_t ids_size = ids.size();
1445 3481134 : data.resize(ids_size);
1446 :
1447 48965385 : for (std::size_t i=0; i != ids_size; ++i)
1448 : {
1449 45484251 : T * obj = objects[ids[i]];
1450 16251 : libmesh_assert(obj);
1451 16251 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1452 90968502 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1453 : }
1454 : };
1455 :
1456 1860305 : auto unique_action_functor =
1457 48931963 : [&objects]
1458 : (processor_id_type libmesh_dbg_var(pid),
1459 : const std::vector<dof_id_type> & ids,
1460 920 : const std::vector<unique_id_type> & data)
1461 : {
1462 48965385 : for (auto i : index_range(ids))
1463 : {
1464 45484251 : T * obj = objects[ids[i]];
1465 16251 : libmesh_assert (obj);
1466 16251 : libmesh_assert_equal_to (obj->processor_id(), pid);
1467 45484251 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1468 0 : obj->set_unique_id(data[i]);
1469 : }
1470 : };
1471 :
1472 800 : const unique_id_type * unique_ex = nullptr;
1473 : Parallel::pull_parallel_vector_data
1474 1843134 : (this->comm(), requested_ids, unique_gather_functor,
1475 : unique_action_functor, unique_ex);
1476 : #endif
1477 :
1478 : // Next set unpartitioned object ids
1479 800 : next_id = 0;
1480 21402636 : for (auto i : make_range(this->n_processors()))
1481 19561102 : next_id += objects_on_proc[i];
1482 110449458 : for (it = objects.begin(); it != end; ++it)
1483 : {
1484 108606324 : T * obj = *it;
1485 108606324 : if (!obj)
1486 0 : continue;
1487 108606324 : if (obj->processor_id() == DofObject::invalid_processor_id)
1488 43119507 : obj->set_id(next_id++);
1489 : }
1490 :
1491 : // Finally shuffle around objects so that container indices
1492 : // match ids
1493 1843134 : it = objects.begin();
1494 800 : end = objects.end();
1495 116948046 : while (it != end)
1496 : {
1497 115104912 : T * obj = *it;
1498 115104912 : if (obj) // don't try shuffling already-nullptr entries
1499 : {
1500 115104912 : T * next = objects[obj->id()];
1501 : // If we have to move this object
1502 115104912 : if (next != obj)
1503 : {
1504 : // nullptr out its original position for now
1505 : // (our shuffling may put another object there shortly)
1506 39995635 : *it = nullptr;
1507 :
1508 : // There may already be another object with this id that
1509 : // needs to be moved itself
1510 51800186 : while (next)
1511 : {
1512 : // We shouldn't be trying to give two objects the
1513 : // same id
1514 9329 : libmesh_assert_not_equal_to (next->id(), obj->id());
1515 11804551 : objects[obj->id()] = obj;
1516 9329 : obj = next;
1517 11804551 : next = objects[obj->id()];
1518 : }
1519 39995635 : objects[obj->id()] = obj;
1520 : }
1521 : }
1522 :
1523 : // Remove any container entries that were left as nullptr.
1524 115296 : if (!obj)
1525 0 : it = objects.erase(it);
1526 : else
1527 57648 : ++it;
1528 : }
1529 :
1530 1843934 : return first_free_id;
1531 : }
1532 :
1533 :
1534 923237 : void DistributedMesh::renumber_nodes_and_elements ()
1535 : {
1536 402 : parallel_object_only();
1537 :
1538 : #ifdef DEBUG
1539 : // Make sure our ids and flags are consistent
1540 402 : this->libmesh_assert_valid_parallel_ids();
1541 402 : this->libmesh_assert_valid_parallel_flags();
1542 402 : this->libmesh_assert_valid_parallel_p_levels();
1543 : #endif
1544 :
1545 402 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1546 :
1547 : // Nodes not connected to any elements, and nullptr node entries
1548 : // in our container, should be deleted. But wait! If we've deleted coarse
1549 : // local elements on some processor, other processors might have ghosted
1550 : // nodes from it that are now no longer connected to any elements on it, but
1551 : // that are connected to their own semilocal elements. We'll have to
1552 : // communicate to ascertain if that's the case.
1553 923237 : this->comm().max(_deleted_coarse_elements);
1554 :
1555 : // What used nodes do we see on our proc?
1556 402 : std::set<dof_id_type> used_nodes;
1557 :
1558 : // What used node info should we send from our proc? Could we take ownership
1559 : // of each node if we needed to?
1560 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1561 402 : used_nodes_on_proc;
1562 :
1563 : // flag the nodes we need
1564 57818144 : for (auto & elem : this->element_ptr_range())
1565 301176235 : for (const Node & node : elem->node_ref_range())
1566 : {
1567 273160785 : const dof_id_type n = node.id();
1568 273016415 : used_nodes.insert(n);
1569 273160785 : if (_deleted_coarse_elements)
1570 : {
1571 3145701 : const processor_id_type p = node.processor_id();
1572 3145985 : if (p != this->processor_id())
1573 : {
1574 2257624 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1575 2257766 : if (elem->processor_id() == this->processor_id())
1576 216210 : used_nodes_on_p[n] = true;
1577 : else
1578 105 : if (!used_nodes_on_p.count(n))
1579 525626 : used_nodes_on_p[n] = false;
1580 : }
1581 : }
1582 922433 : }
1583 :
1584 923237 : if (_deleted_coarse_elements)
1585 : {
1586 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1587 : typedef unsigned char boolish;
1588 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1589 28 : used_nodes_on_proc_vecs;
1590 73209 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1591 56827 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1592 :
1593 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1594 : std::map<processor_id_type, std::set<dof_id_type>>
1595 28 : repartitioned_node_sets_to_push;
1596 :
1597 : auto ids_action_functor =
1598 56801 : [&used_nodes, &repartitioned_node_pids,
1599 : &repartitioned_node_sets_to_push]
1600 : (processor_id_type pid,
1601 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1602 : {
1603 592596 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1604 : {
1605 : // If we don't see a use for our own node, but someone
1606 : // else does, better figure out who should own it next.
1607 97 : if (!used_nodes.count(n))
1608 : {
1609 2 : if (auto it = repartitioned_node_pids.find(n);
1610 2 : sender_could_become_owner)
1611 : {
1612 1 : if (it != repartitioned_node_pids.end() &&
1613 1 : pid < it->second)
1614 1 : it->second = pid;
1615 : else
1616 0 : repartitioned_node_pids[n] = pid;
1617 : }
1618 : else
1619 1 : if (it == repartitioned_node_pids.end())
1620 1 : repartitioned_node_pids[n] =
1621 : DofObject::invalid_processor_id;
1622 :
1623 2 : repartitioned_node_sets_to_push[pid].insert(n);
1624 : }
1625 : }
1626 16394 : };
1627 :
1628 : // We need two pushes instead of a pull here because we need to
1629 : // know *all* the queries for a particular node before we can
1630 : // respond to *any* of them.
1631 : Parallel::push_parallel_vector_data
1632 16382 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1633 :
1634 : // Repartition (what used to be) our own nodes first
1635 16383 : for (auto & [n, p] : repartitioned_node_pids)
1636 : {
1637 1 : Node & node = this->node_ref(n);
1638 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1639 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1640 1 : node.processor_id() = p;
1641 : }
1642 :
1643 : // Then push to repartition others' ghosted copies.
1644 :
1645 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1646 28 : repartitioned_node_vecs;
1647 :
1648 16384 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1649 : {
1650 2 : auto & rn_vec = repartitioned_node_vecs[p];
1651 4 : for (auto n : nodeset)
1652 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1653 : }
1654 :
1655 : auto repartition_node_functor =
1656 2 : [this]
1657 : (processor_id_type libmesh_dbg_var(pid),
1658 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1659 : {
1660 4 : for (auto [n, p] : ids_and_pids)
1661 : {
1662 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1663 2 : Node & node = this->node_ref(n);
1664 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1665 2 : node.processor_id() = p;
1666 : }
1667 16354 : };
1668 :
1669 : Parallel::push_parallel_vector_data
1670 16382 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1671 : }
1672 :
1673 923237 : _deleted_coarse_elements = false;
1674 :
1675 : // Nodes not connected to any local elements, and nullptr node entries
1676 : // in our container, are deleted
1677 : {
1678 923237 : node_iterator_imp it = _nodes.begin();
1679 402 : node_iterator_imp end = _nodes.end();
1680 :
1681 88444484 : while (it != end)
1682 : {
1683 87521247 : Node * nd = *it;
1684 87521247 : if (!nd)
1685 3491539 : it = _nodes.erase(it);
1686 84029708 : else if (!used_nodes.count(nd->id()))
1687 : {
1688 : // remove any boundary information associated with
1689 : // this node
1690 2405183 : this->get_boundary_info().remove (nd);
1691 2828 : _constraint_rows.erase(nd);
1692 :
1693 : // delete the node
1694 2405183 : delete nd;
1695 :
1696 2405183 : it = _nodes.erase(it);
1697 : }
1698 : else
1699 45269 : ++it;
1700 : }
1701 : }
1702 :
1703 923237 : this->_preparation.has_removed_orphaned_nodes = true;
1704 :
1705 923237 : if (_skip_renumber_nodes_and_elements)
1706 : {
1707 1670 : this->update_parallel_id_counts();
1708 2 : return;
1709 : }
1710 :
1711 : // Finally renumber all the elements
1712 921567 : _n_elem = this->renumber_dof_objects (this->_elements);
1713 :
1714 : // and all the remaining nodes
1715 921567 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1716 :
1717 : // And figure out what IDs we should use when adding new nodes and
1718 : // new elements
1719 921567 : this->update_parallel_id_counts();
1720 :
1721 : // Make sure our caches are up to date and our
1722 : // DofObjects are well packed
1723 : #ifdef DEBUG
1724 400 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1725 400 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1726 400 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1727 400 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1728 400 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1729 400 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1730 400 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1731 400 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1732 :
1733 : // Make sure our ids and flags are consistent
1734 400 : this->libmesh_assert_valid_parallel_ids();
1735 400 : this->libmesh_assert_valid_parallel_flags();
1736 :
1737 : // And make sure we've made our numbering monotonic
1738 400 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1739 : #endif
1740 : }
1741 :
1742 :
1743 :
1744 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1745 : {
1746 : // We can't use range-for here because we need access to the special
1747 : // iterators' methods, not just to their dereferenced values.
1748 :
1749 : // Nodes first
1750 0 : for (auto pr = this->_nodes.begin(),
1751 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1752 : {
1753 5427360 : Node * n = *pr;
1754 5427360 : if (n != nullptr)
1755 : {
1756 0 : const dof_id_type id = pr.index();
1757 5204550 : n->set_id() = id;
1758 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1759 : }
1760 : }
1761 :
1762 : // Elements next
1763 0 : for (auto pr = this->_elements.begin(),
1764 5671231 : end = this->_elements.end(); pr != end; ++pr)
1765 : {
1766 5653821 : Elem * e = *pr;
1767 5653821 : if (e != nullptr)
1768 : {
1769 0 : const dof_id_type id = pr.index();
1770 5590229 : e->set_id() = id;
1771 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1772 : }
1773 : }
1774 17410 : }
1775 :
1776 :
1777 :
1778 673725 : dof_id_type DistributedMesh::n_active_elem () const
1779 : {
1780 550 : parallel_object_only();
1781 :
1782 : // Get local active elements first
1783 : dof_id_type active_elements =
1784 1346900 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1785 2020075 : this->active_local_elements_end()));
1786 673725 : this->comm().sum(active_elements);
1787 :
1788 : // Then add unpartitioned active elements, which should exist on
1789 : // every processor
1790 673725 : active_elements +=
1791 673725 : static_cast<dof_id_type>(std::distance
1792 1346900 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1793 674275 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1794 673725 : return active_elements;
1795 : }
1796 :
1797 :
1798 :
1799 427169 : void DistributedMesh::delete_remote_elements()
1800 : {
1801 : #ifdef DEBUG
1802 : // Make sure our neighbor links are all fine
1803 388 : MeshTools::libmesh_assert_valid_neighbors(*this);
1804 :
1805 : // And our child/parent links, and our flags
1806 388 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1807 :
1808 : // Make sure our ids and flags are consistent
1809 388 : this->libmesh_assert_valid_parallel_ids();
1810 388 : this->libmesh_assert_valid_parallel_flags();
1811 :
1812 388 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1813 388 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1814 388 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1815 388 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1816 388 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1817 388 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1818 : #endif
1819 :
1820 427169 : _is_serial = false;
1821 427169 : _is_serial_on_proc_0 = false;
1822 :
1823 427169 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1824 :
1825 388 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1826 :
1827 : // Now make sure the containers actually shrink - strip
1828 : // any newly-created nullptr voids out of the element array
1829 427169 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1830 388 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1831 53652147 : while (e_it != e_end)
1832 53224978 : if (!*e_it)
1833 39541779 : e_it = _elements.erase(e_it);
1834 : else
1835 17389 : ++e_it;
1836 :
1837 427169 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1838 388 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1839 101021053 : while (n_it != n_end)
1840 100593884 : if (!*n_it)
1841 66859130 : n_it = _nodes.erase(n_it);
1842 : else
1843 48597 : ++n_it;
1844 :
1845 : // We may have deleted no-longer-connected nodes or coarsened-away
1846 : // elements; let's update our caches.
1847 427169 : this->update_parallel_id_counts();
1848 :
1849 : // We may have deleted nodes or elements that were the only local
1850 : // representatives of some particular boundary id(s); let's update
1851 : // those caches.
1852 427169 : this->get_boundary_info().regenerate_id_sets();
1853 :
1854 : #ifdef DEBUG
1855 : // We might not have well-packed objects if the user didn't allow us
1856 : // to renumber
1857 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1858 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1859 :
1860 : // Make sure our neighbor links are all fine
1861 388 : MeshTools::libmesh_assert_valid_neighbors(*this);
1862 :
1863 : // And our child/parent links, and our flags
1864 388 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1865 :
1866 : // Make sure our ids and flags are consistent
1867 388 : this->libmesh_assert_valid_parallel_ids();
1868 388 : this->libmesh_assert_valid_parallel_flags();
1869 : #endif
1870 :
1871 427169 : this->_preparation.has_removed_remote_elements = true;
1872 427169 : }
1873 :
1874 :
1875 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1876 : {
1877 : // First add the elem like normal
1878 0 : add_elem(e);
1879 :
1880 : // Now add it to the set that won't be deleted when we call
1881 : // delete_remote_elements()
1882 0 : _extra_ghost_elems.insert(e);
1883 0 : }
1884 :
1885 : void
1886 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1887 : {
1888 0 : std::set<Elem *> tmp;
1889 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1890 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1891 0 : std::inserter(tmp, tmp.begin()));
1892 0 : _extra_ghost_elems = tmp;
1893 0 : }
1894 :
1895 98732 : void DistributedMesh::allgather()
1896 : {
1897 98732 : if (_is_serial)
1898 0 : return;
1899 98730 : MeshCommunication().allgather(*this);
1900 98730 : _is_serial = true;
1901 98730 : _is_serial_on_proc_0 = true;
1902 :
1903 : // Make sure our caches are up to date and our
1904 : // DofObjects are well packed
1905 : #ifdef DEBUG
1906 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1907 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1908 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1909 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1910 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1911 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1912 :
1913 : // If we've disabled renumbering we can't be sure we're contiguous
1914 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1915 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1916 :
1917 : // Make sure our neighbor links are all fine
1918 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1919 :
1920 : // Make sure our ids and flags are consistent
1921 48 : this->libmesh_assert_valid_parallel_ids();
1922 48 : this->libmesh_assert_valid_parallel_flags();
1923 : #endif
1924 : }
1925 :
1926 18352 : void DistributedMesh::gather_to_zero()
1927 : {
1928 18352 : if (_is_serial_on_proc_0)
1929 4 : return;
1930 :
1931 14114 : _is_serial_on_proc_0 = true;
1932 14114 : MeshCommunication().gather(0, *this);
1933 : }
1934 :
1935 :
1936 : } // namespace libMesh
|
