Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 298807 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 298807 : unsigned char d) :
44 298307 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 298307 : _is_serial_on_proc_0(true),
46 298307 : _deleted_coarse_elements(false),
47 298307 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 299057 : _next_free_local_node_id(this->processor_id()),
49 298807 : _next_free_local_elem_id(this->processor_id()),
50 298807 : _next_free_unpartitioned_node_id(this->n_processors()),
51 298807 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 298807 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 298807 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 299057 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 597614 : default_partitioner);
64 : _partitioner = Partitioner::build
65 597364 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 298807 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 13042 : std::string_view DistributedMesh::subclass_first_difference_from (const MeshBase & other_mesh_base) const
99 : {
100 724 : const DistributedMesh * dist_mesh_ptr =
101 13042 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 13042 : if (!dist_mesh_ptr)
103 0 : return "DistributedMesh class";
104 724 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 : #define CHECK_MEMBER(member_name) \
107 : if (member_name != other_mesh.member_name) \
108 : return #member_name;
109 :
110 13042 : CHECK_MEMBER(_is_serial);
111 13042 : CHECK_MEMBER(_is_serial_on_proc_0);
112 13042 : CHECK_MEMBER(_deleted_coarse_elements);
113 13042 : CHECK_MEMBER(_n_nodes);
114 13042 : CHECK_MEMBER(_n_elem);
115 13042 : CHECK_MEMBER(_max_node_id);
116 13042 : CHECK_MEMBER(_max_elem_id);
117 : // We expect these things to change in a prepare_for_use();
118 : // they're conceptually "mutable"...
119 : /*
120 : CHECK_MEMBER(_next_free_local_node_id);
121 : CHECK_MEMBER(_next_free_local_elem_id);
122 : CHECK_MEMBER(_next_free_unpartitioned_node_id);
123 : CHECK_MEMBER(_next_free_unpartitioned_elem_id);
124 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
125 : CHECK_MEMBER(_next_unpartitioned_unique_id);
126 : #endif
127 : */
128 13042 : if (!this->nodes_and_elements_equal(other_mesh))
129 63 : return "nodes and/or elements";
130 :
131 13701 : if (_extra_ghost_elems.size() !=
132 722 : other_mesh._extra_ghost_elems.size())
133 0 : return "_extra_ghost_elems size";
134 12979 : for (auto & elem : _extra_ghost_elems)
135 : {
136 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
137 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
138 0 : if (!other_elem ||
139 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
140 0 : return "_extra_ghost_elems entry";
141 : }
142 :
143 12979 : return "";
144 : }
145 :
146 341968 : DistributedMesh::~DistributedMesh ()
147 : {
148 319516 : this->DistributedMesh::clear(); // Free nodes and elements
149 341968 : }
150 :
151 :
152 : // This might be specialized later, but right now it's just here to
153 : // make sure the compiler doesn't give us a default (non-deep) copy
154 : // constructor instead.
155 18082 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
156 : DistributedMesh(static_cast<const MeshBase &>(other_mesh),
157 18082 : /*bool other_is_distributed_type=*/true)
158 : {
159 18082 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
160 18082 : _next_free_local_node_id =
161 18082 : other_mesh._next_free_local_node_id;
162 18082 : _next_free_local_elem_id =
163 18082 : other_mesh._next_free_local_elem_id;
164 18082 : _next_free_unpartitioned_node_id =
165 18082 : other_mesh._next_free_unpartitioned_node_id;
166 18082 : _next_free_unpartitioned_elem_id =
167 18082 : other_mesh._next_free_unpartitioned_elem_id;
168 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
169 18082 : _next_unique_id =
170 18082 : other_mesh._next_unique_id;
171 18082 : _next_unpartitioned_unique_id =
172 18082 : other_mesh._next_unpartitioned_unique_id;
173 : #endif
174 :
175 : // Need to copy extra_ghost_elems
176 18082 : for (auto & elem : other_mesh._extra_ghost_elems)
177 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
178 18082 : }
179 :
180 :
181 :
182 20709 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
183 20709 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
184 21151 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
185 20173 : _deleted_coarse_elements(true), // better safe than sorry...
186 20173 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
187 21151 : _next_free_local_node_id(this->processor_id()),
188 21057 : _next_free_local_elem_id(this->processor_id()),
189 21057 : _next_free_unpartitioned_node_id(this->n_processors()),
190 41324 : _next_free_unpartitioned_elem_id(this->n_processors())
191 : {
192 : // Just copy, skipping preparation
193 20709 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
194 :
195 536 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
196 536 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
197 536 : this->allow_renumbering(other_mesh.allow_renumbering());
198 536 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
199 536 : this->skip_partitioning(other_mesh.skip_partitioning());
200 :
201 20709 : this->copy_constraint_rows(other_mesh);
202 :
203 442 : auto & this_boundary_info = this->get_boundary_info();
204 442 : const auto & other_boundary_info = other_mesh.get_boundary_info();
205 :
206 20709 : this_boundary_info = other_boundary_info;
207 :
208 442 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
209 :
210 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
211 : // nodes/elements they consider locally owned (or unpartitioned), so after
212 : // copying only the objects visible to this processor, those counts
213 : // reflect an incomplete local view rather than other_mesh's totals,
214 : // and need to be copied afresh.
215 20709 : _n_nodes = other_mesh.n_nodes();
216 20709 : _n_elem = other_mesh.n_elem();
217 20709 : _max_node_id = other_mesh.max_node_id();
218 20709 : _max_elem_id = other_mesh.max_elem_id();
219 :
220 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
221 : // counters directly, mirroring other_mesh's own bookkeeping exactly. This
222 : // is done in the DistributedMesh copy constructor.
223 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
224 : // raw counterpart fields to copy, so derive the free-id-range
225 : // counters from _max_elem_id/_max_node_id the same way
226 : // update_parallel_id_counts() does via the set_next_ids() method. For
227 : // the unique id counters we copy other_mesh's own local next_unique_id()
228 : // and derive the next available unique ids
229 : // (partitioned and unpartitioned) via set_next_unique_ids().
230 20709 : if (!other_is_distributed_type)
231 : {
232 2627 : this->set_next_ids();
233 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
234 2627 : this->set_next_unique_ids(other_mesh.next_unique_id());
235 : #endif
236 : }
237 :
238 : // Copy other_mesh's actual preparation state, including whatever it
239 : // reports for has_synched_id_counts: the counts above are a faithful copy
240 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
241 : // verified computation, so our synced-ness should match other_mesh's.
242 20709 : this->_preparation = other_mesh.preparation();
243 20709 : }
244 :
245 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
246 : {
247 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
248 :
249 4 : this->_nodes = std::move(other_mesh._nodes);
250 270 : this->_n_nodes = other_mesh.n_nodes();
251 :
252 4 : this->_elements = std::move(other_mesh._elements);
253 270 : this->_n_elem = other_mesh.n_elem();
254 :
255 270 : _is_serial = other_mesh.is_serial();
256 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
257 270 : _deleted_coarse_elements = true; // Better safe than sorry
258 :
259 270 : _max_node_id = other_mesh.max_node_id();
260 270 : _max_elem_id = other_mesh.max_elem_id();
261 :
262 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
263 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
264 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
265 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
266 :
267 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
268 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
269 : #endif
270 270 : }
271 :
272 2035216 : void DistributedMesh::set_next_ids()
273 : {
274 2035216 : _next_free_unpartitioned_elem_id =
275 2036990 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
276 2035216 : (this->n_processors() + 1) + this->n_processors();
277 2035216 : _next_free_local_elem_id =
278 2035216 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
279 2035216 : (this->n_processors() + 1) + this->processor_id();
280 :
281 2035216 : _next_free_unpartitioned_node_id =
282 2036990 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
283 2035216 : (this->n_processors() + 1) + this->n_processors();
284 2035216 : _next_free_local_node_id =
285 2035216 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
286 2035216 : (this->n_processors() + 1) + this->processor_id();
287 2035216 : }
288 :
289 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
290 2035216 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
291 : {
292 : // Unique ids are laid out in repeating groups of (n_processors()+1)
293 : // consecutive integers ("alignment" here means a value's position, i.e.
294 : // its residue mod (n_processors()+1), within one of these groups). In
295 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
296 : // are each permanently earmarked for one processor's locally-owned
297 : // objects, and the last slot (residue n_processors()) is permanently
298 : // earmarked for unpartitioned objects.
299 : //
300 : // parallel_max_unique_id carries no guarantee of landing on any
301 : // particular residue, so rather than simply offsetting it by
302 : // processor_id()/n_processors(), we have to explicitly round up to the
303 : // start of the next full, entirely-unused group before adding this
304 : // processor's (or the unpartitioned pool's) fixed offset into that group.
305 : //
306 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
307 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
308 : // the number of complete groups needed to reach at least past the
309 : // current max id. Multiplying back by (n_processors()+1) gives the first
310 : // id of the next group that is guaranteed to be entirely unused; adding
311 : // n_processors(), the unpartitioned pool's fixed offset within a group,
312 : // lands on that pool's reserved slot in that group.
313 2035216 : _next_unpartitioned_unique_id =
314 2036990 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
315 2035216 : (this->n_processors() + 1) +
316 2035216 : this->n_processors();
317 :
318 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
319 : // the rounding accounts for processor_id() possibly being less than the
320 : // residue the raw max id already occupies within its group; this can
321 : // land this processor's slot in the same group as the unpartitioned
322 : // pool's above, or push it into the following group, depending on where
323 : // the max id and this processor's offset happen to fall.
324 2035216 : _next_unique_id =
325 2035216 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
326 2035216 : (this->n_processors() + 1) +
327 2035216 : this->processor_id();
328 2035216 : }
329 : #endif
330 :
331 : // We use cached values for these so they can be called
332 : // from one processor without bothering the rest, but
333 : // we may need to update those caches before doing a full
334 : // renumbering
335 2032589 : void DistributedMesh::update_parallel_id_counts()
336 : {
337 : // This function must be run on all processors at once
338 2048 : parallel_object_only();
339 :
340 2032589 : _n_elem = this->parallel_n_elem();
341 2032589 : _n_nodes = this->parallel_n_nodes();
342 2032589 : _max_node_id = this->parallel_max_node_id();
343 2032589 : _max_elem_id = this->parallel_max_elem_id();
344 :
345 2032589 : this->set_next_ids();
346 :
347 : // set_next_unique_ids() needs the true parallel max unique id (not just
348 : // this processor's possibly-stale cached _next_unique_id) since the whole
349 : // point of this function is to recompute counters that may have drifted
350 : // out of sync.
351 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
352 2032589 : this->set_next_unique_ids(this->parallel_max_unique_id());
353 : #endif
354 :
355 2032589 : this->_preparation.has_synched_id_counts = true;
356 2032589 : }
357 :
358 :
359 : // Or in debug mode we may want to test the uncached values without
360 : // changing the cache
361 2040320 : dof_id_type DistributedMesh::parallel_n_elem() const
362 : {
363 : // This function must be run on all processors at once
364 2902 : parallel_object_only();
365 :
366 2040320 : dof_id_type n_local = this->n_local_elem();
367 2040320 : this->comm().sum(n_local);
368 2040320 : n_local += this->n_unpartitioned_elem();
369 2040320 : return n_local;
370 : }
371 :
372 :
373 :
374 2039635 : dof_id_type DistributedMesh::parallel_max_elem_id() const
375 : {
376 : // This function must be run on all processors at once
377 9094 : parallel_object_only();
378 :
379 2039635 : dof_id_type max_local = 0;
380 :
381 : dofobject_container<Elem>::const_reverse_veclike_iterator
382 9094 : rit = _elements.rbegin();
383 :
384 : const dofobject_container<Elem>::const_reverse_veclike_iterator
385 9094 : rend = _elements.rend();
386 :
387 : // Look for the maximum element id. Search backwards through
388 : // elements so we can break out early. Beware of nullptr entries that
389 : // haven't yet been cleared from _elements.
390 10673659 : for (; rit != rend; ++rit)
391 : {
392 10110550 : const DofObject *d = *rit;
393 19375 : if (d)
394 : {
395 8868 : libmesh_assert(_elements[d->id()] == d);
396 1476526 : max_local = d->id() + 1;
397 1476526 : break;
398 : }
399 : }
400 :
401 2039635 : this->comm().max(max_local);
402 2039635 : return max_local;
403 : }
404 :
405 :
406 :
407 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
408 2118413 : unique_id_type DistributedMesh::parallel_max_unique_id() const
409 : {
410 : // This function must be run on all processors at once
411 2498 : parallel_object_only();
412 :
413 4236826 : unique_id_type max_local = std::max(_next_unique_id,
414 2430426 : _next_unpartitioned_unique_id);
415 2118413 : this->comm().max(max_local);
416 2118413 : return max_local;
417 : }
418 :
419 :
420 :
421 80865 : void DistributedMesh::set_next_unique_id(unique_id_type id)
422 : {
423 496 : _next_unique_id = id;
424 80865 : _next_unpartitioned_unique_id =
425 81013 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
426 80865 : (this->n_processors() + 1) + this->n_processors();
427 80865 : _next_unique_id =
428 80865 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
429 80865 : (this->n_processors() + 1) + this->processor_id();
430 80865 : }
431 : #endif
432 :
433 :
434 :
435 2040768 : dof_id_type DistributedMesh::parallel_n_nodes() const
436 : {
437 : // This function must be run on all processors at once
438 2902 : parallel_object_only();
439 :
440 2040768 : dof_id_type n_local = this->n_local_nodes();
441 2040768 : this->comm().sum(n_local);
442 2040768 : n_local += this->n_unpartitioned_nodes();
443 2040768 : return n_local;
444 : }
445 :
446 :
447 :
448 2037201 : dof_id_type DistributedMesh::parallel_max_node_id() const
449 : {
450 : // This function must be run on all processors at once
451 6660 : parallel_object_only();
452 :
453 2037201 : dof_id_type max_local = 0;
454 :
455 : dofobject_container<Node>::const_reverse_veclike_iterator
456 6660 : rit = _nodes.rbegin();
457 :
458 : const dofobject_container<Node>::const_reverse_veclike_iterator
459 6660 : rend = _nodes.rend();
460 :
461 : // Look for the maximum node id. Search backwards through
462 : // nodes so we can break out early. Beware of nullptr entries that
463 : // haven't yet been cleared from _nodes
464 38857374 : for (; rit != rend; ++rit)
465 : {
466 38294335 : const DofObject *d = *rit;
467 63736 : if (d)
468 : {
469 6504 : libmesh_assert(_nodes[d->id()] == d);
470 1474162 : max_local = d->id() + 1;
471 1474162 : break;
472 : }
473 : }
474 :
475 2037201 : this->comm().max(max_local);
476 2037201 : return max_local;
477 : }
478 :
479 :
480 :
481 43030112 : const Point & DistributedMesh::point (const dof_id_type i) const
482 : {
483 43030112 : return this->node_ref(i);
484 : }
485 :
486 :
487 :
488 44147145 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
489 : {
490 137681 : libmesh_assert(_nodes[i]);
491 137681 : libmesh_assert_equal_to (_nodes[i]->id(), i);
492 :
493 44147145 : return _nodes[i];
494 : }
495 :
496 :
497 :
498 :
499 648238241 : Node * DistributedMesh::node_ptr (const dof_id_type i)
500 : {
501 654169 : libmesh_assert(_nodes[i]);
502 654169 : libmesh_assert_equal_to (_nodes[i]->id(), i);
503 :
504 648238241 : return _nodes[i];
505 : }
506 :
507 :
508 :
509 :
510 3532224 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
511 : {
512 3532224 : if (const auto it = _nodes.find(i);
513 1080501 : it != _nodes.end())
514 : {
515 3104804 : const Node * n = *it;
516 653081 : libmesh_assert (!n || n->id() == i);
517 653081 : return n;
518 : }
519 :
520 427420 : return nullptr;
521 : }
522 :
523 :
524 :
525 :
526 371749761 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
527 : {
528 371749761 : if (auto it = _nodes.find(i);
529 245619 : it != _nodes.end())
530 : {
531 288258346 : Node * n = *it;
532 157670 : libmesh_assert (!n || n->id() == i);
533 157670 : return n;
534 : }
535 :
536 87949 : return nullptr;
537 : }
538 :
539 :
540 :
541 :
542 3058423 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
543 : {
544 10105 : libmesh_assert(_elements[i]);
545 10105 : libmesh_assert_equal_to (_elements[i]->id(), i);
546 :
547 3058423 : return _elements[i];
548 : }
549 :
550 :
551 :
552 :
553 288522824 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
554 : {
555 178241 : libmesh_assert(_elements[i]);
556 178241 : libmesh_assert_equal_to (_elements[i]->id(), i);
557 :
558 288522824 : return _elements[i];
559 : }
560 :
561 :
562 :
563 :
564 1766517 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
565 : {
566 1766517 : if (const auto it = _elements.find(i);
567 927547 : it != _elements.end())
568 : {
569 1248384 : const Elem * e = *it;
570 418329 : libmesh_assert (!e || e->id() == i);
571 418329 : return e;
572 : }
573 :
574 509218 : return nullptr;
575 : }
576 :
577 :
578 :
579 :
580 608719431 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
581 : {
582 608719431 : if (auto it = _elements.find(i);
583 217474 : it != _elements.end())
584 : {
585 523135823 : Elem * e = *it;
586 213794 : libmesh_assert (!e || e->id() == i);
587 213794 : return e;
588 : }
589 :
590 3680 : return nullptr;
591 : }
592 :
593 :
594 :
595 :
596 44217705 : Elem * DistributedMesh::add_elem (Elem * e)
597 : {
598 : // Don't try to add nullptrs!
599 35493 : libmesh_assert(e);
600 :
601 : // Trying to add an existing element is a no-op
602 44217705 : if (e->valid_id() && _elements[e->id()] == e)
603 0 : return e;
604 :
605 44217705 : const processor_id_type elem_procid = e->processor_id();
606 :
607 44217705 : if (!e->valid_id())
608 : {
609 : // We should only be creating new ids past the end of the range
610 : // of existing ids
611 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
612 : _max_elem_id);
613 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
614 :
615 : // Use the unpartitioned ids for unpartitioned elems, and
616 : // temporarily for ghost elems
617 3860793 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
618 3864361 : if (elem_procid == this->processor_id())
619 1133696 : next_id = &_next_free_local_elem_id;
620 3860793 : e->set_id (*next_id);
621 : }
622 :
623 : {
624 : // Advance next_ids up high enough that each is pointing to an
625 : // unused id and any subsequent increments will still point us
626 : // to unused ids
627 88435410 : _max_elem_id = std::max(_max_elem_id,
628 44217705 : static_cast<dof_id_type>(e->id()+1));
629 :
630 44217705 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
631 5186369 : _next_free_unpartitioned_elem_id =
632 5186369 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
633 5186369 : (this->n_processors() + 1) + this->n_processors();
634 44217705 : if (_next_free_local_elem_id < _max_elem_id)
635 3483569 : _next_free_local_elem_id =
636 3489852 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
637 3483569 : (this->n_processors() + 1) + this->processor_id();
638 :
639 : #ifndef NDEBUG
640 : // We need a const dofobject_container so we don't inadvertently create
641 : // nullptr entries when testing for non-nullptr ones
642 35493 : const dofobject_container<Elem> & const_elements = _elements;
643 : #endif
644 35493 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
645 35493 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
646 : }
647 :
648 : // Don't try to overwrite existing elems
649 35493 : libmesh_assert (!_elements[e->id()]);
650 :
651 44217705 : _elements[e->id()] = e;
652 :
653 : // We actually added a new element. Some of our caches might still
654 : // be valid, but we should clear the ones which definitely are not.
655 44217705 : this->clear_point_locator();
656 44217705 : this->clear_stored_ranges();
657 :
658 : // Try to make the cached elem data more accurate
659 44236762 : if (elem_procid == this->processor_id() ||
660 : elem_procid == DofObject::invalid_processor_id)
661 13069232 : _n_elem++;
662 :
663 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
664 44217705 : if (!e->valid_unique_id())
665 : {
666 13653839 : if (processor_id() == e->processor_id())
667 : {
668 1133696 : e->set_unique_id(_next_unique_id);
669 1133696 : _next_unique_id += this->n_processors() + 1;
670 : }
671 : else
672 : {
673 12520143 : e->set_unique_id(_next_unpartitioned_unique_id);
674 12520143 : _next_unpartitioned_unique_id += this->n_processors() + 1;
675 : }
676 : }
677 : else
678 : {
679 30588243 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
680 30563866 : _next_unique_id =
681 30563866 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
682 30563866 : (this->n_processors() + 1) + this->processor_id();
683 : }
684 : #endif
685 :
686 : // Unpartitioned elems should be added on every processor
687 : // And shouldn't be added in the same batch as ghost elems
688 : // But we might be just adding on processor 0 to
689 : // broadcast later
690 : // #ifdef DEBUG
691 : // if (elem_procid == DofObject::invalid_processor_id)
692 : // {
693 : // dof_id_type elem_id = e->id();
694 : // this->comm().max(elem_id);
695 : // libmesh_assert_equal_to (elem_id, e->id());
696 : // }
697 : // #endif
698 :
699 : // Make sure any new element is given space for any extra integers
700 : // we've requested
701 44217705 : e->add_extra_integers(_elem_integer_names.size(),
702 44217705 : _elem_integer_default_values);
703 :
704 : // And set mapping type and data on any new element
705 54550 : e->set_mapping_type(this->default_mapping_type());
706 54550 : e->set_mapping_data(this->default_mapping_data());
707 :
708 44217705 : return e;
709 : }
710 :
711 :
712 :
713 14910314 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
714 : {
715 : // The mesh now takes ownership of the Elem. Eventually the guts of
716 : // add_elem() will get moved to a private helper function, and
717 : // calling add_elem() directly will be deprecated.
718 14910314 : return add_elem(e.release());
719 : }
720 :
721 :
722 :
723 3423532 : Elem * DistributedMesh::insert_elem (Elem * e)
724 : {
725 3423532 : if (_elements[e->id()])
726 3423532 : this->delete_elem(_elements[e->id()]);
727 :
728 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
729 3423532 : if (!e->valid_unique_id())
730 : {
731 0 : if (processor_id() == e->processor_id())
732 : {
733 0 : e->set_unique_id(_next_unique_id);
734 0 : _next_unique_id += this->n_processors() + 1;
735 : }
736 : else
737 : {
738 0 : e->set_unique_id(_next_unpartitioned_unique_id);
739 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
740 : }
741 : }
742 : else
743 : {
744 3423532 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
745 3423532 : _next_unique_id =
746 3427228 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
747 3423532 : (this->n_processors() + 1) + this->processor_id();
748 : }
749 : #endif
750 :
751 : // Try to make the cached elem data more accurate
752 3423532 : processor_id_type elem_procid = e->processor_id();
753 3427228 : if (elem_procid == this->processor_id() ||
754 : elem_procid == DofObject::invalid_processor_id)
755 3292664 : _n_elem++;
756 :
757 3423532 : _elements[e->id()] = e;
758 :
759 : // We actually added a new element. Some of our caches might still
760 : // be valid, but we should clear the ones which definitely are not.
761 3423532 : this->clear_point_locator();
762 3423532 : this->clear_stored_ranges();
763 :
764 : // Make sure any new element is given space for any extra integers
765 : // we've requested
766 3423532 : e->add_extra_integers(_elem_integer_names.size(),
767 3423532 : _elem_integer_default_values);
768 :
769 : // And set mapping type and data on any new element
770 7392 : e->set_mapping_type(this->default_mapping_type());
771 7392 : e->set_mapping_data(this->default_mapping_data());
772 :
773 3423532 : return e;
774 : }
775 :
776 3423532 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
777 : {
778 : // The mesh now takes ownership of the Elem. Eventually the guts of
779 : // insert_elem(Elem*) will get moved to a private helper function, and
780 : // calling insert_elem(Elem*) directly will be deprecated.
781 3423532 : return insert_elem(e.release());
782 : }
783 :
784 :
785 40222240 : void DistributedMesh::delete_elem(Elem * e)
786 : {
787 7594 : libmesh_assert (e);
788 :
789 : // Try to make the cached elem data more accurate
790 40222240 : _n_elem--;
791 :
792 : // Was this a coarse element, not just a coarsening where we still
793 : // have some ancestor structure? Was it a *local* element, that we
794 : // might have been depending on as an owner of local nodes? We'll
795 : // have to be more careful with our nodes in contract() later; no
796 : // telling if we just locally orphaned a node that should be
797 : // globally retained.
798 40231654 : if (e->processor_id() == this->processor_id() &&
799 3640 : !e->parent())
800 80793 : _deleted_coarse_elements = true;
801 :
802 : // Delete the element from the BoundaryInfo object
803 40222240 : this->get_boundary_info().remove(e);
804 :
805 : // But not yet from the container; we might invalidate
806 : // an iterator that way!
807 :
808 : //_elements.erase(e->id());
809 :
810 : // Instead, we set it to nullptr for now
811 :
812 40222240 : _elements[e->id()] = nullptr;
813 :
814 : // delete the element
815 40222240 : delete e;
816 :
817 : // Some of our caches might still be valid, but we should clear the
818 : // ones which definitely are not.
819 40222240 : this->clear_point_locator();
820 40222240 : this->clear_stored_ranges();
821 40222240 : }
822 :
823 :
824 :
825 3208120 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
826 : const dof_id_type new_id)
827 : {
828 : // This could be a no-op
829 3208120 : if (old_id == new_id)
830 0 : return;
831 :
832 3208120 : Elem * el = _elements[old_id];
833 1160 : libmesh_assert (el);
834 1160 : libmesh_assert_equal_to (el->id(), old_id);
835 :
836 3208120 : el->set_id(new_id);
837 1160 : libmesh_assert (!_elements[new_id]);
838 3208120 : _elements[new_id] = el;
839 3208120 : _elements.erase(old_id);
840 :
841 : // Should we delete any caches here? Our point locator indexes by
842 : // element pointer and should be fine with an id change. Our stored
843 : // ranges are no longer sorted, which is *probably* fine, but let's
844 : // just be safe.
845 3208120 : this->clear_stored_ranges();
846 : }
847 :
848 :
849 :
850 41644964 : Node * DistributedMesh::add_point (const Point & p,
851 : const dof_id_type id,
852 : const processor_id_type proc_id)
853 : {
854 41644964 : Node * old_n = this->query_node_ptr(id);
855 :
856 41644964 : if (old_n)
857 : {
858 0 : *old_n = p;
859 0 : old_n->processor_id() = proc_id;
860 :
861 0 : return old_n;
862 : }
863 :
864 41604915 : Node * n = Node::build(p, id).release();
865 41644964 : n->processor_id() = proc_id;
866 :
867 41644964 : return DistributedMesh::add_node(n);
868 : }
869 :
870 :
871 4130 : void DistributedMesh::own_node (Node & n)
872 : {
873 : // This had better be a node in our mesh
874 0 : libmesh_assert(_nodes[n.id()] == &n);
875 :
876 4130 : _nodes[n.id()] = nullptr;
877 4130 : _n_nodes--;
878 :
879 0 : n.set_id(DofObject::invalid_id);
880 4130 : n.processor_id() = this->processor_id();
881 :
882 4130 : this->add_node(&n);
883 4130 : }
884 :
885 :
886 72435612 : Node * DistributedMesh::add_node (Node * n)
887 : {
888 : // Don't try to add nullptrs!
889 82163 : libmesh_assert(n);
890 :
891 : // Trying to add an existing node is a no-op
892 72435612 : if (n->valid_id() && _nodes[n->id()] == n)
893 0 : return n;
894 :
895 72435612 : const processor_id_type node_procid = n->processor_id();
896 :
897 72435612 : if (!n->valid_id())
898 : {
899 : // We should only be creating new ids past the end of the range
900 : // of existing ids
901 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
902 : _max_node_id);
903 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
904 :
905 : // Use the unpartitioned ids for unpartitioned nodes,
906 : // and temporarily for ghost nodes
907 17428175 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
908 17446004 : if (node_procid == this->processor_id())
909 2128755 : next_id = &_next_free_local_node_id;
910 17428175 : n->set_id (*next_id);
911 : }
912 :
913 : {
914 : // Advance next_ids up high enough that each is pointing to an
915 : // unused id and any subsequent increments will still point us
916 : // to unused ids
917 144871224 : _max_node_id = std::max(_max_node_id,
918 72435612 : static_cast<dof_id_type>(n->id()+1));
919 :
920 72435612 : if (_next_free_unpartitioned_node_id < _max_node_id)
921 20405086 : _next_free_unpartitioned_node_id =
922 20405086 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
923 20405086 : (this->n_processors() + 1) + this->n_processors();
924 72435612 : if (_next_free_local_node_id < _max_node_id)
925 11895678 : _next_free_local_node_id =
926 11913883 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
927 11895678 : (this->n_processors() + 1) + this->processor_id();
928 :
929 : #ifndef NDEBUG
930 : // We need a const dofobject_container so we don't inadvertently create
931 : // nullptr entries when testing for non-nullptr ones
932 82163 : const dofobject_container<Node> & const_nodes = _nodes;
933 : #endif
934 82163 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
935 82163 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
936 : }
937 :
938 : // Don't try to overwrite existing nodes
939 82163 : libmesh_assert (!_nodes[n->id()]);
940 :
941 72435612 : _nodes[n->id()] = n;
942 :
943 : // Try to make the cached node data more accurate
944 72477373 : if (node_procid == this->processor_id() ||
945 : node_procid == DofObject::invalid_processor_id)
946 39210840 : _n_nodes++;
947 :
948 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
949 72435612 : if (!n->valid_unique_id())
950 : {
951 41647036 : if (processor_id() == n->processor_id())
952 : {
953 2570878 : n->set_unique_id(_next_unique_id);
954 2570878 : _next_unique_id += this->n_processors() + 1;
955 : }
956 : else
957 : {
958 39076158 : n->set_unique_id(_next_unpartitioned_unique_id);
959 39076158 : _next_unpartitioned_unique_id += this->n_processors() + 1;
960 : }
961 : }
962 : else
963 : {
964 30919071 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
965 30788576 : _next_unique_id =
966 30788576 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
967 30788576 : (this->n_processors() + 1) + this->processor_id();
968 : }
969 : #endif
970 :
971 72435612 : n->add_extra_integers(_node_integer_names.size(),
972 72435612 : _node_integer_default_values);
973 :
974 : // Unpartitioned nodes should be added on every processor
975 : // And shouldn't be added in the same batch as ghost nodes
976 : // But we might be just adding on processor 0 to
977 : // broadcast later
978 : // #ifdef DEBUG
979 : // if (node_procid == DofObject::invalid_processor_id)
980 : // {
981 : // dof_id_type node_id = n->id();
982 : // this->comm().max(node_id);
983 : // libmesh_assert_equal_to (node_id, n->id());
984 : // }
985 : // #endif
986 :
987 72435612 : return n;
988 : }
989 :
990 30786518 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
991 : {
992 : // The mesh now takes ownership of the Node. Eventually the guts of
993 : // add_node() will get moved to a private helper function, and
994 : // calling add_node() directly will be deprecated.
995 30786518 : return add_node(n.release());
996 : }
997 :
998 50733035 : void DistributedMesh::delete_node(Node * n)
999 : {
1000 4469 : libmesh_assert(n);
1001 4469 : libmesh_assert(_nodes[n->id()]);
1002 :
1003 : // Try to make the cached elem data more accurate
1004 50733035 : _n_nodes--;
1005 :
1006 : // Delete the node from the BoundaryInfo object
1007 50733035 : this->get_boundary_info().remove(n);
1008 8938 : _constraint_rows.erase(n);
1009 :
1010 : // But not yet from the container; we might invalidate
1011 : // an iterator that way!
1012 :
1013 : //_nodes.erase(n->id());
1014 :
1015 : // Instead, we set it to nullptr for now
1016 :
1017 50733035 : _nodes[n->id()] = nullptr;
1018 :
1019 : // delete the node
1020 50733035 : delete n;
1021 50733035 : }
1022 :
1023 :
1024 :
1025 6899090 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1026 : const dof_id_type new_id)
1027 : {
1028 : // This could be a no-op
1029 6899090 : if (old_id == new_id)
1030 0 : return;
1031 :
1032 6898485 : Node * nd = _nodes[old_id];
1033 3234 : libmesh_assert (nd);
1034 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1035 :
1036 : // If we have nodes shipped to this processor for NodeConstraints
1037 : // use, then those nodes will exist in _nodes, but may not be
1038 : // locatable via a TopologyMap due to the insufficiency of elements
1039 : // connecting to them. If local refinement then wants to create a
1040 : // *new* node in the same location, it will initially get a temporary
1041 : // id, and then make_node_ids_parallel_consistent() will try to move
1042 : // it to the canonical id. We need to account for this case to
1043 : // avoid false positives and memory leaks.
1044 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1045 6468 : if (_nodes[new_id])
1046 : {
1047 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1048 : *(Point *)_nodes[old_id]);
1049 0 : _nodes.erase(new_id);
1050 : }
1051 : #else
1052 : // If we aren't shipping nodes for NodeConstraints, there should be
1053 : // no reason for renumbering one node onto another.
1054 : libmesh_assert (!_nodes[new_id]);
1055 : #endif
1056 6898485 : _nodes[new_id] = nd;
1057 6898485 : nd->set_id(new_id);
1058 :
1059 6898485 : _nodes.erase(old_id);
1060 : }
1061 :
1062 :
1063 :
1064 597641 : void DistributedMesh::clear ()
1065 : {
1066 : // Call parent clear function
1067 597641 : MeshBase::clear();
1068 :
1069 : // Clear our elements and nodes
1070 : // There is no need to remove them from
1071 : // the BoundaryInfo data structure since we
1072 : // already cleared it.
1073 597641 : this->DistributedMesh::clear_elems();
1074 :
1075 23034224 : for (auto & node : _nodes)
1076 41653567 : delete node;
1077 :
1078 932 : _nodes.clear();
1079 :
1080 : // We're no longer distributed if we were before
1081 597641 : _is_serial = true;
1082 597641 : _is_serial_on_proc_0 = true;
1083 :
1084 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1085 : // all is copacetic.
1086 597641 : _deleted_coarse_elements = false;
1087 :
1088 : // Correct our caches
1089 597641 : _n_nodes = 0;
1090 597641 : _max_node_id = 0;
1091 598225 : _next_free_local_node_id = this->processor_id();
1092 597641 : _next_free_unpartitioned_node_id = this->n_processors();
1093 597641 : }
1094 :
1095 :
1096 :
1097 604041 : void DistributedMesh::clear_elems ()
1098 : {
1099 9375287 : for (auto & elem : _elements)
1100 8771246 : delete elem;
1101 :
1102 932 : _elements.clear();
1103 :
1104 : // Correct our caches
1105 604041 : _n_elem = 0;
1106 604041 : _max_elem_id = 0;
1107 604625 : _next_free_local_elem_id = this->processor_id();
1108 604041 : _next_free_unpartitioned_elem_id = this->n_processors();
1109 604041 : }
1110 :
1111 :
1112 :
1113 595135 : void DistributedMesh::redistribute ()
1114 : {
1115 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1116 595135 : if (!this->is_serial())
1117 : {
1118 : // Construct a MeshCommunication object to actually redistribute the nodes
1119 : // and elements according to the partitioner, and then to re-gather the neighbors.
1120 : MeshCommunication mc;
1121 70338 : mc.redistribute(*this);
1122 :
1123 70338 : this->update_parallel_id_counts();
1124 :
1125 : // We communicate valid neighbor links for newly-redistributed
1126 : // elements, but we may still have remote_elem links that should
1127 : // be corrected but whose correction wasn't communicated.
1128 70610 : this->find_neighbors(/*reset_remote_elements*/ false,
1129 : /*reset_current_list*/ false /*non-default*/,
1130 : /*assert_valid*/ false,
1131 272 : /*check_non_remote*/ false /*non-default!*/);
1132 :
1133 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1134 : // anyway... but users can always call partition directly, in which case we do need
1135 : // to call delete_remote_elements()...
1136 : //
1137 : // Regardless of whether it's necessary, it isn't safe. We
1138 : // haven't communicated new node processor_ids yet, and we can't
1139 : // delete nodes until we do.
1140 : // this->delete_remote_elements();
1141 : }
1142 : else
1143 : // The base class can handle non-distributed things, like
1144 : // notifying any GhostingFunctors of changes
1145 524797 : MeshBase::redistribute();
1146 595135 : }
1147 :
1148 :
1149 :
1150 507735 : void DistributedMesh::update_post_partitioning ()
1151 : {
1152 : // this->recalculate_n_partitions();
1153 :
1154 : // Let's do the base class cache clearing first, just in case our
1155 : // later computations are ever changed to make use of a local
1156 : // elements range cache
1157 507735 : this->MeshBase::update_post_partitioning();
1158 :
1159 : // Partitioning changes our numbers of unpartitioned objects
1160 507735 : this->update_parallel_id_counts();
1161 507735 : }
1162 :
1163 :
1164 :
1165 : template <typename T>
1166 3380 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1167 : {
1168 : // This function must be run on all processors at once
1169 3380 : parallel_object_only();
1170 :
1171 3380 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1172 3380 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1173 3380 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1174 :
1175 894412 : for (dof_id_type i=0; i != pmax_id; ++i)
1176 : {
1177 891032 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1178 :
1179 : // Local lookups by id should return the requested object
1180 891032 : libmesh_assert(!obj || obj->id() == i);
1181 :
1182 : // All processors with an object should agree on id
1183 : #ifndef NDEBUG
1184 891032 : const dof_id_type dofid = obj && obj->valid_id() ?
1185 : obj->id() : DofObject::invalid_id;
1186 891032 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1187 : #endif
1188 :
1189 : // All processors with an object should agree on processor id
1190 891032 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1191 : obj->processor_id() : DofObject::invalid_processor_id;
1192 891032 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1193 :
1194 891032 : dof_id_type min_procid = procid;
1195 891032 : this->comm().min(min_procid);
1196 :
1197 : // Either:
1198 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1199 : // or
1200 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1201 :
1202 : // Original assert logic
1203 : // libmesh_assert (min_procid != this->processor_id() || obj);
1204 :
1205 : // More human-understandable logic...
1206 891032 : libmesh_assert (
1207 : ((min_procid == this->processor_id()) && obj)
1208 : ||
1209 : (min_procid != this->processor_id())
1210 : );
1211 :
1212 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1213 : // All processors with an object should agree on unique id
1214 891032 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1215 891032 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1216 : #endif
1217 : }
1218 3380 : }
1219 :
1220 :
1221 :
1222 1690 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1223 : {
1224 1690 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1225 1690 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1226 1690 : }
1227 :
1228 :
1229 :
1230 406 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1231 : {
1232 : #ifndef NDEBUG
1233 : // This function must be run on all processors at once
1234 406 : parallel_object_only();
1235 :
1236 406 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1237 :
1238 21970 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1239 : {
1240 21564 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1241 :
1242 21564 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1243 :
1244 : // All processors with an active element should agree on p level
1245 21564 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1246 : }
1247 : #endif
1248 406 : }
1249 :
1250 :
1251 :
1252 :
1253 1638 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1254 : {
1255 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1256 : // This function must be run on all processors at once
1257 1638 : parallel_object_only();
1258 :
1259 1638 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1260 :
1261 144704 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1262 : {
1263 143066 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1264 :
1265 143066 : unsigned int refinement_flag = el ?
1266 65765 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1267 143066 : unsigned int p_refinement_flag = el ?
1268 65765 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1269 :
1270 143066 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1271 :
1272 : // p refinement flags aren't always kept correct on inactive
1273 : // ghost elements
1274 143066 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1275 : }
1276 : #endif // LIBMESH_ENABLE_AMR
1277 1638 : }
1278 :
1279 :
1280 :
1281 : template <typename T>
1282 : dof_id_type
1283 1843418 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1284 : {
1285 : // This function must be run on all processors at once
1286 808 : parallel_object_only();
1287 :
1288 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1289 :
1290 : // In parallel we may not know what objects other processors have.
1291 : // Start by figuring out how many
1292 808 : dof_id_type unpartitioned_objects = 0;
1293 :
1294 : std::unordered_map<processor_id_type, dof_id_type>
1295 1616 : ghost_objects_from_proc;
1296 :
1297 1843418 : object_iterator it = objects.begin();
1298 808 : object_iterator end = objects.end();
1299 :
1300 111877924 : while (it != end)
1301 : {
1302 110034506 : T * obj = *it;
1303 :
1304 : // Remove any nullptr container entries while we're here.
1305 110034506 : if (!obj)
1306 1424841 : it = objects.erase(it);
1307 : else
1308 : {
1309 108609665 : processor_id_type obj_procid = obj->processor_id();
1310 108609665 : if (obj_procid == DofObject::invalid_processor_id)
1311 43120004 : unpartitioned_objects++;
1312 : else
1313 65489661 : ghost_objects_from_proc[obj_procid]++;
1314 :
1315 : // Finally, increment the iterator
1316 56464 : ++it;
1317 : }
1318 : }
1319 :
1320 1845034 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1321 1844226 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1322 808 : this->comm().allgather
1323 1843913 : ((this_it == ghost_objects_from_proc.end()) ?
1324 495 : dof_id_type(0) : this_it->second, objects_on_proc);
1325 :
1326 : #ifndef NDEBUG
1327 808 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1328 2424 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1329 1616 : if (ghost_objects_from_proc.count(p))
1330 962 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1331 : else
1332 654 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1333 : #endif
1334 :
1335 : // We'll renumber objects in blocks by processor id
1336 1845034 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1337 19562274 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1338 17722088 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1339 808 : objects_on_proc[i-1];
1340 1844226 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1341 1843418 : dof_id_type first_free_id =
1342 1845034 : first_object_on_proc[this->n_processors()-1] +
1343 808 : objects_on_proc[this->n_processors()-1] +
1344 : unpartitioned_objects;
1345 :
1346 : // First set new local object ids and build request sets
1347 : // for non-local object ids
1348 :
1349 : // Request sets to send to each processor
1350 : std::map<processor_id_type, std::vector<dof_id_type>>
1351 808 : requested_ids;
1352 :
1353 : // We know how many objects live on each processor, so reserve() space for
1354 : // each.
1355 808 : auto ghost_end = ghost_objects_from_proc.end();
1356 21405692 : for (auto p : make_range(this->n_processors()))
1357 19563890 : if (p != this->processor_id())
1358 : {
1359 17719664 : if (const auto p_it = ghost_objects_from_proc.find(p);
1360 808 : p_it != ghost_end)
1361 3482141 : requested_ids[p].reserve(p_it->second);
1362 : }
1363 :
1364 808 : end = objects.end();
1365 110453083 : for (it = objects.begin(); it != end; ++it)
1366 : {
1367 108609665 : T * obj = *it;
1368 108609665 : if (!obj)
1369 0 : continue;
1370 108666129 : if (obj->processor_id() == this->processor_id())
1371 20002755 : obj->set_id(next_id++);
1372 88606910 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1373 45486906 : requested_ids[obj->processor_id()].push_back(obj->id());
1374 : }
1375 :
1376 : // Next set ghost object ids from other processors
1377 :
1378 1860632 : auto gather_functor =
1379 48935553 : [
1380 : #ifndef NDEBUG
1381 : this,
1382 : &first_object_on_proc,
1383 : &objects_on_proc,
1384 : #endif
1385 : &objects]
1386 : (processor_id_type, const std::vector<dof_id_type> & ids,
1387 934 : std::vector<dof_id_type> & new_ids)
1388 : {
1389 934 : std::size_t ids_size = ids.size();
1390 3482141 : new_ids.resize(ids_size);
1391 :
1392 48969047 : for (std::size_t i=0; i != ids_size; ++i)
1393 : {
1394 45486906 : T * obj = objects[ids[i]];
1395 16280 : libmesh_assert(obj);
1396 16280 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1397 45503186 : new_ids[i] = obj->id();
1398 :
1399 16280 : libmesh_assert_greater_equal (new_ids[i],
1400 : first_object_on_proc[this->processor_id()]);
1401 16280 : libmesh_assert_less (new_ids[i],
1402 : first_object_on_proc[this->processor_id()] +
1403 : objects_on_proc[this->processor_id()]);
1404 : }
1405 : };
1406 :
1407 1860632 : auto action_functor =
1408 48935553 : [
1409 : #ifndef NDEBUG
1410 : &first_object_on_proc,
1411 : &objects_on_proc,
1412 : #endif
1413 : &objects]
1414 : (processor_id_type libmesh_dbg_var(pid),
1415 : const std::vector<dof_id_type> & ids,
1416 934 : const std::vector<dof_id_type> & data)
1417 : {
1418 : // Copy the id changes we've now been informed of
1419 48969047 : for (auto i : index_range(ids))
1420 : {
1421 45486906 : T * obj = objects[ids[i]];
1422 16280 : libmesh_assert (obj);
1423 16280 : libmesh_assert_equal_to (obj->processor_id(), pid);
1424 16280 : libmesh_assert_greater_equal (data[i],
1425 : first_object_on_proc[pid]);
1426 16280 : libmesh_assert_less (data[i],
1427 : first_object_on_proc[pid] +
1428 : objects_on_proc[pid]);
1429 45503186 : obj->set_id(data[i]);
1430 : }
1431 : };
1432 :
1433 808 : const dof_id_type * ex = nullptr;
1434 : Parallel::pull_parallel_vector_data
1435 1843418 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1436 :
1437 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1438 1860632 : auto unique_gather_functor =
1439 48935553 : [
1440 : #ifndef NDEBUG
1441 : this,
1442 : #endif
1443 : &objects]
1444 : (processor_id_type, const std::vector<dof_id_type> & ids,
1445 934 : std::vector<unique_id_type> & data)
1446 : {
1447 934 : std::size_t ids_size = ids.size();
1448 3482141 : data.resize(ids_size);
1449 :
1450 48969047 : for (std::size_t i=0; i != ids_size; ++i)
1451 : {
1452 45486906 : T * obj = objects[ids[i]];
1453 16280 : libmesh_assert(obj);
1454 16280 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1455 90973812 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1456 : }
1457 : };
1458 :
1459 1860632 : auto unique_action_functor =
1460 48935553 : [&objects]
1461 : (processor_id_type libmesh_dbg_var(pid),
1462 : const std::vector<dof_id_type> & ids,
1463 934 : const std::vector<unique_id_type> & data)
1464 : {
1465 48969047 : for (auto i : index_range(ids))
1466 : {
1467 45486906 : T * obj = objects[ids[i]];
1468 16280 : libmesh_assert (obj);
1469 16280 : libmesh_assert_equal_to (obj->processor_id(), pid);
1470 45486906 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1471 0 : obj->set_unique_id(data[i]);
1472 : }
1473 : };
1474 :
1475 808 : const unique_id_type * unique_ex = nullptr;
1476 : Parallel::pull_parallel_vector_data
1477 1843418 : (this->comm(), requested_ids, unique_gather_functor,
1478 : unique_action_functor, unique_ex);
1479 : #endif
1480 :
1481 : // Next set unpartitioned object ids
1482 808 : next_id = 0;
1483 21405692 : for (auto i : make_range(this->n_processors()))
1484 19563890 : next_id += objects_on_proc[i];
1485 110453083 : for (it = objects.begin(); it != end; ++it)
1486 : {
1487 108609665 : T * obj = *it;
1488 108609665 : if (!obj)
1489 0 : continue;
1490 108609665 : if (obj->processor_id() == DofObject::invalid_processor_id)
1491 43120004 : obj->set_id(next_id++);
1492 : }
1493 :
1494 : // Finally shuffle around objects so that container indices
1495 : // match ids
1496 1843418 : it = objects.begin();
1497 808 : end = objects.end();
1498 116951363 : while (it != end)
1499 : {
1500 115107945 : T * obj = *it;
1501 115107945 : if (obj) // don't try shuffling already-nullptr entries
1502 : {
1503 115107945 : T * next = objects[obj->id()];
1504 : // If we have to move this object
1505 115107945 : if (next != obj)
1506 : {
1507 : // nullptr out its original position for now
1508 : // (our shuffling may put another object there shortly)
1509 39995456 : *it = nullptr;
1510 :
1511 : // There may already be another object with this id that
1512 : // needs to be moved itself
1513 51802825 : while (next)
1514 : {
1515 : // We shouldn't be trying to give two objects the
1516 : // same id
1517 9385 : libmesh_assert_not_equal_to (next->id(), obj->id());
1518 11807369 : objects[obj->id()] = obj;
1519 9385 : obj = next;
1520 11807369 : next = objects[obj->id()];
1521 : }
1522 39995456 : objects[obj->id()] = obj;
1523 : }
1524 : }
1525 :
1526 : // Remove any container entries that were left as nullptr.
1527 115380 : if (!obj)
1528 0 : it = objects.erase(it);
1529 : else
1530 57690 : ++it;
1531 : }
1532 :
1533 1844226 : return first_free_id;
1534 : }
1535 :
1536 :
1537 923379 : void DistributedMesh::renumber_nodes_and_elements ()
1538 : {
1539 406 : parallel_object_only();
1540 :
1541 : #ifdef DEBUG
1542 : // Make sure our ids and flags are consistent
1543 406 : this->libmesh_assert_valid_parallel_ids();
1544 406 : this->libmesh_assert_valid_parallel_flags();
1545 406 : this->libmesh_assert_valid_parallel_p_levels();
1546 : #endif
1547 :
1548 406 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1549 :
1550 : // Nodes not connected to any elements, and nullptr node entries
1551 : // in our container, should be deleted. But wait! If we've deleted coarse
1552 : // local elements on some processor, other processors might have ghosted
1553 : // nodes from it that are now no longer connected to any elements on it, but
1554 : // that are connected to their own semilocal elements. We'll have to
1555 : // communicate to ascertain if that's the case.
1556 923379 : this->comm().max(_deleted_coarse_elements);
1557 :
1558 : // What used nodes do we see on our proc?
1559 406 : std::set<dof_id_type> used_nodes;
1560 :
1561 : // What used node info should we send from our proc? Could we take ownership
1562 : // of each node if we needed to?
1563 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1564 406 : used_nodes_on_proc;
1565 :
1566 : // flag the nodes we need
1567 57820280 : for (auto & elem : this->element_ptr_range())
1568 301183815 : for (const Node & node : elem->node_ref_range())
1569 : {
1570 273167405 : const dof_id_type n = node.id();
1571 273022939 : used_nodes.insert(n);
1572 273167405 : if (_deleted_coarse_elements)
1573 : {
1574 3145701 : const processor_id_type p = node.processor_id();
1575 3145985 : if (p != this->processor_id())
1576 : {
1577 2257624 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1578 2257766 : if (elem->processor_id() == this->processor_id())
1579 216210 : used_nodes_on_p[n] = true;
1580 : else
1581 105 : if (!used_nodes_on_p.count(n))
1582 525626 : used_nodes_on_p[n] = false;
1583 : }
1584 : }
1585 922567 : }
1586 :
1587 923379 : if (_deleted_coarse_elements)
1588 : {
1589 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1590 : typedef unsigned char boolish;
1591 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1592 28 : used_nodes_on_proc_vecs;
1593 73209 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1594 56827 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1595 :
1596 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1597 : std::map<processor_id_type, std::set<dof_id_type>>
1598 28 : repartitioned_node_sets_to_push;
1599 :
1600 : auto ids_action_functor =
1601 56801 : [&used_nodes, &repartitioned_node_pids,
1602 : &repartitioned_node_sets_to_push]
1603 : (processor_id_type pid,
1604 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1605 : {
1606 592596 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1607 : {
1608 : // If we don't see a use for our own node, but someone
1609 : // else does, better figure out who should own it next.
1610 97 : if (!used_nodes.count(n))
1611 : {
1612 2 : if (auto it = repartitioned_node_pids.find(n);
1613 2 : sender_could_become_owner)
1614 : {
1615 1 : if (it != repartitioned_node_pids.end() &&
1616 0 : pid < it->second)
1617 0 : it->second = pid;
1618 : else
1619 1 : repartitioned_node_pids[n] = pid;
1620 : }
1621 : else
1622 1 : if (it == repartitioned_node_pids.end())
1623 0 : repartitioned_node_pids[n] =
1624 : DofObject::invalid_processor_id;
1625 :
1626 2 : repartitioned_node_sets_to_push[pid].insert(n);
1627 : }
1628 : }
1629 16394 : };
1630 :
1631 : // We need two pushes instead of a pull here because we need to
1632 : // know *all* the queries for a particular node before we can
1633 : // respond to *any* of them.
1634 : Parallel::push_parallel_vector_data
1635 16382 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1636 :
1637 : // Repartition (what used to be) our own nodes first
1638 16383 : for (auto & [n, p] : repartitioned_node_pids)
1639 : {
1640 1 : Node & node = this->node_ref(n);
1641 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1642 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1643 1 : node.processor_id() = p;
1644 : }
1645 :
1646 : // Then push to repartition others' ghosted copies.
1647 :
1648 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1649 28 : repartitioned_node_vecs;
1650 :
1651 16384 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1652 : {
1653 2 : auto & rn_vec = repartitioned_node_vecs[p];
1654 4 : for (auto n : nodeset)
1655 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1656 : }
1657 :
1658 : auto repartition_node_functor =
1659 2 : [this]
1660 : (processor_id_type libmesh_dbg_var(pid),
1661 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1662 : {
1663 4 : for (auto [n, p] : ids_and_pids)
1664 : {
1665 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1666 2 : Node & node = this->node_ref(n);
1667 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1668 2 : node.processor_id() = p;
1669 : }
1670 16354 : };
1671 :
1672 : Parallel::push_parallel_vector_data
1673 16382 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1674 : }
1675 :
1676 923379 : _deleted_coarse_elements = false;
1677 :
1678 : // Nodes not connected to any local elements, and nullptr node entries
1679 : // in our container, are deleted
1680 : {
1681 923379 : node_iterator_imp it = _nodes.begin();
1682 406 : node_iterator_imp end = _nodes.end();
1683 :
1684 88447045 : while (it != end)
1685 : {
1686 87523666 : Node * nd = *it;
1687 87523666 : if (!nd)
1688 3491561 : it = _nodes.erase(it);
1689 84032105 : else if (!used_nodes.count(nd->id()))
1690 : {
1691 : // remove any boundary information associated with
1692 : // this node
1693 2405183 : this->get_boundary_info().remove (nd);
1694 2828 : _constraint_rows.erase(nd);
1695 :
1696 : // delete the node
1697 2405183 : delete nd;
1698 :
1699 2405183 : it = _nodes.erase(it);
1700 : }
1701 : else
1702 45303 : ++it;
1703 : }
1704 : }
1705 :
1706 923379 : this->_preparation.has_removed_orphaned_nodes = true;
1707 :
1708 923379 : if (_skip_renumber_nodes_and_elements)
1709 : {
1710 1670 : this->update_parallel_id_counts();
1711 2 : return;
1712 : }
1713 :
1714 : // Finally renumber all the elements
1715 921709 : _n_elem = this->renumber_dof_objects (this->_elements);
1716 :
1717 : // and all the remaining nodes
1718 921709 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1719 :
1720 : // And figure out what IDs we should use when adding new nodes and
1721 : // new elements
1722 921709 : this->update_parallel_id_counts();
1723 :
1724 : // Make sure our caches are up to date and our
1725 : // DofObjects are well packed
1726 : #ifdef DEBUG
1727 404 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1728 404 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1729 404 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1730 404 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1731 404 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1732 404 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1733 404 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1734 404 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1735 :
1736 : // Make sure our ids and flags are consistent
1737 404 : this->libmesh_assert_valid_parallel_ids();
1738 404 : this->libmesh_assert_valid_parallel_flags();
1739 :
1740 : // And make sure we've made our numbering monotonic
1741 404 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1742 : #endif
1743 : }
1744 :
1745 :
1746 :
1747 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1748 : {
1749 : // We can't use range-for here because we need access to the special
1750 : // iterators' methods, not just to their dereferenced values.
1751 :
1752 : // Nodes first
1753 0 : for (auto pr = this->_nodes.begin(),
1754 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1755 : {
1756 5427360 : Node * n = *pr;
1757 5427360 : if (n != nullptr)
1758 : {
1759 0 : const dof_id_type id = pr.index();
1760 5204550 : n->set_id() = id;
1761 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1762 : }
1763 : }
1764 :
1765 : // Elements next
1766 0 : for (auto pr = this->_elements.begin(),
1767 5671231 : end = this->_elements.end(); pr != end; ++pr)
1768 : {
1769 5653821 : Elem * e = *pr;
1770 5653821 : if (e != nullptr)
1771 : {
1772 0 : const dof_id_type id = pr.index();
1773 5590229 : e->set_id() = id;
1774 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1775 : }
1776 : }
1777 17410 : }
1778 :
1779 :
1780 :
1781 673988 : dof_id_type DistributedMesh::n_active_elem () const
1782 : {
1783 554 : parallel_object_only();
1784 :
1785 : // Get local active elements first
1786 : dof_id_type active_elements =
1787 1347422 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1788 2020856 : this->active_local_elements_end()));
1789 673988 : this->comm().sum(active_elements);
1790 :
1791 : // Then add unpartitioned active elements, which should exist on
1792 : // every processor
1793 673988 : active_elements +=
1794 673988 : static_cast<dof_id_type>(std::distance
1795 1347422 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1796 674542 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1797 673988 : return active_elements;
1798 : }
1799 :
1800 :
1801 :
1802 427240 : void DistributedMesh::delete_remote_elements()
1803 : {
1804 : #ifdef DEBUG
1805 : // Make sure our neighbor links are all fine
1806 390 : MeshTools::libmesh_assert_valid_neighbors(*this);
1807 :
1808 : // And our child/parent links, and our flags
1809 390 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1810 :
1811 : // Make sure our ids and flags are consistent
1812 390 : this->libmesh_assert_valid_parallel_ids();
1813 390 : this->libmesh_assert_valid_parallel_flags();
1814 :
1815 390 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1816 390 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1817 390 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1818 390 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1819 390 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1820 390 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1821 : #endif
1822 :
1823 427240 : _is_serial = false;
1824 427240 : _is_serial_on_proc_0 = false;
1825 :
1826 427240 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1827 :
1828 390 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1829 :
1830 : // Now make sure the containers actually shrink - strip
1831 : // any newly-created nullptr voids out of the element array
1832 427240 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1833 390 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1834 53652139 : while (e_it != e_end)
1835 53224899 : if (!*e_it)
1836 39540658 : e_it = _elements.erase(e_it);
1837 : else
1838 17409 : ++e_it;
1839 :
1840 427240 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1841 390 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1842 101021196 : while (n_it != n_end)
1843 100593956 : if (!*n_it)
1844 66856367 : n_it = _nodes.erase(n_it);
1845 : else
1846 48635 : ++n_it;
1847 :
1848 : // We may have deleted no-longer-connected nodes or coarsened-away
1849 : // elements; let's update our caches.
1850 427240 : this->update_parallel_id_counts();
1851 :
1852 : // We may have deleted nodes or elements that were the only local
1853 : // representatives of some particular boundary id(s); let's update
1854 : // those caches.
1855 427240 : this->get_boundary_info().regenerate_id_sets();
1856 :
1857 : #ifdef DEBUG
1858 : // We might not have well-packed objects if the user didn't allow us
1859 : // to renumber
1860 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1861 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1862 :
1863 : // Make sure our neighbor links are all fine
1864 390 : MeshTools::libmesh_assert_valid_neighbors(*this);
1865 :
1866 : // And our child/parent links, and our flags
1867 390 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1868 :
1869 : // Make sure our ids and flags are consistent
1870 390 : this->libmesh_assert_valid_parallel_ids();
1871 390 : this->libmesh_assert_valid_parallel_flags();
1872 : #endif
1873 :
1874 427240 : this->_preparation.has_removed_remote_elements = true;
1875 427240 : }
1876 :
1877 :
1878 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1879 : {
1880 : // First add the elem like normal
1881 0 : add_elem(e);
1882 :
1883 : // Now add it to the set that won't be deleted when we call
1884 : // delete_remote_elements()
1885 0 : _extra_ghost_elems.insert(e);
1886 0 : }
1887 :
1888 : void
1889 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1890 : {
1891 0 : std::set<Elem *> tmp;
1892 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1893 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1894 0 : std::inserter(tmp, tmp.begin()));
1895 0 : _extra_ghost_elems = tmp;
1896 0 : }
1897 :
1898 98725 : void DistributedMesh::allgather()
1899 : {
1900 98725 : if (_is_serial)
1901 0 : return;
1902 98723 : MeshCommunication().allgather(*this);
1903 98723 : _is_serial = true;
1904 98723 : _is_serial_on_proc_0 = true;
1905 :
1906 : // Make sure our caches are up to date and our
1907 : // DofObjects are well packed
1908 : #ifdef DEBUG
1909 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1910 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1911 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1912 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1913 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1914 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1915 :
1916 : // If we've disabled renumbering we can't be sure we're contiguous
1917 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1918 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1919 :
1920 : // Make sure our neighbor links are all fine
1921 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1922 :
1923 : // Make sure our ids and flags are consistent
1924 48 : this->libmesh_assert_valid_parallel_ids();
1925 48 : this->libmesh_assert_valid_parallel_flags();
1926 : #endif
1927 : }
1928 :
1929 18352 : void DistributedMesh::gather_to_zero()
1930 : {
1931 18352 : if (_is_serial_on_proc_0)
1932 4 : return;
1933 :
1934 14114 : _is_serial_on_proc_0 = true;
1935 14114 : MeshCommunication().gather(0, *this);
1936 : }
1937 :
1938 :
1939 : } // namespace libMesh
|
