Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 400346 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 400346 : unsigned char d) :
44 399826 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 399826 : _is_serial_on_proc_0(true),
46 399826 : _deleted_coarse_elements(false),
47 399826 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 400606 : _next_free_local_node_id(this->processor_id()),
49 400346 : _next_free_local_elem_id(this->processor_id()),
50 400346 : _next_free_unpartitioned_node_id(this->n_processors()),
51 400346 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 400346 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 400346 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 400606 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 800692 : default_partitioner);
64 : _partitioner = Partitioner::build
65 800432 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 400346 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 8 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 442811 : DistributedMesh::~DistributedMesh ()
143 : {
144 420707 : this->DistributedMesh::clear(); // Free nodes and elements
145 442811 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17734 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 : DistributedMesh(static_cast<const MeshBase &>(other_mesh),
153 17734 : /*bool other_is_distributed_type=*/true)
154 : {
155 17734 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
156 17734 : _next_free_local_node_id =
157 17734 : other_mesh._next_free_local_node_id;
158 17734 : _next_free_local_elem_id =
159 17734 : other_mesh._next_free_local_elem_id;
160 17734 : _next_free_unpartitioned_node_id =
161 17734 : other_mesh._next_free_unpartitioned_node_id;
162 17734 : _next_free_unpartitioned_elem_id =
163 17734 : other_mesh._next_free_unpartitioned_elem_id;
164 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
165 17734 : _next_unique_id =
166 17734 : other_mesh._next_unique_id;
167 17734 : _next_unpartitioned_unique_id =
168 17734 : other_mesh._next_unpartitioned_unique_id;
169 : #endif
170 :
171 : // Need to copy extra_ghost_elems
172 17734 : for (auto & elem : other_mesh._extra_ghost_elems)
173 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
174 17734 : }
175 :
176 :
177 :
178 20361 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
179 20361 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
180 20455 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
181 20173 : _deleted_coarse_elements(true), // better safe than sorry...
182 20173 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
183 20455 : _next_free_local_node_id(this->processor_id()),
184 20361 : _next_free_local_elem_id(this->processor_id()),
185 20361 : _next_free_unpartitioned_node_id(this->n_processors()),
186 40628 : _next_free_unpartitioned_elem_id(this->n_processors())
187 : {
188 : // Just copy, skipping preparation
189 20361 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
190 :
191 188 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
192 188 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
193 188 : this->allow_renumbering(other_mesh.allow_renumbering());
194 188 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
195 188 : this->skip_partitioning(other_mesh.skip_partitioning());
196 :
197 20361 : this->copy_constraint_rows(other_mesh);
198 :
199 94 : auto & this_boundary_info = this->get_boundary_info();
200 94 : const auto & other_boundary_info = other_mesh.get_boundary_info();
201 :
202 20361 : this_boundary_info = other_boundary_info;
203 :
204 94 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
205 :
206 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
207 : // nodes/elements they consider locally owned (or unpartitioned), so after
208 : // copying only the objects visible to this processor, those counts
209 : // reflect an incomplete local view rather than other_mesh's totals,
210 : // and need to be copied afresh.
211 20361 : _n_nodes = other_mesh.n_nodes();
212 20361 : _n_elem = other_mesh.n_elem();
213 20361 : _max_node_id = other_mesh.max_node_id();
214 20361 : _max_elem_id = other_mesh.max_elem_id();
215 :
216 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
217 : // counters directly, mirroring other_mesh's own bookkeeping exactly. This
218 : // is done in the DistributedMesh copy constructor.
219 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
220 : // raw counterpart fields to copy, so derive the free-id-range
221 : // counters from _max_elem_id/_max_node_id the same way
222 : // update_parallel_id_counts() does via the set_next_ids() method. For
223 : // the unique id counters we copy other_mesh's own local next_unique_id()
224 : // and derive the next available unique ids
225 : // (partitioned and unpartitioned) via set_next_unique_ids().
226 20361 : if (!other_is_distributed_type)
227 : {
228 2627 : this->set_next_ids();
229 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
230 2627 : this->set_next_unique_ids(other_mesh.next_unique_id());
231 : #endif
232 : }
233 :
234 : // Copy other_mesh's actual preparation state, including whatever it
235 : // reports for has_synched_id_counts: the counts above are a faithful copy
236 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
237 : // verified computation, so our synced-ness should match other_mesh's.
238 20361 : this->_preparation = other_mesh.preparation();
239 20361 : }
240 :
241 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
242 : {
243 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
244 :
245 4 : this->_nodes = std::move(other_mesh._nodes);
246 270 : this->_n_nodes = other_mesh.n_nodes();
247 :
248 4 : this->_elements = std::move(other_mesh._elements);
249 270 : this->_n_elem = other_mesh.n_elem();
250 :
251 270 : _is_serial = other_mesh.is_serial();
252 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
253 270 : _deleted_coarse_elements = true; // Better safe than sorry
254 :
255 270 : _max_node_id = other_mesh.max_node_id();
256 270 : _max_elem_id = other_mesh.max_elem_id();
257 :
258 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
259 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
260 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
261 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
262 :
263 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
264 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
265 : #endif
266 270 : }
267 :
268 3972984 : void DistributedMesh::set_next_ids()
269 : {
270 3972984 : _next_free_unpartitioned_elem_id =
271 3975588 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
272 3972984 : (this->n_processors() + 1) + this->n_processors();
273 3972984 : _next_free_local_elem_id =
274 3972984 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
275 3972984 : (this->n_processors() + 1) + this->processor_id();
276 :
277 3972984 : _next_free_unpartitioned_node_id =
278 3975588 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
279 3972984 : (this->n_processors() + 1) + this->n_processors();
280 3972984 : _next_free_local_node_id =
281 3972984 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
282 3972984 : (this->n_processors() + 1) + this->processor_id();
283 3972984 : }
284 :
285 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
286 3972984 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
287 : {
288 : // Unique ids are laid out in repeating groups of (n_processors()+1)
289 : // consecutive integers ("alignment" here means a value's position, i.e.
290 : // its residue mod (n_processors()+1), within one of these groups). In
291 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
292 : // are each permanently earmarked for one processor's locally-owned
293 : // objects, and the last slot (residue n_processors()) is permanently
294 : // earmarked for unpartitioned objects.
295 : //
296 : // parallel_max_unique_id carries no guarantee of landing on any
297 : // particular residue, so rather than simply offsetting it by
298 : // processor_id()/n_processors(), we have to explicitly round up to the
299 : // start of the next full, entirely-unused group before adding this
300 : // processor's (or the unpartitioned pool's) fixed offset into that group.
301 : //
302 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
303 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
304 : // the number of complete groups needed to reach at least past the
305 : // current max id. Multiplying back by (n_processors()+1) gives the first
306 : // id of the next group that is guaranteed to be entirely unused; adding
307 : // n_processors(), the unpartitioned pool's fixed offset within a group,
308 : // lands on that pool's reserved slot in that group.
309 3972984 : _next_unpartitioned_unique_id =
310 3975588 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
311 3972984 : (this->n_processors() + 1) +
312 3972984 : this->n_processors();
313 :
314 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
315 : // the rounding accounts for processor_id() possibly being less than the
316 : // residue the raw max id already occupies within its group; this can
317 : // land this processor's slot in the same group as the unpartitioned
318 : // pool's above, or push it into the following group, depending on where
319 : // the max id and this processor's offset happen to fall.
320 3972984 : _next_unique_id =
321 3972984 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
322 3972984 : (this->n_processors() + 1) +
323 3972984 : this->processor_id();
324 3972984 : }
325 : #endif
326 :
327 : // We use cached values for these so they can be called
328 : // from one processor without bothering the rest, but
329 : // we may need to update those caches before doing a full
330 : // renumbering
331 3970357 : void DistributedMesh::update_parallel_id_counts()
332 : {
333 : // This function must be run on all processors at once
334 2530 : parallel_object_only();
335 :
336 3970357 : _n_elem = this->parallel_n_elem();
337 3970357 : _n_nodes = this->parallel_n_nodes();
338 3970357 : _max_node_id = this->parallel_max_node_id();
339 3970357 : _max_elem_id = this->parallel_max_elem_id();
340 :
341 3970357 : this->set_next_ids();
342 :
343 : // set_next_unique_ids() needs the true parallel max unique id (not just
344 : // this processor's possibly-stale cached _next_unique_id) since the whole
345 : // point of this function is to recompute counters that may have drifted
346 : // out of sync.
347 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
348 3970357 : this->set_next_unique_ids(this->parallel_max_unique_id());
349 : #endif
350 :
351 3970357 : _n_constraint_rows = this->count_constraint_rows();
352 :
353 3970357 : this->_preparation.has_synched_id_counts = true;
354 3970357 : }
355 :
356 :
357 : // Or in debug mode we may want to test the uncached values without
358 : // changing the cache
359 3978118 : dof_id_type DistributedMesh::parallel_n_elem() const
360 : {
361 : // This function must be run on all processors at once
362 3414 : parallel_object_only();
363 :
364 3978118 : dof_id_type n_local = this->n_local_elem();
365 3978118 : this->comm().sum(n_local);
366 3978118 : n_local += this->n_unpartitioned_elem();
367 3978118 : return n_local;
368 : }
369 :
370 :
371 :
372 3977653 : dof_id_type DistributedMesh::parallel_max_elem_id() const
373 : {
374 : // This function must be run on all processors at once
375 9826 : parallel_object_only();
376 :
377 3977653 : dof_id_type max_local = 0;
378 :
379 : dofobject_container<Elem>::const_reverse_veclike_iterator
380 9826 : rit = _elements.rbegin();
381 :
382 : const dofobject_container<Elem>::const_reverse_veclike_iterator
383 9826 : rend = _elements.rend();
384 :
385 : // Look for the maximum element id. Search backwards through
386 : // elements so we can break out early. Beware of nullptr entries that
387 : // haven't yet been cleared from _elements.
388 16292286 : for (; rit != rend; ++rit)
389 : {
390 15503135 : const DofObject *d = *rit;
391 21068 : if (d)
392 : {
393 9609 : libmesh_assert(_elements[d->id()] == d);
394 3188502 : max_local = d->id() + 1;
395 3188502 : break;
396 : }
397 : }
398 :
399 3977653 : this->comm().max(max_local);
400 3977653 : return max_local;
401 : }
402 :
403 :
404 :
405 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
406 4063897 : unique_id_type DistributedMesh::parallel_max_unique_id() const
407 : {
408 : // This function must be run on all processors at once
409 2632 : parallel_object_only();
410 :
411 8127794 : unique_id_type max_local = std::max(_next_unique_id,
412 4495325 : _next_unpartitioned_unique_id);
413 4063897 : this->comm().max(max_local);
414 4063897 : return max_local;
415 : }
416 :
417 :
418 :
419 88581 : void DistributedMesh::set_next_unique_id(unique_id_type id)
420 : {
421 148 : _next_unique_id = id;
422 88581 : _next_unpartitioned_unique_id =
423 88729 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
424 88581 : (this->n_processors() + 1) + this->n_processors();
425 88581 : _next_unique_id =
426 88581 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
427 88581 : (this->n_processors() + 1) + this->processor_id();
428 88581 : }
429 : #endif
430 :
431 :
432 :
433 3978566 : dof_id_type DistributedMesh::parallel_n_nodes() const
434 : {
435 : // This function must be run on all processors at once
436 3414 : parallel_object_only();
437 :
438 3978566 : dof_id_type n_local = this->n_local_nodes();
439 3978566 : this->comm().sum(n_local);
440 3978566 : n_local += this->n_unpartitioned_nodes();
441 3978566 : return n_local;
442 : }
443 :
444 :
445 :
446 3975129 : dof_id_type DistributedMesh::parallel_max_node_id() const
447 : {
448 : // This function must be run on all processors at once
449 7302 : parallel_object_only();
450 :
451 3975129 : dof_id_type max_local = 0;
452 :
453 : dofobject_container<Node>::const_reverse_veclike_iterator
454 7302 : rit = _nodes.rbegin();
455 :
456 : const dofobject_container<Node>::const_reverse_veclike_iterator
457 7302 : rend = _nodes.rend();
458 :
459 : // Look for the maximum node id. Search backwards through
460 : // nodes so we can break out early. Beware of nullptr entries that
461 : // haven't yet been cleared from _nodes
462 76762454 : for (; rit != rend; ++rit)
463 : {
464 75973373 : const DofObject *d = *rit;
465 74384 : if (d)
466 : {
467 7155 : libmesh_assert(_nodes[d->id()] == d);
468 3186048 : max_local = d->id() + 1;
469 3186048 : break;
470 : }
471 : }
472 :
473 3975129 : this->comm().max(max_local);
474 3975129 : return max_local;
475 : }
476 :
477 :
478 :
479 45958489 : const Point & DistributedMesh::point (const dof_id_type i) const
480 : {
481 45958489 : return this->node_ref(i);
482 : }
483 :
484 :
485 :
486 47075566 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
487 : {
488 137685 : libmesh_assert(_nodes[i]);
489 137685 : libmesh_assert_equal_to (_nodes[i]->id(), i);
490 :
491 47075566 : return _nodes[i];
492 : }
493 :
494 :
495 :
496 :
497 669909800 : Node * DistributedMesh::node_ptr (const dof_id_type i)
498 : {
499 491226 : libmesh_assert(_nodes[i]);
500 491226 : libmesh_assert_equal_to (_nodes[i]->id(), i);
501 :
502 669909800 : return _nodes[i];
503 : }
504 :
505 :
506 :
507 :
508 3110532 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
509 : {
510 3110532 : if (const auto it = _nodes.find(i);
511 658809 : it != _nodes.end())
512 : {
513 2821957 : const Node * n = *it;
514 370234 : libmesh_assert (!n || n->id() == i);
515 370234 : return n;
516 : }
517 :
518 288575 : return nullptr;
519 : }
520 :
521 :
522 :
523 :
524 388880497 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
525 : {
526 388880497 : if (auto it = _nodes.find(i);
527 159782 : it != _nodes.end())
528 : {
529 301896407 : Node * n = *it;
530 112018 : libmesh_assert (!n || n->id() == i);
531 112018 : return n;
532 : }
533 :
534 47764 : return nullptr;
535 : }
536 :
537 :
538 :
539 :
540 3128151 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
541 : {
542 10137 : libmesh_assert(_elements[i]);
543 10137 : libmesh_assert_equal_to (_elements[i]->id(), i);
544 :
545 3128151 : return _elements[i];
546 : }
547 :
548 :
549 :
550 :
551 296295686 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
552 : {
553 167573 : libmesh_assert(_elements[i]);
554 167573 : libmesh_assert_equal_to (_elements[i]->id(), i);
555 :
556 296295686 : return _elements[i];
557 : }
558 :
559 :
560 :
561 :
562 1458971 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
563 : {
564 1458971 : if (const auto it = _elements.find(i);
565 620001 : it != _elements.end())
566 : {
567 1101298 : const Elem * e = *it;
568 271243 : libmesh_assert (!e || e->id() == i);
569 271243 : return e;
570 : }
571 :
572 348758 : return nullptr;
573 : }
574 :
575 :
576 :
577 :
578 630126293 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
579 : {
580 630126293 : if (auto it = _elements.find(i);
581 194380 : it != _elements.end())
582 : {
583 543985351 : Elem * e = *it;
584 190696 : libmesh_assert (!e || e->id() == i);
585 190696 : return e;
586 : }
587 :
588 3684 : return nullptr;
589 : }
590 :
591 :
592 :
593 :
594 45006564 : Elem * DistributedMesh::add_elem (Elem * e)
595 : {
596 : // Don't try to add nullptrs!
597 19155 : libmesh_assert(e);
598 :
599 : // Trying to add an existing element is a no-op
600 45006564 : if (e->valid_id() && _elements[e->id()] == e)
601 0 : return e;
602 :
603 45006564 : const processor_id_type elem_procid = e->processor_id();
604 :
605 45006564 : if (!e->valid_id())
606 : {
607 : // We should only be creating new ids past the end of the range
608 : // of existing ids
609 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
610 : _max_elem_id);
611 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
612 :
613 : // Use the unpartitioned ids for unpartitioned elems, and
614 : // temporarily for ghost elems
615 3889365 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
616 3892933 : if (elem_procid == this->processor_id())
617 1133704 : next_id = &_next_free_local_elem_id;
618 3889365 : e->set_id (*next_id);
619 : }
620 :
621 : {
622 : // Advance next_ids up high enough that each is pointing to an
623 : // unused id and any subsequent increments will still point us
624 : // to unused ids
625 90013128 : _max_elem_id = std::max(_max_elem_id,
626 45006564 : static_cast<dof_id_type>(e->id()+1));
627 :
628 45006564 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
629 5263279 : _next_free_unpartitioned_elem_id =
630 5263279 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
631 5263279 : (this->n_processors() + 1) + this->n_processors();
632 45006564 : if (_next_free_local_elem_id < _max_elem_id)
633 3545634 : _next_free_local_elem_id =
634 3551946 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
635 3545634 : (this->n_processors() + 1) + this->processor_id();
636 :
637 : #ifndef NDEBUG
638 : // We need a const dofobject_container so we don't inadvertently create
639 : // nullptr entries when testing for non-nullptr ones
640 19155 : const dofobject_container<Elem> & const_elements = _elements;
641 : #endif
642 19155 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
643 19155 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
644 : }
645 :
646 : // Don't try to overwrite existing elems
647 19155 : libmesh_assert (!_elements[e->id()]);
648 :
649 45006564 : _elements[e->id()] = e;
650 :
651 : // We actually added a new element. Some of our caches might still
652 : // be valid, but we should clear the ones which definitely are not.
653 45006564 : this->clear_point_locator();
654 45006564 : this->clear_stored_ranges();
655 :
656 : // Try to make the cached elem data more accurate
657 45025719 : if (elem_procid == this->processor_id() ||
658 : elem_procid == DofObject::invalid_processor_id)
659 13821287 : _n_elem++;
660 :
661 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
662 45006564 : if (!e->valid_unique_id())
663 : {
664 14414991 : if (processor_id() == e->processor_id())
665 : {
666 1133704 : e->set_unique_id(_next_unique_id);
667 1133704 : _next_unique_id += this->n_processors() + 1;
668 : }
669 : else
670 : {
671 13281287 : e->set_unique_id(_next_unpartitioned_unique_id);
672 13281287 : _next_unpartitioned_unique_id += this->n_processors() + 1;
673 : }
674 : }
675 : else
676 : {
677 30616013 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
678 30591573 : _next_unique_id =
679 30591573 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
680 30591573 : (this->n_processors() + 1) + this->processor_id();
681 : }
682 : #endif
683 :
684 : // Unpartitioned elems should be added on every processor
685 : // And shouldn't be added in the same batch as ghost elems
686 : // But we might be just adding on processor 0 to
687 : // broadcast later
688 : // #ifdef DEBUG
689 : // if (elem_procid == DofObject::invalid_processor_id)
690 : // {
691 : // dof_id_type elem_id = e->id();
692 : // this->comm().max(elem_id);
693 : // libmesh_assert_equal_to (elem_id, e->id());
694 : // }
695 : // #endif
696 :
697 : // Make sure any new element is given space for any extra integers
698 : // we've requested
699 45006564 : e->add_extra_integers(_elem_integer_names.size(),
700 45006564 : _elem_integer_default_values);
701 :
702 : // And set mapping type and data on any new element
703 38310 : e->set_mapping_type(this->default_mapping_type());
704 38310 : e->set_mapping_data(this->default_mapping_data());
705 :
706 45006564 : return e;
707 : }
708 :
709 :
710 :
711 15654972 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
712 : {
713 : // The mesh now takes ownership of the Elem. Eventually the guts of
714 : // add_elem() will get moved to a private helper function, and
715 : // calling add_elem() directly will be deprecated.
716 15654972 : return add_elem(e.release());
717 : }
718 :
719 :
720 :
721 3785494 : Elem * DistributedMesh::insert_elem (Elem * e)
722 : {
723 3785494 : if (_elements[e->id()])
724 3785494 : this->delete_elem(_elements[e->id()]);
725 :
726 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
727 3785494 : if (!e->valid_unique_id())
728 : {
729 0 : if (processor_id() == e->processor_id())
730 : {
731 0 : e->set_unique_id(_next_unique_id);
732 0 : _next_unique_id += this->n_processors() + 1;
733 : }
734 : else
735 : {
736 0 : e->set_unique_id(_next_unpartitioned_unique_id);
737 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
738 : }
739 : }
740 : else
741 : {
742 3785494 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
743 3785494 : _next_unique_id =
744 3789190 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
745 3785494 : (this->n_processors() + 1) + this->processor_id();
746 : }
747 : #endif
748 :
749 : // Try to make the cached elem data more accurate
750 3785494 : processor_id_type elem_procid = e->processor_id();
751 3789190 : if (elem_procid == this->processor_id() ||
752 : elem_procid == DofObject::invalid_processor_id)
753 3654520 : _n_elem++;
754 :
755 3785494 : _elements[e->id()] = e;
756 :
757 : // We actually added a new element. Some of our caches might still
758 : // be valid, but we should clear the ones which definitely are not.
759 3785494 : this->clear_point_locator();
760 3785494 : this->clear_stored_ranges();
761 :
762 : // Make sure any new element is given space for any extra integers
763 : // we've requested
764 3785494 : e->add_extra_integers(_elem_integer_names.size(),
765 3785494 : _elem_integer_default_values);
766 :
767 : // And set mapping type and data on any new element
768 7392 : e->set_mapping_type(this->default_mapping_type());
769 7392 : e->set_mapping_data(this->default_mapping_data());
770 :
771 3785494 : return e;
772 : }
773 :
774 3785494 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
775 : {
776 : // The mesh now takes ownership of the Elem. Eventually the guts of
777 : // insert_elem(Elem*) will get moved to a private helper function, and
778 : // calling insert_elem(Elem*) directly will be deprecated.
779 3785494 : return insert_elem(e.release());
780 : }
781 :
782 :
783 40920870 : void DistributedMesh::delete_elem(Elem * e)
784 : {
785 7640 : libmesh_assert (e);
786 :
787 : // Try to make the cached elem data more accurate
788 40920870 : _n_elem--;
789 :
790 : // Was this a coarse element, not just a coarsening where we still
791 : // have some ancestor structure? Was it a *local* element, that we
792 : // might have been depending on as an owner of local nodes? We'll
793 : // have to be more careful with our nodes in contract() later; no
794 : // telling if we just locally orphaned a node that should be
795 : // globally retained.
796 40930330 : if (e->processor_id() == this->processor_id() &&
797 3640 : !e->parent())
798 80921 : _deleted_coarse_elements = true;
799 :
800 : // Delete the element from the BoundaryInfo object
801 40920870 : this->get_boundary_info().remove(e);
802 :
803 : // But not yet from the container; we might invalidate
804 : // an iterator that way!
805 :
806 : //_elements.erase(e->id());
807 :
808 : // Instead, we set it to nullptr for now
809 :
810 40920870 : _elements[e->id()] = nullptr;
811 :
812 : // delete the element
813 40920870 : delete e;
814 :
815 : // Some of our caches might still be valid, but we should clear the
816 : // ones which definitely are not.
817 40920870 : this->clear_point_locator();
818 40920870 : this->clear_stored_ranges();
819 40920870 : }
820 :
821 :
822 :
823 3207180 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
824 : const dof_id_type new_id)
825 : {
826 : // This could be a no-op
827 3207180 : if (old_id == new_id)
828 0 : return;
829 :
830 3207180 : Elem * el = _elements[old_id];
831 1160 : libmesh_assert (el);
832 1160 : libmesh_assert_equal_to (el->id(), old_id);
833 :
834 3207180 : el->set_id(new_id);
835 1160 : libmesh_assert (!_elements[new_id]);
836 3207180 : _elements[new_id] = el;
837 3207180 : _elements.erase(old_id);
838 :
839 : // Should we delete any caches here? Our point locator indexes by
840 : // element pointer and should be fine with an id change. Our stored
841 : // ranges are no longer sorted, which is *probably* fine, but let's
842 : // just be safe.
843 3207180 : this->clear_stored_ranges();
844 : }
845 :
846 :
847 :
848 45032946 : Node * DistributedMesh::add_point (const Point & p,
849 : const dof_id_type id,
850 : const processor_id_type proc_id)
851 : {
852 45032946 : Node * old_n = this->query_node_ptr(id);
853 :
854 45032946 : if (old_n)
855 : {
856 2 : *old_n = p;
857 60 : old_n->processor_id() = proc_id;
858 :
859 60 : return old_n;
860 : }
861 :
862 44992661 : Node * n = Node::build(p, id).release();
863 45032886 : n->processor_id() = proc_id;
864 :
865 45032886 : return DistributedMesh::add_node(n);
866 : }
867 :
868 :
869 4130 : void DistributedMesh::own_node (Node & n)
870 : {
871 : // This had better be a node in our mesh
872 0 : libmesh_assert(_nodes[n.id()] == &n);
873 :
874 4130 : _nodes[n.id()] = nullptr;
875 4130 : _n_nodes--;
876 :
877 0 : n.set_id(DofObject::invalid_id);
878 4130 : n.processor_id() = this->processor_id();
879 :
880 4130 : this->add_node(&n);
881 4130 : }
882 :
883 :
884 75938286 : Node * DistributedMesh::add_node (Node * n)
885 : {
886 : // Don't try to add nullptrs!
887 41993 : libmesh_assert(n);
888 :
889 : // Trying to add an existing node is a no-op
890 75938286 : if (n->valid_id() && _nodes[n->id()] == n)
891 0 : return n;
892 :
893 75938286 : const processor_id_type node_procid = n->processor_id();
894 :
895 75938286 : if (!n->valid_id())
896 : {
897 : // We should only be creating new ids past the end of the range
898 : // of existing ids
899 17832 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
900 : _max_node_id);
901 17832 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
902 :
903 : // Use the unpartitioned ids for unpartitioned nodes,
904 : // and temporarily for ghost nodes
905 18669076 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
906 18686908 : if (node_procid == this->processor_id())
907 2130206 : next_id = &_next_free_local_node_id;
908 18669076 : n->set_id (*next_id);
909 : }
910 :
911 : {
912 : // Advance next_ids up high enough that each is pointing to an
913 : // unused id and any subsequent increments will still point us
914 : // to unused ids
915 151876572 : _max_node_id = std::max(_max_node_id,
916 75938286 : static_cast<dof_id_type>(n->id()+1));
917 :
918 75938286 : if (_next_free_unpartitioned_node_id < _max_node_id)
919 21802765 : _next_free_unpartitioned_node_id =
920 21802765 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
921 21802765 : (this->n_processors() + 1) + this->n_processors();
922 75938286 : if (_next_free_local_node_id < _max_node_id)
923 12661950 : _next_free_local_node_id =
924 12680209 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
925 12661950 : (this->n_processors() + 1) + this->processor_id();
926 :
927 : #ifndef NDEBUG
928 : // We need a const dofobject_container so we don't inadvertently create
929 : // nullptr entries when testing for non-nullptr ones
930 41993 : const dofobject_container<Node> & const_nodes = _nodes;
931 : #endif
932 41993 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
933 41993 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
934 : }
935 :
936 : // Don't try to overwrite existing nodes
937 41993 : libmesh_assert (!_nodes[n->id()]);
938 :
939 75938286 : _nodes[n->id()] = n;
940 :
941 : // Try to make the cached node data more accurate
942 75980279 : if (node_procid == this->processor_id() ||
943 : node_procid == DofObject::invalid_processor_id)
944 42619494 : _n_nodes++;
945 :
946 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
947 75938286 : if (!n->valid_unique_id())
948 : {
949 45035617 : if (processor_id() == n->processor_id())
950 : {
951 2547856 : n->set_unique_id(_next_unique_id);
952 2547856 : _next_unique_id += this->n_processors() + 1;
953 : }
954 : else
955 : {
956 42487761 : n->set_unique_id(_next_unpartitioned_unique_id);
957 42487761 : _next_unpartitioned_unique_id += this->n_processors() + 1;
958 : }
959 : }
960 : else
961 : {
962 31033254 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
963 30902669 : _next_unique_id =
964 30902669 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
965 30902669 : (this->n_processors() + 1) + this->processor_id();
966 : }
967 : #endif
968 :
969 : // A new node invalidates any cached node ranges
970 75938286 : this->clear_stored_ranges();
971 :
972 75938286 : n->add_extra_integers(_node_integer_names.size(),
973 75938286 : _node_integer_default_values);
974 :
975 : // Unpartitioned nodes should be added on every processor
976 : // And shouldn't be added in the same batch as ghost nodes
977 : // But we might be just adding on processor 0 to
978 : // broadcast later
979 : // #ifdef DEBUG
980 : // if (node_procid == DofObject::invalid_processor_id)
981 : // {
982 : // dof_id_type node_id = n->id();
983 : // this->comm().max(node_id);
984 : // libmesh_assert_equal_to (node_id, n->id());
985 : // }
986 : // #endif
987 :
988 75938286 : return n;
989 : }
990 :
991 30901270 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
992 : {
993 : // The mesh now takes ownership of the Node. Eventually the guts of
994 : // add_node() will get moved to a private helper function, and
995 : // calling add_node() directly will be deprecated.
996 30901270 : return add_node(n.release());
997 : }
998 :
999 52109063 : void DistributedMesh::delete_node(Node * n)
1000 : {
1001 4571 : libmesh_assert(n);
1002 4571 : libmesh_assert(_nodes[n->id()]);
1003 :
1004 : // Try to make the cached elem data more accurate
1005 52109063 : _n_nodes--;
1006 :
1007 : // Delete the node from the BoundaryInfo object
1008 52109063 : this->get_boundary_info().remove(n);
1009 9142 : _constraint_rows.erase(n);
1010 :
1011 : // But not yet from the container; we might invalidate
1012 : // an iterator that way!
1013 :
1014 : //_nodes.erase(n->id());
1015 :
1016 : // Instead, we set it to nullptr for now
1017 :
1018 52109063 : _nodes[n->id()] = nullptr;
1019 :
1020 : // delete the node
1021 52109063 : delete n;
1022 :
1023 : // We've invalidated any ranges that contained this node
1024 52109063 : this->clear_stored_ranges();
1025 52109063 : }
1026 :
1027 :
1028 :
1029 6897369 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1030 : const dof_id_type new_id)
1031 : {
1032 : // This could be a no-op
1033 6897369 : if (old_id == new_id)
1034 0 : return;
1035 :
1036 6896780 : Node * nd = _nodes[old_id];
1037 3234 : libmesh_assert (nd);
1038 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1039 :
1040 : // If we have nodes shipped to this processor for NodeConstraints
1041 : // use, then those nodes will exist in _nodes, but may not be
1042 : // locatable via a TopologyMap due to the insufficiency of elements
1043 : // connecting to them. If local refinement then wants to create a
1044 : // *new* node in the same location, it will initially get a temporary
1045 : // id, and then make_node_ids_parallel_consistent() will try to move
1046 : // it to the canonical id. We need to account for this case to
1047 : // avoid false positives and memory leaks.
1048 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1049 6468 : if (_nodes[new_id])
1050 : {
1051 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1052 : *(Point *)_nodes[old_id]);
1053 0 : _nodes.erase(new_id);
1054 : }
1055 : #else
1056 : // If we aren't shipping nodes for NodeConstraints, there should be
1057 : // no reason for renumbering one node onto another.
1058 : libmesh_assert (!_nodes[new_id]);
1059 : #endif
1060 6896780 : _nodes[new_id] = nd;
1061 6896780 : nd->set_id(new_id);
1062 :
1063 6896780 : _nodes.erase(old_id);
1064 : }
1065 :
1066 :
1067 :
1068 796837 : void DistributedMesh::clear ()
1069 : {
1070 : // Call parent clear function
1071 796837 : MeshBase::clear();
1072 :
1073 : // Clear our elements and nodes
1074 : // There is no need to remove them from
1075 : // the BoundaryInfo data structure since we
1076 : // already cleared it.
1077 796837 : this->DistributedMesh::clear_elems();
1078 :
1079 25058869 : for (auto & node : _nodes)
1080 45248291 : delete node;
1081 :
1082 600 : _nodes.clear();
1083 :
1084 : // We're no longer distributed if we were before
1085 796837 : _is_serial = true;
1086 796837 : _is_serial_on_proc_0 = true;
1087 :
1088 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1089 : // all is copacetic.
1090 796837 : _deleted_coarse_elements = false;
1091 :
1092 : // Correct our caches
1093 796837 : _n_nodes = 0;
1094 796837 : _max_node_id = 0;
1095 797437 : _next_free_local_node_id = this->processor_id();
1096 796837 : _next_free_unpartitioned_node_id = this->n_processors();
1097 796837 : }
1098 :
1099 :
1100 :
1101 803237 : void DistributedMesh::clear_elems ()
1102 : {
1103 10073594 : for (auto & elem : _elements)
1104 9270357 : delete elem;
1105 :
1106 600 : _elements.clear();
1107 :
1108 : // Correct our caches
1109 803237 : _n_elem = 0;
1110 803237 : _max_elem_id = 0;
1111 803837 : _next_free_local_elem_id = this->processor_id();
1112 803237 : _next_free_unpartitioned_elem_id = this->n_processors();
1113 803237 : }
1114 :
1115 :
1116 :
1117 750788 : void DistributedMesh::redistribute ()
1118 : {
1119 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1120 750788 : if (!this->is_serial())
1121 : {
1122 : // Construct a MeshCommunication object to actually redistribute the nodes
1123 : // and elements according to the partitioner, and then to re-gather the neighbors.
1124 : MeshCommunication mc;
1125 78402 : mc.redistribute(*this);
1126 :
1127 78402 : this->update_parallel_id_counts();
1128 :
1129 : // We communicate valid neighbor links for newly-redistributed
1130 : // elements, but we may still have remote_elem links that should
1131 : // be corrected but whose correction wasn't communicated.
1132 78674 : this->find_neighbors(/*reset_remote_elements*/ false,
1133 : /*reset_current_list*/ false /*non-default*/,
1134 : /*assert_valid*/ false,
1135 272 : /*check_non_remote*/ false /*non-default!*/);
1136 :
1137 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1138 : // anyway... but users can always call partition directly, in which case we do need
1139 : // to call delete_remote_elements()...
1140 : //
1141 : // Regardless of whether it's necessary, it isn't safe. We
1142 : // haven't communicated new node processor_ids yet, and we can't
1143 : // delete nodes until we do.
1144 : // this->delete_remote_elements();
1145 : }
1146 : else
1147 : // The base class can handle non-distributed things, like
1148 : // notifying any GhostingFunctors of changes
1149 672386 : MeshBase::redistribute();
1150 750788 : }
1151 :
1152 :
1153 :
1154 2014696 : void DistributedMesh::update_post_partitioning ()
1155 : {
1156 : // this->recalculate_n_partitions();
1157 :
1158 : // Let's do the base class cache clearing first, just in case our
1159 : // later computations are ever changed to make use of a local
1160 : // elements range cache
1161 2014696 : this->MeshBase::update_post_partitioning();
1162 :
1163 : // Partitioning changes our numbers of unpartitioned objects
1164 2014696 : this->update_parallel_id_counts();
1165 2014696 : }
1166 :
1167 :
1168 :
1169 : template <typename T>
1170 3500 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1171 : {
1172 : // This function must be run on all processors at once
1173 3500 : parallel_object_only();
1174 :
1175 3500 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1176 3500 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1177 3500 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1178 :
1179 897076 : for (dof_id_type i=0; i != pmax_id; ++i)
1180 : {
1181 893576 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1182 :
1183 : // Local lookups by id should return the requested object
1184 893576 : libmesh_assert(!obj || obj->id() == i);
1185 :
1186 : // All processors with an object should agree on id
1187 : #ifndef NDEBUG
1188 893576 : const dof_id_type dofid = obj && obj->valid_id() ?
1189 : obj->id() : DofObject::invalid_id;
1190 893576 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1191 : #endif
1192 :
1193 : // All processors with an object should agree on processor id
1194 893576 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1195 : obj->processor_id() : DofObject::invalid_processor_id;
1196 893576 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1197 :
1198 893576 : dof_id_type min_procid = procid;
1199 893576 : this->comm().min(min_procid);
1200 :
1201 : // Either:
1202 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1203 : // or
1204 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1205 :
1206 : // Original assert logic
1207 : // libmesh_assert (min_procid != this->processor_id() || obj);
1208 :
1209 : // More human-understandable logic...
1210 893576 : libmesh_assert (
1211 : ((min_procid == this->processor_id()) && obj)
1212 : ||
1213 : (min_procid != this->processor_id())
1214 : );
1215 :
1216 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1217 : // All processors with an object should agree on unique id
1218 893576 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1219 893576 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1220 : #endif
1221 : }
1222 3500 : }
1223 :
1224 :
1225 :
1226 1750 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1227 : {
1228 1750 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1229 1750 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1230 1750 : }
1231 :
1232 :
1233 :
1234 426 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1235 : {
1236 : #ifndef NDEBUG
1237 : // This function must be run on all processors at once
1238 426 : parallel_object_only();
1239 :
1240 426 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1241 :
1242 22182 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1243 : {
1244 21756 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1245 :
1246 21756 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1247 :
1248 : // All processors with an active element should agree on p level
1249 21756 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1250 : }
1251 : #endif
1252 426 : }
1253 :
1254 :
1255 :
1256 :
1257 1698 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1258 : {
1259 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1260 : // This function must be run on all processors at once
1261 1698 : parallel_object_only();
1262 :
1263 1698 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1264 :
1265 145340 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1266 : {
1267 143642 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1268 :
1269 143642 : unsigned int refinement_flag = el ?
1270 66203 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1271 143642 : unsigned int p_refinement_flag = el ?
1272 66203 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1273 :
1274 143642 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1275 :
1276 : // p refinement flags aren't always kept correct on inactive
1277 : // ghost elements
1278 143642 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1279 : }
1280 : #endif // LIBMESH_ENABLE_AMR
1281 1698 : }
1282 :
1283 :
1284 :
1285 : template <typename T>
1286 : dof_id_type
1287 2453478 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1288 : {
1289 : // This function must be run on all processors at once
1290 848 : parallel_object_only();
1291 :
1292 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1293 :
1294 : // In parallel we may not know what objects other processors have.
1295 : // Start by figuring out how many
1296 848 : dof_id_type unpartitioned_objects = 0;
1297 :
1298 : std::unordered_map<processor_id_type, dof_id_type>
1299 1696 : ghost_objects_from_proc;
1300 :
1301 2453478 : object_iterator it = objects.begin();
1302 848 : object_iterator end = objects.end();
1303 :
1304 122232105 : while (it != end)
1305 : {
1306 119778627 : T * obj = *it;
1307 :
1308 : // Remove any nullptr container entries while we're here.
1309 119778627 : if (!obj)
1310 1480233 : it = objects.erase(it);
1311 : else
1312 : {
1313 118298394 : processor_id_type obj_procid = obj->processor_id();
1314 118298394 : if (obj_procid == DofObject::invalid_processor_id)
1315 46883781 : unpartitioned_objects++;
1316 : else
1317 71414613 : ghost_objects_from_proc[obj_procid]++;
1318 :
1319 : // Finally, increment the iterator
1320 56932 : ++it;
1321 : }
1322 : }
1323 :
1324 2455174 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1325 2454326 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1326 848 : this->comm().allgather
1327 2453993 : ((this_it == ghost_objects_from_proc.end()) ?
1328 515 : dof_id_type(0) : this_it->second, objects_on_proc);
1329 :
1330 : #ifndef NDEBUG
1331 848 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1332 2544 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1333 1696 : if (ghost_objects_from_proc.count(p))
1334 987 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1335 : else
1336 709 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1337 : #endif
1338 :
1339 : // We'll renumber objects in blocks by processor id
1340 2455174 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1341 26042934 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1342 23592848 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1343 848 : objects_on_proc[i-1];
1344 2454326 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1345 2453478 : dof_id_type first_free_id =
1346 2455174 : first_object_on_proc[this->n_processors()-1] +
1347 848 : objects_on_proc[this->n_processors()-1] +
1348 : unpartitioned_objects;
1349 :
1350 : // First set new local object ids and build request sets
1351 : // for non-local object ids
1352 :
1353 : // Request sets to send to each processor
1354 : std::map<processor_id_type, std::vector<dof_id_type>>
1355 848 : requested_ids;
1356 :
1357 : // We know how many objects live on each processor, so reserve() space for
1358 : // each.
1359 848 : auto ghost_end = ghost_objects_from_proc.end();
1360 28496412 : for (auto p : make_range(this->n_processors()))
1361 26044630 : if (p != this->processor_id())
1362 : {
1363 23590304 : if (const auto p_it = ghost_objects_from_proc.find(p);
1364 848 : p_it != ghost_end)
1365 4092763 : requested_ids[p].reserve(p_it->second);
1366 : }
1367 :
1368 848 : end = objects.end();
1369 120751872 : for (it = objects.begin(); it != end; ++it)
1370 : {
1371 118298394 : T * obj = *it;
1372 118298394 : if (!obj)
1373 0 : continue;
1374 118355326 : if (obj->processor_id() == this->processor_id())
1375 21169086 : obj->set_id(next_id++);
1376 97129308 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1377 50245527 : requested_ids[obj->processor_id()].push_back(obj->id());
1378 : }
1379 :
1380 : // Next set ghost object ids from other processors
1381 :
1382 2470708 : auto gather_functor =
1383 54304774 : [
1384 : #ifndef NDEBUG
1385 : this,
1386 : &first_object_on_proc,
1387 : &objects_on_proc,
1388 : #endif
1389 : &objects]
1390 : (processor_id_type, const std::vector<dof_id_type> & ids,
1391 944 : std::vector<dof_id_type> & new_ids)
1392 : {
1393 944 : std::size_t ids_size = ids.size();
1394 4092763 : new_ids.resize(ids_size);
1395 :
1396 54338290 : for (std::size_t i=0; i != ids_size; ++i)
1397 : {
1398 50245527 : T * obj = objects[ids[i]];
1399 16286 : libmesh_assert(obj);
1400 16286 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1401 50261813 : new_ids[i] = obj->id();
1402 :
1403 16286 : libmesh_assert_greater_equal (new_ids[i],
1404 : first_object_on_proc[this->processor_id()]);
1405 16286 : libmesh_assert_less (new_ids[i],
1406 : first_object_on_proc[this->processor_id()] +
1407 : objects_on_proc[this->processor_id()]);
1408 : }
1409 : };
1410 :
1411 2470708 : auto action_functor =
1412 54304774 : [
1413 : #ifndef NDEBUG
1414 : &first_object_on_proc,
1415 : &objects_on_proc,
1416 : #endif
1417 : &objects]
1418 : (processor_id_type libmesh_dbg_var(pid),
1419 : const std::vector<dof_id_type> & ids,
1420 944 : const std::vector<dof_id_type> & data)
1421 : {
1422 : // Copy the id changes we've now been informed of
1423 54338290 : for (auto i : index_range(ids))
1424 : {
1425 50245527 : T * obj = objects[ids[i]];
1426 16286 : libmesh_assert (obj);
1427 16286 : libmesh_assert_equal_to (obj->processor_id(), pid);
1428 16286 : libmesh_assert_greater_equal (data[i],
1429 : first_object_on_proc[pid]);
1430 16286 : libmesh_assert_less (data[i],
1431 : first_object_on_proc[pid] +
1432 : objects_on_proc[pid]);
1433 50261813 : obj->set_id(data[i]);
1434 : }
1435 : };
1436 :
1437 848 : const dof_id_type * ex = nullptr;
1438 : Parallel::pull_parallel_vector_data
1439 2453478 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1440 :
1441 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1442 2470708 : auto unique_gather_functor =
1443 54304774 : [
1444 : #ifndef NDEBUG
1445 : this,
1446 : #endif
1447 : &objects]
1448 : (processor_id_type, const std::vector<dof_id_type> & ids,
1449 944 : std::vector<unique_id_type> & data)
1450 : {
1451 944 : std::size_t ids_size = ids.size();
1452 4092763 : data.resize(ids_size);
1453 :
1454 54338290 : for (std::size_t i=0; i != ids_size; ++i)
1455 : {
1456 50245527 : T * obj = objects[ids[i]];
1457 16286 : libmesh_assert(obj);
1458 16286 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1459 100491054 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1460 : }
1461 : };
1462 :
1463 2470708 : auto unique_action_functor =
1464 54304774 : [&objects]
1465 : (processor_id_type libmesh_dbg_var(pid),
1466 : const std::vector<dof_id_type> & ids,
1467 944 : const std::vector<unique_id_type> & data)
1468 : {
1469 54338290 : for (auto i : index_range(ids))
1470 : {
1471 50245527 : T * obj = objects[ids[i]];
1472 16286 : libmesh_assert (obj);
1473 16286 : libmesh_assert_equal_to (obj->processor_id(), pid);
1474 50245527 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1475 0 : obj->set_unique_id(data[i]);
1476 : }
1477 : };
1478 :
1479 848 : const unique_id_type * unique_ex = nullptr;
1480 : Parallel::pull_parallel_vector_data
1481 2453478 : (this->comm(), requested_ids, unique_gather_functor,
1482 : unique_action_functor, unique_ex);
1483 : #endif
1484 :
1485 : // Next set unpartitioned object ids
1486 848 : next_id = 0;
1487 28496412 : for (auto i : make_range(this->n_processors()))
1488 26044630 : next_id += objects_on_proc[i];
1489 120751872 : for (it = objects.begin(); it != end; ++it)
1490 : {
1491 118298394 : T * obj = *it;
1492 118298394 : if (!obj)
1493 0 : continue;
1494 118298394 : if (obj->processor_id() == DofObject::invalid_processor_id)
1495 46883781 : obj->set_id(next_id++);
1496 : }
1497 :
1498 : // Finally shuffle around objects so that container indices
1499 : // match ids
1500 2453478 : it = objects.begin();
1501 848 : end = objects.end();
1502 127481601 : while (it != end)
1503 : {
1504 125028123 : T * obj = *it;
1505 125028123 : if (obj) // don't try shuffling already-nullptr entries
1506 : {
1507 125028123 : T * next = objects[obj->id()];
1508 : // If we have to move this object
1509 125028123 : if (next != obj)
1510 : {
1511 : // nullptr out its original position for now
1512 : // (our shuffling may put another object there shortly)
1513 42511938 : *it = nullptr;
1514 :
1515 : // There may already be another object with this id that
1516 : // needs to be moved itself
1517 55861821 : while (next)
1518 : {
1519 : // We shouldn't be trying to give two objects the
1520 : // same id
1521 9394 : libmesh_assert_not_equal_to (next->id(), obj->id());
1522 13349883 : objects[obj->id()] = obj;
1523 9394 : obj = next;
1524 13349883 : next = objects[obj->id()];
1525 : }
1526 42511938 : objects[obj->id()] = obj;
1527 : }
1528 : }
1529 :
1530 : // Remove any container entries that were left as nullptr.
1531 116342 : if (!obj)
1532 0 : it = objects.erase(it);
1533 : else
1534 58171 : ++it;
1535 : }
1536 :
1537 2454326 : return first_free_id;
1538 : }
1539 :
1540 :
1541 1228409 : void DistributedMesh::renumber_nodes_and_elements ()
1542 : {
1543 426 : parallel_object_only();
1544 :
1545 : #ifdef DEBUG
1546 : // Make sure our ids and flags are consistent
1547 426 : this->libmesh_assert_valid_parallel_ids();
1548 426 : this->libmesh_assert_valid_parallel_flags();
1549 426 : this->libmesh_assert_valid_parallel_p_levels();
1550 : #endif
1551 :
1552 850 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1553 :
1554 : // Nodes not connected to any elements, and nullptr node entries
1555 : // in our container, should be deleted. But wait! If we've deleted coarse
1556 : // local elements on some processor, other processors might have ghosted
1557 : // nodes from it that are now no longer connected to any elements on it, but
1558 : // that are connected to their own semilocal elements. We'll have to
1559 : // communicate to ascertain if that's the case.
1560 1228409 : this->comm().max(_deleted_coarse_elements);
1561 :
1562 : // What used nodes do we see on our proc?
1563 426 : std::set<dof_id_type> used_nodes;
1564 :
1565 : // What used node info should we send from our proc? Could we take ownership
1566 : // of each node if we needed to?
1567 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1568 426 : used_nodes_on_proc;
1569 :
1570 : // flag the nodes we need
1571 62220348 : for (auto & elem : this->element_ptr_range())
1572 324443176 : for (const Node & node : elem->node_ref_range())
1573 : {
1574 294531460 : const dof_id_type n = node.id();
1575 294386552 : used_nodes.insert(n);
1576 294531460 : if (_deleted_coarse_elements)
1577 : {
1578 3152735 : const processor_id_type p = node.processor_id();
1579 3153019 : if (p != this->processor_id())
1580 : {
1581 2262002 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1582 2262144 : if (elem->processor_id() == this->processor_id())
1583 216782 : used_nodes_on_p[n] = true;
1584 : else
1585 105 : if (!used_nodes_on_p.count(n))
1586 528761 : used_nodes_on_p[n] = false;
1587 : }
1588 : }
1589 1227557 : }
1590 :
1591 1228409 : if (_deleted_coarse_elements)
1592 : {
1593 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1594 : typedef unsigned char boolish;
1595 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1596 28 : used_nodes_on_proc_vecs;
1597 73624 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1598 56986 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1599 :
1600 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1601 : std::map<processor_id_type, std::set<dof_id_type>>
1602 28 : repartitioned_node_sets_to_push;
1603 :
1604 : auto ids_action_functor =
1605 56960 : [&used_nodes, &repartitioned_node_pids,
1606 : &repartitioned_node_sets_to_push]
1607 : (processor_id_type pid,
1608 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1609 : {
1610 595934 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1611 : {
1612 : // If we don't see a use for our own node, but someone
1613 : // else does, better figure out who should own it next.
1614 97 : if (!used_nodes.count(n))
1615 : {
1616 2 : if (auto it = repartitioned_node_pids.find(n);
1617 2 : sender_could_become_owner)
1618 : {
1619 1 : if (it != repartitioned_node_pids.end() &&
1620 0 : pid < it->second)
1621 0 : it->second = pid;
1622 : else
1623 1 : repartitioned_node_pids[n] = pid;
1624 : }
1625 : else
1626 1 : if (it == repartitioned_node_pids.end())
1627 0 : repartitioned_node_pids[n] =
1628 : DofObject::invalid_processor_id;
1629 :
1630 2 : repartitioned_node_sets_to_push[pid].insert(n);
1631 : }
1632 : }
1633 16650 : };
1634 :
1635 : // We need two pushes instead of a pull here because we need to
1636 : // know *all* the queries for a particular node before we can
1637 : // respond to *any* of them.
1638 : Parallel::push_parallel_vector_data
1639 16638 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1640 :
1641 : // Repartition (what used to be) our own nodes first
1642 16639 : for (auto & [n, p] : repartitioned_node_pids)
1643 : {
1644 1 : Node & node = this->node_ref(n);
1645 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1646 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1647 1 : node.processor_id() = p;
1648 : }
1649 :
1650 : // Then push to repartition others' ghosted copies.
1651 :
1652 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1653 28 : repartitioned_node_vecs;
1654 :
1655 16640 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1656 : {
1657 2 : auto & rn_vec = repartitioned_node_vecs[p];
1658 4 : for (auto n : nodeset)
1659 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1660 : }
1661 :
1662 : auto repartition_node_functor =
1663 2 : [this]
1664 : (processor_id_type libmesh_dbg_var(pid),
1665 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1666 : {
1667 4 : for (auto [n, p] : ids_and_pids)
1668 : {
1669 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1670 2 : Node & node = this->node_ref(n);
1671 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1672 2 : node.processor_id() = p;
1673 : }
1674 16610 : };
1675 :
1676 : Parallel::push_parallel_vector_data
1677 16638 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1678 : }
1679 :
1680 1228409 : _deleted_coarse_elements = false;
1681 :
1682 : // Nodes not connected to any local elements, and nullptr node entries
1683 : // in our container, are deleted
1684 : {
1685 1228409 : node_iterator_imp it = _nodes.begin();
1686 426 : node_iterator_imp end = _nodes.end();
1687 :
1688 97125960 : while (it != end)
1689 : {
1690 95897551 : Node * nd = *it;
1691 95897551 : if (!nd)
1692 3674234 : it = _nodes.erase(it);
1693 92223317 : else if (!used_nodes.count(nd->id()))
1694 : {
1695 : // remove any boundary information associated with
1696 : // this node
1697 2802826 : this->get_boundary_info().remove (nd);
1698 2828 : _constraint_rows.erase(nd);
1699 :
1700 : // delete the node
1701 2802826 : delete nd;
1702 :
1703 2802826 : it = _nodes.erase(it);
1704 : }
1705 : else
1706 45625 : ++it;
1707 : }
1708 : }
1709 :
1710 1228409 : this->_preparation.has_removed_orphaned_nodes = true;
1711 :
1712 1228409 : if (_skip_renumber_nodes_and_elements)
1713 : {
1714 1670 : this->update_parallel_id_counts();
1715 2 : return;
1716 : }
1717 :
1718 : // Finally renumber all the elements
1719 1226739 : _n_elem = this->renumber_dof_objects (this->_elements);
1720 :
1721 : // and all the remaining nodes
1722 1226739 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1723 :
1724 : // And figure out what IDs we should use when adding new nodes and
1725 : // new elements
1726 1226739 : this->update_parallel_id_counts();
1727 :
1728 : // We've probably done *some* node removal and/or renumbering; let's
1729 : // clear caches to be safe.
1730 1226739 : this->clear_stored_ranges();
1731 :
1732 : // Make sure our caches are up to date and our
1733 : // DofObjects are well packed
1734 : #ifdef DEBUG
1735 424 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1736 424 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1737 424 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1738 424 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1739 424 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1740 424 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1741 424 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1742 424 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1743 :
1744 : // Make sure our ids and flags are consistent
1745 424 : this->libmesh_assert_valid_parallel_ids();
1746 424 : this->libmesh_assert_valid_parallel_flags();
1747 :
1748 : // And make sure we've made our numbering monotonic
1749 424 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1750 : #endif
1751 : }
1752 :
1753 :
1754 :
1755 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1756 : {
1757 : // We can't use range-for here because we need access to the special
1758 : // iterators' methods, not just to their dereferenced values.
1759 :
1760 : // Nodes first
1761 0 : for (auto pr = this->_nodes.begin(),
1762 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1763 : {
1764 5427360 : Node * n = *pr;
1765 5427360 : if (n != nullptr)
1766 : {
1767 0 : const dof_id_type id = pr.index();
1768 5204550 : n->set_id() = id;
1769 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1770 : }
1771 : }
1772 :
1773 : // Elements next
1774 0 : for (auto pr = this->_elements.begin(),
1775 5671231 : end = this->_elements.end(); pr != end; ++pr)
1776 : {
1777 5653821 : Elem * e = *pr;
1778 5653821 : if (e != nullptr)
1779 : {
1780 0 : const dof_id_type id = pr.index();
1781 5590229 : e->set_id() = id;
1782 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1783 : }
1784 : }
1785 17410 : }
1786 :
1787 :
1788 :
1789 830366 : dof_id_type DistributedMesh::n_active_elem () const
1790 : {
1791 578 : parallel_object_only();
1792 :
1793 : // Get local active elements first
1794 : dof_id_type active_elements =
1795 1660154 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1796 2489942 : this->active_local_elements_end()));
1797 830366 : this->comm().sum(active_elements);
1798 :
1799 : // Then add unpartitioned active elements, which should exist on
1800 : // every processor
1801 830366 : active_elements +=
1802 830366 : static_cast<dof_id_type>(std::distance
1803 1660154 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1804 830944 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1805 830366 : return active_elements;
1806 : }
1807 :
1808 :
1809 :
1810 536843 : void DistributedMesh::delete_remote_elements()
1811 : {
1812 : #ifdef DEBUG
1813 : // Make sure our neighbor links are all fine
1814 400 : MeshTools::libmesh_assert_valid_neighbors(*this);
1815 :
1816 : // And our child/parent links, and our flags
1817 400 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1818 :
1819 : // Make sure our ids and flags are consistent
1820 400 : this->libmesh_assert_valid_parallel_ids();
1821 400 : this->libmesh_assert_valid_parallel_flags();
1822 :
1823 400 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1824 400 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1825 400 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1826 400 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1827 400 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1828 400 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1829 : #endif
1830 :
1831 536843 : _is_serial = false;
1832 536843 : _is_serial_on_proc_0 = false;
1833 :
1834 536843 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1835 :
1836 400 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1837 :
1838 : // Now make sure the containers actually shrink - strip
1839 : // any newly-created nullptr voids out of the element array
1840 536843 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1841 400 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1842 54799368 : while (e_it != e_end)
1843 54262525 : if (!*e_it)
1844 39872776 : e_it = _elements.erase(e_it);
1845 : else
1846 17459 : ++e_it;
1847 :
1848 536843 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1849 400 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1850 106586265 : while (n_it != n_end)
1851 106049422 : if (!*n_it)
1852 69479563 : n_it = _nodes.erase(n_it);
1853 : else
1854 48745 : ++n_it;
1855 :
1856 : // We may have deleted no-longer-connected nodes or coarsened-away
1857 : // elements; let's update our caches.
1858 536843 : this->update_parallel_id_counts();
1859 :
1860 : // We may have deleted nodes or elements that were the only local
1861 : // representatives of some particular boundary id(s); let's update
1862 : // those caches.
1863 536843 : this->get_boundary_info().regenerate_id_sets();
1864 :
1865 : #ifdef DEBUG
1866 : // We might not have well-packed objects if the user didn't allow us
1867 : // to renumber
1868 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1869 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1870 :
1871 : // Make sure our neighbor links are all fine
1872 400 : MeshTools::libmesh_assert_valid_neighbors(*this);
1873 :
1874 : // And our child/parent links, and our flags
1875 400 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1876 :
1877 : // Make sure our ids and flags are consistent
1878 400 : this->libmesh_assert_valid_parallel_ids();
1879 400 : this->libmesh_assert_valid_parallel_flags();
1880 : #endif
1881 :
1882 536843 : this->_preparation.has_removed_remote_elements = true;
1883 536843 : }
1884 :
1885 :
1886 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1887 : {
1888 : // First add the elem like normal
1889 0 : add_elem(e);
1890 :
1891 : // Now add it to the set that won't be deleted when we call
1892 : // delete_remote_elements()
1893 0 : _extra_ghost_elems.insert(e);
1894 0 : }
1895 :
1896 : void
1897 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1898 : {
1899 0 : std::set<Elem *> tmp;
1900 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1901 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1902 0 : std::inserter(tmp, tmp.begin()));
1903 0 : _extra_ghost_elems = tmp;
1904 0 : }
1905 :
1906 104603 : void DistributedMesh::allgather()
1907 : {
1908 104603 : if (_is_serial)
1909 0 : return;
1910 104601 : MeshCommunication().allgather(*this);
1911 104601 : _is_serial = true;
1912 104601 : _is_serial_on_proc_0 = true;
1913 :
1914 : // Make sure our caches are up to date and our
1915 : // DofObjects are well packed
1916 : #ifdef DEBUG
1917 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1918 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1919 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1920 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1921 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1922 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1923 :
1924 : // If we've disabled renumbering we can't be sure we're contiguous
1925 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1926 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1927 :
1928 : // Make sure our neighbor links are all fine
1929 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1930 :
1931 : // Make sure our ids and flags are consistent
1932 48 : this->libmesh_assert_valid_parallel_ids();
1933 48 : this->libmesh_assert_valid_parallel_flags();
1934 : #endif
1935 : }
1936 :
1937 18885 : void DistributedMesh::gather_to_zero()
1938 : {
1939 18885 : if (_is_serial_on_proc_0)
1940 6 : return;
1941 :
1942 14576 : _is_serial_on_proc_0 = true;
1943 14576 : MeshCommunication().gather(0, *this);
1944 : }
1945 :
1946 :
1947 : } // namespace libMesh
|
