Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 348158 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 348158 : unsigned char d) :
44 347654 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 347654 : _is_serial_on_proc_0(true),
46 347654 : _deleted_coarse_elements(false),
47 347654 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 348410 : _next_free_local_node_id(this->processor_id()),
49 348158 : _next_free_local_elem_id(this->processor_id()),
50 348158 : _next_free_unpartitioned_node_id(this->n_processors()),
51 348158 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 348158 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 348158 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 348410 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 696316 : default_partitioner);
64 : _partitioner = Partitioner::build
65 696064 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 348158 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 8 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 390623 : DistributedMesh::~DistributedMesh ()
143 : {
144 368519 : this->DistributedMesh::clear(); // Free nodes and elements
145 390623 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17734 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 : DistributedMesh(static_cast<const MeshBase &>(other_mesh),
153 17734 : /*bool other_is_distributed_type=*/true)
154 : {
155 17734 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
156 17734 : _next_free_local_node_id =
157 17734 : other_mesh._next_free_local_node_id;
158 17734 : _next_free_local_elem_id =
159 17734 : other_mesh._next_free_local_elem_id;
160 17734 : _next_free_unpartitioned_node_id =
161 17734 : other_mesh._next_free_unpartitioned_node_id;
162 17734 : _next_free_unpartitioned_elem_id =
163 17734 : other_mesh._next_free_unpartitioned_elem_id;
164 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
165 17734 : _next_unique_id =
166 17734 : other_mesh._next_unique_id;
167 17734 : _next_unpartitioned_unique_id =
168 17734 : other_mesh._next_unpartitioned_unique_id;
169 : #endif
170 :
171 : // Need to copy extra_ghost_elems
172 17734 : for (auto & elem : other_mesh._extra_ghost_elems)
173 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
174 17734 : }
175 :
176 :
177 :
178 20361 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
179 20361 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
180 20455 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
181 20173 : _deleted_coarse_elements(true), // better safe than sorry...
182 20173 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
183 20455 : _next_free_local_node_id(this->processor_id()),
184 20361 : _next_free_local_elem_id(this->processor_id()),
185 20361 : _next_free_unpartitioned_node_id(this->n_processors()),
186 40628 : _next_free_unpartitioned_elem_id(this->n_processors())
187 : {
188 : // Just copy, skipping preparation
189 20361 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
190 :
191 188 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
192 188 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
193 188 : this->allow_renumbering(other_mesh.allow_renumbering());
194 188 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
195 188 : this->skip_partitioning(other_mesh.skip_partitioning());
196 :
197 20361 : this->copy_constraint_rows(other_mesh);
198 :
199 94 : auto & this_boundary_info = this->get_boundary_info();
200 94 : const auto & other_boundary_info = other_mesh.get_boundary_info();
201 :
202 20361 : this_boundary_info = other_boundary_info;
203 :
204 94 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
205 :
206 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
207 : // nodes/elements they consider locally owned (or unpartitioned), so after
208 : // copying only the objects visible to this processor, those counts
209 : // reflect an incomplete local view rather than other_mesh's totals,
210 : // and need to be copied afresh.
211 20361 : _n_nodes = other_mesh.n_nodes();
212 20361 : _n_elem = other_mesh.n_elem();
213 20361 : _max_node_id = other_mesh.max_node_id();
214 20361 : _max_elem_id = other_mesh.max_elem_id();
215 :
216 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
217 : // counters directly, mirroring other_mesh's own bookkeeping exactly. This
218 : // is done in the DistributedMesh copy constructor.
219 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
220 : // raw counterpart fields to copy, so derive the free-id-range
221 : // counters from _max_elem_id/_max_node_id the same way
222 : // update_parallel_id_counts() does via the set_next_ids() method. For
223 : // the unique id counters we copy other_mesh's own local next_unique_id()
224 : // and derive the next available unique ids
225 : // (partitioned and unpartitioned) via set_next_unique_ids().
226 20361 : if (!other_is_distributed_type)
227 : {
228 2627 : this->set_next_ids();
229 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
230 2627 : this->set_next_unique_ids(other_mesh.next_unique_id());
231 : #endif
232 : }
233 :
234 : // Copy other_mesh's actual preparation state, including whatever it
235 : // reports for has_synched_id_counts: the counts above are a faithful copy
236 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
237 : // verified computation, so our synced-ness should match other_mesh's.
238 20361 : this->_preparation = other_mesh.preparation();
239 20361 : }
240 :
241 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
242 : {
243 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
244 :
245 4 : this->_nodes = std::move(other_mesh._nodes);
246 270 : this->_n_nodes = other_mesh.n_nodes();
247 :
248 4 : this->_elements = std::move(other_mesh._elements);
249 270 : this->_n_elem = other_mesh.n_elem();
250 :
251 270 : _is_serial = other_mesh.is_serial();
252 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
253 270 : _deleted_coarse_elements = true; // Better safe than sorry
254 :
255 270 : _max_node_id = other_mesh.max_node_id();
256 270 : _max_elem_id = other_mesh.max_elem_id();
257 :
258 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
259 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
260 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
261 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
262 :
263 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
264 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
265 : #endif
266 270 : }
267 :
268 2316662 : void DistributedMesh::set_next_ids()
269 : {
270 2316662 : _next_free_unpartitioned_elem_id =
271 2318444 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
272 2316662 : (this->n_processors() + 1) + this->n_processors();
273 2316662 : _next_free_local_elem_id =
274 2316662 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
275 2316662 : (this->n_processors() + 1) + this->processor_id();
276 :
277 2316662 : _next_free_unpartitioned_node_id =
278 2318444 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
279 2316662 : (this->n_processors() + 1) + this->n_processors();
280 2316662 : _next_free_local_node_id =
281 2316662 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
282 2316662 : (this->n_processors() + 1) + this->processor_id();
283 2316662 : }
284 :
285 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
286 2316662 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
287 : {
288 : // Unique ids are laid out in repeating groups of (n_processors()+1)
289 : // consecutive integers ("alignment" here means a value's position, i.e.
290 : // its residue mod (n_processors()+1), within one of these groups). In
291 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
292 : // are each permanently earmarked for one processor's locally-owned
293 : // objects, and the last slot (residue n_processors()) is permanently
294 : // earmarked for unpartitioned objects.
295 : //
296 : // parallel_max_unique_id carries no guarantee of landing on any
297 : // particular residue, so rather than simply offsetting it by
298 : // processor_id()/n_processors(), we have to explicitly round up to the
299 : // start of the next full, entirely-unused group before adding this
300 : // processor's (or the unpartitioned pool's) fixed offset into that group.
301 : //
302 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
303 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
304 : // the number of complete groups needed to reach at least past the
305 : // current max id. Multiplying back by (n_processors()+1) gives the first
306 : // id of the next group that is guaranteed to be entirely unused; adding
307 : // n_processors(), the unpartitioned pool's fixed offset within a group,
308 : // lands on that pool's reserved slot in that group.
309 2316662 : _next_unpartitioned_unique_id =
310 2318444 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
311 2316662 : (this->n_processors() + 1) +
312 2316662 : this->n_processors();
313 :
314 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
315 : // the rounding accounts for processor_id() possibly being less than the
316 : // residue the raw max id already occupies within its group; this can
317 : // land this processor's slot in the same group as the unpartitioned
318 : // pool's above, or push it into the following group, depending on where
319 : // the max id and this processor's offset happen to fall.
320 2316662 : _next_unique_id =
321 2316662 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
322 2316662 : (this->n_processors() + 1) +
323 2316662 : this->processor_id();
324 2316662 : }
325 : #endif
326 :
327 : // We use cached values for these so they can be called
328 : // from one processor without bothering the rest, but
329 : // we may need to update those caches before doing a full
330 : // renumbering
331 2314035 : void DistributedMesh::update_parallel_id_counts()
332 : {
333 : // This function must be run on all processors at once
334 1708 : parallel_object_only();
335 :
336 2314035 : _n_elem = this->parallel_n_elem();
337 2314035 : _n_nodes = this->parallel_n_nodes();
338 2314035 : _max_node_id = this->parallel_max_node_id();
339 2314035 : _max_elem_id = this->parallel_max_elem_id();
340 :
341 2314035 : this->set_next_ids();
342 :
343 : // set_next_unique_ids() needs the true parallel max unique id (not just
344 : // this processor's possibly-stale cached _next_unique_id) since the whole
345 : // point of this function is to recompute counters that may have drifted
346 : // out of sync.
347 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
348 2314035 : this->set_next_unique_ids(this->parallel_max_unique_id());
349 : #endif
350 :
351 2314035 : _n_constraint_rows = this->count_constraint_rows();
352 :
353 2314035 : this->_preparation.has_synched_id_counts = true;
354 2314035 : }
355 :
356 :
357 : // Or in debug mode we may want to test the uncached values without
358 : // changing the cache
359 2321772 : dof_id_type DistributedMesh::parallel_n_elem() const
360 : {
361 : // This function must be run on all processors at once
362 2568 : parallel_object_only();
363 :
364 2321772 : dof_id_type n_local = this->n_local_elem();
365 2321772 : this->comm().sum(n_local);
366 2321772 : n_local += this->n_unpartitioned_elem();
367 2321772 : return n_local;
368 : }
369 :
370 :
371 :
372 2321131 : dof_id_type DistributedMesh::parallel_max_elem_id() const
373 : {
374 : // This function must be run on all processors at once
375 8804 : parallel_object_only();
376 :
377 2321131 : dof_id_type max_local = 0;
378 :
379 : dofobject_container<Elem>::const_reverse_veclike_iterator
380 8804 : rit = _elements.rbegin();
381 :
382 : const dofobject_container<Elem>::const_reverse_veclike_iterator
383 8804 : rend = _elements.rend();
384 :
385 : // Look for the maximum element id. Search backwards through
386 : // elements so we can break out early. Beware of nullptr entries that
387 : // haven't yet been cleared from _elements.
388 10963637 : for (; rit != rend; ++rit)
389 : {
390 10311237 : const DofObject *d = *rit;
391 19115 : if (d)
392 : {
393 8589 : libmesh_assert(_elements[d->id()] == d);
394 1668731 : max_local = d->id() + 1;
395 1668731 : break;
396 : }
397 : }
398 :
399 2321131 : this->comm().max(max_local);
400 2321131 : return max_local;
401 : }
402 :
403 :
404 :
405 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
406 2405143 : unique_id_type DistributedMesh::parallel_max_unique_id() const
407 : {
408 : // This function must be run on all processors at once
409 1810 : parallel_object_only();
410 :
411 4810286 : unique_id_type max_local = std::max(_next_unique_id,
412 2769093 : _next_unpartitioned_unique_id);
413 2405143 : this->comm().max(max_local);
414 2405143 : return max_local;
415 : }
416 :
417 :
418 :
419 86149 : void DistributedMesh::set_next_unique_id(unique_id_type id)
420 : {
421 148 : _next_unique_id = id;
422 86149 : _next_unpartitioned_unique_id =
423 86297 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
424 86149 : (this->n_processors() + 1) + this->n_processors();
425 86149 : _next_unique_id =
426 86149 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
427 86149 : (this->n_processors() + 1) + this->processor_id();
428 86149 : }
429 : #endif
430 :
431 :
432 :
433 2322220 : dof_id_type DistributedMesh::parallel_n_nodes() const
434 : {
435 : // This function must be run on all processors at once
436 2568 : parallel_object_only();
437 :
438 2322220 : dof_id_type n_local = this->n_local_nodes();
439 2322220 : this->comm().sum(n_local);
440 2322220 : n_local += this->n_unpartitioned_nodes();
441 2322220 : return n_local;
442 : }
443 :
444 :
445 :
446 2318679 : dof_id_type DistributedMesh::parallel_max_node_id() const
447 : {
448 : // This function must be run on all processors at once
449 6352 : parallel_object_only();
450 :
451 2318679 : dof_id_type max_local = 0;
452 :
453 : dofobject_container<Node>::const_reverse_veclike_iterator
454 6352 : rit = _nodes.rbegin();
455 :
456 : const dofobject_container<Node>::const_reverse_veclike_iterator
457 6352 : rend = _nodes.rend();
458 :
459 : // Look for the maximum node id. Search backwards through
460 : // nodes so we can break out early. Beware of nullptr entries that
461 : // haven't yet been cleared from _nodes
462 41512602 : for (; rit != rend; ++rit)
463 : {
464 40860272 : const DofObject *d = *rit;
465 63439 : if (d)
466 : {
467 6207 : libmesh_assert(_nodes[d->id()] == d);
468 1666349 : max_local = d->id() + 1;
469 1666349 : break;
470 : }
471 : }
472 :
473 2318679 : this->comm().max(max_local);
474 2318679 : return max_local;
475 : }
476 :
477 :
478 :
479 45583208 : const Point & DistributedMesh::point (const dof_id_type i) const
480 : {
481 45583208 : return this->node_ref(i);
482 : }
483 :
484 :
485 :
486 46700285 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
487 : {
488 137685 : libmesh_assert(_nodes[i]);
489 137685 : libmesh_assert_equal_to (_nodes[i]->id(), i);
490 :
491 46700285 : return _nodes[i];
492 : }
493 :
494 :
495 :
496 :
497 666566952 : Node * DistributedMesh::node_ptr (const dof_id_type i)
498 : {
499 490956 : libmesh_assert(_nodes[i]);
500 490956 : libmesh_assert_equal_to (_nodes[i]->id(), i);
501 :
502 666566952 : return _nodes[i];
503 : }
504 :
505 :
506 :
507 :
508 3110012 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
509 : {
510 3110012 : if (const auto it = _nodes.find(i);
511 658289 : it != _nodes.end())
512 : {
513 2821478 : const Node * n = *it;
514 369755 : libmesh_assert (!n || n->id() == i);
515 369755 : return n;
516 : }
517 :
518 288534 : return nullptr;
519 : }
520 :
521 :
522 :
523 :
524 385119713 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
525 : {
526 385119713 : if (auto it = _nodes.find(i);
527 159599 : it != _nodes.end())
528 : {
529 299051185 : Node * n = *it;
530 111970 : libmesh_assert (!n || n->id() == i);
531 111970 : return n;
532 : }
533 :
534 47629 : return nullptr;
535 : }
536 :
537 :
538 :
539 :
540 3074903 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
541 : {
542 10107 : libmesh_assert(_elements[i]);
543 10107 : libmesh_assert_equal_to (_elements[i]->id(), i);
544 :
545 3074903 : return _elements[i];
546 : }
547 :
548 :
549 :
550 :
551 295398238 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
552 : {
553 167573 : libmesh_assert(_elements[i]);
554 167573 : libmesh_assert_equal_to (_elements[i]->id(), i);
555 :
556 295398238 : return _elements[i];
557 : }
558 :
559 :
560 :
561 :
562 1458763 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
563 : {
564 1458763 : if (const auto it = _elements.find(i);
565 619793 : it != _elements.end())
566 : {
567 1101106 : const Elem * e = *it;
568 271051 : libmesh_assert (!e || e->id() == i);
569 271051 : return e;
570 : }
571 :
572 348742 : return nullptr;
573 : }
574 :
575 :
576 :
577 :
578 625916287 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
579 : {
580 625916287 : if (auto it = _elements.find(i);
581 194370 : it != _elements.end())
582 : {
583 539912356 : Elem * e = *it;
584 190690 : libmesh_assert (!e || e->id() == i);
585 190690 : return e;
586 : }
587 :
588 3680 : return nullptr;
589 : }
590 :
591 :
592 :
593 :
594 44832221 : Elem * DistributedMesh::add_elem (Elem * e)
595 : {
596 : // Don't try to add nullptrs!
597 19137 : libmesh_assert(e);
598 :
599 : // Trying to add an existing element is a no-op
600 44832221 : if (e->valid_id() && _elements[e->id()] == e)
601 0 : return e;
602 :
603 44832221 : const processor_id_type elem_procid = e->processor_id();
604 :
605 44832221 : if (!e->valid_id())
606 : {
607 : // We should only be creating new ids past the end of the range
608 : // of existing ids
609 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
610 : _max_elem_id);
611 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
612 :
613 : // Use the unpartitioned ids for unpartitioned elems, and
614 : // temporarily for ghost elems
615 3863637 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
616 3867205 : if (elem_procid == this->processor_id())
617 1133684 : next_id = &_next_free_local_elem_id;
618 3863637 : e->set_id (*next_id);
619 : }
620 :
621 : {
622 : // Advance next_ids up high enough that each is pointing to an
623 : // unused id and any subsequent increments will still point us
624 : // to unused ids
625 89664442 : _max_elem_id = std::max(_max_elem_id,
626 44832221 : static_cast<dof_id_type>(e->id()+1));
627 :
628 44832221 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
629 5232644 : _next_free_unpartitioned_elem_id =
630 5232644 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
631 5232644 : (this->n_processors() + 1) + this->n_processors();
632 44832221 : if (_next_free_local_elem_id < _max_elem_id)
633 3520842 : _next_free_local_elem_id =
634 3527146 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
635 3520842 : (this->n_processors() + 1) + this->processor_id();
636 :
637 : #ifndef NDEBUG
638 : // We need a const dofobject_container so we don't inadvertently create
639 : // nullptr entries when testing for non-nullptr ones
640 19137 : const dofobject_container<Elem> & const_elements = _elements;
641 : #endif
642 19137 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
643 19137 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
644 : }
645 :
646 : // Don't try to overwrite existing elems
647 19137 : libmesh_assert (!_elements[e->id()]);
648 :
649 44832221 : _elements[e->id()] = e;
650 :
651 : // We actually added a new element. Some of our caches might still
652 : // be valid, but we should clear the ones which definitely are not.
653 44832221 : this->clear_point_locator();
654 44832221 : this->clear_stored_ranges();
655 :
656 : // Try to make the cached elem data more accurate
657 44851358 : if (elem_procid == this->processor_id() ||
658 : elem_procid == DofObject::invalid_processor_id)
659 13674119 : _n_elem++;
660 :
661 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
662 44832221 : if (!e->valid_unique_id())
663 : {
664 14269443 : if (processor_id() == e->processor_id())
665 : {
666 1133684 : e->set_unique_id(_next_unique_id);
667 1133684 : _next_unique_id += this->n_processors() + 1;
668 : }
669 : else
670 : {
671 13135759 : e->set_unique_id(_next_unpartitioned_unique_id);
672 13135759 : _next_unpartitioned_unique_id += this->n_processors() + 1;
673 : }
674 : }
675 : else
676 : {
677 30587155 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
678 30562778 : _next_unique_id =
679 30562778 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
680 30562778 : (this->n_processors() + 1) + this->processor_id();
681 : }
682 : #endif
683 :
684 : // Unpartitioned elems should be added on every processor
685 : // And shouldn't be added in the same batch as ghost elems
686 : // But we might be just adding on processor 0 to
687 : // broadcast later
688 : // #ifdef DEBUG
689 : // if (elem_procid == DofObject::invalid_processor_id)
690 : // {
691 : // dof_id_type elem_id = e->id();
692 : // this->comm().max(elem_id);
693 : // libmesh_assert_equal_to (elem_id, e->id());
694 : // }
695 : // #endif
696 :
697 : // Make sure any new element is given space for any extra integers
698 : // we've requested
699 44832221 : e->add_extra_integers(_elem_integer_names.size(),
700 44832221 : _elem_integer_default_values);
701 :
702 : // And set mapping type and data on any new element
703 38274 : e->set_mapping_type(this->default_mapping_type());
704 38274 : e->set_mapping_data(this->default_mapping_data());
705 :
706 44832221 : return e;
707 : }
708 :
709 :
710 :
711 15509482 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
712 : {
713 : // The mesh now takes ownership of the Elem. Eventually the guts of
714 : // add_elem() will get moved to a private helper function, and
715 : // calling add_elem() directly will be deprecated.
716 15509482 : return add_elem(e.release());
717 : }
718 :
719 :
720 :
721 3745676 : Elem * DistributedMesh::insert_elem (Elem * e)
722 : {
723 3745676 : if (_elements[e->id()])
724 3745676 : this->delete_elem(_elements[e->id()]);
725 :
726 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
727 3745676 : if (!e->valid_unique_id())
728 : {
729 0 : if (processor_id() == e->processor_id())
730 : {
731 0 : e->set_unique_id(_next_unique_id);
732 0 : _next_unique_id += this->n_processors() + 1;
733 : }
734 : else
735 : {
736 0 : e->set_unique_id(_next_unpartitioned_unique_id);
737 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
738 : }
739 : }
740 : else
741 : {
742 3745676 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
743 3745676 : _next_unique_id =
744 3749372 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
745 3745676 : (this->n_processors() + 1) + this->processor_id();
746 : }
747 : #endif
748 :
749 : // Try to make the cached elem data more accurate
750 3745676 : processor_id_type elem_procid = e->processor_id();
751 3749372 : if (elem_procid == this->processor_id() ||
752 : elem_procid == DofObject::invalid_processor_id)
753 3614584 : _n_elem++;
754 :
755 3745676 : _elements[e->id()] = e;
756 :
757 : // We actually added a new element. Some of our caches might still
758 : // be valid, but we should clear the ones which definitely are not.
759 3745676 : this->clear_point_locator();
760 3745676 : this->clear_stored_ranges();
761 :
762 : // Make sure any new element is given space for any extra integers
763 : // we've requested
764 3745676 : e->add_extra_integers(_elem_integer_names.size(),
765 3745676 : _elem_integer_default_values);
766 :
767 : // And set mapping type and data on any new element
768 7392 : e->set_mapping_type(this->default_mapping_type());
769 7392 : e->set_mapping_data(this->default_mapping_data());
770 :
771 3745676 : return e;
772 : }
773 :
774 3745676 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
775 : {
776 : // The mesh now takes ownership of the Elem. Eventually the guts of
777 : // insert_elem(Elem*) will get moved to a private helper function, and
778 : // calling insert_elem(Elem*) directly will be deprecated.
779 3745676 : return insert_elem(e.release());
780 : }
781 :
782 :
783 40798955 : void DistributedMesh::delete_elem(Elem * e)
784 : {
785 7632 : libmesh_assert (e);
786 :
787 : // Try to make the cached elem data more accurate
788 40798955 : _n_elem--;
789 :
790 : // Was this a coarse element, not just a coarsening where we still
791 : // have some ancestor structure? Was it a *local* element, that we
792 : // might have been depending on as an owner of local nodes? We'll
793 : // have to be more careful with our nodes in contract() later; no
794 : // telling if we just locally orphaned a node that should be
795 : // globally retained.
796 40808407 : if (e->processor_id() == this->processor_id() &&
797 3640 : !e->parent())
798 80921 : _deleted_coarse_elements = true;
799 :
800 : // Delete the element from the BoundaryInfo object
801 40798955 : this->get_boundary_info().remove(e);
802 :
803 : // But not yet from the container; we might invalidate
804 : // an iterator that way!
805 :
806 : //_elements.erase(e->id());
807 :
808 : // Instead, we set it to nullptr for now
809 :
810 40798955 : _elements[e->id()] = nullptr;
811 :
812 : // delete the element
813 40798955 : delete e;
814 :
815 : // Some of our caches might still be valid, but we should clear the
816 : // ones which definitely are not.
817 40798955 : this->clear_point_locator();
818 40798955 : this->clear_stored_ranges();
819 40798955 : }
820 :
821 :
822 :
823 3208096 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
824 : const dof_id_type new_id)
825 : {
826 : // This could be a no-op
827 3208096 : if (old_id == new_id)
828 0 : return;
829 :
830 3208096 : Elem * el = _elements[old_id];
831 1160 : libmesh_assert (el);
832 1160 : libmesh_assert_equal_to (el->id(), old_id);
833 :
834 3208096 : el->set_id(new_id);
835 1160 : libmesh_assert (!_elements[new_id]);
836 3208096 : _elements[new_id] = el;
837 3208096 : _elements.erase(old_id);
838 :
839 : // Should we delete any caches here? Our point locator indexes by
840 : // element pointer and should be fine with an id change. Our stored
841 : // ranges are no longer sorted, which is *probably* fine, but let's
842 : // just be safe.
843 3208096 : this->clear_stored_ranges();
844 : }
845 :
846 :
847 :
848 44170526 : Node * DistributedMesh::add_point (const Point & p,
849 : const dof_id_type id,
850 : const processor_id_type proc_id)
851 : {
852 44170526 : Node * old_n = this->query_node_ptr(id);
853 :
854 44170526 : if (old_n)
855 : {
856 0 : *old_n = p;
857 0 : old_n->processor_id() = proc_id;
858 :
859 0 : return old_n;
860 : }
861 :
862 44130395 : Node * n = Node::build(p, id).release();
863 44170526 : n->processor_id() = proc_id;
864 :
865 44170526 : return DistributedMesh::add_node(n);
866 : }
867 :
868 :
869 4130 : void DistributedMesh::own_node (Node & n)
870 : {
871 : // This had better be a node in our mesh
872 0 : libmesh_assert(_nodes[n.id()] == &n);
873 :
874 4130 : _nodes[n.id()] = nullptr;
875 4130 : _n_nodes--;
876 :
877 0 : n.set_id(DofObject::invalid_id);
878 4130 : n.processor_id() = this->processor_id();
879 :
880 4130 : this->add_node(&n);
881 4130 : }
882 :
883 :
884 75012962 : Node * DistributedMesh::add_node (Node * n)
885 : {
886 : // Don't try to add nullptrs!
887 41843 : libmesh_assert(n);
888 :
889 : // Trying to add an existing node is a no-op
890 75012962 : if (n->valid_id() && _nodes[n->id()] == n)
891 0 : return n;
892 :
893 75012962 : const processor_id_type node_procid = n->processor_id();
894 :
895 75012962 : if (!n->valid_id())
896 : {
897 : // We should only be creating new ids past the end of the range
898 : // of existing ids
899 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
900 : _max_node_id);
901 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
902 :
903 : // Use the unpartitioned ids for unpartitioned nodes,
904 : // and temporarily for ghost nodes
905 18470524 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
906 18488353 : if (node_procid == this->processor_id())
907 2130168 : next_id = &_next_free_local_node_id;
908 18470524 : n->set_id (*next_id);
909 : }
910 :
911 : {
912 : // Advance next_ids up high enough that each is pointing to an
913 : // unused id and any subsequent increments will still point us
914 : // to unused ids
915 150025924 : _max_node_id = std::max(_max_node_id,
916 75012962 : static_cast<dof_id_type>(n->id()+1));
917 :
918 75012962 : if (_next_free_unpartitioned_node_id < _max_node_id)
919 21559690 : _next_free_unpartitioned_node_id =
920 21559690 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
921 21559690 : (this->n_processors() + 1) + this->n_processors();
922 75012962 : if (_next_free_local_node_id < _max_node_id)
923 12507741 : _next_free_local_node_id =
924 12525971 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
925 12507741 : (this->n_processors() + 1) + this->processor_id();
926 :
927 : #ifndef NDEBUG
928 : // We need a const dofobject_container so we don't inadvertently create
929 : // nullptr entries when testing for non-nullptr ones
930 41843 : const dofobject_container<Node> & const_nodes = _nodes;
931 : #endif
932 41843 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
933 41843 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
934 : }
935 :
936 : // Don't try to overwrite existing nodes
937 41843 : libmesh_assert (!_nodes[n->id()]);
938 :
939 75012962 : _nodes[n->id()] = n;
940 :
941 : // Try to make the cached node data more accurate
942 75054805 : if (node_procid == this->processor_id() ||
943 : node_procid == DofObject::invalid_processor_id)
944 41750115 : _n_nodes++;
945 :
946 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
947 75012962 : if (!n->valid_unique_id())
948 : {
949 44172982 : if (processor_id() == n->processor_id())
950 : {
951 2547690 : n->set_unique_id(_next_unique_id);
952 2547690 : _next_unique_id += this->n_processors() + 1;
953 : }
954 : else
955 : {
956 41625292 : n->set_unique_id(_next_unpartitioned_unique_id);
957 41625292 : _next_unpartitioned_unique_id += this->n_processors() + 1;
958 : }
959 : }
960 : else
961 : {
962 30970484 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
963 30839980 : _next_unique_id =
964 30839980 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
965 30839980 : (this->n_processors() + 1) + this->processor_id();
966 : }
967 : #endif
968 :
969 75012962 : n->add_extra_integers(_node_integer_names.size(),
970 75012962 : _node_integer_default_values);
971 :
972 : // Unpartitioned nodes should be added on every processor
973 : // And shouldn't be added in the same batch as ghost nodes
974 : // But we might be just adding on processor 0 to
975 : // broadcast later
976 : // #ifdef DEBUG
977 : // if (node_procid == DofObject::invalid_processor_id)
978 : // {
979 : // dof_id_type node_id = n->id();
980 : // this->comm().max(node_id);
981 : // libmesh_assert_equal_to (node_id, n->id());
982 : // }
983 : // #endif
984 :
985 75012962 : return n;
986 : }
987 :
988 30838306 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
989 : {
990 : // The mesh now takes ownership of the Node. Eventually the guts of
991 : // add_node() will get moved to a private helper function, and
992 : // calling add_node() directly will be deprecated.
993 30838306 : return add_node(n.release());
994 : }
995 :
996 51575869 : void DistributedMesh::delete_node(Node * n)
997 : {
998 4507 : libmesh_assert(n);
999 4507 : libmesh_assert(_nodes[n->id()]);
1000 :
1001 : // Try to make the cached elem data more accurate
1002 51575869 : _n_nodes--;
1003 :
1004 : // Delete the node from the BoundaryInfo object
1005 51575869 : this->get_boundary_info().remove(n);
1006 9014 : _constraint_rows.erase(n);
1007 :
1008 : // But not yet from the container; we might invalidate
1009 : // an iterator that way!
1010 :
1011 : //_nodes.erase(n->id());
1012 :
1013 : // Instead, we set it to nullptr for now
1014 :
1015 51575869 : _nodes[n->id()] = nullptr;
1016 :
1017 : // delete the node
1018 51575869 : delete n;
1019 51575869 : }
1020 :
1021 :
1022 :
1023 6901524 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1024 : const dof_id_type new_id)
1025 : {
1026 : // This could be a no-op
1027 6901524 : if (old_id == new_id)
1028 0 : return;
1029 :
1030 6900985 : Node * nd = _nodes[old_id];
1031 3234 : libmesh_assert (nd);
1032 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1033 :
1034 : // If we have nodes shipped to this processor for NodeConstraints
1035 : // use, then those nodes will exist in _nodes, but may not be
1036 : // locatable via a TopologyMap due to the insufficiency of elements
1037 : // connecting to them. If local refinement then wants to create a
1038 : // *new* node in the same location, it will initially get a temporary
1039 : // id, and then make_node_ids_parallel_consistent() will try to move
1040 : // it to the canonical id. We need to account for this case to
1041 : // avoid false positives and memory leaks.
1042 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1043 6468 : if (_nodes[new_id])
1044 : {
1045 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1046 : *(Point *)_nodes[old_id]);
1047 0 : _nodes.erase(new_id);
1048 : }
1049 : #else
1050 : // If we aren't shipping nodes for NodeConstraints, there should be
1051 : // no reason for renumbering one node onto another.
1052 : libmesh_assert (!_nodes[new_id]);
1053 : #endif
1054 6900985 : _nodes[new_id] = nd;
1055 6900985 : nd->set_id(new_id);
1056 :
1057 6900985 : _nodes.erase(old_id);
1058 : }
1059 :
1060 :
1061 :
1062 693563 : void DistributedMesh::clear ()
1063 : {
1064 : // Call parent clear function
1065 693563 : MeshBase::clear();
1066 :
1067 : // Clear our elements and nodes
1068 : // There is no need to remove them from
1069 : // the BoundaryInfo data structure since we
1070 : // already cleared it.
1071 693563 : this->DistributedMesh::clear_elems();
1072 :
1073 24629354 : for (auto & node : _nodes)
1074 44608161 : delete node;
1075 :
1076 588 : _nodes.clear();
1077 :
1078 : // We're no longer distributed if we were before
1079 693563 : _is_serial = true;
1080 693563 : _is_serial_on_proc_0 = true;
1081 :
1082 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1083 : // all is copacetic.
1084 693563 : _deleted_coarse_elements = false;
1085 :
1086 : // Correct our caches
1087 693563 : _n_nodes = 0;
1088 693563 : _max_node_id = 0;
1089 694151 : _next_free_local_node_id = this->processor_id();
1090 693563 : _next_free_unpartitioned_node_id = this->n_processors();
1091 693563 : }
1092 :
1093 :
1094 :
1095 699963 : void DistributedMesh::clear_elems ()
1096 : {
1097 9872153 : for (auto & elem : _elements)
1098 9172190 : delete elem;
1099 :
1100 588 : _elements.clear();
1101 :
1102 : // Correct our caches
1103 699963 : _n_elem = 0;
1104 699963 : _max_elem_id = 0;
1105 700551 : _next_free_local_elem_id = this->processor_id();
1106 699963 : _next_free_unpartitioned_elem_id = this->n_processors();
1107 699963 : }
1108 :
1109 :
1110 :
1111 681909 : void DistributedMesh::redistribute ()
1112 : {
1113 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1114 681909 : if (!this->is_serial())
1115 : {
1116 : // Construct a MeshCommunication object to actually redistribute the nodes
1117 : // and elements according to the partitioner, and then to re-gather the neighbors.
1118 : MeshCommunication mc;
1119 75882 : mc.redistribute(*this);
1120 :
1121 75882 : this->update_parallel_id_counts();
1122 :
1123 : // We communicate valid neighbor links for newly-redistributed
1124 : // elements, but we may still have remote_elem links that should
1125 : // be corrected but whose correction wasn't communicated.
1126 76154 : this->find_neighbors(/*reset_remote_elements*/ false,
1127 : /*reset_current_list*/ false /*non-default*/,
1128 : /*assert_valid*/ false,
1129 272 : /*check_non_remote*/ false /*non-default!*/);
1130 :
1131 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1132 : // anyway... but users can always call partition directly, in which case we do need
1133 : // to call delete_remote_elements()...
1134 : //
1135 : // Regardless of whether it's necessary, it isn't safe. We
1136 : // haven't communicated new node processor_ids yet, and we can't
1137 : // delete nodes until we do.
1138 : // this->delete_remote_elements();
1139 : }
1140 : else
1141 : // The base class can handle non-distributed things, like
1142 : // notifying any GhostingFunctors of changes
1143 606027 : MeshBase::redistribute();
1144 681909 : }
1145 :
1146 :
1147 :
1148 576412 : void DistributedMesh::update_post_partitioning ()
1149 : {
1150 : // this->recalculate_n_partitions();
1151 :
1152 : // Let's do the base class cache clearing first, just in case our
1153 : // later computations are ever changed to make use of a local
1154 : // elements range cache
1155 576412 : this->MeshBase::update_post_partitioning();
1156 :
1157 : // Partitioning changes our numbers of unpartitioned objects
1158 576412 : this->update_parallel_id_counts();
1159 576412 : }
1160 :
1161 :
1162 :
1163 : template <typename T>
1164 3404 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1165 : {
1166 : // This function must be run on all processors at once
1167 3404 : parallel_object_only();
1168 :
1169 3404 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1170 3404 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1171 3404 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1172 :
1173 895420 : for (dof_id_type i=0; i != pmax_id; ++i)
1174 : {
1175 892016 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1176 :
1177 : // Local lookups by id should return the requested object
1178 892016 : libmesh_assert(!obj || obj->id() == i);
1179 :
1180 : // All processors with an object should agree on id
1181 : #ifndef NDEBUG
1182 892016 : const dof_id_type dofid = obj && obj->valid_id() ?
1183 : obj->id() : DofObject::invalid_id;
1184 892016 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1185 : #endif
1186 :
1187 : // All processors with an object should agree on processor id
1188 892016 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1189 : obj->processor_id() : DofObject::invalid_processor_id;
1190 892016 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1191 :
1192 892016 : dof_id_type min_procid = procid;
1193 892016 : this->comm().min(min_procid);
1194 :
1195 : // Either:
1196 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1197 : // or
1198 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1199 :
1200 : // Original assert logic
1201 : // libmesh_assert (min_procid != this->processor_id() || obj);
1202 :
1203 : // More human-understandable logic...
1204 892016 : libmesh_assert (
1205 : ((min_procid == this->processor_id()) && obj)
1206 : ||
1207 : (min_procid != this->processor_id())
1208 : );
1209 :
1210 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1211 : // All processors with an object should agree on unique id
1212 892016 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1213 892016 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1214 : #endif
1215 : }
1216 3404 : }
1217 :
1218 :
1219 :
1220 1702 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1221 : {
1222 1702 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1223 1702 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1224 1702 : }
1225 :
1226 :
1227 :
1228 410 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1229 : {
1230 : #ifndef NDEBUG
1231 : // This function must be run on all processors at once
1232 410 : parallel_object_only();
1233 :
1234 410 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1235 :
1236 22134 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1237 : {
1238 21724 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1239 :
1240 21724 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1241 :
1242 : // All processors with an active element should agree on p level
1243 21724 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1244 : }
1245 : #endif
1246 410 : }
1247 :
1248 :
1249 :
1250 :
1251 1650 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1252 : {
1253 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1254 : // This function must be run on all processors at once
1255 1650 : parallel_object_only();
1256 :
1257 1650 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1258 :
1259 145196 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1260 : {
1261 143546 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1262 :
1263 143546 : unsigned int refinement_flag = el ?
1264 66131 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1265 143546 : unsigned int p_refinement_flag = el ?
1266 66131 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1267 :
1268 143546 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1269 :
1270 : // p refinement flags aren't always kept correct on inactive
1271 : // ghost elements
1272 143546 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1273 : }
1274 : #endif // LIBMESH_ENABLE_AMR
1275 1650 : }
1276 :
1277 :
1278 :
1279 : template <typename T>
1280 : dof_id_type
1281 2137334 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1282 : {
1283 : // This function must be run on all processors at once
1284 816 : parallel_object_only();
1285 :
1286 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1287 :
1288 : // In parallel we may not know what objects other processors have.
1289 : // Start by figuring out how many
1290 816 : dof_id_type unpartitioned_objects = 0;
1291 :
1292 : std::unordered_map<processor_id_type, dof_id_type>
1293 1632 : ghost_objects_from_proc;
1294 :
1295 2137334 : object_iterator it = objects.begin();
1296 816 : object_iterator end = objects.end();
1297 :
1298 119949682 : while (it != end)
1299 : {
1300 117812348 : T * obj = *it;
1301 :
1302 : // Remove any nullptr container entries while we're here.
1303 117812348 : if (!obj)
1304 1482108 : it = objects.erase(it);
1305 : else
1306 : {
1307 116330240 : processor_id_type obj_procid = obj->processor_id();
1308 116330240 : if (obj_procid == DofObject::invalid_processor_id)
1309 45950459 : unpartitioned_objects++;
1310 : else
1311 70379781 : ghost_objects_from_proc[obj_procid]++;
1312 :
1313 : // Finally, increment the iterator
1314 56712 : ++it;
1315 : }
1316 : }
1317 :
1318 2138966 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1319 2138150 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1320 816 : this->comm().allgather
1321 2137833 : ((this_it == ghost_objects_from_proc.end()) ?
1322 499 : dof_id_type(0) : this_it->second, objects_on_proc);
1323 :
1324 : #ifndef NDEBUG
1325 816 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1326 2448 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1327 1632 : if (ghost_objects_from_proc.count(p))
1328 970 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1329 : else
1330 662 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1331 : #endif
1332 :
1333 : // We'll renumber objects in blocks by processor id
1334 2138966 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1335 22684886 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1336 20550816 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1337 816 : objects_on_proc[i-1];
1338 2138150 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1339 2137334 : dof_id_type first_free_id =
1340 2138966 : first_object_on_proc[this->n_processors()-1] +
1341 816 : objects_on_proc[this->n_processors()-1] +
1342 : unpartitioned_objects;
1343 :
1344 : // First set new local object ids and build request sets
1345 : // for non-local object ids
1346 :
1347 : // Request sets to send to each processor
1348 : std::map<processor_id_type, std::vector<dof_id_type>>
1349 816 : requested_ids;
1350 :
1351 : // We know how many objects live on each processor, so reserve() space for
1352 : // each.
1353 816 : auto ghost_end = ghost_objects_from_proc.end();
1354 24822220 : for (auto p : make_range(this->n_processors()))
1355 22686518 : if (p != this->processor_id())
1356 : {
1357 20548368 : if (const auto p_it = ghost_objects_from_proc.find(p);
1358 816 : p_it != ghost_end)
1359 3945074 : requested_ids[p].reserve(p_it->second);
1360 : }
1361 :
1362 816 : end = objects.end();
1363 118467574 : for (it = objects.begin(); it != end; ++it)
1364 : {
1365 116330240 : T * obj = *it;
1366 116330240 : if (!obj)
1367 0 : continue;
1368 116386952 : if (obj->processor_id() == this->processor_id())
1369 20897248 : obj->set_id(next_id++);
1370 95432992 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1371 49482533 : requested_ids[obj->processor_id()].push_back(obj->id());
1372 : }
1373 :
1374 : // Next set ghost object ids from other processors
1375 :
1376 2154561 : auto gather_functor =
1377 53394095 : [
1378 : #ifndef NDEBUG
1379 : this,
1380 : &first_object_on_proc,
1381 : &objects_on_proc,
1382 : #endif
1383 : &objects]
1384 : (processor_id_type, const std::vector<dof_id_type> & ids,
1385 942 : std::vector<dof_id_type> & new_ids)
1386 : {
1387 942 : std::size_t ids_size = ids.size();
1388 3945074 : new_ids.resize(ids_size);
1389 :
1390 53427607 : for (std::size_t i=0; i != ids_size; ++i)
1391 : {
1392 49482533 : T * obj = objects[ids[i]];
1393 16285 : libmesh_assert(obj);
1394 16285 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1395 49498818 : new_ids[i] = obj->id();
1396 :
1397 16285 : libmesh_assert_greater_equal (new_ids[i],
1398 : first_object_on_proc[this->processor_id()]);
1399 16285 : libmesh_assert_less (new_ids[i],
1400 : first_object_on_proc[this->processor_id()] +
1401 : objects_on_proc[this->processor_id()]);
1402 : }
1403 : };
1404 :
1405 2154561 : auto action_functor =
1406 53394095 : [
1407 : #ifndef NDEBUG
1408 : &first_object_on_proc,
1409 : &objects_on_proc,
1410 : #endif
1411 : &objects]
1412 : (processor_id_type libmesh_dbg_var(pid),
1413 : const std::vector<dof_id_type> & ids,
1414 942 : const std::vector<dof_id_type> & data)
1415 : {
1416 : // Copy the id changes we've now been informed of
1417 53427607 : for (auto i : index_range(ids))
1418 : {
1419 49482533 : T * obj = objects[ids[i]];
1420 16285 : libmesh_assert (obj);
1421 16285 : libmesh_assert_equal_to (obj->processor_id(), pid);
1422 16285 : libmesh_assert_greater_equal (data[i],
1423 : first_object_on_proc[pid]);
1424 16285 : libmesh_assert_less (data[i],
1425 : first_object_on_proc[pid] +
1426 : objects_on_proc[pid]);
1427 49498818 : obj->set_id(data[i]);
1428 : }
1429 : };
1430 :
1431 816 : const dof_id_type * ex = nullptr;
1432 : Parallel::pull_parallel_vector_data
1433 2137334 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1434 :
1435 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1436 2154561 : auto unique_gather_functor =
1437 53394095 : [
1438 : #ifndef NDEBUG
1439 : this,
1440 : #endif
1441 : &objects]
1442 : (processor_id_type, const std::vector<dof_id_type> & ids,
1443 942 : std::vector<unique_id_type> & data)
1444 : {
1445 942 : std::size_t ids_size = ids.size();
1446 3945074 : data.resize(ids_size);
1447 :
1448 53427607 : for (std::size_t i=0; i != ids_size; ++i)
1449 : {
1450 49482533 : T * obj = objects[ids[i]];
1451 16285 : libmesh_assert(obj);
1452 16285 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1453 98965066 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1454 : }
1455 : };
1456 :
1457 2154561 : auto unique_action_functor =
1458 53394095 : [&objects]
1459 : (processor_id_type libmesh_dbg_var(pid),
1460 : const std::vector<dof_id_type> & ids,
1461 942 : const std::vector<unique_id_type> & data)
1462 : {
1463 53427607 : for (auto i : index_range(ids))
1464 : {
1465 49482533 : T * obj = objects[ids[i]];
1466 16285 : libmesh_assert (obj);
1467 16285 : libmesh_assert_equal_to (obj->processor_id(), pid);
1468 49482533 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1469 0 : obj->set_unique_id(data[i]);
1470 : }
1471 : };
1472 :
1473 816 : const unique_id_type * unique_ex = nullptr;
1474 : Parallel::pull_parallel_vector_data
1475 2137334 : (this->comm(), requested_ids, unique_gather_functor,
1476 : unique_action_functor, unique_ex);
1477 : #endif
1478 :
1479 : // Next set unpartitioned object ids
1480 816 : next_id = 0;
1481 24822220 : for (auto i : make_range(this->n_processors()))
1482 22686518 : next_id += objects_on_proc[i];
1483 118467574 : for (it = objects.begin(); it != end; ++it)
1484 : {
1485 116330240 : T * obj = *it;
1486 116330240 : if (!obj)
1487 0 : continue;
1488 116330240 : if (obj->processor_id() == DofObject::invalid_processor_id)
1489 45950459 : obj->set_id(next_id++);
1490 : }
1491 :
1492 : // Finally shuffle around objects so that container indices
1493 : // match ids
1494 2137334 : it = objects.begin();
1495 816 : end = objects.end();
1496 125180048 : while (it != end)
1497 : {
1498 123042714 : T * obj = *it;
1499 123042714 : if (obj) // don't try shuffling already-nullptr entries
1500 : {
1501 123042714 : T * next = objects[obj->id()];
1502 : // If we have to move this object
1503 123042714 : if (next != obj)
1504 : {
1505 : // nullptr out its original position for now
1506 : // (our shuffling may put another object there shortly)
1507 42139564 : *it = nullptr;
1508 :
1509 : // There may already be another object with this id that
1510 : // needs to be moved itself
1511 55231344 : while (next)
1512 : {
1513 : // We shouldn't be trying to give two objects the
1514 : // same id
1515 9385 : libmesh_assert_not_equal_to (next->id(), obj->id());
1516 13091780 : objects[obj->id()] = obj;
1517 9385 : obj = next;
1518 13091780 : next = objects[obj->id()];
1519 : }
1520 42139564 : objects[obj->id()] = obj;
1521 : }
1522 : }
1523 :
1524 : // Remove any container entries that were left as nullptr.
1525 115876 : if (!obj)
1526 0 : it = objects.erase(it);
1527 : else
1528 57938 : ++it;
1529 : }
1530 :
1531 2138150 : return first_free_id;
1532 : }
1533 :
1534 :
1535 1070337 : void DistributedMesh::renumber_nodes_and_elements ()
1536 : {
1537 410 : parallel_object_only();
1538 :
1539 : #ifdef DEBUG
1540 : // Make sure our ids and flags are consistent
1541 410 : this->libmesh_assert_valid_parallel_ids();
1542 410 : this->libmesh_assert_valid_parallel_flags();
1543 410 : this->libmesh_assert_valid_parallel_p_levels();
1544 : #endif
1545 :
1546 818 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1547 :
1548 : // Nodes not connected to any elements, and nullptr node entries
1549 : // in our container, should be deleted. But wait! If we've deleted coarse
1550 : // local elements on some processor, other processors might have ghosted
1551 : // nodes from it that are now no longer connected to any elements on it, but
1552 : // that are connected to their own semilocal elements. We'll have to
1553 : // communicate to ascertain if that's the case.
1554 1070337 : this->comm().max(_deleted_coarse_elements);
1555 :
1556 : // What used nodes do we see on our proc?
1557 410 : std::set<dof_id_type> used_nodes;
1558 :
1559 : // What used node info should we send from our proc? Could we take ownership
1560 : // of each node if we needed to?
1561 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1562 410 : used_nodes_on_proc;
1563 :
1564 : // flag the nodes we need
1565 61224080 : for (auto & elem : this->element_ptr_range())
1566 320608219 : for (const Node & node : elem->node_ref_range())
1567 : {
1568 291036621 : const dof_id_type n = node.id();
1569 290891911 : used_nodes.insert(n);
1570 291036621 : if (_deleted_coarse_elements)
1571 : {
1572 3153005 : const processor_id_type p = node.processor_id();
1573 3153289 : if (p != this->processor_id())
1574 : {
1575 2262272 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1576 2262414 : if (elem->processor_id() == this->processor_id())
1577 216806 : used_nodes_on_p[n] = true;
1578 : else
1579 105 : if (!used_nodes_on_p.count(n))
1580 528874 : used_nodes_on_p[n] = false;
1581 : }
1582 : }
1583 1069517 : }
1584 :
1585 1070337 : if (_deleted_coarse_elements)
1586 : {
1587 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1588 : typedef unsigned char boolish;
1589 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1590 28 : used_nodes_on_proc_vecs;
1591 73633 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1592 56995 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1593 :
1594 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1595 : std::map<processor_id_type, std::set<dof_id_type>>
1596 28 : repartitioned_node_sets_to_push;
1597 :
1598 : auto ids_action_functor =
1599 56969 : [&used_nodes, &repartitioned_node_pids,
1600 : &repartitioned_node_sets_to_push]
1601 : (processor_id_type pid,
1602 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1603 : {
1604 596012 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1605 : {
1606 : // If we don't see a use for our own node, but someone
1607 : // else does, better figure out who should own it next.
1608 97 : if (!used_nodes.count(n))
1609 : {
1610 2 : if (auto it = repartitioned_node_pids.find(n);
1611 2 : sender_could_become_owner)
1612 : {
1613 1 : if (it != repartitioned_node_pids.end() &&
1614 0 : pid < it->second)
1615 0 : it->second = pid;
1616 : else
1617 1 : repartitioned_node_pids[n] = pid;
1618 : }
1619 : else
1620 1 : if (it == repartitioned_node_pids.end())
1621 0 : repartitioned_node_pids[n] =
1622 : DofObject::invalid_processor_id;
1623 :
1624 2 : repartitioned_node_sets_to_push[pid].insert(n);
1625 : }
1626 : }
1627 16650 : };
1628 :
1629 : // We need two pushes instead of a pull here because we need to
1630 : // know *all* the queries for a particular node before we can
1631 : // respond to *any* of them.
1632 : Parallel::push_parallel_vector_data
1633 16638 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1634 :
1635 : // Repartition (what used to be) our own nodes first
1636 16639 : for (auto & [n, p] : repartitioned_node_pids)
1637 : {
1638 1 : Node & node = this->node_ref(n);
1639 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1640 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1641 1 : node.processor_id() = p;
1642 : }
1643 :
1644 : // Then push to repartition others' ghosted copies.
1645 :
1646 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1647 28 : repartitioned_node_vecs;
1648 :
1649 16640 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1650 : {
1651 2 : auto & rn_vec = repartitioned_node_vecs[p];
1652 4 : for (auto n : nodeset)
1653 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1654 : }
1655 :
1656 : auto repartition_node_functor =
1657 2 : [this]
1658 : (processor_id_type libmesh_dbg_var(pid),
1659 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1660 : {
1661 4 : for (auto [n, p] : ids_and_pids)
1662 : {
1663 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1664 2 : Node & node = this->node_ref(n);
1665 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1666 2 : node.processor_id() = p;
1667 : }
1668 16610 : };
1669 :
1670 : Parallel::push_parallel_vector_data
1671 16638 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1672 : }
1673 :
1674 1070337 : _deleted_coarse_elements = false;
1675 :
1676 : // Nodes not connected to any local elements, and nullptr node entries
1677 : // in our container, are deleted
1678 : {
1679 1070337 : node_iterator_imp it = _nodes.begin();
1680 410 : node_iterator_imp end = _nodes.end();
1681 :
1682 95266360 : while (it != end)
1683 : {
1684 94196023 : Node * nd = *it;
1685 94196023 : if (!nd)
1686 3678921 : it = _nodes.erase(it);
1687 90517102 : else if (!used_nodes.count(nd->id()))
1688 : {
1689 : // remove any boundary information associated with
1690 : // this node
1691 2724671 : this->get_boundary_info().remove (nd);
1692 2828 : _constraint_rows.erase(nd);
1693 :
1694 : // delete the node
1695 2724671 : delete nd;
1696 :
1697 2724671 : it = _nodes.erase(it);
1698 : }
1699 : else
1700 45429 : ++it;
1701 : }
1702 : }
1703 :
1704 1070337 : this->_preparation.has_removed_orphaned_nodes = true;
1705 :
1706 1070337 : if (_skip_renumber_nodes_and_elements)
1707 : {
1708 1670 : this->update_parallel_id_counts();
1709 2 : return;
1710 : }
1711 :
1712 : // Finally renumber all the elements
1713 1068667 : _n_elem = this->renumber_dof_objects (this->_elements);
1714 :
1715 : // and all the remaining nodes
1716 1068667 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1717 :
1718 : // And figure out what IDs we should use when adding new nodes and
1719 : // new elements
1720 1068667 : this->update_parallel_id_counts();
1721 :
1722 : // Make sure our caches are up to date and our
1723 : // DofObjects are well packed
1724 : #ifdef DEBUG
1725 408 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1726 408 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1727 408 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1728 408 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1729 408 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1730 408 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1731 408 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1732 408 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1733 :
1734 : // Make sure our ids and flags are consistent
1735 408 : this->libmesh_assert_valid_parallel_ids();
1736 408 : this->libmesh_assert_valid_parallel_flags();
1737 :
1738 : // And make sure we've made our numbering monotonic
1739 408 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1740 : #endif
1741 : }
1742 :
1743 :
1744 :
1745 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1746 : {
1747 : // We can't use range-for here because we need access to the special
1748 : // iterators' methods, not just to their dereferenced values.
1749 :
1750 : // Nodes first
1751 0 : for (auto pr = this->_nodes.begin(),
1752 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1753 : {
1754 5427360 : Node * n = *pr;
1755 5427360 : if (n != nullptr)
1756 : {
1757 0 : const dof_id_type id = pr.index();
1758 5204550 : n->set_id() = id;
1759 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1760 : }
1761 : }
1762 :
1763 : // Elements next
1764 0 : for (auto pr = this->_elements.begin(),
1765 5671231 : end = this->_elements.end(); pr != end; ++pr)
1766 : {
1767 5653821 : Elem * e = *pr;
1768 5653821 : if (e != nullptr)
1769 : {
1770 0 : const dof_id_type id = pr.index();
1771 5590229 : e->set_id() = id;
1772 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1773 : }
1774 : }
1775 17410 : }
1776 :
1777 :
1778 :
1779 760978 : dof_id_type DistributedMesh::n_active_elem () const
1780 : {
1781 556 : parallel_object_only();
1782 :
1783 : // Get local active elements first
1784 : dof_id_type active_elements =
1785 1521400 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1786 2281822 : this->active_local_elements_end()));
1787 760978 : this->comm().sum(active_elements);
1788 :
1789 : // Then add unpartitioned active elements, which should exist on
1790 : // every processor
1791 760978 : active_elements +=
1792 760978 : static_cast<dof_id_type>(std::distance
1793 1521400 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1794 761534 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1795 760978 : return active_elements;
1796 : }
1797 :
1798 :
1799 :
1800 482223 : void DistributedMesh::delete_remote_elements()
1801 : {
1802 : #ifdef DEBUG
1803 : // Make sure our neighbor links are all fine
1804 392 : MeshTools::libmesh_assert_valid_neighbors(*this);
1805 :
1806 : // And our child/parent links, and our flags
1807 392 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1808 :
1809 : // Make sure our ids and flags are consistent
1810 392 : this->libmesh_assert_valid_parallel_ids();
1811 392 : this->libmesh_assert_valid_parallel_flags();
1812 :
1813 392 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1814 392 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1815 392 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1816 392 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1817 392 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1818 392 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1819 : #endif
1820 :
1821 482223 : _is_serial = false;
1822 482223 : _is_serial_on_proc_0 = false;
1823 :
1824 482223 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1825 :
1826 392 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1827 :
1828 : // Now make sure the containers actually shrink - strip
1829 : // any newly-created nullptr voids out of the element array
1830 482223 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1831 392 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1832 54538282 : while (e_it != e_end)
1833 54056059 : if (!*e_it)
1834 39769838 : e_it = _elements.erase(e_it);
1835 : else
1836 17451 : ++e_it;
1837 :
1838 482223 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1839 392 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1840 105336293 : while (n_it != n_end)
1841 104854070 : if (!*n_it)
1842 68726047 : n_it = _nodes.erase(n_it);
1843 : else
1844 48679 : ++n_it;
1845 :
1846 : // We may have deleted no-longer-connected nodes or coarsened-away
1847 : // elements; let's update our caches.
1848 482223 : this->update_parallel_id_counts();
1849 :
1850 : // We may have deleted nodes or elements that were the only local
1851 : // representatives of some particular boundary id(s); let's update
1852 : // those caches.
1853 482223 : this->get_boundary_info().regenerate_id_sets();
1854 :
1855 : #ifdef DEBUG
1856 : // We might not have well-packed objects if the user didn't allow us
1857 : // to renumber
1858 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1859 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1860 :
1861 : // Make sure our neighbor links are all fine
1862 392 : MeshTools::libmesh_assert_valid_neighbors(*this);
1863 :
1864 : // And our child/parent links, and our flags
1865 392 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1866 :
1867 : // Make sure our ids and flags are consistent
1868 392 : this->libmesh_assert_valid_parallel_ids();
1869 392 : this->libmesh_assert_valid_parallel_flags();
1870 : #endif
1871 :
1872 482223 : this->_preparation.has_removed_remote_elements = true;
1873 482223 : }
1874 :
1875 :
1876 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1877 : {
1878 : // First add the elem like normal
1879 0 : add_elem(e);
1880 :
1881 : // Now add it to the set that won't be deleted when we call
1882 : // delete_remote_elements()
1883 0 : _extra_ghost_elems.insert(e);
1884 0 : }
1885 :
1886 : void
1887 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1888 : {
1889 0 : std::set<Elem *> tmp;
1890 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1891 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1892 0 : std::inserter(tmp, tmp.begin()));
1893 0 : _extra_ghost_elems = tmp;
1894 0 : }
1895 :
1896 102286 : void DistributedMesh::allgather()
1897 : {
1898 102286 : if (_is_serial)
1899 0 : return;
1900 102284 : MeshCommunication().allgather(*this);
1901 102284 : _is_serial = true;
1902 102284 : _is_serial_on_proc_0 = true;
1903 :
1904 : // Make sure our caches are up to date and our
1905 : // DofObjects are well packed
1906 : #ifdef DEBUG
1907 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1908 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1909 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1910 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1911 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1912 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1913 :
1914 : // If we've disabled renumbering we can't be sure we're contiguous
1915 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1916 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1917 :
1918 : // Make sure our neighbor links are all fine
1919 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1920 :
1921 : // Make sure our ids and flags are consistent
1922 48 : this->libmesh_assert_valid_parallel_ids();
1923 48 : this->libmesh_assert_valid_parallel_flags();
1924 : #endif
1925 : }
1926 :
1927 18352 : void DistributedMesh::gather_to_zero()
1928 : {
1929 18352 : if (_is_serial_on_proc_0)
1930 4 : return;
1931 :
1932 14114 : _is_serial_on_proc_0 = true;
1933 14114 : MeshCommunication().gather(0, *this);
1934 : }
1935 :
1936 :
1937 : } // namespace libMesh
|
