Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 298807 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 298807 : unsigned char d) :
44 298307 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 298307 : _is_serial_on_proc_0(true),
46 298307 : _deleted_coarse_elements(false),
47 298307 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 299057 : _next_free_local_node_id(this->processor_id()),
49 298807 : _next_free_local_elem_id(this->processor_id()),
50 298807 : _next_free_unpartitioned_node_id(this->n_processors()),
51 298807 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 298807 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 298807 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 299057 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 597614 : default_partitioner);
64 : _partitioner = Partitioner::build
65 597364 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 298807 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 341272 : DistributedMesh::~DistributedMesh ()
143 : {
144 319168 : this->DistributedMesh::clear(); // Free nodes and elements
145 341272 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17734 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 : DistributedMesh(static_cast<const MeshBase &>(other_mesh),
153 17734 : /*bool other_is_distributed_type=*/true)
154 : {
155 17734 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
156 17734 : _next_free_local_node_id =
157 17734 : other_mesh._next_free_local_node_id;
158 17734 : _next_free_local_elem_id =
159 17734 : other_mesh._next_free_local_elem_id;
160 17734 : _next_free_unpartitioned_node_id =
161 17734 : other_mesh._next_free_unpartitioned_node_id;
162 17734 : _next_free_unpartitioned_elem_id =
163 17734 : other_mesh._next_free_unpartitioned_elem_id;
164 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
165 17734 : _next_unique_id =
166 17734 : other_mesh._next_unique_id;
167 17734 : _next_unpartitioned_unique_id =
168 17734 : other_mesh._next_unpartitioned_unique_id;
169 : #endif
170 :
171 : // Need to copy extra_ghost_elems
172 17734 : for (auto & elem : other_mesh._extra_ghost_elems)
173 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
174 17734 : }
175 :
176 :
177 :
178 20361 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
179 20361 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
180 20455 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
181 20173 : _deleted_coarse_elements(true), // better safe than sorry...
182 20173 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
183 20455 : _next_free_local_node_id(this->processor_id()),
184 20361 : _next_free_local_elem_id(this->processor_id()),
185 20361 : _next_free_unpartitioned_node_id(this->n_processors()),
186 40628 : _next_free_unpartitioned_elem_id(this->n_processors())
187 : {
188 : // Just copy, skipping preparation
189 20361 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
190 :
191 188 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
192 188 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
193 188 : this->allow_renumbering(other_mesh.allow_renumbering());
194 188 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
195 188 : this->skip_partitioning(other_mesh.skip_partitioning());
196 :
197 20361 : this->copy_constraint_rows(other_mesh);
198 :
199 94 : auto & this_boundary_info = this->get_boundary_info();
200 94 : const auto & other_boundary_info = other_mesh.get_boundary_info();
201 :
202 20361 : this_boundary_info = other_boundary_info;
203 :
204 94 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
205 :
206 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
207 : // nodes/elements they consider locally owned (or unpartitioned), so after
208 : // copying only the objects visible to this processor, those counts
209 : // reflect an incomplete local view rather than other_mesh's totals,
210 : // and need to be copied afresh.
211 20361 : _n_nodes = other_mesh.n_nodes();
212 20361 : _n_elem = other_mesh.n_elem();
213 20361 : _max_node_id = other_mesh.max_node_id();
214 20361 : _max_elem_id = other_mesh.max_elem_id();
215 :
216 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
217 : // counters directly, mirroring other_mesh's own bookkeeping exactly. This
218 : // is done in the DistributedMesh copy constructor.
219 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
220 : // raw counterpart fields to copy, so derive the free-id-range
221 : // counters from _max_elem_id/_max_node_id the same way
222 : // update_parallel_id_counts() does via the set_next_ids() method. For
223 : // the unique id counters we copy other_mesh's own local next_unique_id()
224 : // and derive the next available unique ids
225 : // (partitioned and unpartitioned) via set_next_unique_ids().
226 20361 : if (!other_is_distributed_type)
227 : {
228 2627 : this->set_next_ids();
229 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
230 2627 : this->set_next_unique_ids(other_mesh.next_unique_id());
231 : #endif
232 : }
233 :
234 : // Copy other_mesh's actual preparation state, including whatever it
235 : // reports for has_synched_id_counts: the counts above are a faithful copy
236 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
237 : // verified computation, so our synced-ness should match other_mesh's.
238 20361 : this->_preparation = other_mesh.preparation();
239 20361 : }
240 :
241 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
242 : {
243 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
244 :
245 4 : this->_nodes = std::move(other_mesh._nodes);
246 270 : this->_n_nodes = other_mesh.n_nodes();
247 :
248 4 : this->_elements = std::move(other_mesh._elements);
249 270 : this->_n_elem = other_mesh.n_elem();
250 :
251 270 : _is_serial = other_mesh.is_serial();
252 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
253 270 : _deleted_coarse_elements = true; // Better safe than sorry
254 :
255 270 : _max_node_id = other_mesh.max_node_id();
256 270 : _max_elem_id = other_mesh.max_elem_id();
257 :
258 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
259 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
260 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
261 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
262 :
263 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
264 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
265 : #endif
266 270 : }
267 :
268 2034875 : void DistributedMesh::set_next_ids()
269 : {
270 2034875 : _next_free_unpartitioned_elem_id =
271 2036649 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
272 2034875 : (this->n_processors() + 1) + this->n_processors();
273 2034875 : _next_free_local_elem_id =
274 2034875 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
275 2034875 : (this->n_processors() + 1) + this->processor_id();
276 :
277 2034875 : _next_free_unpartitioned_node_id =
278 2036649 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
279 2034875 : (this->n_processors() + 1) + this->n_processors();
280 2034875 : _next_free_local_node_id =
281 2034875 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
282 2034875 : (this->n_processors() + 1) + this->processor_id();
283 2034875 : }
284 :
285 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
286 2034875 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
287 : {
288 : // Unique ids are laid out in repeating groups of (n_processors()+1)
289 : // consecutive integers ("alignment" here means a value's position, i.e.
290 : // its residue mod (n_processors()+1), within one of these groups). In
291 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
292 : // are each permanently earmarked for one processor's locally-owned
293 : // objects, and the last slot (residue n_processors()) is permanently
294 : // earmarked for unpartitioned objects.
295 : //
296 : // parallel_max_unique_id carries no guarantee of landing on any
297 : // particular residue, so rather than simply offsetting it by
298 : // processor_id()/n_processors(), we have to explicitly round up to the
299 : // start of the next full, entirely-unused group before adding this
300 : // processor's (or the unpartitioned pool's) fixed offset into that group.
301 : //
302 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
303 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
304 : // the number of complete groups needed to reach at least past the
305 : // current max id. Multiplying back by (n_processors()+1) gives the first
306 : // id of the next group that is guaranteed to be entirely unused; adding
307 : // n_processors(), the unpartitioned pool's fixed offset within a group,
308 : // lands on that pool's reserved slot in that group.
309 2034875 : _next_unpartitioned_unique_id =
310 2036649 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
311 2034875 : (this->n_processors() + 1) +
312 2034875 : this->n_processors();
313 :
314 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
315 : // the rounding accounts for processor_id() possibly being less than the
316 : // residue the raw max id already occupies within its group; this can
317 : // land this processor's slot in the same group as the unpartitioned
318 : // pool's above, or push it into the following group, depending on where
319 : // the max id and this processor's offset happen to fall.
320 2034875 : _next_unique_id =
321 2034875 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
322 2034875 : (this->n_processors() + 1) +
323 2034875 : this->processor_id();
324 2034875 : }
325 : #endif
326 :
327 : // We use cached values for these so they can be called
328 : // from one processor without bothering the rest, but
329 : // we may need to update those caches before doing a full
330 : // renumbering
331 2032248 : void DistributedMesh::update_parallel_id_counts()
332 : {
333 : // This function must be run on all processors at once
334 1700 : parallel_object_only();
335 :
336 2032248 : _n_elem = this->parallel_n_elem();
337 2032248 : _n_nodes = this->parallel_n_nodes();
338 2032248 : _max_node_id = this->parallel_max_node_id();
339 2032248 : _max_elem_id = this->parallel_max_elem_id();
340 :
341 2032248 : this->set_next_ids();
342 :
343 : // set_next_unique_ids() needs the true parallel max unique id (not just
344 : // this processor's possibly-stale cached _next_unique_id) since the whole
345 : // point of this function is to recompute counters that may have drifted
346 : // out of sync.
347 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
348 2032248 : this->set_next_unique_ids(this->parallel_max_unique_id());
349 : #endif
350 :
351 2032248 : this->_preparation.has_synched_id_counts = true;
352 2032248 : }
353 :
354 :
355 : // Or in debug mode we may want to test the uncached values without
356 : // changing the cache
357 2039979 : dof_id_type DistributedMesh::parallel_n_elem() const
358 : {
359 : // This function must be run on all processors at once
360 2554 : parallel_object_only();
361 :
362 2039979 : dof_id_type n_local = this->n_local_elem();
363 2039979 : this->comm().sum(n_local);
364 2039979 : n_local += this->n_unpartitioned_elem();
365 2039979 : return n_local;
366 : }
367 :
368 :
369 :
370 2039294 : dof_id_type DistributedMesh::parallel_max_elem_id() const
371 : {
372 : // This function must be run on all processors at once
373 8746 : parallel_object_only();
374 :
375 2039294 : dof_id_type max_local = 0;
376 :
377 : dofobject_container<Elem>::const_reverse_veclike_iterator
378 8746 : rit = _elements.rbegin();
379 :
380 : const dofobject_container<Elem>::const_reverse_veclike_iterator
381 8746 : rend = _elements.rend();
382 :
383 : // Look for the maximum element id. Search backwards through
384 : // elements so we can break out early. Beware of nullptr entries that
385 : // haven't yet been cleared from _elements.
386 10672929 : for (; rit != rend; ++rit)
387 : {
388 10109831 : const DofObject *d = *rit;
389 19038 : if (d)
390 : {
391 8531 : libmesh_assert(_elements[d->id()] == d);
392 1476196 : max_local = d->id() + 1;
393 1476196 : break;
394 : }
395 : }
396 :
397 2039294 : this->comm().max(max_local);
398 2039294 : return max_local;
399 : }
400 :
401 :
402 :
403 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
404 2117724 : unique_id_type DistributedMesh::parallel_max_unique_id() const
405 : {
406 : // This function must be run on all processors at once
407 1802 : parallel_object_only();
408 :
409 4235448 : unique_id_type max_local = std::max(_next_unique_id,
410 2429736 : _next_unpartitioned_unique_id);
411 2117724 : this->comm().max(max_local);
412 2117724 : return max_local;
413 : }
414 :
415 :
416 :
417 80517 : void DistributedMesh::set_next_unique_id(unique_id_type id)
418 : {
419 148 : _next_unique_id = id;
420 80517 : _next_unpartitioned_unique_id =
421 80665 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
422 80517 : (this->n_processors() + 1) + this->n_processors();
423 80517 : _next_unique_id =
424 80517 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
425 80517 : (this->n_processors() + 1) + this->processor_id();
426 80517 : }
427 : #endif
428 :
429 :
430 :
431 2040427 : dof_id_type DistributedMesh::parallel_n_nodes() const
432 : {
433 : // This function must be run on all processors at once
434 2554 : parallel_object_only();
435 :
436 2040427 : dof_id_type n_local = this->n_local_nodes();
437 2040427 : this->comm().sum(n_local);
438 2040427 : n_local += this->n_unpartitioned_nodes();
439 2040427 : return n_local;
440 : }
441 :
442 :
443 :
444 2036860 : dof_id_type DistributedMesh::parallel_max_node_id() const
445 : {
446 : // This function must be run on all processors at once
447 6312 : parallel_object_only();
448 :
449 2036860 : dof_id_type max_local = 0;
450 :
451 : dofobject_container<Node>::const_reverse_veclike_iterator
452 6312 : rit = _nodes.rbegin();
453 :
454 : const dofobject_container<Node>::const_reverse_veclike_iterator
455 6312 : rend = _nodes.rend();
456 :
457 : // Look for the maximum node id. Search backwards through
458 : // nodes so we can break out early. Beware of nullptr entries that
459 : // haven't yet been cleared from _nodes
460 38856309 : for (; rit != rend; ++rit)
461 : {
462 38293281 : const DofObject *d = *rit;
463 63399 : if (d)
464 : {
465 6167 : libmesh_assert(_nodes[d->id()] == d);
466 1473832 : max_local = d->id() + 1;
467 1473832 : break;
468 : }
469 : }
470 :
471 2036860 : this->comm().max(max_local);
472 2036860 : return max_local;
473 : }
474 :
475 :
476 :
477 43030112 : const Point & DistributedMesh::point (const dof_id_type i) const
478 : {
479 43030112 : return this->node_ref(i);
480 : }
481 :
482 :
483 :
484 44147145 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
485 : {
486 137681 : libmesh_assert(_nodes[i]);
487 137681 : libmesh_assert_equal_to (_nodes[i]->id(), i);
488 :
489 44147145 : return _nodes[i];
490 : }
491 :
492 :
493 :
494 :
495 648072178 : Node * DistributedMesh::node_ptr (const dof_id_type i)
496 : {
497 490796 : libmesh_assert(_nodes[i]);
498 490796 : libmesh_assert_equal_to (_nodes[i]->id(), i);
499 :
500 648072178 : return _nodes[i];
501 : }
502 :
503 :
504 :
505 :
506 3109766 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
507 : {
508 3109766 : if (const auto it = _nodes.find(i);
509 658043 : it != _nodes.end())
510 : {
511 2821270 : const Node * n = *it;
512 369547 : libmesh_assert (!n || n->id() == i);
513 369547 : return n;
514 : }
515 :
516 288496 : return nullptr;
517 : }
518 :
519 :
520 :
521 :
522 371664553 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
523 : {
524 371664553 : if (auto it = _nodes.find(i);
525 159516 : it != _nodes.end())
526 : {
527 288213458 : Node * n = *it;
528 111969 : libmesh_assert (!n || n->id() == i);
529 111969 : return n;
530 : }
531 :
532 47547 : return nullptr;
533 : }
534 :
535 :
536 :
537 :
538 3058465 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
539 : {
540 10105 : libmesh_assert(_elements[i]);
541 10105 : libmesh_assert_equal_to (_elements[i]->id(), i);
542 :
543 3058465 : return _elements[i];
544 : }
545 :
546 :
547 :
548 :
549 288506630 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
550 : {
551 167573 : libmesh_assert(_elements[i]);
552 167573 : libmesh_assert_equal_to (_elements[i]->id(), i);
553 :
554 288506630 : return _elements[i];
555 : }
556 :
557 :
558 :
559 :
560 1458003 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
561 : {
562 1458003 : if (const auto it = _elements.find(i);
563 619033 : it != _elements.end())
564 : {
565 1100460 : const Elem * e = *it;
566 270405 : libmesh_assert (!e || e->id() == i);
567 270405 : return e;
568 : }
569 :
570 348628 : return nullptr;
571 : }
572 :
573 :
574 :
575 :
576 608698339 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
577 : {
578 608698339 : if (auto it = _elements.find(i);
579 194370 : it != _elements.end())
580 : {
581 523114316 : Elem * e = *it;
582 190690 : libmesh_assert (!e || e->id() == i);
583 190690 : return e;
584 : }
585 :
586 3680 : return nullptr;
587 : }
588 :
589 :
590 :
591 :
592 44201393 : Elem * DistributedMesh::add_elem (Elem * e)
593 : {
594 : // Don't try to add nullptrs!
595 19057 : libmesh_assert(e);
596 :
597 : // Trying to add an existing element is a no-op
598 44201393 : if (e->valid_id() && _elements[e->id()] == e)
599 0 : return e;
600 :
601 44201393 : const processor_id_type elem_procid = e->processor_id();
602 :
603 44201393 : if (!e->valid_id())
604 : {
605 : // We should only be creating new ids past the end of the range
606 : // of existing ids
607 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
608 : _max_elem_id);
609 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
610 :
611 : // Use the unpartitioned ids for unpartitioned elems, and
612 : // temporarily for ghost elems
613 3860721 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
614 3864289 : if (elem_procid == this->processor_id())
615 1133676 : next_id = &_next_free_local_elem_id;
616 3860721 : e->set_id (*next_id);
617 : }
618 :
619 : {
620 : // Advance next_ids up high enough that each is pointing to an
621 : // unused id and any subsequent increments will still point us
622 : // to unused ids
623 88402786 : _max_elem_id = std::max(_max_elem_id,
624 44201393 : static_cast<dof_id_type>(e->id()+1));
625 :
626 44201393 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
627 5178748 : _next_free_unpartitioned_elem_id =
628 5178748 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
629 5178748 : (this->n_processors() + 1) + this->n_processors();
630 44201393 : if (_next_free_local_elem_id < _max_elem_id)
631 3475432 : _next_free_local_elem_id =
632 3481715 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
633 3475432 : (this->n_processors() + 1) + this->processor_id();
634 :
635 : #ifndef NDEBUG
636 : // We need a const dofobject_container so we don't inadvertently create
637 : // nullptr entries when testing for non-nullptr ones
638 19057 : const dofobject_container<Elem> & const_elements = _elements;
639 : #endif
640 19057 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
641 19057 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
642 : }
643 :
644 : // Don't try to overwrite existing elems
645 19057 : libmesh_assert (!_elements[e->id()]);
646 :
647 44201393 : _elements[e->id()] = e;
648 :
649 : // We actually added a new element. Some of our caches might still
650 : // be valid, but we should clear the ones which definitely are not.
651 44201393 : this->clear_point_locator();
652 44201393 : this->clear_stored_ranges();
653 :
654 : // Try to make the cached elem data more accurate
655 44220450 : if (elem_procid == this->processor_id() ||
656 : elem_procid == DofObject::invalid_processor_id)
657 13058218 : _n_elem++;
658 :
659 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
660 44201393 : if (!e->valid_unique_id())
661 : {
662 13653767 : if (processor_id() == e->processor_id())
663 : {
664 1133676 : e->set_unique_id(_next_unique_id);
665 1133676 : _next_unique_id += this->n_processors() + 1;
666 : }
667 : else
668 : {
669 12520091 : e->set_unique_id(_next_unpartitioned_unique_id);
670 12520091 : _next_unpartitioned_unique_id += this->n_processors() + 1;
671 : }
672 : }
673 : else
674 : {
675 30572003 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
676 30547626 : _next_unique_id =
677 30547626 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
678 30547626 : (this->n_processors() + 1) + this->processor_id();
679 : }
680 : #endif
681 :
682 : // Unpartitioned elems should be added on every processor
683 : // And shouldn't be added in the same batch as ghost elems
684 : // But we might be just adding on processor 0 to
685 : // broadcast later
686 : // #ifdef DEBUG
687 : // if (elem_procid == DofObject::invalid_processor_id)
688 : // {
689 : // dof_id_type elem_id = e->id();
690 : // this->comm().max(elem_id);
691 : // libmesh_assert_equal_to (elem_id, e->id());
692 : // }
693 : // #endif
694 :
695 : // Make sure any new element is given space for any extra integers
696 : // we've requested
697 44201393 : e->add_extra_integers(_elem_integer_names.size(),
698 44201393 : _elem_integer_default_values);
699 :
700 : // And set mapping type and data on any new element
701 38114 : e->set_mapping_type(this->default_mapping_type());
702 38114 : e->set_mapping_data(this->default_mapping_data());
703 :
704 44201393 : return e;
705 : }
706 :
707 :
708 :
709 14893806 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
710 : {
711 : // The mesh now takes ownership of the Elem. Eventually the guts of
712 : // add_elem() will get moved to a private helper function, and
713 : // calling add_elem() directly will be deprecated.
714 14893806 : return add_elem(e.release());
715 : }
716 :
717 :
718 :
719 3423532 : Elem * DistributedMesh::insert_elem (Elem * e)
720 : {
721 3423532 : if (_elements[e->id()])
722 3423532 : this->delete_elem(_elements[e->id()]);
723 :
724 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
725 3423532 : if (!e->valid_unique_id())
726 : {
727 0 : if (processor_id() == e->processor_id())
728 : {
729 0 : e->set_unique_id(_next_unique_id);
730 0 : _next_unique_id += this->n_processors() + 1;
731 : }
732 : else
733 : {
734 0 : e->set_unique_id(_next_unpartitioned_unique_id);
735 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
736 : }
737 : }
738 : else
739 : {
740 3423532 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
741 3423532 : _next_unique_id =
742 3427228 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
743 3423532 : (this->n_processors() + 1) + this->processor_id();
744 : }
745 : #endif
746 :
747 : // Try to make the cached elem data more accurate
748 3423532 : processor_id_type elem_procid = e->processor_id();
749 3427228 : if (elem_procid == this->processor_id() ||
750 : elem_procid == DofObject::invalid_processor_id)
751 3292664 : _n_elem++;
752 :
753 3423532 : _elements[e->id()] = e;
754 :
755 : // We actually added a new element. Some of our caches might still
756 : // be valid, but we should clear the ones which definitely are not.
757 3423532 : this->clear_point_locator();
758 3423532 : this->clear_stored_ranges();
759 :
760 : // Make sure any new element is given space for any extra integers
761 : // we've requested
762 3423532 : e->add_extra_integers(_elem_integer_names.size(),
763 3423532 : _elem_integer_default_values);
764 :
765 : // And set mapping type and data on any new element
766 7392 : e->set_mapping_type(this->default_mapping_type());
767 7392 : e->set_mapping_data(this->default_mapping_data());
768 :
769 3423532 : return e;
770 : }
771 :
772 3423532 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
773 : {
774 : // The mesh now takes ownership of the Elem. Eventually the guts of
775 : // insert_elem(Elem*) will get moved to a private helper function, and
776 : // calling insert_elem(Elem*) directly will be deprecated.
777 3423532 : return insert_elem(e.release());
778 : }
779 :
780 :
781 40222387 : void DistributedMesh::delete_elem(Elem * e)
782 : {
783 7594 : libmesh_assert (e);
784 :
785 : // Try to make the cached elem data more accurate
786 40222387 : _n_elem--;
787 :
788 : // Was this a coarse element, not just a coarsening where we still
789 : // have some ancestor structure? Was it a *local* element, that we
790 : // might have been depending on as an owner of local nodes? We'll
791 : // have to be more careful with our nodes in contract() later; no
792 : // telling if we just locally orphaned a node that should be
793 : // globally retained.
794 40231801 : if (e->processor_id() == this->processor_id() &&
795 3640 : !e->parent())
796 80793 : _deleted_coarse_elements = true;
797 :
798 : // Delete the element from the BoundaryInfo object
799 40222387 : this->get_boundary_info().remove(e);
800 :
801 : // But not yet from the container; we might invalidate
802 : // an iterator that way!
803 :
804 : //_elements.erase(e->id());
805 :
806 : // Instead, we set it to nullptr for now
807 :
808 40222387 : _elements[e->id()] = nullptr;
809 :
810 : // delete the element
811 40222387 : delete e;
812 :
813 : // Some of our caches might still be valid, but we should clear the
814 : // ones which definitely are not.
815 40222387 : this->clear_point_locator();
816 40222387 : this->clear_stored_ranges();
817 40222387 : }
818 :
819 :
820 :
821 3208068 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
822 : const dof_id_type new_id)
823 : {
824 : // This could be a no-op
825 3208068 : if (old_id == new_id)
826 0 : return;
827 :
828 3208068 : Elem * el = _elements[old_id];
829 1160 : libmesh_assert (el);
830 1160 : libmesh_assert_equal_to (el->id(), old_id);
831 :
832 3208068 : el->set_id(new_id);
833 1160 : libmesh_assert (!_elements[new_id]);
834 3208068 : _elements[new_id] = el;
835 3208068 : _elements.erase(old_id);
836 :
837 : // Should we delete any caches here? Our point locator indexes by
838 : // element pointer and should be fine with an id change. Our stored
839 : // ranges are no longer sorted, which is *probably* fine, but let's
840 : // just be safe.
841 3208068 : this->clear_stored_ranges();
842 : }
843 :
844 :
845 :
846 41604422 : Node * DistributedMesh::add_point (const Point & p,
847 : const dof_id_type id,
848 : const processor_id_type proc_id)
849 : {
850 41604422 : Node * old_n = this->query_node_ptr(id);
851 :
852 41604422 : if (old_n)
853 : {
854 0 : *old_n = p;
855 0 : old_n->processor_id() = proc_id;
856 :
857 0 : return old_n;
858 : }
859 :
860 41564373 : Node * n = Node::build(p, id).release();
861 41604422 : n->processor_id() = proc_id;
862 :
863 41604422 : return DistributedMesh::add_node(n);
864 : }
865 :
866 :
867 4130 : void DistributedMesh::own_node (Node & n)
868 : {
869 : // This had better be a node in our mesh
870 0 : libmesh_assert(_nodes[n.id()] == &n);
871 :
872 4130 : _nodes[n.id()] = nullptr;
873 4130 : _n_nodes--;
874 :
875 0 : n.set_id(DofObject::invalid_id);
876 4130 : n.processor_id() = this->processor_id();
877 :
878 4130 : this->add_node(&n);
879 4130 : }
880 :
881 :
882 72395431 : Node * DistributedMesh::add_node (Node * n)
883 : {
884 : // Don't try to add nullptrs!
885 41761 : libmesh_assert(n);
886 :
887 : // Trying to add an existing node is a no-op
888 72395431 : if (n->valid_id() && _nodes[n->id()] == n)
889 0 : return n;
890 :
891 72395431 : const processor_id_type node_procid = n->processor_id();
892 :
893 72395431 : if (!n->valid_id())
894 : {
895 : // We should only be creating new ids past the end of the range
896 : // of existing ids
897 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
898 : _max_node_id);
899 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
900 :
901 : // Use the unpartitioned ids for unpartitioned nodes,
902 : // and temporarily for ghost nodes
903 17428035 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
904 17445864 : if (node_procid == this->processor_id())
905 2128712 : next_id = &_next_free_local_node_id;
906 17428035 : n->set_id (*next_id);
907 : }
908 :
909 : {
910 : // Advance next_ids up high enough that each is pointing to an
911 : // unused id and any subsequent increments will still point us
912 : // to unused ids
913 144790862 : _max_node_id = std::max(_max_node_id,
914 72395431 : static_cast<dof_id_type>(n->id()+1));
915 :
916 72395431 : if (_next_free_unpartitioned_node_id < _max_node_id)
917 20384919 : _next_free_unpartitioned_node_id =
918 20384919 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
919 20384919 : (this->n_processors() + 1) + this->n_processors();
920 72395431 : if (_next_free_local_node_id < _max_node_id)
921 11877192 : _next_free_local_node_id =
922 11895397 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
923 11877192 : (this->n_processors() + 1) + this->processor_id();
924 :
925 : #ifndef NDEBUG
926 : // We need a const dofobject_container so we don't inadvertently create
927 : // nullptr entries when testing for non-nullptr ones
928 41761 : const dofobject_container<Node> & const_nodes = _nodes;
929 : #endif
930 41761 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
931 41761 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
932 : }
933 :
934 : // Don't try to overwrite existing nodes
935 41761 : libmesh_assert (!_nodes[n->id()]);
936 :
937 72395431 : _nodes[n->id()] = n;
938 :
939 : // Try to make the cached node data more accurate
940 72437192 : if (node_procid == this->processor_id() ||
941 : node_procid == DofObject::invalid_processor_id)
942 39186196 : _n_nodes++;
943 :
944 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
945 72395431 : if (!n->valid_unique_id())
946 : {
947 41606494 : if (processor_id() == n->processor_id())
948 : {
949 2546234 : n->set_unique_id(_next_unique_id);
950 2546234 : _next_unique_id += this->n_processors() + 1;
951 : }
952 : else
953 : {
954 39060260 : n->set_unique_id(_next_unpartitioned_unique_id);
955 39060260 : _next_unpartitioned_unique_id += this->n_processors() + 1;
956 : }
957 : }
958 : else
959 : {
960 30919429 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
961 30788937 : _next_unique_id =
962 30788937 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
963 30788937 : (this->n_processors() + 1) + this->processor_id();
964 : }
965 : #endif
966 :
967 72395431 : n->add_extra_integers(_node_integer_names.size(),
968 72395431 : _node_integer_default_values);
969 :
970 : // Unpartitioned nodes should be added on every processor
971 : // And shouldn't be added in the same batch as ghost nodes
972 : // But we might be just adding on processor 0 to
973 : // broadcast later
974 : // #ifdef DEBUG
975 : // if (node_procid == DofObject::invalid_processor_id)
976 : // {
977 : // dof_id_type node_id = n->id();
978 : // this->comm().max(node_id);
979 : // libmesh_assert_equal_to (node_id, n->id());
980 : // }
981 : // #endif
982 :
983 72395431 : return n;
984 : }
985 :
986 30786879 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
987 : {
988 : // The mesh now takes ownership of the Node. Eventually the guts of
989 : // add_node() will get moved to a private helper function, and
990 : // calling add_node() directly will be deprecated.
991 30786879 : return add_node(n.release());
992 : }
993 :
994 50733298 : void DistributedMesh::delete_node(Node * n)
995 : {
996 4469 : libmesh_assert(n);
997 4469 : libmesh_assert(_nodes[n->id()]);
998 :
999 : // Try to make the cached elem data more accurate
1000 50733298 : _n_nodes--;
1001 :
1002 : // Delete the node from the BoundaryInfo object
1003 50733298 : this->get_boundary_info().remove(n);
1004 8938 : _constraint_rows.erase(n);
1005 :
1006 : // But not yet from the container; we might invalidate
1007 : // an iterator that way!
1008 :
1009 : //_nodes.erase(n->id());
1010 :
1011 : // Instead, we set it to nullptr for now
1012 :
1013 50733298 : _nodes[n->id()] = nullptr;
1014 :
1015 : // delete the node
1016 50733298 : delete n;
1017 50733298 : }
1018 :
1019 :
1020 :
1021 6898992 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1022 : const dof_id_type new_id)
1023 : {
1024 : // This could be a no-op
1025 6898992 : if (old_id == new_id)
1026 0 : return;
1027 :
1028 6898400 : Node * nd = _nodes[old_id];
1029 3234 : libmesh_assert (nd);
1030 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1031 :
1032 : // If we have nodes shipped to this processor for NodeConstraints
1033 : // use, then those nodes will exist in _nodes, but may not be
1034 : // locatable via a TopologyMap due to the insufficiency of elements
1035 : // connecting to them. If local refinement then wants to create a
1036 : // *new* node in the same location, it will initially get a temporary
1037 : // id, and then make_node_ids_parallel_consistent() will try to move
1038 : // it to the canonical id. We need to account for this case to
1039 : // avoid false positives and memory leaks.
1040 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1041 6468 : if (_nodes[new_id])
1042 : {
1043 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1044 : *(Point *)_nodes[old_id]);
1045 0 : _nodes.erase(new_id);
1046 : }
1047 : #else
1048 : // If we aren't shipping nodes for NodeConstraints, there should be
1049 : // no reason for renumbering one node onto another.
1050 : libmesh_assert (!_nodes[new_id]);
1051 : #endif
1052 6898400 : _nodes[new_id] = nd;
1053 6898400 : nd->set_id(new_id);
1054 :
1055 6898400 : _nodes.erase(old_id);
1056 : }
1057 :
1058 :
1059 :
1060 597293 : void DistributedMesh::clear ()
1061 : {
1062 : // Call parent clear function
1063 597293 : MeshBase::clear();
1064 :
1065 : // Clear our elements and nodes
1066 : // There is no need to remove them from
1067 : // the BoundaryInfo data structure since we
1068 : // already cleared it.
1069 597293 : this->DistributedMesh::clear_elems();
1070 :
1071 22993432 : for (auto & node : _nodes)
1072 41613081 : delete node;
1073 :
1074 584 : _nodes.clear();
1075 :
1076 : // We're no longer distributed if we were before
1077 597293 : _is_serial = true;
1078 597293 : _is_serial_on_proc_0 = true;
1079 :
1080 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1081 : // all is copacetic.
1082 597293 : _deleted_coarse_elements = false;
1083 :
1084 : // Correct our caches
1085 597293 : _n_nodes = 0;
1086 597293 : _max_node_id = 0;
1087 597877 : _next_free_local_node_id = this->processor_id();
1088 597293 : _next_free_unpartitioned_node_id = this->n_processors();
1089 597293 : }
1090 :
1091 :
1092 :
1093 603693 : void DistributedMesh::clear_elems ()
1094 : {
1095 9358480 : for (auto & elem : _elements)
1096 8754787 : delete elem;
1097 :
1098 584 : _elements.clear();
1099 :
1100 : // Correct our caches
1101 603693 : _n_elem = 0;
1102 603693 : _max_elem_id = 0;
1103 604277 : _next_free_local_elem_id = this->processor_id();
1104 603693 : _next_free_unpartitioned_elem_id = this->n_processors();
1105 603693 : }
1106 :
1107 :
1108 :
1109 595142 : void DistributedMesh::redistribute ()
1110 : {
1111 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1112 595142 : if (!this->is_serial())
1113 : {
1114 : // Construct a MeshCommunication object to actually redistribute the nodes
1115 : // and elements according to the partitioner, and then to re-gather the neighbors.
1116 : MeshCommunication mc;
1117 70338 : mc.redistribute(*this);
1118 :
1119 70338 : this->update_parallel_id_counts();
1120 :
1121 : // We communicate valid neighbor links for newly-redistributed
1122 : // elements, but we may still have remote_elem links that should
1123 : // be corrected but whose correction wasn't communicated.
1124 70610 : this->find_neighbors(/*reset_remote_elements*/ false,
1125 : /*reset_current_list*/ false /*non-default*/,
1126 : /*assert_valid*/ false,
1127 272 : /*check_non_remote*/ false /*non-default!*/);
1128 :
1129 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1130 : // anyway... but users can always call partition directly, in which case we do need
1131 : // to call delete_remote_elements()...
1132 : //
1133 : // Regardless of whether it's necessary, it isn't safe. We
1134 : // haven't communicated new node processor_ids yet, and we can't
1135 : // delete nodes until we do.
1136 : // this->delete_remote_elements();
1137 : }
1138 : else
1139 : // The base class can handle non-distributed things, like
1140 : // notifying any GhostingFunctors of changes
1141 524804 : MeshBase::redistribute();
1142 595142 : }
1143 :
1144 :
1145 :
1146 507742 : void DistributedMesh::update_post_partitioning ()
1147 : {
1148 : // this->recalculate_n_partitions();
1149 :
1150 : // Let's do the base class cache clearing first, just in case our
1151 : // later computations are ever changed to make use of a local
1152 : // elements range cache
1153 507742 : this->MeshBase::update_post_partitioning();
1154 :
1155 : // Partitioning changes our numbers of unpartitioned objects
1156 507742 : this->update_parallel_id_counts();
1157 507742 : }
1158 :
1159 :
1160 :
1161 : template <typename T>
1162 3380 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1163 : {
1164 : // This function must be run on all processors at once
1165 3380 : parallel_object_only();
1166 :
1167 3380 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1168 3380 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1169 3380 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1170 :
1171 894412 : for (dof_id_type i=0; i != pmax_id; ++i)
1172 : {
1173 891032 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1174 :
1175 : // Local lookups by id should return the requested object
1176 891032 : libmesh_assert(!obj || obj->id() == i);
1177 :
1178 : // All processors with an object should agree on id
1179 : #ifndef NDEBUG
1180 891032 : const dof_id_type dofid = obj && obj->valid_id() ?
1181 : obj->id() : DofObject::invalid_id;
1182 891032 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1183 : #endif
1184 :
1185 : // All processors with an object should agree on processor id
1186 891032 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1187 : obj->processor_id() : DofObject::invalid_processor_id;
1188 891032 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1189 :
1190 891032 : dof_id_type min_procid = procid;
1191 891032 : this->comm().min(min_procid);
1192 :
1193 : // Either:
1194 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1195 : // or
1196 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1197 :
1198 : // Original assert logic
1199 : // libmesh_assert (min_procid != this->processor_id() || obj);
1200 :
1201 : // More human-understandable logic...
1202 891032 : libmesh_assert (
1203 : ((min_procid == this->processor_id()) && obj)
1204 : ||
1205 : (min_procid != this->processor_id())
1206 : );
1207 :
1208 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1209 : // All processors with an object should agree on unique id
1210 891032 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1211 891032 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1212 : #endif
1213 : }
1214 3380 : }
1215 :
1216 :
1217 :
1218 1690 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1219 : {
1220 1690 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1221 1690 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1222 1690 : }
1223 :
1224 :
1225 :
1226 406 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1227 : {
1228 : #ifndef NDEBUG
1229 : // This function must be run on all processors at once
1230 406 : parallel_object_only();
1231 :
1232 406 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1233 :
1234 21970 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1235 : {
1236 21564 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1237 :
1238 21564 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1239 :
1240 : // All processors with an active element should agree on p level
1241 21564 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1242 : }
1243 : #endif
1244 406 : }
1245 :
1246 :
1247 :
1248 :
1249 1638 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1250 : {
1251 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1252 : // This function must be run on all processors at once
1253 1638 : parallel_object_only();
1254 :
1255 1638 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1256 :
1257 144704 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1258 : {
1259 143066 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1260 :
1261 143066 : unsigned int refinement_flag = el ?
1262 65765 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1263 143066 : unsigned int p_refinement_flag = el ?
1264 65765 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1265 :
1266 143066 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1267 :
1268 : // p refinement flags aren't always kept correct on inactive
1269 : // ghost elements
1270 143066 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1271 : }
1272 : #endif // LIBMESH_ENABLE_AMR
1273 1638 : }
1274 :
1275 :
1276 :
1277 : template <typename T>
1278 : dof_id_type
1279 1843418 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1280 : {
1281 : // This function must be run on all processors at once
1282 808 : parallel_object_only();
1283 :
1284 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1285 :
1286 : // In parallel we may not know what objects other processors have.
1287 : // Start by figuring out how many
1288 808 : dof_id_type unpartitioned_objects = 0;
1289 :
1290 : std::unordered_map<processor_id_type, dof_id_type>
1291 1616 : ghost_objects_from_proc;
1292 :
1293 1843418 : object_iterator it = objects.begin();
1294 808 : object_iterator end = objects.end();
1295 :
1296 111877875 : while (it != end)
1297 : {
1298 110034457 : T * obj = *it;
1299 :
1300 : // Remove any nullptr container entries while we're here.
1301 110034457 : if (!obj)
1302 1424826 : it = objects.erase(it);
1303 : else
1304 : {
1305 108609631 : processor_id_type obj_procid = obj->processor_id();
1306 108609631 : if (obj_procid == DofObject::invalid_processor_id)
1307 43120004 : unpartitioned_objects++;
1308 : else
1309 65489627 : ghost_objects_from_proc[obj_procid]++;
1310 :
1311 : // Finally, increment the iterator
1312 56464 : ++it;
1313 : }
1314 : }
1315 :
1316 1845034 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1317 1844226 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1318 808 : this->comm().allgather
1319 1843913 : ((this_it == ghost_objects_from_proc.end()) ?
1320 495 : dof_id_type(0) : this_it->second, objects_on_proc);
1321 :
1322 : #ifndef NDEBUG
1323 808 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1324 2424 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1325 1616 : if (ghost_objects_from_proc.count(p))
1326 962 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1327 : else
1328 654 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1329 : #endif
1330 :
1331 : // We'll renumber objects in blocks by processor id
1332 1845034 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1333 19562274 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1334 17722088 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1335 808 : objects_on_proc[i-1];
1336 1844226 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1337 1843418 : dof_id_type first_free_id =
1338 1845034 : first_object_on_proc[this->n_processors()-1] +
1339 808 : objects_on_proc[this->n_processors()-1] +
1340 : unpartitioned_objects;
1341 :
1342 : // First set new local object ids and build request sets
1343 : // for non-local object ids
1344 :
1345 : // Request sets to send to each processor
1346 : std::map<processor_id_type, std::vector<dof_id_type>>
1347 808 : requested_ids;
1348 :
1349 : // We know how many objects live on each processor, so reserve() space for
1350 : // each.
1351 808 : auto ghost_end = ghost_objects_from_proc.end();
1352 21405692 : for (auto p : make_range(this->n_processors()))
1353 19563890 : if (p != this->processor_id())
1354 : {
1355 17719664 : if (const auto p_it = ghost_objects_from_proc.find(p);
1356 808 : p_it != ghost_end)
1357 3482185 : requested_ids[p].reserve(p_it->second);
1358 : }
1359 :
1360 808 : end = objects.end();
1361 110453049 : for (it = objects.begin(); it != end; ++it)
1362 : {
1363 108609631 : T * obj = *it;
1364 108609631 : if (!obj)
1365 0 : continue;
1366 108666095 : if (obj->processor_id() == this->processor_id())
1367 20002578 : obj->set_id(next_id++);
1368 88607053 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1369 45487049 : requested_ids[obj->processor_id()].push_back(obj->id());
1370 : }
1371 :
1372 : // Next set ghost object ids from other processors
1373 :
1374 1860632 : auto gather_functor =
1375 48935740 : [
1376 : #ifndef NDEBUG
1377 : this,
1378 : &first_object_on_proc,
1379 : &objects_on_proc,
1380 : #endif
1381 : &objects]
1382 : (processor_id_type, const std::vector<dof_id_type> & ids,
1383 934 : std::vector<dof_id_type> & new_ids)
1384 : {
1385 934 : std::size_t ids_size = ids.size();
1386 3482185 : new_ids.resize(ids_size);
1387 :
1388 48969234 : for (std::size_t i=0; i != ids_size; ++i)
1389 : {
1390 45487049 : T * obj = objects[ids[i]];
1391 16280 : libmesh_assert(obj);
1392 16280 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1393 45503329 : new_ids[i] = obj->id();
1394 :
1395 16280 : libmesh_assert_greater_equal (new_ids[i],
1396 : first_object_on_proc[this->processor_id()]);
1397 16280 : libmesh_assert_less (new_ids[i],
1398 : first_object_on_proc[this->processor_id()] +
1399 : objects_on_proc[this->processor_id()]);
1400 : }
1401 : };
1402 :
1403 1860632 : auto action_functor =
1404 48935740 : [
1405 : #ifndef NDEBUG
1406 : &first_object_on_proc,
1407 : &objects_on_proc,
1408 : #endif
1409 : &objects]
1410 : (processor_id_type libmesh_dbg_var(pid),
1411 : const std::vector<dof_id_type> & ids,
1412 934 : const std::vector<dof_id_type> & data)
1413 : {
1414 : // Copy the id changes we've now been informed of
1415 48969234 : for (auto i : index_range(ids))
1416 : {
1417 45487049 : T * obj = objects[ids[i]];
1418 16280 : libmesh_assert (obj);
1419 16280 : libmesh_assert_equal_to (obj->processor_id(), pid);
1420 16280 : libmesh_assert_greater_equal (data[i],
1421 : first_object_on_proc[pid]);
1422 16280 : libmesh_assert_less (data[i],
1423 : first_object_on_proc[pid] +
1424 : objects_on_proc[pid]);
1425 45503329 : obj->set_id(data[i]);
1426 : }
1427 : };
1428 :
1429 808 : const dof_id_type * ex = nullptr;
1430 : Parallel::pull_parallel_vector_data
1431 1843418 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1432 :
1433 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1434 1860632 : auto unique_gather_functor =
1435 48935740 : [
1436 : #ifndef NDEBUG
1437 : this,
1438 : #endif
1439 : &objects]
1440 : (processor_id_type, const std::vector<dof_id_type> & ids,
1441 934 : std::vector<unique_id_type> & data)
1442 : {
1443 934 : std::size_t ids_size = ids.size();
1444 3482185 : data.resize(ids_size);
1445 :
1446 48969234 : for (std::size_t i=0; i != ids_size; ++i)
1447 : {
1448 45487049 : T * obj = objects[ids[i]];
1449 16280 : libmesh_assert(obj);
1450 16280 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1451 90974098 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1452 : }
1453 : };
1454 :
1455 1860632 : auto unique_action_functor =
1456 48935740 : [&objects]
1457 : (processor_id_type libmesh_dbg_var(pid),
1458 : const std::vector<dof_id_type> & ids,
1459 934 : const std::vector<unique_id_type> & data)
1460 : {
1461 48969234 : for (auto i : index_range(ids))
1462 : {
1463 45487049 : T * obj = objects[ids[i]];
1464 16280 : libmesh_assert (obj);
1465 16280 : libmesh_assert_equal_to (obj->processor_id(), pid);
1466 45487049 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1467 0 : obj->set_unique_id(data[i]);
1468 : }
1469 : };
1470 :
1471 808 : const unique_id_type * unique_ex = nullptr;
1472 : Parallel::pull_parallel_vector_data
1473 1843418 : (this->comm(), requested_ids, unique_gather_functor,
1474 : unique_action_functor, unique_ex);
1475 : #endif
1476 :
1477 : // Next set unpartitioned object ids
1478 808 : next_id = 0;
1479 21405692 : for (auto i : make_range(this->n_processors()))
1480 19563890 : next_id += objects_on_proc[i];
1481 110453049 : for (it = objects.begin(); it != end; ++it)
1482 : {
1483 108609631 : T * obj = *it;
1484 108609631 : if (!obj)
1485 0 : continue;
1486 108609631 : if (obj->processor_id() == DofObject::invalid_processor_id)
1487 43120004 : obj->set_id(next_id++);
1488 : }
1489 :
1490 : // Finally shuffle around objects so that container indices
1491 : // match ids
1492 1843418 : it = objects.begin();
1493 808 : end = objects.end();
1494 116951304 : while (it != end)
1495 : {
1496 115107886 : T * obj = *it;
1497 115107886 : if (obj) // don't try shuffling already-nullptr entries
1498 : {
1499 115107886 : T * next = objects[obj->id()];
1500 : // If we have to move this object
1501 115107886 : if (next != obj)
1502 : {
1503 : // nullptr out its original position for now
1504 : // (our shuffling may put another object there shortly)
1505 39995152 : *it = nullptr;
1506 :
1507 : // There may already be another object with this id that
1508 : // needs to be moved itself
1509 51802699 : while (next)
1510 : {
1511 : // We shouldn't be trying to give two objects the
1512 : // same id
1513 9385 : libmesh_assert_not_equal_to (next->id(), obj->id());
1514 11807547 : objects[obj->id()] = obj;
1515 9385 : obj = next;
1516 11807547 : next = objects[obj->id()];
1517 : }
1518 39995152 : objects[obj->id()] = obj;
1519 : }
1520 : }
1521 :
1522 : // Remove any container entries that were left as nullptr.
1523 115380 : if (!obj)
1524 0 : it = objects.erase(it);
1525 : else
1526 57690 : ++it;
1527 : }
1528 :
1529 1844226 : return first_free_id;
1530 : }
1531 :
1532 :
1533 923379 : void DistributedMesh::renumber_nodes_and_elements ()
1534 : {
1535 406 : parallel_object_only();
1536 :
1537 : #ifdef DEBUG
1538 : // Make sure our ids and flags are consistent
1539 406 : this->libmesh_assert_valid_parallel_ids();
1540 406 : this->libmesh_assert_valid_parallel_flags();
1541 406 : this->libmesh_assert_valid_parallel_p_levels();
1542 : #endif
1543 :
1544 406 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1545 :
1546 : // Nodes not connected to any elements, and nullptr node entries
1547 : // in our container, should be deleted. But wait! If we've deleted coarse
1548 : // local elements on some processor, other processors might have ghosted
1549 : // nodes from it that are now no longer connected to any elements on it, but
1550 : // that are connected to their own semilocal elements. We'll have to
1551 : // communicate to ascertain if that's the case.
1552 923379 : this->comm().max(_deleted_coarse_elements);
1553 :
1554 : // What used nodes do we see on our proc?
1555 406 : std::set<dof_id_type> used_nodes;
1556 :
1557 : // What used node info should we send from our proc? Could we take ownership
1558 : // of each node if we needed to?
1559 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1560 406 : used_nodes_on_proc;
1561 :
1562 : // flag the nodes we need
1563 57820272 : for (auto & elem : this->element_ptr_range())
1564 301183787 : for (const Node & node : elem->node_ref_range())
1565 : {
1566 273167381 : const dof_id_type n = node.id();
1567 273022915 : used_nodes.insert(n);
1568 273167381 : if (_deleted_coarse_elements)
1569 : {
1570 3145701 : const processor_id_type p = node.processor_id();
1571 3145985 : if (p != this->processor_id())
1572 : {
1573 2257624 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1574 2257766 : if (elem->processor_id() == this->processor_id())
1575 216210 : used_nodes_on_p[n] = true;
1576 : else
1577 105 : if (!used_nodes_on_p.count(n))
1578 525626 : used_nodes_on_p[n] = false;
1579 : }
1580 : }
1581 922567 : }
1582 :
1583 923379 : if (_deleted_coarse_elements)
1584 : {
1585 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1586 : typedef unsigned char boolish;
1587 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1588 28 : used_nodes_on_proc_vecs;
1589 73209 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1590 56827 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1591 :
1592 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1593 : std::map<processor_id_type, std::set<dof_id_type>>
1594 28 : repartitioned_node_sets_to_push;
1595 :
1596 : auto ids_action_functor =
1597 56801 : [&used_nodes, &repartitioned_node_pids,
1598 : &repartitioned_node_sets_to_push]
1599 : (processor_id_type pid,
1600 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1601 : {
1602 592596 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1603 : {
1604 : // If we don't see a use for our own node, but someone
1605 : // else does, better figure out who should own it next.
1606 97 : if (!used_nodes.count(n))
1607 : {
1608 2 : if (auto it = repartitioned_node_pids.find(n);
1609 2 : sender_could_become_owner)
1610 : {
1611 1 : if (it != repartitioned_node_pids.end() &&
1612 0 : pid < it->second)
1613 0 : it->second = pid;
1614 : else
1615 1 : repartitioned_node_pids[n] = pid;
1616 : }
1617 : else
1618 1 : if (it == repartitioned_node_pids.end())
1619 0 : repartitioned_node_pids[n] =
1620 : DofObject::invalid_processor_id;
1621 :
1622 2 : repartitioned_node_sets_to_push[pid].insert(n);
1623 : }
1624 : }
1625 16394 : };
1626 :
1627 : // We need two pushes instead of a pull here because we need to
1628 : // know *all* the queries for a particular node before we can
1629 : // respond to *any* of them.
1630 : Parallel::push_parallel_vector_data
1631 16382 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1632 :
1633 : // Repartition (what used to be) our own nodes first
1634 16383 : for (auto & [n, p] : repartitioned_node_pids)
1635 : {
1636 1 : Node & node = this->node_ref(n);
1637 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1638 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1639 1 : node.processor_id() = p;
1640 : }
1641 :
1642 : // Then push to repartition others' ghosted copies.
1643 :
1644 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1645 28 : repartitioned_node_vecs;
1646 :
1647 16384 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1648 : {
1649 2 : auto & rn_vec = repartitioned_node_vecs[p];
1650 4 : for (auto n : nodeset)
1651 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1652 : }
1653 :
1654 : auto repartition_node_functor =
1655 2 : [this]
1656 : (processor_id_type libmesh_dbg_var(pid),
1657 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1658 : {
1659 4 : for (auto [n, p] : ids_and_pids)
1660 : {
1661 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1662 2 : Node & node = this->node_ref(n);
1663 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1664 2 : node.processor_id() = p;
1665 : }
1666 16354 : };
1667 :
1668 : Parallel::push_parallel_vector_data
1669 16382 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1670 : }
1671 :
1672 923379 : _deleted_coarse_elements = false;
1673 :
1674 : // Nodes not connected to any local elements, and nullptr node entries
1675 : // in our container, are deleted
1676 : {
1677 923379 : node_iterator_imp it = _nodes.begin();
1678 406 : node_iterator_imp end = _nodes.end();
1679 :
1680 88446973 : while (it != end)
1681 : {
1682 87523594 : Node * nd = *it;
1683 87523594 : if (!nd)
1684 3491519 : it = _nodes.erase(it);
1685 84032075 : else if (!used_nodes.count(nd->id()))
1686 : {
1687 : // remove any boundary information associated with
1688 : // this node
1689 2405183 : this->get_boundary_info().remove (nd);
1690 2828 : _constraint_rows.erase(nd);
1691 :
1692 : // delete the node
1693 2405183 : delete nd;
1694 :
1695 2405183 : it = _nodes.erase(it);
1696 : }
1697 : else
1698 45303 : ++it;
1699 : }
1700 : }
1701 :
1702 923379 : this->_preparation.has_removed_orphaned_nodes = true;
1703 :
1704 923379 : if (_skip_renumber_nodes_and_elements)
1705 : {
1706 1670 : this->update_parallel_id_counts();
1707 2 : return;
1708 : }
1709 :
1710 : // Finally renumber all the elements
1711 921709 : _n_elem = this->renumber_dof_objects (this->_elements);
1712 :
1713 : // and all the remaining nodes
1714 921709 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1715 :
1716 : // And figure out what IDs we should use when adding new nodes and
1717 : // new elements
1718 921709 : this->update_parallel_id_counts();
1719 :
1720 : // Make sure our caches are up to date and our
1721 : // DofObjects are well packed
1722 : #ifdef DEBUG
1723 404 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1724 404 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1725 404 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1726 404 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1727 404 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1728 404 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1729 404 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1730 404 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1731 :
1732 : // Make sure our ids and flags are consistent
1733 404 : this->libmesh_assert_valid_parallel_ids();
1734 404 : this->libmesh_assert_valid_parallel_flags();
1735 :
1736 : // And make sure we've made our numbering monotonic
1737 404 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1738 : #endif
1739 : }
1740 :
1741 :
1742 :
1743 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1744 : {
1745 : // We can't use range-for here because we need access to the special
1746 : // iterators' methods, not just to their dereferenced values.
1747 :
1748 : // Nodes first
1749 0 : for (auto pr = this->_nodes.begin(),
1750 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1751 : {
1752 5427360 : Node * n = *pr;
1753 5427360 : if (n != nullptr)
1754 : {
1755 0 : const dof_id_type id = pr.index();
1756 5204550 : n->set_id() = id;
1757 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1758 : }
1759 : }
1760 :
1761 : // Elements next
1762 0 : for (auto pr = this->_elements.begin(),
1763 5671231 : end = this->_elements.end(); pr != end; ++pr)
1764 : {
1765 5653821 : Elem * e = *pr;
1766 5653821 : if (e != nullptr)
1767 : {
1768 0 : const dof_id_type id = pr.index();
1769 5590229 : e->set_id() = id;
1770 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1771 : }
1772 : }
1773 17410 : }
1774 :
1775 :
1776 :
1777 673995 : dof_id_type DistributedMesh::n_active_elem () const
1778 : {
1779 554 : parallel_object_only();
1780 :
1781 : // Get local active elements first
1782 : dof_id_type active_elements =
1783 1347436 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1784 2020877 : this->active_local_elements_end()));
1785 673995 : this->comm().sum(active_elements);
1786 :
1787 : // Then add unpartitioned active elements, which should exist on
1788 : // every processor
1789 673995 : active_elements +=
1790 673995 : static_cast<dof_id_type>(std::distance
1791 1347436 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1792 674549 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1793 673995 : return active_elements;
1794 : }
1795 :
1796 :
1797 :
1798 427240 : void DistributedMesh::delete_remote_elements()
1799 : {
1800 : #ifdef DEBUG
1801 : // Make sure our neighbor links are all fine
1802 390 : MeshTools::libmesh_assert_valid_neighbors(*this);
1803 :
1804 : // And our child/parent links, and our flags
1805 390 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1806 :
1807 : // Make sure our ids and flags are consistent
1808 390 : this->libmesh_assert_valid_parallel_ids();
1809 390 : this->libmesh_assert_valid_parallel_flags();
1810 :
1811 390 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1812 390 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1813 390 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1814 390 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1815 390 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1816 390 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1817 : #endif
1818 :
1819 427240 : _is_serial = false;
1820 427240 : _is_serial_on_proc_0 = false;
1821 :
1822 427240 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1823 :
1824 390 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1825 :
1826 : // Now make sure the containers actually shrink - strip
1827 : // any newly-created nullptr voids out of the element array
1828 427240 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1829 390 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1830 53652197 : while (e_it != e_end)
1831 53224957 : if (!*e_it)
1832 39540700 : e_it = _elements.erase(e_it);
1833 : else
1834 17409 : ++e_it;
1835 :
1836 427240 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1837 390 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1838 101021280 : while (n_it != n_end)
1839 100594040 : if (!*n_it)
1840 66856422 : n_it = _nodes.erase(n_it);
1841 : else
1842 48635 : ++n_it;
1843 :
1844 : // We may have deleted no-longer-connected nodes or coarsened-away
1845 : // elements; let's update our caches.
1846 427240 : this->update_parallel_id_counts();
1847 :
1848 : // We may have deleted nodes or elements that were the only local
1849 : // representatives of some particular boundary id(s); let's update
1850 : // those caches.
1851 427240 : this->get_boundary_info().regenerate_id_sets();
1852 :
1853 : #ifdef DEBUG
1854 : // We might not have well-packed objects if the user didn't allow us
1855 : // to renumber
1856 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1857 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1858 :
1859 : // Make sure our neighbor links are all fine
1860 390 : MeshTools::libmesh_assert_valid_neighbors(*this);
1861 :
1862 : // And our child/parent links, and our flags
1863 390 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1864 :
1865 : // Make sure our ids and flags are consistent
1866 390 : this->libmesh_assert_valid_parallel_ids();
1867 390 : this->libmesh_assert_valid_parallel_flags();
1868 : #endif
1869 :
1870 427240 : this->_preparation.has_removed_remote_elements = true;
1871 427240 : }
1872 :
1873 :
1874 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1875 : {
1876 : // First add the elem like normal
1877 0 : add_elem(e);
1878 :
1879 : // Now add it to the set that won't be deleted when we call
1880 : // delete_remote_elements()
1881 0 : _extra_ghost_elems.insert(e);
1882 0 : }
1883 :
1884 : void
1885 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1886 : {
1887 0 : std::set<Elem *> tmp;
1888 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1889 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1890 0 : std::inserter(tmp, tmp.begin()));
1891 0 : _extra_ghost_elems = tmp;
1892 0 : }
1893 :
1894 98732 : void DistributedMesh::allgather()
1895 : {
1896 98732 : if (_is_serial)
1897 0 : return;
1898 98730 : MeshCommunication().allgather(*this);
1899 98730 : _is_serial = true;
1900 98730 : _is_serial_on_proc_0 = true;
1901 :
1902 : // Make sure our caches are up to date and our
1903 : // DofObjects are well packed
1904 : #ifdef DEBUG
1905 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1906 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1907 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1908 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1909 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1910 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1911 :
1912 : // If we've disabled renumbering we can't be sure we're contiguous
1913 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1914 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1915 :
1916 : // Make sure our neighbor links are all fine
1917 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1918 :
1919 : // Make sure our ids and flags are consistent
1920 48 : this->libmesh_assert_valid_parallel_ids();
1921 48 : this->libmesh_assert_valid_parallel_flags();
1922 : #endif
1923 : }
1924 :
1925 18352 : void DistributedMesh::gather_to_zero()
1926 : {
1927 18352 : if (_is_serial_on_proc_0)
1928 4 : return;
1929 :
1930 14114 : _is_serial_on_proc_0 = true;
1931 14114 : MeshCommunication().gather(0, *this);
1932 : }
1933 :
1934 :
1935 : } // namespace libMesh
|
