Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 419167 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 419167 : unsigned char d) :
44 418643 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 418643 : _is_serial_on_proc_0(true),
46 418643 : _deleted_coarse_elements(false),
47 418643 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 419429 : _next_free_local_node_id(this->processor_id()),
49 419167 : _next_free_local_elem_id(this->processor_id()),
50 419167 : _next_free_unpartitioned_node_id(this->n_processors()),
51 419167 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 419167 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 419167 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 419429 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 838334 : default_partitioner);
64 : _partitioner = Partitioner::build
65 838072 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 419167 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 8 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 13066 : std::string_view DistributedMesh::subclass_first_difference_from (const MeshBase & other_mesh_base) const
99 : {
100 748 : const DistributedMesh * dist_mesh_ptr =
101 13066 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 13066 : if (!dist_mesh_ptr)
103 0 : return "DistributedMesh class";
104 748 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 : #define CHECK_MEMBER(member_name) \
107 : if (member_name != other_mesh.member_name) \
108 : return #member_name;
109 :
110 13066 : CHECK_MEMBER(_is_serial);
111 13066 : CHECK_MEMBER(_is_serial_on_proc_0);
112 13066 : CHECK_MEMBER(_deleted_coarse_elements);
113 13066 : CHECK_MEMBER(_n_nodes);
114 13066 : CHECK_MEMBER(_n_elem);
115 13066 : CHECK_MEMBER(_max_node_id);
116 13066 : CHECK_MEMBER(_max_elem_id);
117 : // We expect these things to change in a prepare_for_use();
118 : // they're conceptually "mutable"...
119 : /*
120 : CHECK_MEMBER(_next_free_local_node_id);
121 : CHECK_MEMBER(_next_free_local_elem_id);
122 : CHECK_MEMBER(_next_free_unpartitioned_node_id);
123 : CHECK_MEMBER(_next_free_unpartitioned_elem_id);
124 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
125 : CHECK_MEMBER(_next_unpartitioned_unique_id);
126 : #endif
127 : */
128 13066 : if (!this->nodes_and_elements_equal(other_mesh))
129 63 : return "nodes and/or elements";
130 :
131 13749 : if (_extra_ghost_elems.size() !=
132 746 : other_mesh._extra_ghost_elems.size())
133 0 : return "_extra_ghost_elems size";
134 13003 : for (auto & elem : _extra_ghost_elems)
135 : {
136 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
137 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
138 0 : if (!other_elem ||
139 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
140 0 : return "_extra_ghost_elems entry";
141 : }
142 :
143 13003 : return "";
144 : }
145 :
146 462352 : DistributedMesh::~DistributedMesh ()
147 : {
148 439888 : this->DistributedMesh::clear(); // Free nodes and elements
149 462352 : }
150 :
151 :
152 : // This might be specialized later, but right now it's just here to
153 : // make sure the compiler doesn't give us a default (non-deep) copy
154 : // constructor instead.
155 18094 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
156 : DistributedMesh(static_cast<const MeshBase &>(other_mesh),
157 18094 : /*bool other_is_distributed_type=*/true)
158 : {
159 18094 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
160 18094 : _next_free_local_node_id =
161 18094 : other_mesh._next_free_local_node_id;
162 18094 : _next_free_local_elem_id =
163 18094 : other_mesh._next_free_local_elem_id;
164 18094 : _next_free_unpartitioned_node_id =
165 18094 : other_mesh._next_free_unpartitioned_node_id;
166 18094 : _next_free_unpartitioned_elem_id =
167 18094 : other_mesh._next_free_unpartitioned_elem_id;
168 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
169 18094 : _next_unique_id =
170 18094 : other_mesh._next_unique_id;
171 18094 : _next_unpartitioned_unique_id =
172 18094 : other_mesh._next_unpartitioned_unique_id;
173 : #endif
174 :
175 : // Need to copy extra_ghost_elems
176 18094 : for (auto & elem : other_mesh._extra_ghost_elems)
177 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
178 18094 : }
179 :
180 :
181 :
182 20721 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
183 20721 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
184 21175 : _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
185 20173 : _deleted_coarse_elements(true), // better safe than sorry...
186 20173 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
187 21175 : _next_free_local_node_id(this->processor_id()),
188 21081 : _next_free_local_elem_id(this->processor_id()),
189 21081 : _next_free_unpartitioned_node_id(this->n_processors()),
190 41348 : _next_free_unpartitioned_elem_id(this->n_processors())
191 : {
192 : // Just copy, skipping preparation
193 20721 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
194 :
195 548 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
196 548 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
197 548 : this->allow_renumbering(other_mesh.allow_renumbering());
198 548 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
199 548 : this->skip_partitioning(other_mesh.skip_partitioning());
200 :
201 20721 : this->copy_constraint_rows(other_mesh);
202 :
203 454 : auto & this_boundary_info = this->get_boundary_info();
204 454 : const auto & other_boundary_info = other_mesh.get_boundary_info();
205 :
206 20721 : this_boundary_info = other_boundary_info;
207 :
208 454 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
209 :
210 : // add_node()/add_elem() only increment _n_nodes/_n_elem for
211 : // nodes/elements they consider locally owned (or unpartitioned), so after
212 : // copying only the objects visible to this processor, those counts
213 : // reflect an incomplete local view rather than other_mesh's totals,
214 : // and need to be copied afresh.
215 20721 : _n_nodes = other_mesh.n_nodes();
216 20721 : _n_elem = other_mesh.n_elem();
217 20721 : _max_node_id = other_mesh.max_node_id();
218 20721 : _max_elem_id = other_mesh.max_elem_id();
219 :
220 : // If other_mesh is actually a DistributedMesh, we can just copy its raw
221 : // counters directly, mirroring other_mesh's own bookkeeping exactly. This
222 : // is done in the DistributedMesh copy constructor.
223 : // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
224 : // raw counterpart fields to copy, so derive the free-id-range
225 : // counters from _max_elem_id/_max_node_id the same way
226 : // update_parallel_id_counts() does via the set_next_ids() method. For
227 : // the unique id counters we copy other_mesh's own local next_unique_id()
228 : // and derive the next available unique ids
229 : // (partitioned and unpartitioned) via set_next_unique_ids().
230 20721 : if (!other_is_distributed_type)
231 : {
232 2627 : this->set_next_ids();
233 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
234 2627 : this->set_next_unique_ids(other_mesh.next_unique_id());
235 : #endif
236 : }
237 :
238 : // Copy other_mesh's actual preparation state, including whatever it
239 : // reports for has_synched_id_counts: the counts above are a faithful copy
240 : // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
241 : // verified computation, so our synced-ness should match other_mesh's.
242 20721 : this->_preparation = other_mesh.preparation();
243 20721 : }
244 :
245 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
246 : {
247 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
248 :
249 4 : this->_nodes = std::move(other_mesh._nodes);
250 270 : this->_n_nodes = other_mesh.n_nodes();
251 :
252 4 : this->_elements = std::move(other_mesh._elements);
253 270 : this->_n_elem = other_mesh.n_elem();
254 :
255 270 : _is_serial = other_mesh.is_serial();
256 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
257 270 : _deleted_coarse_elements = true; // Better safe than sorry
258 :
259 270 : _max_node_id = other_mesh.max_node_id();
260 270 : _max_elem_id = other_mesh.max_elem_id();
261 :
262 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
263 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
264 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
265 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
266 :
267 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
268 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
269 : #endif
270 270 : }
271 :
272 4122757 : void DistributedMesh::set_next_ids()
273 : {
274 4122757 : _next_free_unpartitioned_elem_id =
275 4125379 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
276 4122757 : (this->n_processors() + 1) + this->n_processors();
277 4122757 : _next_free_local_elem_id =
278 4122757 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
279 4122757 : (this->n_processors() + 1) + this->processor_id();
280 :
281 4122757 : _next_free_unpartitioned_node_id =
282 4125379 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
283 4122757 : (this->n_processors() + 1) + this->n_processors();
284 4122757 : _next_free_local_node_id =
285 4122757 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
286 4122757 : (this->n_processors() + 1) + this->processor_id();
287 4122757 : }
288 :
289 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
290 4122757 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
291 : {
292 : // Unique ids are laid out in repeating groups of (n_processors()+1)
293 : // consecutive integers ("alignment" here means a value's position, i.e.
294 : // its residue mod (n_processors()+1), within one of these groups). In
295 : // every group, the first n_processors() slots (residues 0..n_processors()-1)
296 : // are each permanently earmarked for one processor's locally-owned
297 : // objects, and the last slot (residue n_processors()) is permanently
298 : // earmarked for unpartitioned objects.
299 : //
300 : // parallel_max_unique_id carries no guarantee of landing on any
301 : // particular residue, so rather than simply offsetting it by
302 : // processor_id()/n_processors(), we have to explicitly round up to the
303 : // start of the next full, entirely-unused group before adding this
304 : // processor's (or the unpartitioned pool's) fixed offset into that group.
305 : //
306 : // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
307 : // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
308 : // the number of complete groups needed to reach at least past the
309 : // current max id. Multiplying back by (n_processors()+1) gives the first
310 : // id of the next group that is guaranteed to be entirely unused; adding
311 : // n_processors(), the unpartitioned pool's fixed offset within a group,
312 : // lands on that pool's reserved slot in that group.
313 4122757 : _next_unpartitioned_unique_id =
314 4125379 : ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
315 4122757 : (this->n_processors() + 1) +
316 4122757 : this->n_processors();
317 :
318 : // Same idiom, but shifted by (n_processors()-1) before dividing so that
319 : // the rounding accounts for processor_id() possibly being less than the
320 : // residue the raw max id already occupies within its group; this can
321 : // land this processor's slot in the same group as the unpartitioned
322 : // pool's above, or push it into the following group, depending on where
323 : // the max id and this processor's offset happen to fall.
324 4122757 : _next_unique_id =
325 4122757 : ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
326 4122757 : (this->n_processors() + 1) +
327 4122757 : this->processor_id();
328 4122757 : }
329 : #endif
330 :
331 : // We use cached values for these so they can be called
332 : // from one processor without bothering the rest, but
333 : // we may need to update those caches before doing a full
334 : // renumbering
335 4120130 : void DistributedMesh::update_parallel_id_counts()
336 : {
337 : // This function must be run on all processors at once
338 2908 : parallel_object_only();
339 :
340 4120130 : _n_elem = this->parallel_n_elem();
341 4120130 : _n_nodes = this->parallel_n_nodes();
342 4120130 : _max_node_id = this->parallel_max_node_id();
343 4120130 : _max_elem_id = this->parallel_max_elem_id();
344 :
345 4120130 : this->set_next_ids();
346 :
347 : // set_next_unique_ids() needs the true parallel max unique id (not just
348 : // this processor's possibly-stale cached _next_unique_id) since the whole
349 : // point of this function is to recompute counters that may have drifted
350 : // out of sync.
351 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
352 4120130 : this->set_next_unique_ids(this->parallel_max_unique_id());
353 : #endif
354 :
355 4120130 : _n_constraint_rows = this->count_constraint_rows();
356 :
357 4120130 : this->_preparation.has_synched_id_counts = true;
358 4120130 : }
359 :
360 :
361 : // Or in debug mode we may want to test the uncached values without
362 : // changing the cache
363 4127897 : dof_id_type DistributedMesh::parallel_n_elem() const
364 : {
365 : // This function must be run on all processors at once
366 3798 : parallel_object_only();
367 :
368 4127897 : dof_id_type n_local = this->n_local_elem();
369 4127897 : this->comm().sum(n_local);
370 4127897 : n_local += this->n_unpartitioned_elem();
371 4127897 : return n_local;
372 : }
373 :
374 :
375 :
376 4127476 : dof_id_type DistributedMesh::parallel_max_elem_id() const
377 : {
378 : // This function must be run on all processors at once
379 10254 : parallel_object_only();
380 :
381 4127476 : dof_id_type max_local = 0;
382 :
383 : dofobject_container<Elem>::const_reverse_veclike_iterator
384 10254 : rit = _elements.rbegin();
385 :
386 : const dofobject_container<Elem>::const_reverse_veclike_iterator
387 10254 : rend = _elements.rend();
388 :
389 : // Look for the maximum element id. Search backwards through
390 : // elements so we can break out early. Beware of nullptr entries that
391 : // haven't yet been cleared from _elements.
392 16468912 : for (; rit != rend; ++rit)
393 : {
394 15639909 : const DofObject *d = *rit;
395 21497 : if (d)
396 : {
397 10026 : libmesh_assert(_elements[d->id()] == d);
398 3298473 : max_local = d->id() + 1;
399 3298473 : break;
400 : }
401 : }
402 :
403 4127476 : this->comm().max(max_local);
404 4127476 : return max_local;
405 : }
406 :
407 :
408 :
409 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
410 4215182 : unique_id_type DistributedMesh::parallel_max_unique_id() const
411 : {
412 : // This function must be run on all processors at once
413 3370 : parallel_object_only();
414 :
415 8430364 : unique_id_type max_local = std::max(_next_unique_id,
416 4666181 : _next_unpartitioned_unique_id);
417 4215182 : this->comm().max(max_local);
418 4215182 : return max_local;
419 : }
420 :
421 :
422 :
423 90093 : void DistributedMesh::set_next_unique_id(unique_id_type id)
424 : {
425 508 : _next_unique_id = id;
426 90093 : _next_unpartitioned_unique_id =
427 90241 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
428 90093 : (this->n_processors() + 1) + this->n_processors();
429 90093 : _next_unique_id =
430 90093 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
431 90093 : (this->n_processors() + 1) + this->processor_id();
432 90093 : }
433 : #endif
434 :
435 :
436 :
437 4128345 : dof_id_type DistributedMesh::parallel_n_nodes() const
438 : {
439 : // This function must be run on all processors at once
440 3798 : parallel_object_only();
441 :
442 4128345 : dof_id_type n_local = this->n_local_nodes();
443 4128345 : this->comm().sum(n_local);
444 4128345 : n_local += this->n_unpartitioned_nodes();
445 4128345 : return n_local;
446 : }
447 :
448 :
449 :
450 4124934 : dof_id_type DistributedMesh::parallel_max_node_id() const
451 : {
452 : // This function must be run on all processors at once
453 7712 : parallel_object_only();
454 :
455 4124934 : dof_id_type max_local = 0;
456 :
457 : dofobject_container<Node>::const_reverse_veclike_iterator
458 7712 : rit = _nodes.rbegin();
459 :
460 : const dofobject_container<Node>::const_reverse_veclike_iterator
461 7712 : rend = _nodes.rend();
462 :
463 : // Look for the maximum node id. Search backwards through
464 : // nodes so we can break out early. Beware of nullptr entries that
465 : // haven't yet been cleared from _nodes
466 77839300 : for (; rit != rend; ++rit)
467 : {
468 77010367 : const DofObject *d = *rit;
469 74783 : if (d)
470 : {
471 7554 : libmesh_assert(_nodes[d->id()] == d);
472 3296001 : max_local = d->id() + 1;
473 3296001 : break;
474 : }
475 : }
476 :
477 4124934 : this->comm().max(max_local);
478 4124934 : return max_local;
479 : }
480 :
481 :
482 :
483 46397113 : const Point & DistributedMesh::point (const dof_id_type i) const
484 : {
485 46397113 : return this->node_ref(i);
486 : }
487 :
488 :
489 :
490 47514190 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
491 : {
492 137685 : libmesh_assert(_nodes[i]);
493 137685 : libmesh_assert_equal_to (_nodes[i]->id(), i);
494 :
495 47514190 : return _nodes[i];
496 : }
497 :
498 :
499 :
500 :
501 675570237 : Node * DistributedMesh::node_ptr (const dof_id_type i)
502 : {
503 656010 : libmesh_assert(_nodes[i]);
504 656010 : libmesh_assert_equal_to (_nodes[i]->id(), i);
505 :
506 675570237 : return _nodes[i];
507 : }
508 :
509 :
510 :
511 :
512 3535266 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
513 : {
514 3535266 : if (const auto it = _nodes.find(i);
515 1083543 : it != _nodes.end())
516 : {
517 3107379 : const Node * n = *it;
518 655656 : libmesh_assert (!n || n->id() == i);
519 655656 : return n;
520 : }
521 :
522 427887 : return nullptr;
523 : }
524 :
525 :
526 :
527 :
528 394254893 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
529 : {
530 394254893 : if (auto it = _nodes.find(i);
531 246425 : it != _nodes.end())
532 : {
533 306401485 : Node * n = *it;
534 157969 : libmesh_assert (!n || n->id() == i);
535 157969 : return n;
536 : }
537 :
538 88456 : return nullptr;
539 : }
540 :
541 :
542 :
543 :
544 3145478 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
545 : {
546 10249 : libmesh_assert(_elements[i]);
547 10249 : libmesh_assert_equal_to (_elements[i]->id(), i);
548 :
549 3145478 : return _elements[i];
550 : }
551 :
552 :
553 :
554 :
555 298651292 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
556 : {
557 178609 : libmesh_assert(_elements[i]);
558 178609 : libmesh_assert_equal_to (_elements[i]->id(), i);
559 :
560 298651292 : return _elements[i];
561 : }
562 :
563 :
564 :
565 :
566 1769577 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
567 : {
568 1769577 : if (const auto it = _elements.find(i);
569 930607 : it != _elements.end())
570 : {
571 1250980 : const Elem * e = *it;
572 420925 : libmesh_assert (!e || e->id() == i);
573 420925 : return e;
574 : }
575 :
576 509682 : return nullptr;
577 : }
578 :
579 :
580 :
581 :
582 634052291 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
583 : {
584 634052291 : if (auto it = _elements.find(i);
585 218068 : it != _elements.end())
586 : {
587 547792203 : Elem * e = *it;
588 214366 : libmesh_assert (!e || e->id() == i);
589 214366 : return e;
590 : }
591 :
592 3702 : return nullptr;
593 : }
594 :
595 :
596 :
597 :
598 45201381 : Elem * DistributedMesh::add_elem (Elem * e)
599 : {
600 : // Don't try to add nullptrs!
601 35765 : libmesh_assert(e);
602 :
603 : // Trying to add an existing element is a no-op
604 45201381 : if (e->valid_id() && _elements[e->id()] == e)
605 0 : return e;
606 :
607 45201381 : const processor_id_type elem_procid = e->processor_id();
608 :
609 45201381 : if (!e->valid_id())
610 : {
611 : // We should only be creating new ids past the end of the range
612 : // of existing ids
613 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
614 : _max_elem_id);
615 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
616 :
617 : // Use the unpartitioned ids for unpartitioned elems, and
618 : // temporarily for ghost elems
619 3891621 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
620 3895189 : if (elem_procid == this->processor_id())
621 1135136 : next_id = &_next_free_local_elem_id;
622 3891621 : e->set_id (*next_id);
623 : }
624 :
625 : {
626 : // Advance next_ids up high enough that each is pointing to an
627 : // unused id and any subsequent increments will still point us
628 : // to unused ids
629 90402762 : _max_elem_id = std::max(_max_elem_id,
630 45201381 : static_cast<dof_id_type>(e->id()+1));
631 :
632 45201381 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
633 5302238 : _next_free_unpartitioned_elem_id =
634 5302238 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
635 5302238 : (this->n_processors() + 1) + this->n_processors();
636 45201381 : if (_next_free_local_elem_id < _max_elem_id)
637 3577053 : _next_free_local_elem_id =
638 3583382 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
639 3577053 : (this->n_processors() + 1) + this->processor_id();
640 :
641 : #ifndef NDEBUG
642 : // We need a const dofobject_container so we don't inadvertently create
643 : // nullptr entries when testing for non-nullptr ones
644 35765 : const dofobject_container<Elem> & const_elements = _elements;
645 : #endif
646 35765 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
647 35765 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
648 : }
649 :
650 : // Don't try to overwrite existing elems
651 35765 : libmesh_assert (!_elements[e->id()]);
652 :
653 45201381 : _elements[e->id()] = e;
654 :
655 : // We actually added a new element. Some of our caches might still
656 : // be valid, but we should clear the ones which definitely are not.
657 45201381 : this->clear_point_locator();
658 45201381 : this->clear_stored_ranges();
659 :
660 : // Try to make the cached elem data more accurate
661 45220608 : if (elem_procid == this->processor_id() ||
662 : elem_procid == DofObject::invalid_processor_id)
663 13994444 : _n_elem++;
664 :
665 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
666 45201381 : if (!e->valid_unique_id())
667 : {
668 14556543 : if (processor_id() == e->processor_id())
669 : {
670 1135136 : e->set_unique_id(_next_unique_id);
671 1135136 : _next_unique_id += this->n_processors() + 1;
672 : }
673 : else
674 : {
675 13421407 : e->set_unique_id(_next_unpartitioned_unique_id);
676 13421407 : _next_unpartitioned_unique_id += this->n_processors() + 1;
677 : }
678 : }
679 : else
680 : {
681 30669278 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
682 30644838 : _next_unique_id =
683 30644838 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
684 30644838 : (this->n_processors() + 1) + this->processor_id();
685 : }
686 : #endif
687 :
688 : // Unpartitioned elems should be added on every processor
689 : // And shouldn't be added in the same batch as ghost elems
690 : // But we might be just adding on processor 0 to
691 : // broadcast later
692 : // #ifdef DEBUG
693 : // if (elem_procid == DofObject::invalid_processor_id)
694 : // {
695 : // dof_id_type elem_id = e->id();
696 : // this->comm().max(elem_id);
697 : // libmesh_assert_equal_to (elem_id, e->id());
698 : // }
699 : // #endif
700 :
701 : // Make sure any new element is given space for any extra integers
702 : // we've requested
703 45201381 : e->add_extra_integers(_elem_integer_names.size(),
704 45201381 : _elem_integer_default_values);
705 :
706 : // And set mapping type and data on any new element
707 54992 : e->set_mapping_type(this->default_mapping_type());
708 54992 : e->set_mapping_data(this->default_mapping_data());
709 :
710 45201381 : return e;
711 : }
712 :
713 :
714 :
715 15813062 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
716 : {
717 : // The mesh now takes ownership of the Elem. Eventually the guts of
718 : // add_elem() will get moved to a private helper function, and
719 : // calling add_elem() directly will be deprecated.
720 15813062 : return add_elem(e.release());
721 : }
722 :
723 :
724 :
725 3838118 : Elem * DistributedMesh::insert_elem (Elem * e)
726 : {
727 3838118 : if (_elements[e->id()])
728 3838118 : this->delete_elem(_elements[e->id()]);
729 :
730 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
731 3838118 : if (!e->valid_unique_id())
732 : {
733 0 : if (processor_id() == e->processor_id())
734 : {
735 0 : e->set_unique_id(_next_unique_id);
736 0 : _next_unique_id += this->n_processors() + 1;
737 : }
738 : else
739 : {
740 0 : e->set_unique_id(_next_unpartitioned_unique_id);
741 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
742 : }
743 : }
744 : else
745 : {
746 3838118 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
747 3838118 : _next_unique_id =
748 3841814 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
749 3838118 : (this->n_processors() + 1) + this->processor_id();
750 : }
751 : #endif
752 :
753 : // Try to make the cached elem data more accurate
754 3838118 : processor_id_type elem_procid = e->processor_id();
755 3841814 : if (elem_procid == this->processor_id() ||
756 : elem_procid == DofObject::invalid_processor_id)
757 3704344 : _n_elem++;
758 :
759 3838118 : _elements[e->id()] = e;
760 :
761 : // We actually added a new element. Some of our caches might still
762 : // be valid, but we should clear the ones which definitely are not.
763 3838118 : this->clear_point_locator();
764 3838118 : this->clear_stored_ranges();
765 :
766 : // Make sure any new element is given space for any extra integers
767 : // we've requested
768 3838118 : e->add_extra_integers(_elem_integer_names.size(),
769 3838118 : _elem_integer_default_values);
770 :
771 : // And set mapping type and data on any new element
772 7392 : e->set_mapping_type(this->default_mapping_type());
773 7392 : e->set_mapping_data(this->default_mapping_data());
774 :
775 3838118 : return e;
776 : }
777 :
778 3838118 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
779 : {
780 : // The mesh now takes ownership of the Elem. Eventually the guts of
781 : // insert_elem(Elem*) will get moved to a private helper function, and
782 : // calling insert_elem(Elem*) directly will be deprecated.
783 3838118 : return insert_elem(e.release());
784 : }
785 :
786 :
787 41044102 : void DistributedMesh::delete_elem(Elem * e)
788 : {
789 7652 : libmesh_assert (e);
790 :
791 : // Try to make the cached elem data more accurate
792 41044102 : _n_elem--;
793 :
794 : // Was this a coarse element, not just a coarsening where we still
795 : // have some ancestor structure? Was it a *local* element, that we
796 : // might have been depending on as an owner of local nodes? We'll
797 : // have to be more careful with our nodes in contract() later; no
798 : // telling if we just locally orphaned a node that should be
799 : // globally retained.
800 41053574 : if (e->processor_id() == this->processor_id() &&
801 3640 : !e->parent())
802 81593 : _deleted_coarse_elements = true;
803 :
804 : // Delete the element from the BoundaryInfo object
805 41044102 : this->get_boundary_info().remove(e);
806 :
807 : // But not yet from the container; we might invalidate
808 : // an iterator that way!
809 :
810 : //_elements.erase(e->id());
811 :
812 : // Instead, we set it to nullptr for now
813 :
814 41044102 : _elements[e->id()] = nullptr;
815 :
816 : // delete the element
817 41044102 : delete e;
818 :
819 : // Some of our caches might still be valid, but we should clear the
820 : // ones which definitely are not.
821 41044102 : this->clear_point_locator();
822 41044102 : this->clear_stored_ranges();
823 41044102 : }
824 :
825 :
826 :
827 3212724 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
828 : const dof_id_type new_id)
829 : {
830 : // This could be a no-op
831 3212724 : if (old_id == new_id)
832 0 : return;
833 :
834 3212724 : Elem * el = _elements[old_id];
835 1160 : libmesh_assert (el);
836 1160 : libmesh_assert_equal_to (el->id(), old_id);
837 :
838 3212724 : el->set_id(new_id);
839 1160 : libmesh_assert (!_elements[new_id]);
840 3212724 : _elements[new_id] = el;
841 3212724 : _elements.erase(old_id);
842 :
843 : // Should we delete any caches here? Our point locator indexes by
844 : // element pointer and should be fine with an id change. Our stored
845 : // ranges are no longer sorted, which is *probably* fine, but let's
846 : // just be safe.
847 3212724 : this->clear_stored_ranges();
848 : }
849 :
850 :
851 :
852 45766867 : Node * DistributedMesh::add_point (const Point & p,
853 : const dof_id_type id,
854 : const processor_id_type proc_id)
855 : {
856 45766867 : Node * old_n = this->query_node_ptr(id);
857 :
858 45766867 : if (old_n)
859 : {
860 2 : *old_n = p;
861 60 : old_n->processor_id() = proc_id;
862 :
863 60 : return old_n;
864 : }
865 :
866 45726492 : Node * n = Node::build(p, id).release();
867 45766807 : n->processor_id() = proc_id;
868 :
869 45766807 : return DistributedMesh::add_node(n);
870 : }
871 :
872 :
873 4130 : void DistributedMesh::own_node (Node & n)
874 : {
875 : // This had better be a node in our mesh
876 0 : libmesh_assert(_nodes[n.id()] == &n);
877 :
878 4130 : _nodes[n.id()] = nullptr;
879 4130 : _n_nodes--;
880 :
881 0 : n.set_id(DofObject::invalid_id);
882 4130 : n.processor_id() = this->processor_id();
883 :
884 4130 : this->add_node(&n);
885 4130 : }
886 :
887 :
888 76763493 : Node * DistributedMesh::add_node (Node * n)
889 : {
890 : // Don't try to add nullptrs!
891 82685 : libmesh_assert(n);
892 :
893 : // Trying to add an existing node is a no-op
894 76763493 : if (n->valid_id() && _nodes[n->id()] == n)
895 0 : return n;
896 :
897 76763493 : const processor_id_type node_procid = n->processor_id();
898 :
899 76763493 : if (!n->valid_id())
900 : {
901 : // We should only be creating new ids past the end of the range
902 : // of existing ids
903 17832 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
904 : _max_node_id);
905 17832 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
906 :
907 : // Use the unpartitioned ids for unpartitioned nodes,
908 : // and temporarily for ghost nodes
909 18903241 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
910 18921073 : if (node_procid == this->processor_id())
911 2149042 : next_id = &_next_free_local_node_id;
912 18903241 : n->set_id (*next_id);
913 : }
914 :
915 : {
916 : // Advance next_ids up high enough that each is pointing to an
917 : // unused id and any subsequent increments will still point us
918 : // to unused ids
919 153526986 : _max_node_id = std::max(_max_node_id,
920 76763493 : static_cast<dof_id_type>(n->id()+1));
921 :
922 76763493 : if (_next_free_unpartitioned_node_id < _max_node_id)
923 22098875 : _next_free_unpartitioned_node_id =
924 22098875 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
925 22098875 : (this->n_processors() + 1) + this->n_processors();
926 76763493 : if (_next_free_local_node_id < _max_node_id)
927 12844556 : _next_free_local_node_id =
928 12862837 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
929 12844556 : (this->n_processors() + 1) + this->processor_id();
930 :
931 : #ifndef NDEBUG
932 : // We need a const dofobject_container so we don't inadvertently create
933 : // nullptr entries when testing for non-nullptr ones
934 82685 : const dofobject_container<Node> & const_nodes = _nodes;
935 : #endif
936 82685 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
937 82685 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
938 : }
939 :
940 : // Don't try to overwrite existing nodes
941 82685 : libmesh_assert (!_nodes[n->id()]);
942 :
943 76763493 : _nodes[n->id()] = n;
944 :
945 : // Try to make the cached node data more accurate
946 76805591 : if (node_procid == this->processor_id() ||
947 : node_procid == DofObject::invalid_processor_id)
948 43266059 : _n_nodes++;
949 :
950 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
951 76763493 : if (!n->valid_unique_id())
952 : {
953 45769538 : if (processor_id() == n->processor_id())
954 : {
955 2591444 : n->set_unique_id(_next_unique_id);
956 2591444 : _next_unique_id += this->n_processors() + 1;
957 : }
958 : else
959 : {
960 43178094 : n->set_unique_id(_next_unpartitioned_unique_id);
961 43178094 : _next_unpartitioned_unique_id += this->n_processors() + 1;
962 : }
963 : }
964 : else
965 : {
966 31126328 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
967 30993955 : _next_unique_id =
968 30993955 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
969 30993955 : (this->n_processors() + 1) + this->processor_id();
970 : }
971 : #endif
972 :
973 : // A new node invalidates any cached node ranges
974 76763493 : this->clear_stored_ranges();
975 :
976 76763493 : n->add_extra_integers(_node_integer_names.size(),
977 76763493 : _node_integer_default_values);
978 :
979 : // Unpartitioned nodes should be added on every processor
980 : // And shouldn't be added in the same batch as ghost nodes
981 : // But we might be just adding on processor 0 to
982 : // broadcast later
983 : // #ifdef DEBUG
984 : // if (node_procid == DofObject::invalid_processor_id)
985 : // {
986 : // dof_id_type node_id = n->id();
987 : // this->comm().max(node_id);
988 : // libmesh_assert_equal_to (node_id, n->id());
989 : // }
990 : // #endif
991 :
992 76763493 : return n;
993 : }
994 :
995 30992556 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
996 : {
997 : // The mesh now takes ownership of the Node. Eventually the guts of
998 : // add_node() will get moved to a private helper function, and
999 : // calling add_node() directly will be deprecated.
1000 30992556 : return add_node(n.release());
1001 : }
1002 :
1003 52427661 : void DistributedMesh::delete_node(Node * n)
1004 : {
1005 4586 : libmesh_assert(n);
1006 4586 : libmesh_assert(_nodes[n->id()]);
1007 :
1008 : // Try to make the cached elem data more accurate
1009 52427661 : _n_nodes--;
1010 :
1011 : // Delete the node from the BoundaryInfo object
1012 52427661 : this->get_boundary_info().remove(n);
1013 9172 : _constraint_rows.erase(n);
1014 :
1015 : // But not yet from the container; we might invalidate
1016 : // an iterator that way!
1017 :
1018 : //_nodes.erase(n->id());
1019 :
1020 : // Instead, we set it to nullptr for now
1021 :
1022 52427661 : _nodes[n->id()] = nullptr;
1023 :
1024 : // delete the node
1025 52427661 : delete n;
1026 :
1027 : // We've invalidated any ranges that contained this node
1028 52427661 : this->clear_stored_ranges();
1029 52427661 : }
1030 :
1031 :
1032 :
1033 6984483 : void DistributedMesh::renumber_node(const dof_id_type old_id,
1034 : const dof_id_type new_id)
1035 : {
1036 : // This could be a no-op
1037 6984483 : if (old_id == new_id)
1038 0 : return;
1039 :
1040 6983896 : Node * nd = _nodes[old_id];
1041 3234 : libmesh_assert (nd);
1042 3234 : libmesh_assert_equal_to (nd->id(), old_id);
1043 :
1044 : // If we have nodes shipped to this processor for NodeConstraints
1045 : // use, then those nodes will exist in _nodes, but may not be
1046 : // locatable via a TopologyMap due to the insufficiency of elements
1047 : // connecting to them. If local refinement then wants to create a
1048 : // *new* node in the same location, it will initially get a temporary
1049 : // id, and then make_node_ids_parallel_consistent() will try to move
1050 : // it to the canonical id. We need to account for this case to
1051 : // avoid false positives and memory leaks.
1052 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
1053 6468 : if (_nodes[new_id])
1054 : {
1055 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
1056 : *(Point *)_nodes[old_id]);
1057 0 : _nodes.erase(new_id);
1058 : }
1059 : #else
1060 : // If we aren't shipping nodes for NodeConstraints, there should be
1061 : // no reason for renumbering one node onto another.
1062 : libmesh_assert (!_nodes[new_id]);
1063 : #endif
1064 6983896 : _nodes[new_id] = nd;
1065 6983896 : nd->set_id(new_id);
1066 :
1067 6983896 : _nodes.erase(old_id);
1068 : }
1069 :
1070 :
1071 :
1072 834775 : void DistributedMesh::clear ()
1073 : {
1074 : // Call parent clear function
1075 834775 : MeshBase::clear();
1076 :
1077 : // Clear our elements and nodes
1078 : // There is no need to remove them from
1079 : // the BoundaryInfo data structure since we
1080 : // already cleared it.
1081 834775 : this->DistributedMesh::clear_elems();
1082 :
1083 25629896 : for (auto & node : _nodes)
1084 46215504 : delete node;
1085 :
1086 964 : _nodes.clear();
1087 :
1088 : // We're no longer distributed if we were before
1089 834775 : _is_serial = true;
1090 834775 : _is_serial_on_proc_0 = true;
1091 :
1092 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1093 : // all is copacetic.
1094 834775 : _deleted_coarse_elements = false;
1095 :
1096 : // Correct our caches
1097 834775 : _n_nodes = 0;
1098 834775 : _max_node_id = 0;
1099 835379 : _next_free_local_node_id = this->processor_id();
1100 834775 : _next_free_unpartitioned_node_id = this->n_processors();
1101 834775 : }
1102 :
1103 :
1104 :
1105 841239 : void DistributedMesh::clear_elems ()
1106 : {
1107 10250103 : for (auto & elem : _elements)
1108 9408864 : delete elem;
1109 :
1110 964 : _elements.clear();
1111 :
1112 : // Correct our caches
1113 841239 : _n_elem = 0;
1114 841239 : _max_elem_id = 0;
1115 841843 : _next_free_local_elem_id = this->processor_id();
1116 841239 : _next_free_unpartitioned_elem_id = this->n_processors();
1117 841239 : }
1118 :
1119 :
1120 :
1121 779457 : void DistributedMesh::redistribute ()
1122 : {
1123 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1124 779457 : if (!this->is_serial())
1125 : {
1126 : // Construct a MeshCommunication object to actually redistribute the nodes
1127 : // and elements according to the partitioner, and then to re-gather the neighbors.
1128 : MeshCommunication mc;
1129 80676 : mc.redistribute(*this);
1130 :
1131 80676 : this->update_parallel_id_counts();
1132 :
1133 : // We communicate valid neighbor links for newly-redistributed
1134 : // elements, but we may still have remote_elem links that should
1135 : // be corrected but whose correction wasn't communicated.
1136 80952 : this->find_neighbors(/*reset_remote_elements*/ false,
1137 : /*reset_current_list*/ false /*non-default*/,
1138 : /*assert_valid*/ false,
1139 276 : /*check_non_remote*/ false /*non-default!*/);
1140 :
1141 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1142 : // anyway... but users can always call partition directly, in which case we do need
1143 : // to call delete_remote_elements()...
1144 : //
1145 : // Regardless of whether it's necessary, it isn't safe. We
1146 : // haven't communicated new node processor_ids yet, and we can't
1147 : // delete nodes until we do.
1148 : // this->delete_remote_elements();
1149 : }
1150 : else
1151 : // The base class can handle non-distributed things, like
1152 : // notifying any GhostingFunctors of changes
1153 698781 : MeshBase::redistribute();
1154 779457 : }
1155 :
1156 :
1157 :
1158 2085790 : void DistributedMesh::update_post_partitioning ()
1159 : {
1160 : // this->recalculate_n_partitions();
1161 :
1162 : // Let's do the base class cache clearing first, just in case our
1163 : // later computations are ever changed to make use of a local
1164 : // elements range cache
1165 2085790 : this->MeshBase::update_post_partitioning();
1166 :
1167 : // Partitioning changes our numbers of unpartitioned objects
1168 2085790 : this->update_parallel_id_counts();
1169 2085790 : }
1170 :
1171 :
1172 :
1173 : template <typename T>
1174 3524 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1175 : {
1176 : // This function must be run on all processors at once
1177 3524 : parallel_object_only();
1178 :
1179 3524 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1180 3524 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1181 3524 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1182 :
1183 898180 : for (dof_id_type i=0; i != pmax_id; ++i)
1184 : {
1185 894656 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1186 :
1187 : // Local lookups by id should return the requested object
1188 894656 : libmesh_assert(!obj || obj->id() == i);
1189 :
1190 : // All processors with an object should agree on id
1191 : #ifndef NDEBUG
1192 894656 : const dof_id_type dofid = obj && obj->valid_id() ?
1193 : obj->id() : DofObject::invalid_id;
1194 894656 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1195 : #endif
1196 :
1197 : // All processors with an object should agree on processor id
1198 894656 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1199 : obj->processor_id() : DofObject::invalid_processor_id;
1200 894656 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1201 :
1202 894656 : dof_id_type min_procid = procid;
1203 894656 : this->comm().min(min_procid);
1204 :
1205 : // Either:
1206 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1207 : // or
1208 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1209 :
1210 : // Original assert logic
1211 : // libmesh_assert (min_procid != this->processor_id() || obj);
1212 :
1213 : // More human-understandable logic...
1214 894656 : libmesh_assert (
1215 : ((min_procid == this->processor_id()) && obj)
1216 : ||
1217 : (min_procid != this->processor_id())
1218 : );
1219 :
1220 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1221 : // All processors with an object should agree on unique id
1222 894656 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1223 894656 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1224 : #endif
1225 : }
1226 3524 : }
1227 :
1228 :
1229 :
1230 1762 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1231 : {
1232 1762 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1233 1762 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1234 1762 : }
1235 :
1236 :
1237 :
1238 430 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1239 : {
1240 : #ifndef NDEBUG
1241 : // This function must be run on all processors at once
1242 430 : parallel_object_only();
1243 :
1244 430 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1245 :
1246 22314 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1247 : {
1248 21884 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1249 :
1250 21884 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1251 :
1252 : // All processors with an active element should agree on p level
1253 21884 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1254 : }
1255 : #endif
1256 430 : }
1257 :
1258 :
1259 :
1260 :
1261 1710 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1262 : {
1263 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1264 : // This function must be run on all processors at once
1265 1710 : parallel_object_only();
1266 :
1267 1710 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1268 :
1269 145736 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1270 : {
1271 144026 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1272 :
1273 144026 : unsigned int refinement_flag = el ?
1274 66551 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1275 144026 : unsigned int p_refinement_flag = el ?
1276 66551 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1277 :
1278 144026 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1279 :
1280 : // p refinement flags aren't always kept correct on inactive
1281 : // ghost elements
1282 144026 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1283 : }
1284 : #endif // LIBMESH_ENABLE_AMR
1285 1710 : }
1286 :
1287 :
1288 :
1289 : template <typename T>
1290 : dof_id_type
1291 2563322 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1292 : {
1293 : // This function must be run on all processors at once
1294 856 : parallel_object_only();
1295 :
1296 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1297 :
1298 : // In parallel we may not know what objects other processors have.
1299 : // Start by figuring out how many
1300 856 : dof_id_type unpartitioned_objects = 0;
1301 :
1302 : std::unordered_map<processor_id_type, dof_id_type>
1303 1712 : ghost_objects_from_proc;
1304 :
1305 2563322 : object_iterator it = objects.begin();
1306 856 : object_iterator end = objects.end();
1307 :
1308 124110281 : while (it != end)
1309 : {
1310 121546959 : T * obj = *it;
1311 :
1312 : // Remove any nullptr container entries while we're here.
1313 121546959 : if (!obj)
1314 1491382 : it = objects.erase(it);
1315 : else
1316 : {
1317 120055577 : processor_id_type obj_procid = obj->processor_id();
1318 120055577 : if (obj_procid == DofObject::invalid_processor_id)
1319 47563654 : unpartitioned_objects++;
1320 : else
1321 72491923 : ghost_objects_from_proc[obj_procid]++;
1322 :
1323 : // Finally, increment the iterator
1324 57213 : ++it;
1325 : }
1326 : }
1327 :
1328 2565034 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1329 2564178 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1330 856 : this->comm().allgather
1331 2563840 : ((this_it == ghost_objects_from_proc.end()) ?
1332 518 : dof_id_type(0) : this_it->second, objects_on_proc);
1333 :
1334 : #ifndef NDEBUG
1335 856 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1336 2568 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1337 1712 : if (ghost_objects_from_proc.count(p))
1338 993 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1339 : else
1340 719 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1341 : #endif
1342 :
1343 : // We'll renumber objects in blocks by processor id
1344 2565034 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1345 27209858 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1346 24649960 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1347 856 : objects_on_proc[i-1];
1348 2564178 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1349 2563322 : dof_id_type first_free_id =
1350 2565034 : first_object_on_proc[this->n_processors()-1] +
1351 856 : objects_on_proc[this->n_processors()-1] +
1352 : unpartitioned_objects;
1353 :
1354 : // First set new local object ids and build request sets
1355 : // for non-local object ids
1356 :
1357 : // Request sets to send to each processor
1358 : std::map<processor_id_type, std::vector<dof_id_type>>
1359 856 : requested_ids;
1360 :
1361 : // We know how many objects live on each processor, so reserve() space for
1362 : // each.
1363 856 : auto ghost_end = ghost_objects_from_proc.end();
1364 29773180 : for (auto p : make_range(this->n_processors()))
1365 27211570 : if (p != this->processor_id())
1366 : {
1367 24647392 : if (const auto p_it = ghost_objects_from_proc.find(p);
1368 856 : p_it != ghost_end)
1369 4177680 : requested_ids[p].reserve(p_it->second);
1370 : }
1371 :
1372 856 : end = objects.end();
1373 122618899 : for (it = objects.begin(); it != end; ++it)
1374 : {
1375 120055577 : T * obj = *it;
1376 120055577 : if (!obj)
1377 0 : continue;
1378 120112790 : if (obj->processor_id() == this->processor_id())
1379 21396492 : obj->set_id(next_id++);
1380 98659085 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1381 51095431 : requested_ids[obj->processor_id()].push_back(obj->id());
1382 : }
1383 :
1384 : // Next set ghost object ids from other processors
1385 :
1386 2580608 : auto gather_functor =
1387 55239489 : [
1388 : #ifndef NDEBUG
1389 : this,
1390 : &first_object_on_proc,
1391 : &objects_on_proc,
1392 : #endif
1393 : &objects]
1394 : (processor_id_type, const std::vector<dof_id_type> & ids,
1395 950 : std::vector<dof_id_type> & new_ids)
1396 : {
1397 950 : std::size_t ids_size = ids.size();
1398 4177680 : new_ids.resize(ids_size);
1399 :
1400 55273111 : for (std::size_t i=0; i != ids_size; ++i)
1401 : {
1402 51095431 : T * obj = objects[ids[i]];
1403 16336 : libmesh_assert(obj);
1404 16336 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1405 51111767 : new_ids[i] = obj->id();
1406 :
1407 16336 : libmesh_assert_greater_equal (new_ids[i],
1408 : first_object_on_proc[this->processor_id()]);
1409 16336 : libmesh_assert_less (new_ids[i],
1410 : first_object_on_proc[this->processor_id()] +
1411 : objects_on_proc[this->processor_id()]);
1412 : }
1413 : };
1414 :
1415 2580608 : auto action_functor =
1416 55239489 : [
1417 : #ifndef NDEBUG
1418 : &first_object_on_proc,
1419 : &objects_on_proc,
1420 : #endif
1421 : &objects]
1422 : (processor_id_type libmesh_dbg_var(pid),
1423 : const std::vector<dof_id_type> & ids,
1424 950 : const std::vector<dof_id_type> & data)
1425 : {
1426 : // Copy the id changes we've now been informed of
1427 55273111 : for (auto i : index_range(ids))
1428 : {
1429 51095431 : T * obj = objects[ids[i]];
1430 16336 : libmesh_assert (obj);
1431 16336 : libmesh_assert_equal_to (obj->processor_id(), pid);
1432 16336 : libmesh_assert_greater_equal (data[i],
1433 : first_object_on_proc[pid]);
1434 16336 : libmesh_assert_less (data[i],
1435 : first_object_on_proc[pid] +
1436 : objects_on_proc[pid]);
1437 51111767 : obj->set_id(data[i]);
1438 : }
1439 : };
1440 :
1441 856 : const dof_id_type * ex = nullptr;
1442 : Parallel::pull_parallel_vector_data
1443 2563322 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1444 :
1445 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1446 2580608 : auto unique_gather_functor =
1447 55239489 : [
1448 : #ifndef NDEBUG
1449 : this,
1450 : #endif
1451 : &objects]
1452 : (processor_id_type, const std::vector<dof_id_type> & ids,
1453 950 : std::vector<unique_id_type> & data)
1454 : {
1455 950 : std::size_t ids_size = ids.size();
1456 4177680 : data.resize(ids_size);
1457 :
1458 55273111 : for (std::size_t i=0; i != ids_size; ++i)
1459 : {
1460 51095431 : T * obj = objects[ids[i]];
1461 16336 : libmesh_assert(obj);
1462 16336 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1463 102190862 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1464 : }
1465 : };
1466 :
1467 2580608 : auto unique_action_functor =
1468 55239489 : [&objects]
1469 : (processor_id_type libmesh_dbg_var(pid),
1470 : const std::vector<dof_id_type> & ids,
1471 950 : const std::vector<unique_id_type> & data)
1472 : {
1473 55273111 : for (auto i : index_range(ids))
1474 : {
1475 51095431 : T * obj = objects[ids[i]];
1476 16336 : libmesh_assert (obj);
1477 16336 : libmesh_assert_equal_to (obj->processor_id(), pid);
1478 51095431 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1479 0 : obj->set_unique_id(data[i]);
1480 : }
1481 : };
1482 :
1483 856 : const unique_id_type * unique_ex = nullptr;
1484 : Parallel::pull_parallel_vector_data
1485 2563322 : (this->comm(), requested_ids, unique_gather_functor,
1486 : unique_action_functor, unique_ex);
1487 : #endif
1488 :
1489 : // Next set unpartitioned object ids
1490 856 : next_id = 0;
1491 29773180 : for (auto i : make_range(this->n_processors()))
1492 27211570 : next_id += objects_on_proc[i];
1493 122618899 : for (it = objects.begin(); it != end; ++it)
1494 : {
1495 120055577 : T * obj = *it;
1496 120055577 : if (!obj)
1497 0 : continue;
1498 120055577 : if (obj->processor_id() == DofObject::invalid_processor_id)
1499 47563654 : obj->set_id(next_id++);
1500 : }
1501 :
1502 : // Finally shuffle around objects so that container indices
1503 : // match ids
1504 2563322 : it = objects.begin();
1505 856 : end = objects.end();
1506 129462455 : while (it != end)
1507 : {
1508 126899133 : T * obj = *it;
1509 126899133 : if (obj) // don't try shuffling already-nullptr entries
1510 : {
1511 126899133 : T * next = objects[obj->id()];
1512 : // If we have to move this object
1513 126899133 : if (next != obj)
1514 : {
1515 : // nullptr out its original position for now
1516 : // (our shuffling may put another object there shortly)
1517 43073816 : *it = nullptr;
1518 :
1519 : // There may already be another object with this id that
1520 : // needs to be moved itself
1521 56741475 : while (next)
1522 : {
1523 : // We shouldn't be trying to give two objects the
1524 : // same id
1525 9414 : libmesh_assert_not_equal_to (next->id(), obj->id());
1526 13667659 : objects[obj->id()] = obj;
1527 9414 : obj = next;
1528 13667659 : next = objects[obj->id()];
1529 : }
1530 43073816 : objects[obj->id()] = obj;
1531 : }
1532 : }
1533 :
1534 : // Remove any container entries that were left as nullptr.
1535 116912 : if (!obj)
1536 0 : it = objects.erase(it);
1537 : else
1538 58456 : ++it;
1539 : }
1540 :
1541 2564178 : return first_free_id;
1542 : }
1543 :
1544 :
1545 1283331 : void DistributedMesh::renumber_nodes_and_elements ()
1546 : {
1547 430 : parallel_object_only();
1548 :
1549 : #ifdef DEBUG
1550 : // Make sure our ids and flags are consistent
1551 430 : this->libmesh_assert_valid_parallel_ids();
1552 430 : this->libmesh_assert_valid_parallel_flags();
1553 430 : this->libmesh_assert_valid_parallel_p_levels();
1554 : #endif
1555 :
1556 858 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1557 :
1558 : // Nodes not connected to any elements, and nullptr node entries
1559 : // in our container, should be deleted. But wait! If we've deleted coarse
1560 : // local elements on some processor, other processors might have ghosted
1561 : // nodes from it that are now no longer connected to any elements on it, but
1562 : // that are connected to their own semilocal elements. We'll have to
1563 : // communicate to ascertain if that's the case.
1564 1283331 : this->comm().max(_deleted_coarse_elements);
1565 :
1566 : // What used nodes do we see on our proc?
1567 430 : std::set<dof_id_type> used_nodes;
1568 :
1569 : // What used node info should we send from our proc? Could we take ownership
1570 : // of each node if we needed to?
1571 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1572 430 : used_nodes_on_proc;
1573 :
1574 : // flag the nodes we need
1575 62913586 : for (auto & elem : this->element_ptr_range())
1576 328273897 : for (const Node & node : elem->node_ref_range())
1577 : {
1578 298070250 : const dof_id_type n = node.id();
1579 297924878 : used_nodes.insert(n);
1580 298070250 : if (_deleted_coarse_elements)
1581 : {
1582 3232367 : const processor_id_type p = node.processor_id();
1583 3232651 : if (p != this->processor_id())
1584 : {
1585 2327522 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1586 2327664 : if (elem->processor_id() == this->processor_id())
1587 221409 : used_nodes_on_p[n] = true;
1588 : else
1589 105 : if (!used_nodes_on_p.count(n))
1590 567261 : used_nodes_on_p[n] = false;
1591 : }
1592 : }
1593 1282471 : }
1594 :
1595 1283331 : if (_deleted_coarse_elements)
1596 : {
1597 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1598 : typedef unsigned char boolish;
1599 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1600 28 : used_nodes_on_proc_vecs;
1601 77110 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1602 59576 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1603 :
1604 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1605 : std::map<processor_id_type, std::set<dof_id_type>>
1606 28 : repartitioned_node_sets_to_push;
1607 :
1608 : auto ids_action_functor =
1609 59550 : [&used_nodes, &repartitioned_node_pids,
1610 : &repartitioned_node_sets_to_push]
1611 : (processor_id_type pid,
1612 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1613 : {
1614 637024 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1615 : {
1616 : // If we don't see a use for our own node, but someone
1617 : // else does, better figure out who should own it next.
1618 97 : if (!used_nodes.count(n))
1619 : {
1620 2 : if (auto it = repartitioned_node_pids.find(n);
1621 2 : sender_could_become_owner)
1622 : {
1623 1 : if (it != repartitioned_node_pids.end() &&
1624 0 : pid < it->second)
1625 0 : it->second = pid;
1626 : else
1627 1 : repartitioned_node_pids[n] = pid;
1628 : }
1629 : else
1630 1 : if (it == repartitioned_node_pids.end())
1631 0 : repartitioned_node_pids[n] =
1632 : DofObject::invalid_processor_id;
1633 :
1634 2 : repartitioned_node_sets_to_push[pid].insert(n);
1635 : }
1636 : }
1637 17546 : };
1638 :
1639 : // We need two pushes instead of a pull here because we need to
1640 : // know *all* the queries for a particular node before we can
1641 : // respond to *any* of them.
1642 : Parallel::push_parallel_vector_data
1643 17534 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1644 :
1645 : // Repartition (what used to be) our own nodes first
1646 17535 : for (auto & [n, p] : repartitioned_node_pids)
1647 : {
1648 1 : Node & node = this->node_ref(n);
1649 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1650 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1651 1 : node.processor_id() = p;
1652 : }
1653 :
1654 : // Then push to repartition others' ghosted copies.
1655 :
1656 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1657 28 : repartitioned_node_vecs;
1658 :
1659 17536 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1660 : {
1661 2 : auto & rn_vec = repartitioned_node_vecs[p];
1662 4 : for (auto n : nodeset)
1663 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1664 : }
1665 :
1666 : auto repartition_node_functor =
1667 2 : [this]
1668 : (processor_id_type libmesh_dbg_var(pid),
1669 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1670 : {
1671 4 : for (auto [n, p] : ids_and_pids)
1672 : {
1673 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1674 2 : Node & node = this->node_ref(n);
1675 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1676 2 : node.processor_id() = p;
1677 : }
1678 17506 : };
1679 :
1680 : Parallel::push_parallel_vector_data
1681 17534 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1682 : }
1683 :
1684 1283331 : _deleted_coarse_elements = false;
1685 :
1686 : // Nodes not connected to any local elements, and nullptr node entries
1687 : // in our container, are deleted
1688 : {
1689 1283331 : node_iterator_imp it = _nodes.begin();
1690 430 : node_iterator_imp end = _nodes.end();
1691 :
1692 98712540 : while (it != end)
1693 : {
1694 97429209 : Node * nd = *it;
1695 97429209 : if (!nd)
1696 3708716 : it = _nodes.erase(it);
1697 93720493 : else if (!used_nodes.count(nd->id()))
1698 : {
1699 : // remove any boundary information associated with
1700 : // this node
1701 2834634 : this->get_boundary_info().remove (nd);
1702 2828 : _constraint_rows.erase(nd);
1703 :
1704 : // delete the node
1705 2834634 : delete nd;
1706 :
1707 2834634 : it = _nodes.erase(it);
1708 : }
1709 : else
1710 45790 : ++it;
1711 : }
1712 : }
1713 :
1714 1283331 : this->_preparation.has_removed_orphaned_nodes = true;
1715 :
1716 1283331 : if (_skip_renumber_nodes_and_elements)
1717 : {
1718 1670 : this->update_parallel_id_counts();
1719 2 : return;
1720 : }
1721 :
1722 : // Finally renumber all the elements
1723 1281661 : _n_elem = this->renumber_dof_objects (this->_elements);
1724 :
1725 : // and all the remaining nodes
1726 1281661 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1727 :
1728 : // And figure out what IDs we should use when adding new nodes and
1729 : // new elements
1730 1281661 : this->update_parallel_id_counts();
1731 :
1732 : // We've probably done *some* node removal and/or renumbering; let's
1733 : // clear caches to be safe.
1734 1281661 : this->clear_stored_ranges();
1735 :
1736 : // Make sure our caches are up to date and our
1737 : // DofObjects are well packed
1738 : #ifdef DEBUG
1739 428 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1740 428 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1741 428 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1742 428 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1743 428 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1744 428 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1745 428 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1746 428 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1747 :
1748 : // Make sure our ids and flags are consistent
1749 428 : this->libmesh_assert_valid_parallel_ids();
1750 428 : this->libmesh_assert_valid_parallel_flags();
1751 :
1752 : // And make sure we've made our numbering monotonic
1753 428 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1754 : #endif
1755 : }
1756 :
1757 :
1758 :
1759 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1760 : {
1761 : // We can't use range-for here because we need access to the special
1762 : // iterators' methods, not just to their dereferenced values.
1763 :
1764 : // Nodes first
1765 0 : for (auto pr = this->_nodes.begin(),
1766 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1767 : {
1768 5427360 : Node * n = *pr;
1769 5427360 : if (n != nullptr)
1770 : {
1771 0 : const dof_id_type id = pr.index();
1772 5204550 : n->set_id() = id;
1773 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1774 : }
1775 : }
1776 :
1777 : // Elements next
1778 0 : for (auto pr = this->_elements.begin(),
1779 5671231 : end = this->_elements.end(); pr != end; ++pr)
1780 : {
1781 5653821 : Elem * e = *pr;
1782 5653821 : if (e != nullptr)
1783 : {
1784 0 : const dof_id_type id = pr.index();
1785 5590229 : e->set_id() = id;
1786 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1787 : }
1788 : }
1789 17410 : }
1790 :
1791 :
1792 :
1793 859071 : dof_id_type DistributedMesh::n_active_elem () const
1794 : {
1795 582 : parallel_object_only();
1796 :
1797 : // Get local active elements first
1798 : dof_id_type active_elements =
1799 1717560 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1800 2576049 : this->active_local_elements_end()));
1801 859071 : this->comm().sum(active_elements);
1802 :
1803 : // Then add unpartitioned active elements, which should exist on
1804 : // every processor
1805 859071 : active_elements +=
1806 859071 : static_cast<dof_id_type>(std::distance
1807 1717560 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1808 859653 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1809 859071 : return active_elements;
1810 : }
1811 :
1812 :
1813 :
1814 557776 : void DistributedMesh::delete_remote_elements()
1815 : {
1816 : #ifdef DEBUG
1817 : // Make sure our neighbor links are all fine
1818 402 : MeshTools::libmesh_assert_valid_neighbors(*this);
1819 :
1820 : // And our child/parent links, and our flags
1821 402 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1822 :
1823 : // Make sure our ids and flags are consistent
1824 402 : this->libmesh_assert_valid_parallel_ids();
1825 402 : this->libmesh_assert_valid_parallel_flags();
1826 :
1827 402 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1828 402 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1829 402 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1830 402 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1831 402 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1832 402 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1833 : #endif
1834 :
1835 557776 : _is_serial = false;
1836 557776 : _is_serial_on_proc_0 = false;
1837 :
1838 557776 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1839 :
1840 402 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1841 :
1842 : // Now make sure the containers actually shrink - strip
1843 : // any newly-created nullptr voids out of the element array
1844 557776 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1845 402 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1846 55009024 : while (e_it != e_end)
1847 54451248 : if (!*e_it)
1848 39946807 : e_it = _elements.erase(e_it);
1849 : else
1850 17511 : ++e_it;
1851 :
1852 557776 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1853 402 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1854 107731182 : while (n_it != n_end)
1855 107173406 : if (!*n_it)
1856 70023989 : n_it = _nodes.erase(n_it);
1857 : else
1858 48820 : ++n_it;
1859 :
1860 : // We may have deleted no-longer-connected nodes or coarsened-away
1861 : // elements; let's update our caches if they're expected to be up to
1862 : // date.
1863 557776 : if (_preparation.has_synched_id_counts)
1864 555536 : this->update_parallel_id_counts();
1865 :
1866 : // We may have deleted nodes or elements that were the only local
1867 : // representatives of some particular boundary id(s); let's update
1868 : // those caches if they're expected to be up to date.
1869 557776 : if (_preparation.has_boundary_id_sets)
1870 54599 : this->get_boundary_info().regenerate_id_sets();
1871 :
1872 : #ifdef DEBUG
1873 : // We might not have well-packed objects if the user didn't allow us
1874 : // to renumber
1875 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1876 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1877 :
1878 : // Make sure our neighbor links are all fine
1879 402 : MeshTools::libmesh_assert_valid_neighbors(*this);
1880 :
1881 : // And our child/parent links, and our flags
1882 402 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1883 :
1884 : // Make sure our ids and flags are consistent
1885 402 : this->libmesh_assert_valid_parallel_ids();
1886 402 : this->libmesh_assert_valid_parallel_flags();
1887 : #endif
1888 :
1889 557776 : this->_preparation.has_removed_remote_elements = true;
1890 557776 : }
1891 :
1892 :
1893 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1894 : {
1895 : // First add the elem like normal
1896 0 : add_elem(e);
1897 :
1898 : // Now add it to the set that won't be deleted when we call
1899 : // delete_remote_elements()
1900 0 : _extra_ghost_elems.insert(e);
1901 0 : }
1902 :
1903 : void
1904 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1905 : {
1906 0 : std::set<Elem *> tmp;
1907 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1908 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1909 0 : std::inserter(tmp, tmp.begin()));
1910 0 : _extra_ghost_elems = tmp;
1911 0 : }
1912 :
1913 106609 : void DistributedMesh::allgather()
1914 : {
1915 106609 : if (_is_serial)
1916 0 : return;
1917 106607 : MeshCommunication().allgather(*this);
1918 106607 : _is_serial = true;
1919 106607 : _is_serial_on_proc_0 = true;
1920 :
1921 : // Make sure our caches are up to date and our
1922 : // DofObjects are well packed
1923 : #ifdef DEBUG
1924 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1925 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1926 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1927 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1928 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1929 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1930 :
1931 : // If we've disabled renumbering we can't be sure we're contiguous
1932 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1933 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1934 :
1935 : // Make sure our neighbor links are all fine
1936 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1937 :
1938 : // Make sure our ids and flags are consistent
1939 48 : this->libmesh_assert_valid_parallel_ids();
1940 48 : this->libmesh_assert_valid_parallel_flags();
1941 : #endif
1942 : }
1943 :
1944 18885 : void DistributedMesh::gather_to_zero()
1945 : {
1946 18885 : if (_is_serial_on_proc_0)
1947 6 : return;
1948 :
1949 14576 : _is_serial_on_proc_0 = true;
1950 14576 : MeshCommunication().gather(0, *this);
1951 : }
1952 :
1953 :
1954 : } // namespace libMesh
|