Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 278057 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 278057 : unsigned char d) :
44 277565 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 277565 : _is_serial_on_proc_0(true),
46 277565 : _deleted_coarse_elements(false),
47 277565 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 278303 : _next_free_local_node_id(this->processor_id()),
49 278057 : _next_free_local_elem_id(this->processor_id()),
50 278057 : _next_free_unpartitioned_node_id(this->n_processors()),
51 278057 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 278057 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 278057 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 278303 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 556114 : default_partitioner);
64 : _partitioner = Partitioner::build
65 555868 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 278057 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 320096 : DistributedMesh::~DistributedMesh ()
143 : {
144 298205 : this->DistributedMesh::clear(); // Free nodes and elements
145 320096 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17663 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 17663 : DistributedMesh(static_cast<const MeshBase &>(other_mesh))
153 : {
154 17663 : _is_serial = other_mesh._is_serial;
155 17663 : _is_serial_on_proc_0 = other_mesh._is_serial_on_proc_0;
156 17663 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
157 :
158 17663 : _n_nodes = other_mesh.n_nodes();
159 17663 : _n_elem = other_mesh.n_elem();
160 17663 : _max_node_id = other_mesh.max_node_id();
161 17663 : _max_elem_id = other_mesh.max_elem_id();
162 17663 : _next_free_local_node_id =
163 17663 : other_mesh._next_free_local_node_id;
164 17663 : _next_free_local_elem_id =
165 17663 : other_mesh._next_free_local_elem_id;
166 17663 : _next_free_unpartitioned_node_id =
167 17663 : other_mesh._next_free_unpartitioned_node_id;
168 17663 : _next_free_unpartitioned_elem_id =
169 17663 : other_mesh._next_free_unpartitioned_elem_id;
170 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
171 17663 : _next_unique_id =
172 17663 : other_mesh._next_unique_id;
173 17663 : _next_unpartitioned_unique_id =
174 17663 : other_mesh._next_unpartitioned_unique_id;
175 : #endif
176 :
177 : // Need to copy extra_ghost_elems
178 17663 : for (auto & elem : other_mesh._extra_ghost_elems)
179 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
180 17663 : }
181 :
182 :
183 :
184 20148 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh) :
185 20148 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
186 20236 : _is_serial_on_proc_0(other_mesh.is_serial()),
187 19972 : _deleted_coarse_elements(true), // better safe than sorry...
188 19972 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
189 20236 : _next_free_local_node_id(this->processor_id()),
190 20148 : _next_free_local_elem_id(this->processor_id()),
191 20148 : _next_free_unpartitioned_node_id(this->n_processors()),
192 40208 : _next_free_unpartitioned_elem_id(this->n_processors())
193 : {
194 : // Just copy, skipping preparation
195 20148 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
196 :
197 176 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
198 176 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
199 176 : this->allow_renumbering(other_mesh.allow_renumbering());
200 176 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
201 176 : this->skip_partitioning(other_mesh.skip_partitioning());
202 :
203 20148 : this->copy_constraint_rows(other_mesh);
204 :
205 88 : auto & this_boundary_info = this->get_boundary_info();
206 88 : const auto & other_boundary_info = other_mesh.get_boundary_info();
207 :
208 20148 : this_boundary_info = other_boundary_info;
209 :
210 88 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
211 :
212 20148 : this->_preparation = other_mesh.preparation();
213 :
214 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
215 20148 : _next_unique_id = other_mesh.parallel_max_unique_id() +
216 20236 : this->processor_id();
217 20236 : _next_unpartitioned_unique_id = _next_unique_id +
218 20148 : (this->n_processors() - this->processor_id());
219 : #endif
220 20148 : this->update_parallel_id_counts();
221 20148 : }
222 :
223 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
224 : {
225 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
226 :
227 4 : this->_nodes = std::move(other_mesh._nodes);
228 270 : this->_n_nodes = other_mesh.n_nodes();
229 :
230 4 : this->_elements = std::move(other_mesh._elements);
231 270 : this->_n_elem = other_mesh.n_elem();
232 :
233 270 : _is_serial = other_mesh.is_serial();
234 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
235 270 : _deleted_coarse_elements = true; // Better safe than sorry
236 :
237 270 : _max_node_id = other_mesh.max_node_id();
238 270 : _max_elem_id = other_mesh.max_elem_id();
239 :
240 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
241 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
242 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
243 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
244 :
245 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
246 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
247 : #endif
248 270 : }
249 :
250 : // We use cached values for these so they can be called
251 : // from one processor without bothering the rest, but
252 : // we may need to update those caches before doing a full
253 : // renumbering
254 1935788 : void DistributedMesh::update_parallel_id_counts()
255 : {
256 : // This function must be run on all processors at once
257 1774 : parallel_object_only();
258 :
259 1935788 : _n_elem = this->parallel_n_elem();
260 1935788 : _n_nodes = this->parallel_n_nodes();
261 1935788 : _max_node_id = this->parallel_max_node_id();
262 1935788 : _max_elem_id = this->parallel_max_elem_id();
263 :
264 1935788 : _next_free_unpartitioned_elem_id =
265 1935788 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
266 1935788 : (this->n_processors() + 1) + this->n_processors();
267 1935788 : _next_free_local_elem_id =
268 1935788 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
269 1935788 : (this->n_processors() + 1) + this->processor_id();
270 :
271 1935788 : _next_free_unpartitioned_node_id =
272 1935788 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
273 1935788 : (this->n_processors() + 1) + this->n_processors();
274 1935788 : _next_free_local_node_id =
275 1935788 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
276 1935788 : (this->n_processors() + 1) + this->processor_id();
277 :
278 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
279 1935788 : _next_unique_id = this->parallel_max_unique_id();
280 1935788 : _next_unpartitioned_unique_id =
281 1937562 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
282 1935788 : (this->n_processors() + 1) + this->n_processors();
283 1935788 : _next_unique_id =
284 1935788 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
285 1935788 : (this->n_processors() + 1) + this->processor_id();
286 : #endif
287 :
288 1935788 : this->_preparation.has_synched_id_counts = true;
289 1935788 : }
290 :
291 :
292 : // Or in debug mode we may want to test the uncached values without
293 : // changing the cache
294 1943507 : dof_id_type DistributedMesh::parallel_n_elem() const
295 : {
296 : // This function must be run on all processors at once
297 2616 : parallel_object_only();
298 :
299 1943507 : dof_id_type n_local = this->n_local_elem();
300 1943507 : this->comm().sum(n_local);
301 1943507 : n_local += this->n_unpartitioned_elem();
302 1943507 : return n_local;
303 : }
304 :
305 :
306 :
307 1942734 : dof_id_type DistributedMesh::parallel_max_elem_id() const
308 : {
309 : // This function must be run on all processors at once
310 8720 : parallel_object_only();
311 :
312 1942734 : dof_id_type max_local = 0;
313 :
314 : dofobject_container<Elem>::const_reverse_veclike_iterator
315 8720 : rit = _elements.rbegin();
316 :
317 : const dofobject_container<Elem>::const_reverse_veclike_iterator
318 8720 : rend = _elements.rend();
319 :
320 : // Look for the maximum element id. Search backwards through
321 : // elements so we can break out early. Beware of nullptr entries that
322 : // haven't yet been cleared from _elements.
323 10575506 : for (; rit != rend; ++rit)
324 : {
325 10048484 : const DofObject *d = *rit;
326 19027 : if (d)
327 : {
328 8505 : libmesh_assert(_elements[d->id()] == d);
329 1415712 : max_local = d->id() + 1;
330 1415712 : break;
331 : }
332 : }
333 :
334 1942734 : this->comm().max(max_local);
335 1942734 : return max_local;
336 : }
337 :
338 :
339 :
340 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
341 2038856 : unique_id_type DistributedMesh::parallel_max_unique_id() const
342 : {
343 : // This function must be run on all processors at once
344 1892 : parallel_object_only();
345 :
346 4077712 : unique_id_type max_local = std::max(_next_unique_id,
347 2349160 : _next_unpartitioned_unique_id);
348 2038856 : this->comm().max(max_local);
349 2038856 : return max_local;
350 : }
351 :
352 :
353 :
354 78256 : void DistributedMesh::set_next_unique_id(unique_id_type id)
355 : {
356 142 : _next_unique_id = id;
357 78256 : _next_unpartitioned_unique_id =
358 78398 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
359 78256 : (this->n_processors() + 1) + this->n_processors();
360 78256 : _next_unique_id =
361 78256 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
362 78256 : (this->n_processors() + 1) + this->processor_id();
363 78256 : }
364 : #endif
365 :
366 :
367 :
368 1943955 : dof_id_type DistributedMesh::parallel_n_nodes() const
369 : {
370 : // This function must be run on all processors at once
371 2616 : parallel_object_only();
372 :
373 1943955 : dof_id_type n_local = this->n_local_nodes();
374 1943955 : this->comm().sum(n_local);
375 1943955 : n_local += this->n_unpartitioned_nodes();
376 1943955 : return n_local;
377 : }
378 :
379 :
380 :
381 1940336 : dof_id_type DistributedMesh::parallel_max_node_id() const
382 : {
383 : // This function must be run on all processors at once
384 6322 : parallel_object_only();
385 :
386 1940336 : dof_id_type max_local = 0;
387 :
388 : dofobject_container<Node>::const_reverse_veclike_iterator
389 6322 : rit = _nodes.rbegin();
390 :
391 : const dofobject_container<Node>::const_reverse_veclike_iterator
392 6322 : rend = _nodes.rend();
393 :
394 : // Look for the maximum node id. Search backwards through
395 : // nodes so we can break out early. Beware of nullptr entries that
396 : // haven't yet been cleared from _nodes
397 38153844 : for (; rit != rend; ++rit)
398 : {
399 37626892 : const DofObject *d = *rit;
400 63433 : if (d)
401 : {
402 6177 : libmesh_assert(_nodes[d->id()] == d);
403 1413384 : max_local = d->id() + 1;
404 1413384 : break;
405 : }
406 : }
407 :
408 1940336 : this->comm().max(max_local);
409 1940336 : return max_local;
410 : }
411 :
412 :
413 :
414 42415116 : const Point & DistributedMesh::point (const dof_id_type i) const
415 : {
416 42415116 : return this->node_ref(i);
417 : }
418 :
419 :
420 :
421 43520505 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
422 : {
423 136653 : libmesh_assert(_nodes[i]);
424 136653 : libmesh_assert_equal_to (_nodes[i]->id(), i);
425 :
426 43520505 : return _nodes[i];
427 : }
428 :
429 :
430 :
431 :
432 643656273 : Node * DistributedMesh::node_ptr (const dof_id_type i)
433 : {
434 489230 : libmesh_assert(_nodes[i]);
435 489230 : libmesh_assert_equal_to (_nodes[i]->id(), i);
436 :
437 643656273 : return _nodes[i];
438 : }
439 :
440 :
441 :
442 :
443 3109046 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
444 : {
445 3109046 : if (const auto it = _nodes.find(i);
446 657323 : it != _nodes.end())
447 : {
448 2820550 : const Node * n = *it;
449 368827 : libmesh_assert (!n || n->id() == i);
450 368827 : return n;
451 : }
452 :
453 288496 : return nullptr;
454 : }
455 :
456 :
457 :
458 :
459 367382791 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
460 : {
461 367382791 : if (auto it = _nodes.find(i);
462 159331 : it != _nodes.end())
463 : {
464 284597219 : Node * n = *it;
465 111970 : libmesh_assert (!n || n->id() == i);
466 111970 : return n;
467 : }
468 :
469 47361 : return nullptr;
470 : }
471 :
472 :
473 :
474 :
475 3050122 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
476 : {
477 10097 : libmesh_assert(_elements[i]);
478 10097 : libmesh_assert_equal_to (_elements[i]->id(), i);
479 :
480 3050122 : return _elements[i];
481 : }
482 :
483 :
484 :
485 :
486 286920137 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
487 : {
488 167219 : libmesh_assert(_elements[i]);
489 167219 : libmesh_assert_equal_to (_elements[i]->id(), i);
490 :
491 286920137 : return _elements[i];
492 : }
493 :
494 :
495 :
496 :
497 1457183 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
498 : {
499 1457183 : if (const auto it = _elements.find(i);
500 618213 : it != _elements.end())
501 : {
502 1099628 : const Elem * e = *it;
503 269573 : libmesh_assert (!e || e->id() == i);
504 269573 : return e;
505 : }
506 :
507 348640 : return nullptr;
508 : }
509 :
510 :
511 :
512 :
513 603672041 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
514 : {
515 603672041 : if (auto it = _elements.find(i);
516 194370 : it != _elements.end())
517 : {
518 518167989 : Elem * e = *it;
519 190690 : libmesh_assert (!e || e->id() == i);
520 190690 : return e;
521 : }
522 :
523 3680 : return nullptr;
524 : }
525 :
526 :
527 :
528 :
529 44047414 : Elem * DistributedMesh::add_elem (Elem * e)
530 : {
531 : // Don't try to add nullptrs!
532 18913 : libmesh_assert(e);
533 :
534 : // Trying to add an existing element is a no-op
535 44047414 : if (e->valid_id() && _elements[e->id()] == e)
536 0 : return e;
537 :
538 44047414 : const processor_id_type elem_procid = e->processor_id();
539 :
540 44047414 : if (!e->valid_id())
541 : {
542 : // We should only be creating new ids past the end of the range
543 : // of existing ids
544 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
545 : _max_elem_id);
546 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
547 :
548 : // Use the unpartitioned ids for unpartitioned elems, and
549 : // temporarily for ghost elems
550 3860161 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
551 3863729 : if (elem_procid == this->processor_id())
552 1133680 : next_id = &_next_free_local_elem_id;
553 3860161 : e->set_id (*next_id);
554 : }
555 :
556 : {
557 : // Advance next_ids up high enough that each is pointing to an
558 : // unused id and any subsequent increments will still point us
559 : // to unused ids
560 88094828 : _max_elem_id = std::max(_max_elem_id,
561 44047414 : static_cast<dof_id_type>(e->id()+1));
562 :
563 44047414 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
564 5166969 : _next_free_unpartitioned_elem_id =
565 5166969 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
566 5166969 : (this->n_processors() + 1) + this->n_processors();
567 44047414 : if (_next_free_local_elem_id < _max_elem_id)
568 3463429 : _next_free_local_elem_id =
569 3469673 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
570 3463429 : (this->n_processors() + 1) + this->processor_id();
571 :
572 : #ifndef NDEBUG
573 : // We need a const dofobject_container so we don't inadvertently create
574 : // nullptr entries when testing for non-nullptr ones
575 18913 : const dofobject_container<Elem> & const_elements = _elements;
576 : #endif
577 18913 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
578 18913 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
579 : }
580 :
581 : // Don't try to overwrite existing elems
582 18913 : libmesh_assert (!_elements[e->id()]);
583 :
584 44047414 : _elements[e->id()] = e;
585 :
586 : // We actually added a new element. Some of our caches might still
587 : // be valid, but we should clear the ones which definitely are not.
588 44047414 : this->clear_point_locator();
589 44047414 : this->clear_stored_ranges();
590 :
591 : // Try to make the cached elem data more accurate
592 44066327 : if (elem_procid == this->processor_id() ||
593 : elem_procid == DofObject::invalid_processor_id)
594 12911096 : _n_elem++;
595 :
596 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
597 44047414 : if (!e->valid_unique_id())
598 : {
599 13507459 : if (processor_id() == e->processor_id())
600 : {
601 1133680 : e->set_unique_id(_next_unique_id);
602 1133680 : _next_unique_id += this->n_processors() + 1;
603 : }
604 : else
605 : {
606 12373779 : e->set_unique_id(_next_unpartitioned_unique_id);
607 12373779 : _next_unpartitioned_unique_id += this->n_processors() + 1;
608 : }
609 : }
610 : else
611 : {
612 30564332 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
613 30539955 : _next_unique_id =
614 30539955 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
615 30539955 : (this->n_processors() + 1) + this->processor_id();
616 : }
617 : #endif
618 :
619 : // Unpartitioned elems should be added on every processor
620 : // And shouldn't be added in the same batch as ghost elems
621 : // But we might be just adding on processor 0 to
622 : // broadcast later
623 : // #ifdef DEBUG
624 : // if (elem_procid == DofObject::invalid_processor_id)
625 : // {
626 : // dof_id_type elem_id = e->id();
627 : // this->comm().max(elem_id);
628 : // libmesh_assert_equal_to (elem_id, e->id());
629 : // }
630 : // #endif
631 :
632 : // Make sure any new element is given space for any extra integers
633 : // we've requested
634 44047414 : e->add_extra_integers(_elem_integer_names.size(),
635 44047414 : _elem_integer_default_values);
636 :
637 : // And set mapping type and data on any new element
638 37826 : e->set_mapping_type(this->default_mapping_type());
639 37826 : e->set_mapping_data(this->default_mapping_data());
640 :
641 44047414 : return e;
642 : }
643 :
644 :
645 :
646 14743490 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
647 : {
648 : // The mesh now takes ownership of the Elem. Eventually the guts of
649 : // add_elem() will get moved to a private helper function, and
650 : // calling add_elem() directly will be deprecated.
651 14743490 : return add_elem(e.release());
652 : }
653 :
654 :
655 :
656 3351724 : Elem * DistributedMesh::insert_elem (Elem * e)
657 : {
658 3351724 : if (_elements[e->id()])
659 3351724 : this->delete_elem(_elements[e->id()]);
660 :
661 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
662 3351724 : if (!e->valid_unique_id())
663 : {
664 0 : if (processor_id() == e->processor_id())
665 : {
666 0 : e->set_unique_id(_next_unique_id);
667 0 : _next_unique_id += this->n_processors() + 1;
668 : }
669 : else
670 : {
671 0 : e->set_unique_id(_next_unpartitioned_unique_id);
672 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
673 : }
674 : }
675 : else
676 : {
677 3351724 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
678 3351724 : _next_unique_id =
679 3355420 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
680 3351724 : (this->n_processors() + 1) + this->processor_id();
681 : }
682 : #endif
683 :
684 : // Try to make the cached elem data more accurate
685 3351724 : processor_id_type elem_procid = e->processor_id();
686 3355420 : if (elem_procid == this->processor_id() ||
687 : elem_procid == DofObject::invalid_processor_id)
688 3220856 : _n_elem++;
689 :
690 3351724 : _elements[e->id()] = e;
691 :
692 : // We actually added a new element. Some of our caches might still
693 : // be valid, but we should clear the ones which definitely are not.
694 3351724 : this->clear_point_locator();
695 3351724 : this->clear_stored_ranges();
696 :
697 : // Make sure any new element is given space for any extra integers
698 : // we've requested
699 3351724 : e->add_extra_integers(_elem_integer_names.size(),
700 3351724 : _elem_integer_default_values);
701 :
702 : // And set mapping type and data on any new element
703 7392 : e->set_mapping_type(this->default_mapping_type());
704 7392 : e->set_mapping_data(this->default_mapping_data());
705 :
706 3351724 : return e;
707 : }
708 :
709 3351724 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
710 : {
711 : // The mesh now takes ownership of the Elem. Eventually the guts of
712 : // insert_elem(Elem*) will get moved to a private helper function, and
713 : // calling insert_elem(Elem*) directly will be deprecated.
714 3351724 : return insert_elem(e.release());
715 : }
716 :
717 :
718 40097320 : void DistributedMesh::delete_elem(Elem * e)
719 : {
720 7602 : libmesh_assert (e);
721 :
722 : // Try to make the cached elem data more accurate
723 40097320 : _n_elem--;
724 :
725 : // Was this a coarse element, not just a coarsening where we still
726 : // have some ancestor structure? Was it a *local* element, that we
727 : // might have been depending on as an owner of local nodes? We'll
728 : // have to be more careful with our nodes in contract() later; no
729 : // telling if we just locally orphaned a node that should be
730 : // globally retained.
731 40106742 : if (e->processor_id() == this->processor_id() &&
732 3640 : !e->parent())
733 80793 : _deleted_coarse_elements = true;
734 :
735 : // Delete the element from the BoundaryInfo object
736 40097320 : this->get_boundary_info().remove(e);
737 :
738 : // But not yet from the container; we might invalidate
739 : // an iterator that way!
740 :
741 : //_elements.erase(e->id());
742 :
743 : // Instead, we set it to nullptr for now
744 :
745 40097320 : _elements[e->id()] = nullptr;
746 :
747 : // delete the element
748 40097320 : delete e;
749 :
750 : // Some of our caches might still be valid, but we should clear the
751 : // ones which definitely are not.
752 40097320 : this->clear_point_locator();
753 40097320 : this->clear_stored_ranges();
754 40097320 : }
755 :
756 :
757 :
758 3208080 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
759 : const dof_id_type new_id)
760 : {
761 : // This could be a no-op
762 3208080 : if (old_id == new_id)
763 0 : return;
764 :
765 3208080 : Elem * el = _elements[old_id];
766 1160 : libmesh_assert (el);
767 1160 : libmesh_assert_equal_to (el->id(), old_id);
768 :
769 3208080 : el->set_id(new_id);
770 1160 : libmesh_assert (!_elements[new_id]);
771 3208080 : _elements[new_id] = el;
772 3208080 : _elements.erase(old_id);
773 :
774 : // Should we delete any caches here? Our point locator indexes by
775 : // element pointer and should be fine with an id change. Our stored
776 : // ranges are no longer sorted, which is *probably* fine, but let's
777 : // just be safe.
778 3208080 : this->clear_stored_ranges();
779 : }
780 :
781 :
782 :
783 40951989 : Node * DistributedMesh::add_point (const Point & p,
784 : const dof_id_type id,
785 : const processor_id_type proc_id)
786 : {
787 40951989 : Node * old_n = this->query_node_ptr(id);
788 :
789 40951989 : if (old_n)
790 : {
791 0 : *old_n = p;
792 0 : old_n->processor_id() = proc_id;
793 :
794 0 : return old_n;
795 : }
796 :
797 40912126 : Node * n = Node::build(p, id).release();
798 40951989 : n->processor_id() = proc_id;
799 :
800 40951989 : return DistributedMesh::add_node(n);
801 : }
802 :
803 :
804 4130 : void DistributedMesh::own_node (Node & n)
805 : {
806 : // This had better be a node in our mesh
807 0 : libmesh_assert(_nodes[n.id()] == &n);
808 :
809 4130 : _nodes[n.id()] = nullptr;
810 4130 : _n_nodes--;
811 :
812 0 : n.set_id(DofObject::invalid_id);
813 4130 : n.processor_id() = this->processor_id();
814 :
815 4130 : this->add_node(&n);
816 4130 : }
817 :
818 :
819 71729874 : Node * DistributedMesh::add_node (Node * n)
820 : {
821 : // Don't try to add nullptrs!
822 41575 : libmesh_assert(n);
823 :
824 : // Trying to add an existing node is a no-op
825 71729874 : if (n->valid_id() && _nodes[n->id()] == n)
826 0 : return n;
827 :
828 71729874 : const processor_id_type node_procid = n->processor_id();
829 :
830 71729874 : if (!n->valid_id())
831 : {
832 : // We should only be creating new ids past the end of the range
833 : // of existing ids
834 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
835 : _max_node_id);
836 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
837 :
838 : // Use the unpartitioned ids for unpartitioned nodes,
839 : // and temporarily for ghost nodes
840 17191201 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
841 17209030 : if (node_procid == this->processor_id())
842 2128721 : next_id = &_next_free_local_node_id;
843 17191201 : n->set_id (*next_id);
844 : }
845 :
846 : {
847 : // Advance next_ids up high enough that each is pointing to an
848 : // unused id and any subsequent increments will still point us
849 : // to unused ids
850 143459748 : _max_node_id = std::max(_max_node_id,
851 71729874 : static_cast<dof_id_type>(n->id()+1));
852 :
853 71729874 : if (_next_free_unpartitioned_node_id < _max_node_id)
854 20115602 : _next_free_unpartitioned_node_id =
855 20115602 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
856 20115602 : (this->n_processors() + 1) + this->n_processors();
857 71729874 : if (_next_free_local_node_id < _max_node_id)
858 11727010 : _next_free_local_node_id =
859 11745167 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
860 11727010 : (this->n_processors() + 1) + this->processor_id();
861 :
862 : #ifndef NDEBUG
863 : // We need a const dofobject_container so we don't inadvertently create
864 : // nullptr entries when testing for non-nullptr ones
865 41575 : const dofobject_container<Node> & const_nodes = _nodes;
866 : #endif
867 41575 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
868 41575 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
869 : }
870 :
871 : // Don't try to overwrite existing nodes
872 41575 : libmesh_assert (!_nodes[n->id()]);
873 :
874 71729874 : _nodes[n->id()] = n;
875 :
876 : // Try to make the cached node data more accurate
877 71771449 : if (node_procid == this->processor_id() ||
878 : node_procid == DofObject::invalid_processor_id)
879 38537761 : _n_nodes++;
880 :
881 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
882 71729874 : if (!n->valid_unique_id())
883 : {
884 40953869 : if (processor_id() == n->processor_id())
885 : {
886 2545271 : n->set_unique_id(_next_unique_id);
887 2545271 : _next_unique_id += this->n_processors() + 1;
888 : }
889 : else
890 : {
891 38408598 : n->set_unique_id(_next_unpartitioned_unique_id);
892 38408598 : _next_unpartitioned_unique_id += this->n_processors() + 1;
893 : }
894 : }
895 : else
896 : {
897 30906511 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
898 30776005 : _next_unique_id =
899 30776005 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
900 30776005 : (this->n_processors() + 1) + this->processor_id();
901 : }
902 : #endif
903 :
904 71729874 : n->add_extra_integers(_node_integer_names.size(),
905 71729874 : _node_integer_default_values);
906 :
907 : // Unpartitioned nodes should be added on every processor
908 : // And shouldn't be added in the same batch as ghost nodes
909 : // But we might be just adding on processor 0 to
910 : // broadcast later
911 : // #ifdef DEBUG
912 : // if (node_procid == DofObject::invalid_processor_id)
913 : // {
914 : // dof_id_type node_id = n->id();
915 : // this->comm().max(node_id);
916 : // libmesh_assert_equal_to (node_id, n->id());
917 : // }
918 : // #endif
919 :
920 71729874 : return n;
921 : }
922 :
923 30773755 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
924 : {
925 : // The mesh now takes ownership of the Node. Eventually the guts of
926 : // add_node() will get moved to a private helper function, and
927 : // calling add_node() directly will be deprecated.
928 30773755 : return add_node(n.release());
929 : }
930 :
931 50494628 : void DistributedMesh::delete_node(Node * n)
932 : {
933 4489 : libmesh_assert(n);
934 4489 : libmesh_assert(_nodes[n->id()]);
935 :
936 : // Try to make the cached elem data more accurate
937 50494628 : _n_nodes--;
938 :
939 : // Delete the node from the BoundaryInfo object
940 50494628 : this->get_boundary_info().remove(n);
941 8978 : _constraint_rows.erase(n);
942 :
943 : // But not yet from the container; we might invalidate
944 : // an iterator that way!
945 :
946 : //_nodes.erase(n->id());
947 :
948 : // Instead, we set it to nullptr for now
949 :
950 50494628 : _nodes[n->id()] = nullptr;
951 :
952 : // delete the node
953 50494628 : delete n;
954 50494628 : }
955 :
956 :
957 :
958 6898995 : void DistributedMesh::renumber_node(const dof_id_type old_id,
959 : const dof_id_type new_id)
960 : {
961 : // This could be a no-op
962 6898995 : if (old_id == new_id)
963 0 : return;
964 :
965 6898419 : Node * nd = _nodes[old_id];
966 3234 : libmesh_assert (nd);
967 3234 : libmesh_assert_equal_to (nd->id(), old_id);
968 :
969 : // If we have nodes shipped to this processor for NodeConstraints
970 : // use, then those nodes will exist in _nodes, but may not be
971 : // locatable via a TopologyMap due to the insufficiency of elements
972 : // connecting to them. If local refinement then wants to create a
973 : // *new* node in the same location, it will initially get a temporary
974 : // id, and then make_node_ids_parallel_consistent() will try to move
975 : // it to the canonical id. We need to account for this case to
976 : // avoid false positives and memory leaks.
977 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
978 6468 : if (_nodes[new_id])
979 : {
980 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
981 : *(Point *)_nodes[old_id]);
982 0 : _nodes.erase(new_id);
983 : }
984 : #else
985 : // If we aren't shipping nodes for NodeConstraints, there should be
986 : // no reason for renumbering one node onto another.
987 : libmesh_assert (!_nodes[new_id]);
988 : #endif
989 6898419 : _nodes[new_id] = nd;
990 6898419 : nd->set_id(new_id);
991 :
992 6898419 : _nodes.erase(old_id);
993 : }
994 :
995 :
996 :
997 556611 : void DistributedMesh::clear ()
998 : {
999 : // Call parent clear function
1000 556611 : MeshBase::clear();
1001 :
1002 : // Clear our elements and nodes
1003 : // There is no need to remove them from
1004 : // the BoundaryInfo data structure since we
1005 : // already cleared it.
1006 556611 : this->DistributedMesh::clear_elems();
1007 :
1008 22591466 : for (auto & node : _nodes)
1009 40902108 : delete node;
1010 :
1011 572 : _nodes.clear();
1012 :
1013 : // We're no longer distributed if we were before
1014 556611 : _is_serial = true;
1015 556611 : _is_serial_on_proc_0 = true;
1016 :
1017 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1018 : // all is copacetic.
1019 556611 : _deleted_coarse_elements = false;
1020 :
1021 : // Correct our caches
1022 556611 : _n_nodes = 0;
1023 556611 : _max_node_id = 0;
1024 557183 : _next_free_local_node_id = this->processor_id();
1025 556611 : _next_free_unpartitioned_node_id = this->n_processors();
1026 556611 : }
1027 :
1028 :
1029 :
1030 562883 : void DistributedMesh::clear_elems ()
1031 : {
1032 9210668 : for (auto & elem : _elements)
1033 8647785 : delete elem;
1034 :
1035 572 : _elements.clear();
1036 :
1037 : // Correct our caches
1038 562883 : _n_elem = 0;
1039 562883 : _max_elem_id = 0;
1040 563455 : _next_free_local_elem_id = this->processor_id();
1041 562883 : _next_free_unpartitioned_elem_id = this->n_processors();
1042 562883 : }
1043 :
1044 :
1045 :
1046 559635 : void DistributedMesh::redistribute ()
1047 : {
1048 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1049 559635 : if (!this->is_serial())
1050 : {
1051 : // Construct a MeshCommunication object to actually redistribute the nodes
1052 : // and elements according to the partitioner, and then to re-gather the neighbors.
1053 : MeshCommunication mc;
1054 68322 : mc.redistribute(*this);
1055 :
1056 68322 : this->update_parallel_id_counts();
1057 :
1058 : // We ought to still have valid neighbor links; we communicate
1059 : // them for newly-redistributed elements
1060 : // this->find_neighbors();
1061 :
1062 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1063 : // anyway... but users can always call partition directly, in which case we do need
1064 : // to call delete_remote_elements()...
1065 : //
1066 : // Regardless of whether it's necessary, it isn't safe. We
1067 : // haven't communicated new node processor_ids yet, and we can't
1068 : // delete nodes until we do.
1069 : // this->delete_remote_elements();
1070 : }
1071 : else
1072 : // The base class can handle non-distributed things, like
1073 : // notifying any GhostingFunctors of changes
1074 491313 : MeshBase::redistribute();
1075 559635 : }
1076 :
1077 :
1078 :
1079 481192 : void DistributedMesh::update_post_partitioning ()
1080 : {
1081 : // this->recalculate_n_partitions();
1082 :
1083 : // Let's do the base class cache clearing first, just in case our
1084 : // later computations are ever changed to make use of a local
1085 : // elements range cache
1086 481192 : this->MeshBase::update_post_partitioning();
1087 :
1088 : // Partitioning changes our numbers of unpartitioned objects
1089 481192 : this->update_parallel_id_counts();
1090 481192 : }
1091 :
1092 :
1093 :
1094 : template <typename T>
1095 3332 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1096 : {
1097 : // This function must be run on all processors at once
1098 3332 : parallel_object_only();
1099 :
1100 3332 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1101 3332 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1102 3332 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1103 :
1104 893980 : for (dof_id_type i=0; i != pmax_id; ++i)
1105 : {
1106 890648 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1107 :
1108 : // Local lookups by id should return the requested object
1109 890648 : libmesh_assert(!obj || obj->id() == i);
1110 :
1111 : // All processors with an object should agree on id
1112 : #ifndef NDEBUG
1113 890648 : const dof_id_type dofid = obj && obj->valid_id() ?
1114 : obj->id() : DofObject::invalid_id;
1115 890648 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1116 : #endif
1117 :
1118 : // All processors with an object should agree on processor id
1119 890648 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1120 : obj->processor_id() : DofObject::invalid_processor_id;
1121 890648 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1122 :
1123 890648 : dof_id_type min_procid = procid;
1124 890648 : this->comm().min(min_procid);
1125 :
1126 : // Either:
1127 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1128 : // or
1129 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1130 :
1131 : // Original assert logic
1132 : // libmesh_assert (min_procid != this->processor_id() || obj);
1133 :
1134 : // More human-understandable logic...
1135 890648 : libmesh_assert (
1136 : ((min_procid == this->processor_id()) && obj)
1137 : ||
1138 : (min_procid != this->processor_id())
1139 : );
1140 :
1141 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1142 : // All processors with an object should agree on unique id
1143 890648 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1144 890648 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1145 : #endif
1146 : }
1147 3332 : }
1148 :
1149 :
1150 :
1151 1666 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1152 : {
1153 1666 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1154 1666 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1155 1666 : }
1156 :
1157 :
1158 :
1159 398 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1160 : {
1161 : #ifndef NDEBUG
1162 : // This function must be run on all processors at once
1163 398 : parallel_object_only();
1164 :
1165 398 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1166 :
1167 21914 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1168 : {
1169 21516 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1170 :
1171 21516 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1172 :
1173 : // All processors with an active element should agree on p level
1174 21516 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1175 : }
1176 : #endif
1177 398 : }
1178 :
1179 :
1180 :
1181 :
1182 1614 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1183 : {
1184 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1185 : // This function must be run on all processors at once
1186 1614 : parallel_object_only();
1187 :
1188 1614 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1189 :
1190 144536 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1191 : {
1192 142922 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1193 :
1194 142922 : unsigned int refinement_flag = el ?
1195 65613 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1196 142922 : unsigned int p_refinement_flag = el ?
1197 65613 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1198 :
1199 142922 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1200 :
1201 : // p refinement flags aren't always kept correct on inactive
1202 : // ghost elements
1203 142922 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1204 : }
1205 : #endif // LIBMESH_ENABLE_AMR
1206 1614 : }
1207 :
1208 :
1209 :
1210 : template <typename T>
1211 : dof_id_type
1212 1716770 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1213 : {
1214 : // This function must be run on all processors at once
1215 792 : parallel_object_only();
1216 :
1217 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1218 :
1219 : // In parallel we may not know what objects other processors have.
1220 : // Start by figuring out how many
1221 792 : dof_id_type unpartitioned_objects = 0;
1222 :
1223 : std::unordered_map<processor_id_type, dof_id_type>
1224 1584 : ghost_objects_from_proc;
1225 :
1226 1716770 : object_iterator it = objects.begin();
1227 792 : object_iterator end = objects.end();
1228 :
1229 109765869 : while (it != end)
1230 : {
1231 108049099 : T * obj = *it;
1232 :
1233 : // Remove any nullptr container entries while we're here.
1234 108049099 : if (!obj)
1235 1417854 : it = objects.erase(it);
1236 : else
1237 : {
1238 106631245 : processor_id_type obj_procid = obj->processor_id();
1239 106631245 : if (obj_procid == DofObject::invalid_processor_id)
1240 42409216 : unpartitioned_objects++;
1241 : else
1242 64222029 : ghost_objects_from_proc[obj_procid]++;
1243 :
1244 : // Finally, increment the iterator
1245 56346 : ++it;
1246 : }
1247 : }
1248 :
1249 1718354 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1250 1717562 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1251 792 : this->comm().allgather
1252 1717257 : ((this_it == ghost_objects_from_proc.end()) ?
1253 487 : dof_id_type(0) : this_it->second, objects_on_proc);
1254 :
1255 : #ifndef NDEBUG
1256 792 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1257 2376 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1258 1584 : if (ghost_objects_from_proc.count(p))
1259 943 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1260 : else
1261 641 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1262 : #endif
1263 :
1264 : // We'll renumber objects in blocks by processor id
1265 1718354 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1266 18217130 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1267 16503528 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1268 792 : objects_on_proc[i-1];
1269 1717562 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1270 1716770 : dof_id_type first_free_id =
1271 1718354 : first_object_on_proc[this->n_processors()-1] +
1272 792 : objects_on_proc[this->n_processors()-1] +
1273 : unpartitioned_objects;
1274 :
1275 : // First set new local object ids and build request sets
1276 : // for non-local object ids
1277 :
1278 : // Request sets to send to each processor
1279 : std::map<processor_id_type, std::vector<dof_id_type>>
1280 792 : requested_ids;
1281 :
1282 : // We know how many objects live on each processor, so reserve() space for
1283 : // each.
1284 792 : auto ghost_end = ghost_objects_from_proc.end();
1285 19933900 : for (auto p : make_range(this->n_processors()))
1286 18218714 : if (p != this->processor_id())
1287 : {
1288 16501152 : if (const auto p_it = ghost_objects_from_proc.find(p);
1289 792 : p_it != ghost_end)
1290 3321294 : requested_ids[p].reserve(p_it->second);
1291 : }
1292 :
1293 792 : end = objects.end();
1294 108348015 : for (it = objects.begin(); it != end; ++it)
1295 : {
1296 106631245 : T * obj = *it;
1297 106631245 : if (!obj)
1298 0 : continue;
1299 106687591 : if (obj->processor_id() == this->processor_id())
1300 19772667 : obj->set_id(next_id++);
1301 86858578 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1302 44449362 : requested_ids[obj->processor_id()].push_back(obj->id());
1303 : }
1304 :
1305 : // Next set ghost object ids from other processors
1306 :
1307 1733928 : auto gather_functor =
1308 47737252 : [
1309 : #ifndef NDEBUG
1310 : this,
1311 : &first_object_on_proc,
1312 : &objects_on_proc,
1313 : #endif
1314 : &objects]
1315 : (processor_id_type, const std::vector<dof_id_type> & ids,
1316 912 : std::vector<dof_id_type> & new_ids)
1317 : {
1318 912 : std::size_t ids_size = ids.size();
1319 3321294 : new_ids.resize(ids_size);
1320 :
1321 47770656 : for (std::size_t i=0; i != ids_size; ++i)
1322 : {
1323 44449362 : T * obj = objects[ids[i]];
1324 16246 : libmesh_assert(obj);
1325 16246 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1326 44465608 : new_ids[i] = obj->id();
1327 :
1328 16246 : libmesh_assert_greater_equal (new_ids[i],
1329 : first_object_on_proc[this->processor_id()]);
1330 16246 : libmesh_assert_less (new_ids[i],
1331 : first_object_on_proc[this->processor_id()] +
1332 : objects_on_proc[this->processor_id()]);
1333 : }
1334 : };
1335 :
1336 1733928 : auto action_functor =
1337 47737252 : [
1338 : #ifndef NDEBUG
1339 : &first_object_on_proc,
1340 : &objects_on_proc,
1341 : #endif
1342 : &objects]
1343 : (processor_id_type libmesh_dbg_var(pid),
1344 : const std::vector<dof_id_type> & ids,
1345 912 : const std::vector<dof_id_type> & data)
1346 : {
1347 : // Copy the id changes we've now been informed of
1348 47770656 : for (auto i : index_range(ids))
1349 : {
1350 44449362 : T * obj = objects[ids[i]];
1351 16246 : libmesh_assert (obj);
1352 16246 : libmesh_assert_equal_to (obj->processor_id(), pid);
1353 16246 : libmesh_assert_greater_equal (data[i],
1354 : first_object_on_proc[pid]);
1355 16246 : libmesh_assert_less (data[i],
1356 : first_object_on_proc[pid] +
1357 : objects_on_proc[pid]);
1358 44465608 : obj->set_id(data[i]);
1359 : }
1360 : };
1361 :
1362 792 : const dof_id_type * ex = nullptr;
1363 : Parallel::pull_parallel_vector_data
1364 1716770 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1365 :
1366 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1367 1733928 : auto unique_gather_functor =
1368 47737252 : [
1369 : #ifndef NDEBUG
1370 : this,
1371 : #endif
1372 : &objects]
1373 : (processor_id_type, const std::vector<dof_id_type> & ids,
1374 912 : std::vector<unique_id_type> & data)
1375 : {
1376 912 : std::size_t ids_size = ids.size();
1377 3321294 : data.resize(ids_size);
1378 :
1379 47770656 : for (std::size_t i=0; i != ids_size; ++i)
1380 : {
1381 44449362 : T * obj = objects[ids[i]];
1382 16246 : libmesh_assert(obj);
1383 16246 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1384 88898724 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1385 : }
1386 : };
1387 :
1388 1733928 : auto unique_action_functor =
1389 47737252 : [&objects]
1390 : (processor_id_type libmesh_dbg_var(pid),
1391 : const std::vector<dof_id_type> & ids,
1392 912 : const std::vector<unique_id_type> & data)
1393 : {
1394 47770656 : for (auto i : index_range(ids))
1395 : {
1396 44449362 : T * obj = objects[ids[i]];
1397 16246 : libmesh_assert (obj);
1398 16246 : libmesh_assert_equal_to (obj->processor_id(), pid);
1399 44449362 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1400 0 : obj->set_unique_id(data[i]);
1401 : }
1402 : };
1403 :
1404 792 : const unique_id_type * unique_ex = nullptr;
1405 : Parallel::pull_parallel_vector_data
1406 1716770 : (this->comm(), requested_ids, unique_gather_functor,
1407 : unique_action_functor, unique_ex);
1408 : #endif
1409 :
1410 : // Next set unpartitioned object ids
1411 792 : next_id = 0;
1412 19933900 : for (auto i : make_range(this->n_processors()))
1413 18218714 : next_id += objects_on_proc[i];
1414 108348015 : for (it = objects.begin(); it != end; ++it)
1415 : {
1416 106631245 : T * obj = *it;
1417 106631245 : if (!obj)
1418 0 : continue;
1419 106631245 : if (obj->processor_id() == DofObject::invalid_processor_id)
1420 42409216 : obj->set_id(next_id++);
1421 : }
1422 :
1423 : // Finally shuffle around objects so that container indices
1424 : // match ids
1425 1716770 : it = objects.begin();
1426 792 : end = objects.end();
1427 114819443 : while (it != end)
1428 : {
1429 113102673 : T * obj = *it;
1430 113102673 : if (obj) // don't try shuffling already-nullptr entries
1431 : {
1432 113102673 : T * next = objects[obj->id()];
1433 : // If we have to move this object
1434 113102673 : if (next != obj)
1435 : {
1436 : // nullptr out its original position for now
1437 : // (our shuffling may put another object there shortly)
1438 39559556 : *it = nullptr;
1439 :
1440 : // There may already be another object with this id that
1441 : // needs to be moved itself
1442 51024446 : while (next)
1443 : {
1444 : // We shouldn't be trying to give two objects the
1445 : // same id
1446 9329 : libmesh_assert_not_equal_to (next->id(), obj->id());
1447 11464890 : objects[obj->id()] = obj;
1448 9329 : obj = next;
1449 11464890 : next = objects[obj->id()];
1450 : }
1451 39559556 : objects[obj->id()] = obj;
1452 : }
1453 : }
1454 :
1455 : // Remove any container entries that were left as nullptr.
1456 115160 : if (!obj)
1457 0 : it = objects.erase(it);
1458 : else
1459 57580 : ++it;
1460 : }
1461 :
1462 1717562 : return first_free_id;
1463 : }
1464 :
1465 :
1466 860055 : void DistributedMesh::renumber_nodes_and_elements ()
1467 : {
1468 398 : parallel_object_only();
1469 :
1470 : #ifdef DEBUG
1471 : // Make sure our ids and flags are consistent
1472 398 : this->libmesh_assert_valid_parallel_ids();
1473 398 : this->libmesh_assert_valid_parallel_flags();
1474 398 : this->libmesh_assert_valid_parallel_p_levels();
1475 : #endif
1476 :
1477 398 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1478 :
1479 : // Nodes not connected to any elements, and nullptr node entries
1480 : // in our container, should be deleted. But wait! If we've deleted coarse
1481 : // local elements on some processor, other processors might have ghosted
1482 : // nodes from it that are now no longer connected to any elements on it, but
1483 : // that are connected to their own semilocal elements. We'll have to
1484 : // communicate to ascertain if that's the case.
1485 860055 : this->comm().max(_deleted_coarse_elements);
1486 :
1487 : // What used nodes do we see on our proc?
1488 398 : std::set<dof_id_type> used_nodes;
1489 :
1490 : // What used node info should we send from our proc? Could we take ownership
1491 : // of each node if we needed to?
1492 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1493 398 : used_nodes_on_proc;
1494 :
1495 : // flag the nodes we need
1496 56886160 : for (auto & elem : this->element_ptr_range())
1497 296414339 : for (const Node & node : elem->node_ref_range())
1498 : {
1499 268801765 : const dof_id_type n = node.id();
1500 268657459 : used_nodes.insert(n);
1501 268801765 : if (_deleted_coarse_elements)
1502 : {
1503 3145701 : const processor_id_type p = node.processor_id();
1504 3145985 : if (p != this->processor_id())
1505 : {
1506 2257624 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1507 2257766 : if (elem->processor_id() == this->processor_id())
1508 216210 : used_nodes_on_p[n] = true;
1509 : else
1510 105 : if (!used_nodes_on_p.count(n))
1511 525626 : used_nodes_on_p[n] = false;
1512 : }
1513 : }
1514 859259 : }
1515 :
1516 860055 : if (_deleted_coarse_elements)
1517 : {
1518 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1519 : typedef unsigned char boolish;
1520 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1521 28 : used_nodes_on_proc_vecs;
1522 73209 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1523 56827 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1524 :
1525 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1526 : std::map<processor_id_type, std::set<dof_id_type>>
1527 28 : repartitioned_node_sets_to_push;
1528 :
1529 : auto ids_action_functor =
1530 56801 : [&used_nodes, &repartitioned_node_pids,
1531 : &repartitioned_node_sets_to_push]
1532 : (processor_id_type pid,
1533 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1534 : {
1535 592596 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1536 : {
1537 : // If we don't see a use for our own node, but someone
1538 : // else does, better figure out who should own it next.
1539 97 : if (!used_nodes.count(n))
1540 : {
1541 2 : if (auto it = repartitioned_node_pids.find(n);
1542 2 : sender_could_become_owner)
1543 : {
1544 1 : if (it != repartitioned_node_pids.end() &&
1545 0 : pid < it->second)
1546 0 : it->second = pid;
1547 : else
1548 1 : repartitioned_node_pids[n] = pid;
1549 : }
1550 : else
1551 1 : if (it == repartitioned_node_pids.end())
1552 0 : repartitioned_node_pids[n] =
1553 : DofObject::invalid_processor_id;
1554 :
1555 2 : repartitioned_node_sets_to_push[pid].insert(n);
1556 : }
1557 : }
1558 16394 : };
1559 :
1560 : // We need two pushes instead of a pull here because we need to
1561 : // know *all* the queries for a particular node before we can
1562 : // respond to *any* of them.
1563 : Parallel::push_parallel_vector_data
1564 16382 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1565 :
1566 : // Repartition (what used to be) our own nodes first
1567 16383 : for (auto & [n, p] : repartitioned_node_pids)
1568 : {
1569 1 : Node & node = this->node_ref(n);
1570 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1571 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1572 1 : node.processor_id() = p;
1573 : }
1574 :
1575 : // Then push to repartition others' ghosted copies.
1576 :
1577 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1578 28 : repartitioned_node_vecs;
1579 :
1580 16384 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1581 : {
1582 2 : auto & rn_vec = repartitioned_node_vecs[p];
1583 4 : for (auto n : nodeset)
1584 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1585 : }
1586 :
1587 : auto repartition_node_functor =
1588 2 : [this]
1589 : (processor_id_type libmesh_dbg_var(pid),
1590 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1591 : {
1592 4 : for (auto [n, p] : ids_and_pids)
1593 : {
1594 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1595 2 : Node & node = this->node_ref(n);
1596 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1597 2 : node.processor_id() = p;
1598 : }
1599 16354 : };
1600 :
1601 : Parallel::push_parallel_vector_data
1602 16382 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1603 : }
1604 :
1605 860055 : _deleted_coarse_elements = false;
1606 :
1607 : // Nodes not connected to any local elements, and nullptr node entries
1608 : // in our container, are deleted
1609 : {
1610 860055 : node_iterator_imp it = _nodes.begin();
1611 398 : node_iterator_imp end = _nodes.end();
1612 :
1613 86712153 : while (it != end)
1614 : {
1615 85852098 : Node * nd = *it;
1616 85852098 : if (!nd)
1617 3471617 : it = _nodes.erase(it);
1618 82380481 : else if (!used_nodes.count(nd->id()))
1619 : {
1620 : // remove any boundary information associated with
1621 : // this node
1622 2328191 : this->get_boundary_info().remove (nd);
1623 2828 : _constraint_rows.erase(nd);
1624 :
1625 : // delete the node
1626 2328191 : delete nd;
1627 :
1628 2328191 : it = _nodes.erase(it);
1629 : }
1630 : else
1631 45233 : ++it;
1632 : }
1633 : }
1634 :
1635 860055 : this->_preparation.has_removed_orphaned_nodes = true;
1636 :
1637 860055 : if (_skip_renumber_nodes_and_elements)
1638 : {
1639 1670 : this->update_parallel_id_counts();
1640 2 : return;
1641 : }
1642 :
1643 : // Finally renumber all the elements
1644 858385 : _n_elem = this->renumber_dof_objects (this->_elements);
1645 :
1646 : // and all the remaining nodes
1647 858385 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1648 :
1649 : // And figure out what IDs we should use when adding new nodes and
1650 : // new elements
1651 858385 : this->update_parallel_id_counts();
1652 :
1653 : // Make sure our caches are up to date and our
1654 : // DofObjects are well packed
1655 : #ifdef DEBUG
1656 396 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1657 396 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1658 396 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1659 396 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1660 396 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1661 396 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1662 396 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1663 396 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1664 :
1665 : // Make sure our ids and flags are consistent
1666 396 : this->libmesh_assert_valid_parallel_ids();
1667 396 : this->libmesh_assert_valid_parallel_flags();
1668 :
1669 : // And make sure we've made our numbering monotonic
1670 396 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1671 : #endif
1672 : }
1673 :
1674 :
1675 :
1676 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1677 : {
1678 : // We can't use range-for here because we need access to the special
1679 : // iterators' methods, not just to their dereferenced values.
1680 :
1681 : // Nodes first
1682 0 : for (auto pr = this->_nodes.begin(),
1683 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1684 : {
1685 5427360 : Node * n = *pr;
1686 5427360 : if (n != nullptr)
1687 : {
1688 0 : const dof_id_type id = pr.index();
1689 5204550 : n->set_id() = id;
1690 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1691 : }
1692 : }
1693 :
1694 : // Elements next
1695 0 : for (auto pr = this->_elements.begin(),
1696 5671231 : end = this->_elements.end(); pr != end; ++pr)
1697 : {
1698 5653821 : Elem * e = *pr;
1699 5653821 : if (e != nullptr)
1700 : {
1701 0 : const dof_id_type id = pr.index();
1702 5590229 : e->set_id() = id;
1703 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1704 : }
1705 : }
1706 17410 : }
1707 :
1708 :
1709 :
1710 638314 : dof_id_type DistributedMesh::n_active_elem () const
1711 : {
1712 548 : parallel_object_only();
1713 :
1714 : // Get local active elements first
1715 : dof_id_type active_elements =
1716 1276080 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1717 1913846 : this->active_local_elements_end()));
1718 638314 : this->comm().sum(active_elements);
1719 :
1720 : // Then add unpartitioned active elements, which should exist on
1721 : // every processor
1722 638314 : active_elements +=
1723 638314 : static_cast<dof_id_type>(std::distance
1724 1276080 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1725 638862 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1726 638314 : return active_elements;
1727 : }
1728 :
1729 :
1730 :
1731 404570 : void DistributedMesh::delete_remote_elements()
1732 : {
1733 : #ifdef DEBUG
1734 : // Make sure our neighbor links are all fine
1735 386 : MeshTools::libmesh_assert_valid_neighbors(*this);
1736 :
1737 : // And our child/parent links, and our flags
1738 386 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1739 :
1740 : // Make sure our ids and flags are consistent
1741 386 : this->libmesh_assert_valid_parallel_ids();
1742 386 : this->libmesh_assert_valid_parallel_flags();
1743 :
1744 386 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1745 386 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1746 386 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1747 386 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1748 386 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1749 386 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1750 : #endif
1751 :
1752 404570 : _is_serial = false;
1753 404570 : _is_serial_on_proc_0 = false;
1754 :
1755 404570 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1756 :
1757 386 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1758 :
1759 : // Now make sure the containers actually shrink - strip
1760 : // any newly-created nullptr voids out of the element array
1761 404570 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1762 386 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1763 53429537 : while (e_it != e_end)
1764 53024967 : if (!*e_it)
1765 39492875 : e_it = _elements.erase(e_it);
1766 : else
1767 17377 : ++e_it;
1768 :
1769 404570 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1770 386 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1771 99940981 : while (n_it != n_end)
1772 99536411 : if (!*n_it)
1773 66380105 : n_it = _nodes.erase(n_it);
1774 : else
1775 48583 : ++n_it;
1776 :
1777 : // We may have deleted no-longer-connected nodes or coarsened-away
1778 : // elements; let's update our caches.
1779 404570 : this->update_parallel_id_counts();
1780 :
1781 : // We may have deleted nodes or elements that were the only local
1782 : // representatives of some particular boundary id(s); let's update
1783 : // those caches.
1784 404570 : this->get_boundary_info().regenerate_id_sets();
1785 :
1786 : #ifdef DEBUG
1787 : // We might not have well-packed objects if the user didn't allow us
1788 : // to renumber
1789 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1790 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1791 :
1792 : // Make sure our neighbor links are all fine
1793 386 : MeshTools::libmesh_assert_valid_neighbors(*this);
1794 :
1795 : // And our child/parent links, and our flags
1796 386 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1797 :
1798 : // Make sure our ids and flags are consistent
1799 386 : this->libmesh_assert_valid_parallel_ids();
1800 386 : this->libmesh_assert_valid_parallel_flags();
1801 : #endif
1802 :
1803 404570 : this->_preparation.has_removed_remote_elements = true;
1804 404570 : }
1805 :
1806 :
1807 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1808 : {
1809 : // First add the elem like normal
1810 0 : add_elem(e);
1811 :
1812 : // Now add it to the set that won't be deleted when we call
1813 : // delete_remote_elements()
1814 0 : _extra_ghost_elems.insert(e);
1815 0 : }
1816 :
1817 : void
1818 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1819 : {
1820 0 : std::set<Elem *> tmp;
1821 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1822 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1823 0 : std::inserter(tmp, tmp.begin()));
1824 0 : _extra_ghost_elems = tmp;
1825 0 : }
1826 :
1827 97139 : void DistributedMesh::allgather()
1828 : {
1829 97139 : if (_is_serial)
1830 0 : return;
1831 97137 : MeshCommunication().allgather(*this);
1832 97137 : _is_serial = true;
1833 97137 : _is_serial_on_proc_0 = true;
1834 :
1835 : // Make sure our caches are up to date and our
1836 : // DofObjects are well packed
1837 : #ifdef DEBUG
1838 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1839 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1840 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1841 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1842 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1843 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1844 :
1845 : // If we've disabled renumbering we can't be sure we're contiguous
1846 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1847 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1848 :
1849 : // Make sure our neighbor links are all fine
1850 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1851 :
1852 : // Make sure our ids and flags are consistent
1853 48 : this->libmesh_assert_valid_parallel_ids();
1854 48 : this->libmesh_assert_valid_parallel_flags();
1855 : #endif
1856 : }
1857 :
1858 18352 : void DistributedMesh::gather_to_zero()
1859 : {
1860 18352 : if (_is_serial_on_proc_0)
1861 4 : return;
1862 :
1863 14114 : _is_serial_on_proc_0 = true;
1864 14114 : MeshCommunication().gather(0, *this);
1865 : }
1866 :
1867 :
1868 : } // namespace libMesh
|
