Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 276635 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 276635 : unsigned char d) :
44 276151 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 276151 : _is_serial_on_proc_0(true),
46 276151 : _deleted_coarse_elements(false),
47 276151 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 276877 : _next_free_local_node_id(this->processor_id()),
49 276635 : _next_free_local_elem_id(this->processor_id()),
50 276635 : _next_free_unpartitioned_node_id(this->n_processors()),
51 276635 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 276635 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 276635 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 276877 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 553270 : default_partitioner);
64 : _partitioner = Partitioner::build
65 553028 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 276635 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 12346 : bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 : {
100 28 : const DistributedMesh * dist_mesh_ptr =
101 12346 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 12346 : if (!dist_mesh_ptr)
103 0 : return false;
104 28 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 84 : if (_is_serial != other_mesh._is_serial ||
107 12328 : _is_serial_on_proc_0 != other_mesh._is_serial_on_proc_0 ||
108 12346 : _deleted_coarse_elements != other_mesh._deleted_coarse_elements ||
109 12328 : _n_nodes != other_mesh._n_nodes ||
110 12346 : _n_elem != other_mesh._n_elem ||
111 12346 : _max_node_id != other_mesh._max_node_id ||
112 37038 : _max_elem_id != other_mesh._max_elem_id ||
113 : // We expect these things to change in a prepare_for_use();
114 : // they're conceptually "mutable"...
115 : /*
116 : _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117 : _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118 : _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119 : _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
121 : _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 : #endif
123 : */
124 12346 : !this->nodes_and_elements_equal(other_mesh))
125 63 : return false;
126 :
127 12309 : if (_extra_ghost_elems.size() !=
128 26 : other_mesh._extra_ghost_elems.size())
129 0 : return false;
130 12283 : for (auto & elem : _extra_ghost_elems)
131 : {
132 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134 0 : if (!other_elem ||
135 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136 0 : return false;
137 : }
138 :
139 26 : return true;
140 : }
141 :
142 318532 : DistributedMesh::~DistributedMesh ()
143 : {
144 296783 : this->DistributedMesh::clear(); // Free nodes and elements
145 318532 : }
146 :
147 :
148 : // This might be specialized later, but right now it's just here to
149 : // make sure the compiler doesn't give us a default (non-deep) copy
150 : // constructor instead.
151 17663 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
152 17663 : DistributedMesh(static_cast<const MeshBase &>(other_mesh))
153 : {
154 17663 : _is_serial = other_mesh._is_serial;
155 17663 : _is_serial_on_proc_0 = other_mesh._is_serial_on_proc_0;
156 17663 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
157 :
158 17663 : _n_nodes = other_mesh.n_nodes();
159 17663 : _n_elem = other_mesh.n_elem();
160 17663 : _max_node_id = other_mesh.max_node_id();
161 17663 : _max_elem_id = other_mesh.max_elem_id();
162 17663 : _next_free_local_node_id =
163 17663 : other_mesh._next_free_local_node_id;
164 17663 : _next_free_local_elem_id =
165 17663 : other_mesh._next_free_local_elem_id;
166 17663 : _next_free_unpartitioned_node_id =
167 17663 : other_mesh._next_free_unpartitioned_node_id;
168 17663 : _next_free_unpartitioned_elem_id =
169 17663 : other_mesh._next_free_unpartitioned_elem_id;
170 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
171 17663 : _next_unique_id =
172 17663 : other_mesh._next_unique_id;
173 17663 : _next_unpartitioned_unique_id =
174 17663 : other_mesh._next_unpartitioned_unique_id;
175 : #endif
176 :
177 : // Need to copy extra_ghost_elems
178 17663 : for (auto & elem : other_mesh._extra_ghost_elems)
179 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
180 17663 : }
181 :
182 :
183 :
184 20148 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh) :
185 20148 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
186 20236 : _is_serial_on_proc_0(other_mesh.is_serial()),
187 19972 : _deleted_coarse_elements(true), // better safe than sorry...
188 19972 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
189 20236 : _next_free_local_node_id(this->processor_id()),
190 20148 : _next_free_local_elem_id(this->processor_id()),
191 20148 : _next_free_unpartitioned_node_id(this->n_processors()),
192 40208 : _next_free_unpartitioned_elem_id(this->n_processors())
193 : {
194 : // Just copy, skipping preparation
195 20148 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
196 :
197 176 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
198 176 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
199 176 : this->allow_renumbering(other_mesh.allow_renumbering());
200 176 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
201 176 : this->skip_partitioning(other_mesh.skip_partitioning());
202 :
203 20148 : this->copy_constraint_rows(other_mesh);
204 :
205 20148 : this->_preparation = other_mesh.preparation();
206 :
207 88 : auto & this_boundary_info = this->get_boundary_info();
208 88 : const auto & other_boundary_info = other_mesh.get_boundary_info();
209 :
210 20148 : this_boundary_info = other_boundary_info;
211 :
212 88 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
213 :
214 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
215 20148 : _next_unique_id = other_mesh.parallel_max_unique_id() +
216 20236 : this->processor_id();
217 20236 : _next_unpartitioned_unique_id = _next_unique_id +
218 20148 : (this->n_processors() - this->processor_id());
219 : #endif
220 20148 : this->update_parallel_id_counts();
221 20148 : }
222 :
223 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
224 : {
225 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
226 :
227 4 : this->_nodes = std::move(other_mesh._nodes);
228 270 : this->_n_nodes = other_mesh.n_nodes();
229 :
230 4 : this->_elements = std::move(other_mesh._elements);
231 270 : this->_n_elem = other_mesh.n_elem();
232 :
233 270 : _is_serial = other_mesh.is_serial();
234 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
235 270 : _deleted_coarse_elements = true; // Better safe than sorry
236 :
237 270 : _max_node_id = other_mesh.max_node_id();
238 270 : _max_elem_id = other_mesh.max_elem_id();
239 :
240 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
241 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
242 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
243 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
244 :
245 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
246 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
247 : #endif
248 270 : }
249 :
250 : // We use cached values for these so they can be called
251 : // from one processor without bothering the rest, but
252 : // we may need to update those caches before doing a full
253 : // renumbering
254 1709473 : void DistributedMesh::update_parallel_id_counts()
255 : {
256 : // This function must be run on all processors at once
257 1722 : parallel_object_only();
258 :
259 1709473 : _n_elem = this->parallel_n_elem();
260 1709473 : _n_nodes = this->parallel_n_nodes();
261 1709473 : _max_node_id = this->parallel_max_node_id();
262 1709473 : _max_elem_id = this->parallel_max_elem_id();
263 :
264 1709473 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
265 21546 : _next_free_unpartitioned_elem_id =
266 21546 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
267 21546 : (this->n_processors() + 1) + this->n_processors();
268 1709473 : if (_next_free_local_elem_id < _max_elem_id)
269 21054 : _next_free_local_elem_id =
270 21076 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
271 21054 : (this->n_processors() + 1) + this->processor_id();
272 :
273 1709473 : if (_next_free_unpartitioned_node_id < _max_node_id)
274 28243 : _next_free_unpartitioned_node_id =
275 28243 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
276 28243 : (this->n_processors() + 1) + this->n_processors();
277 1709473 : if (_next_free_local_node_id < _max_node_id)
278 28885 : _next_free_local_node_id =
279 28910 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
280 28885 : (this->n_processors() + 1) + this->processor_id();
281 :
282 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
283 1709473 : _next_unique_id = this->parallel_max_unique_id();
284 1709473 : _next_unpartitioned_unique_id =
285 1711195 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
286 1709473 : (this->n_processors() + 1) + this->n_processors();
287 1709473 : _next_unique_id =
288 1709473 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
289 1709473 : (this->n_processors() + 1) + this->processor_id();
290 : #endif
291 :
292 1709473 : this->_preparation.has_synched_id_counts = true;
293 1709473 : }
294 :
295 :
296 : // Or in debug mode we may want to test the uncached values without
297 : // changing the cache
298 1717184 : dof_id_type DistributedMesh::parallel_n_elem() const
299 : {
300 : // This function must be run on all processors at once
301 2556 : parallel_object_only();
302 :
303 1717184 : dof_id_type n_local = this->n_local_elem();
304 1717184 : this->comm().sum(n_local);
305 1717184 : n_local += this->n_unpartitioned_elem();
306 1717184 : return n_local;
307 : }
308 :
309 :
310 :
311 1716351 : dof_id_type DistributedMesh::parallel_max_elem_id() const
312 : {
313 : // This function must be run on all processors at once
314 8600 : parallel_object_only();
315 :
316 1716351 : dof_id_type max_local = 0;
317 :
318 : dofobject_container<Elem>::const_reverse_veclike_iterator
319 8600 : rit = _elements.rbegin();
320 :
321 : const dofobject_container<Elem>::const_reverse_veclike_iterator
322 8600 : rend = _elements.rend();
323 :
324 : // Look for the maximum element id. Search backwards through
325 : // elements so we can break out early. Beware of nullptr entries that
326 : // haven't yet been cleared from _elements.
327 11135240 : for (; rit != rend; ++rit)
328 : {
329 10732337 : const DofObject *d = *rit;
330 18877 : if (d)
331 : {
332 8385 : libmesh_assert(_elements[d->id()] == d);
333 1313448 : max_local = d->id() + 1;
334 1313448 : break;
335 : }
336 : }
337 :
338 1716351 : this->comm().max(max_local);
339 1716351 : return max_local;
340 : }
341 :
342 :
343 :
344 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
345 1811325 : unique_id_type DistributedMesh::parallel_max_unique_id() const
346 : {
347 : // This function must be run on all processors at once
348 1840 : parallel_object_only();
349 :
350 3622650 : unique_id_type max_local = std::max(_next_unique_id,
351 2119723 : _next_unpartitioned_unique_id);
352 1811325 : this->comm().max(max_local);
353 1811325 : return max_local;
354 : }
355 :
356 :
357 :
358 77744 : void DistributedMesh::set_next_unique_id(unique_id_type id)
359 : {
360 142 : _next_unique_id = id;
361 77744 : _next_unpartitioned_unique_id =
362 77886 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
363 77744 : (this->n_processors() + 1) + this->n_processors();
364 77744 : _next_unique_id =
365 77744 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
366 77744 : (this->n_processors() + 1) + this->processor_id();
367 77744 : }
368 : #endif
369 :
370 :
371 :
372 1717632 : dof_id_type DistributedMesh::parallel_n_nodes() const
373 : {
374 : // This function must be run on all processors at once
375 2556 : parallel_object_only();
376 :
377 1717632 : dof_id_type n_local = this->n_local_nodes();
378 1717632 : this->comm().sum(n_local);
379 1717632 : n_local += this->n_unpartitioned_nodes();
380 1717632 : return n_local;
381 : }
382 :
383 :
384 :
385 1713977 : dof_id_type DistributedMesh::parallel_max_node_id() const
386 : {
387 : // This function must be run on all processors at once
388 6226 : parallel_object_only();
389 :
390 1713977 : dof_id_type max_local = 0;
391 :
392 : dofobject_container<Node>::const_reverse_veclike_iterator
393 6226 : rit = _nodes.rbegin();
394 :
395 : const dofobject_container<Node>::const_reverse_veclike_iterator
396 6226 : rend = _nodes.rend();
397 :
398 : // Look for the maximum node id. Search backwards through
399 : // nodes so we can break out early. Beware of nullptr entries that
400 : // haven't yet been cleared from _nodes
401 39401767 : for (; rit != rend; ++rit)
402 : {
403 38998934 : const DofObject *d = *rit;
404 63161 : if (d)
405 : {
406 6081 : libmesh_assert(_nodes[d->id()] == d);
407 1311144 : max_local = d->id() + 1;
408 1311144 : break;
409 : }
410 : }
411 :
412 1713977 : this->comm().max(max_local);
413 1713977 : return max_local;
414 : }
415 :
416 :
417 :
418 42431212 : const Point & DistributedMesh::point (const dof_id_type i) const
419 : {
420 42431212 : return this->node_ref(i);
421 : }
422 :
423 :
424 :
425 43536601 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
426 : {
427 136653 : libmesh_assert(_nodes[i]);
428 136653 : libmesh_assert_equal_to (_nodes[i]->id(), i);
429 :
430 43536601 : return _nodes[i];
431 : }
432 :
433 :
434 :
435 :
436 734214004 : Node * DistributedMesh::node_ptr (const dof_id_type i)
437 : {
438 541258 : libmesh_assert(_nodes[i]);
439 541258 : libmesh_assert_equal_to (_nodes[i]->id(), i);
440 :
441 734214004 : return _nodes[i];
442 : }
443 :
444 :
445 :
446 :
447 3114230 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
448 : {
449 3114230 : if (const auto it = _nodes.find(i);
450 662507 : it != _nodes.end())
451 : {
452 2820162 : const Node * n = *it;
453 368439 : libmesh_assert (!n || n->id() == i);
454 368439 : return n;
455 : }
456 :
457 294068 : return nullptr;
458 : }
459 :
460 :
461 :
462 :
463 367281329 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
464 : {
465 367281329 : if (auto it = _nodes.find(i);
466 159224 : it != _nodes.end())
467 : {
468 284558487 : Node * n = *it;
469 111971 : libmesh_assert (!n || n->id() == i);
470 111971 : return n;
471 : }
472 :
473 47253 : return nullptr;
474 : }
475 :
476 :
477 :
478 :
479 3049564 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
480 : {
481 10062 : libmesh_assert(_elements[i]);
482 10062 : libmesh_assert_equal_to (_elements[i]->id(), i);
483 :
484 3049564 : return _elements[i];
485 : }
486 :
487 :
488 :
489 :
490 286875120 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
491 : {
492 167219 : libmesh_assert(_elements[i]);
493 167219 : libmesh_assert_equal_to (_elements[i]->id(), i);
494 :
495 286875120 : return _elements[i];
496 : }
497 :
498 :
499 :
500 :
501 1474667 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
502 : {
503 1474667 : if (const auto it = _elements.find(i);
504 636453 : it != _elements.end())
505 : {
506 1098550 : const Elem * e = *it;
507 269217 : libmesh_assert (!e || e->id() == i);
508 269217 : return e;
509 : }
510 :
511 367236 : return nullptr;
512 : }
513 :
514 :
515 :
516 :
517 603557308 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
518 : {
519 603557308 : if (auto it = _elements.find(i);
520 194370 : it != _elements.end())
521 : {
522 518040619 : Elem * e = *it;
523 190691 : libmesh_assert (!e || e->id() == i);
524 190691 : return e;
525 : }
526 :
527 3679 : return nullptr;
528 : }
529 :
530 :
531 :
532 :
533 44036151 : Elem * DistributedMesh::add_elem (Elem * e)
534 : {
535 : // Don't try to add nullptrs!
536 18881 : libmesh_assert(e);
537 :
538 : // Trying to add an existing element is a no-op
539 44036151 : if (e->valid_id() && _elements[e->id()] == e)
540 0 : return e;
541 :
542 44036151 : const processor_id_type elem_procid = e->processor_id();
543 :
544 44036151 : if (!e->valid_id())
545 : {
546 : // We should only be creating new ids past the end of the range
547 : // of existing ids
548 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
549 : _max_elem_id);
550 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
551 :
552 : // Use the unpartitioned ids for unpartitioned elems, and
553 : // temporarily for ghost elems
554 3857269 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
555 3860837 : if (elem_procid == this->processor_id())
556 1133564 : next_id = &_next_free_local_elem_id;
557 3857269 : e->set_id (*next_id);
558 : }
559 :
560 : {
561 : // Advance next_ids up high enough that each is pointing to an
562 : // unused id and any subsequent increments will still point us
563 : // to unused ids
564 88072302 : _max_elem_id = std::max(_max_elem_id,
565 44036151 : static_cast<dof_id_type>(e->id()+1));
566 :
567 44036151 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
568 5138781 : _next_free_unpartitioned_elem_id =
569 5138781 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
570 5138781 : (this->n_processors() + 1) + this->n_processors();
571 44036151 : if (_next_free_local_elem_id < _max_elem_id)
572 3445697 : _next_free_local_elem_id =
573 3451937 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
574 3445697 : (this->n_processors() + 1) + this->processor_id();
575 :
576 : #ifndef NDEBUG
577 : // We need a const dofobject_container so we don't inadvertently create
578 : // nullptr entries when testing for non-nullptr ones
579 18881 : const dofobject_container<Elem> & const_elements = _elements;
580 : #endif
581 18881 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
582 18881 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
583 : }
584 :
585 : // Don't try to overwrite existing elems
586 18881 : libmesh_assert (!_elements[e->id()]);
587 :
588 44036151 : _elements[e->id()] = e;
589 :
590 : // We actually added a new element. Some of our caches might still
591 : // be valid, but we should clear the ones which definitely are not.
592 44036151 : this->clear_point_locator();
593 44036151 : this->clear_stored_ranges();
594 :
595 : // Try to make the cached elem data more accurate
596 44055032 : if (elem_procid == this->processor_id() ||
597 : elem_procid == DofObject::invalid_processor_id)
598 12902933 : _n_elem++;
599 :
600 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
601 44036151 : if (!e->valid_unique_id())
602 : {
603 13500311 : if (processor_id() == e->processor_id())
604 : {
605 1133564 : e->set_unique_id(_next_unique_id);
606 1133564 : _next_unique_id += this->n_processors() + 1;
607 : }
608 : else
609 : {
610 12366747 : e->set_unique_id(_next_unpartitioned_unique_id);
611 12366747 : _next_unpartitioned_unique_id += this->n_processors() + 1;
612 : }
613 : }
614 : else
615 : {
616 30560078 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
617 30535840 : _next_unique_id =
618 30535840 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
619 30535840 : (this->n_processors() + 1) + this->processor_id();
620 : }
621 : #endif
622 :
623 : // Unpartitioned elems should be added on every processor
624 : // And shouldn't be added in the same batch as ghost elems
625 : // But we might be just adding on processor 0 to
626 : // broadcast later
627 : // #ifdef DEBUG
628 : // if (elem_procid == DofObject::invalid_processor_id)
629 : // {
630 : // dof_id_type elem_id = e->id();
631 : // this->comm().max(elem_id);
632 : // libmesh_assert_equal_to (elem_id, e->id());
633 : // }
634 : // #endif
635 :
636 : // Make sure any new element is given space for any extra integers
637 : // we've requested
638 44036151 : e->add_extra_integers(_elem_integer_names.size(),
639 44036151 : _elem_integer_default_values);
640 :
641 : // And set mapping type and data on any new element
642 37762 : e->set_mapping_type(this->default_mapping_type());
643 37762 : e->set_mapping_data(this->default_mapping_data());
644 :
645 44036151 : return e;
646 : }
647 :
648 :
649 :
650 14735595 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
651 : {
652 : // The mesh now takes ownership of the Elem. Eventually the guts of
653 : // add_elem() will get moved to a private helper function, and
654 : // calling add_elem() directly will be deprecated.
655 14735595 : return add_elem(e.release());
656 : }
657 :
658 :
659 :
660 3348460 : Elem * DistributedMesh::insert_elem (Elem * e)
661 : {
662 3348460 : if (_elements[e->id()])
663 3348460 : this->delete_elem(_elements[e->id()]);
664 :
665 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
666 3348460 : if (!e->valid_unique_id())
667 : {
668 0 : if (processor_id() == e->processor_id())
669 : {
670 0 : e->set_unique_id(_next_unique_id);
671 0 : _next_unique_id += this->n_processors() + 1;
672 : }
673 : else
674 : {
675 0 : e->set_unique_id(_next_unpartitioned_unique_id);
676 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
677 : }
678 : }
679 : else
680 : {
681 3373838 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
682 3348460 : _next_unique_id =
683 3352156 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
684 3348460 : (this->n_processors() + 1) + this->processor_id();
685 : }
686 : #endif
687 :
688 : // Try to make the cached elem data more accurate
689 3348460 : processor_id_type elem_procid = e->processor_id();
690 3352156 : if (elem_procid == this->processor_id() ||
691 : elem_procid == DofObject::invalid_processor_id)
692 3219384 : _n_elem++;
693 :
694 3348460 : _elements[e->id()] = e;
695 :
696 : // We actually added a new element. Some of our caches might still
697 : // be valid, but we should clear the ones which definitely are not.
698 3348460 : this->clear_point_locator();
699 3348460 : this->clear_stored_ranges();
700 :
701 : // Make sure any new element is given space for any extra integers
702 : // we've requested
703 3348460 : e->add_extra_integers(_elem_integer_names.size(),
704 3348460 : _elem_integer_default_values);
705 :
706 : // And set mapping type and data on any new element
707 7392 : e->set_mapping_type(this->default_mapping_type());
708 7392 : e->set_mapping_data(this->default_mapping_data());
709 :
710 3348460 : return e;
711 : }
712 :
713 3348460 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
714 : {
715 : // The mesh now takes ownership of the Elem. Eventually the guts of
716 : // insert_elem(Elem*) will get moved to a private helper function, and
717 : // calling insert_elem(Elem*) directly will be deprecated.
718 3348460 : return insert_elem(e.release());
719 : }
720 :
721 :
722 40088130 : void DistributedMesh::delete_elem(Elem * e)
723 : {
724 7586 : libmesh_assert (e);
725 :
726 : // Try to make the cached elem data more accurate
727 40088130 : _n_elem--;
728 :
729 : // Was this a coarse element, not just a coarsening where we still
730 : // have some ancestor structure? Was it a *local* element, that we
731 : // might have been depending on as an owner of local nodes? We'll
732 : // have to be more careful with our nodes in contract() later; no
733 : // telling if we just locally orphaned a node that should be
734 : // globally retained.
735 40097536 : if (e->processor_id() == this->processor_id() &&
736 3640 : !e->parent())
737 80473 : _deleted_coarse_elements = true;
738 :
739 : // Delete the element from the BoundaryInfo object
740 40088130 : this->get_boundary_info().remove(e);
741 :
742 : // But not yet from the container; we might invalidate
743 : // an iterator that way!
744 :
745 : //_elements.erase(e->id());
746 :
747 : // Instead, we set it to nullptr for now
748 :
749 40088130 : _elements[e->id()] = nullptr;
750 :
751 : // delete the element
752 40088130 : delete e;
753 :
754 : // Some of our caches might still be valid, but we should clear the
755 : // ones which definitely are not.
756 40088130 : this->clear_point_locator();
757 40088130 : this->clear_stored_ranges();
758 40088130 : }
759 :
760 :
761 :
762 3207800 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
763 : const dof_id_type new_id)
764 : {
765 : // This could be a no-op
766 3207800 : if (old_id == new_id)
767 0 : return;
768 :
769 3207800 : Elem * el = _elements[old_id];
770 1160 : libmesh_assert (el);
771 1160 : libmesh_assert_equal_to (el->id(), old_id);
772 :
773 3207800 : el->set_id(new_id);
774 1160 : libmesh_assert (!_elements[new_id]);
775 3207800 : _elements[new_id] = el;
776 3207800 : _elements.erase(old_id);
777 :
778 : // Should we delete any caches here? Our point locator indexes by
779 : // element pointer and should be fine with an id change. Our stored
780 : // ranges are no longer sorted, which is *probably* fine, but let's
781 : // just be safe.
782 3207800 : this->clear_stored_ranges();
783 : }
784 :
785 :
786 :
787 40929334 : Node * DistributedMesh::add_point (const Point & p,
788 : const dof_id_type id,
789 : const processor_id_type proc_id)
790 : {
791 40929334 : Node * old_n = this->query_node_ptr(id);
792 :
793 40929334 : if (old_n)
794 : {
795 0 : *old_n = p;
796 0 : old_n->processor_id() = proc_id;
797 :
798 0 : return old_n;
799 : }
800 :
801 40889579 : Node * n = Node::build(p, id).release();
802 40929334 : n->processor_id() = proc_id;
803 :
804 40929334 : return DistributedMesh::add_node(n);
805 : }
806 :
807 :
808 4130 : void DistributedMesh::own_node (Node & n)
809 : {
810 : // This had better be a node in our mesh
811 0 : libmesh_assert(_nodes[n.id()] == &n);
812 :
813 4130 : _nodes[n.id()] = nullptr;
814 4130 : _n_nodes--;
815 :
816 0 : n.set_id(DofObject::invalid_id);
817 4130 : n.processor_id() = this->processor_id();
818 :
819 4130 : this->add_node(&n);
820 4130 : }
821 :
822 :
823 71701873 : Node * DistributedMesh::add_node (Node * n)
824 : {
825 : // Don't try to add nullptrs!
826 41467 : libmesh_assert(n);
827 :
828 : // Trying to add an existing node is a no-op
829 71701873 : if (n->valid_id() && _nodes[n->id()] == n)
830 0 : return n;
831 :
832 71701873 : const processor_id_type node_procid = n->processor_id();
833 :
834 71701873 : if (!n->valid_id())
835 : {
836 : // We should only be creating new ids past the end of the range
837 : // of existing ids
838 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
839 : _max_node_id);
840 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
841 :
842 : // Use the unpartitioned ids for unpartitioned nodes,
843 : // and temporarily for ghost nodes
844 17183188 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
845 17201017 : if (node_procid == this->processor_id())
846 2128337 : next_id = &_next_free_local_node_id;
847 17183188 : n->set_id (*next_id);
848 : }
849 :
850 : {
851 : // Advance next_ids up high enough that each is pointing to an
852 : // unused id and any subsequent increments will still point us
853 : // to unused ids
854 143403746 : _max_node_id = std::max(_max_node_id,
855 71701873 : static_cast<dof_id_type>(n->id()+1));
856 :
857 71701873 : if (_next_free_unpartitioned_node_id < _max_node_id)
858 20102498 : _next_free_unpartitioned_node_id =
859 20102498 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
860 20102498 : (this->n_processors() + 1) + this->n_processors();
861 71701873 : if (_next_free_local_node_id < _max_node_id)
862 11711351 : _next_free_local_node_id =
863 11729147 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
864 11711351 : (this->n_processors() + 1) + this->processor_id();
865 :
866 : #ifndef NDEBUG
867 : // We need a const dofobject_container so we don't inadvertently create
868 : // nullptr entries when testing for non-nullptr ones
869 41467 : const dofobject_container<Node> & const_nodes = _nodes;
870 : #endif
871 41467 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
872 41467 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
873 : }
874 :
875 : // Don't try to overwrite existing nodes
876 41467 : libmesh_assert (!_nodes[n->id()]);
877 :
878 71701873 : _nodes[n->id()] = n;
879 :
880 : // Try to make the cached node data more accurate
881 71743340 : if (node_procid == this->processor_id() ||
882 : node_procid == DofObject::invalid_processor_id)
883 38515799 : _n_nodes++;
884 :
885 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
886 71701873 : if (!n->valid_unique_id())
887 : {
888 40931062 : if (processor_id() == n->processor_id())
889 : {
890 2544599 : n->set_unique_id(_next_unique_id);
891 2544599 : _next_unique_id += this->n_processors() + 1;
892 : }
893 : else
894 : {
895 38386463 : n->set_unique_id(_next_unpartitioned_unique_id);
896 38386463 : _next_unpartitioned_unique_id += this->n_processors() + 1;
897 : }
898 : }
899 : else
900 : {
901 30901256 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
902 30770811 : _next_unique_id =
903 30770811 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
904 30770811 : (this->n_processors() + 1) + this->processor_id();
905 : }
906 : #endif
907 :
908 71701873 : n->add_extra_integers(_node_integer_names.size(),
909 71701873 : _node_integer_default_values);
910 :
911 : // Unpartitioned nodes should be added on every processor
912 : // And shouldn't be added in the same batch as ghost nodes
913 : // But we might be just adding on processor 0 to
914 : // broadcast later
915 : // #ifdef DEBUG
916 : // if (node_procid == DofObject::invalid_processor_id)
917 : // {
918 : // dof_id_type node_id = n->id();
919 : // this->comm().max(node_id);
920 : // libmesh_assert_equal_to (node_id, n->id());
921 : // }
922 : // #endif
923 :
924 71701873 : return n;
925 : }
926 :
927 30768409 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
928 : {
929 : // The mesh now takes ownership of the Node. Eventually the guts of
930 : // add_node() will get moved to a private helper function, and
931 : // calling add_node() directly will be deprecated.
932 30768409 : return add_node(n.release());
933 : }
934 :
935 50479443 : void DistributedMesh::delete_node(Node * n)
936 : {
937 4461 : libmesh_assert(n);
938 4461 : libmesh_assert(_nodes[n->id()]);
939 :
940 : // Try to make the cached elem data more accurate
941 50479443 : _n_nodes--;
942 :
943 : // Delete the node from the BoundaryInfo object
944 50479443 : this->get_boundary_info().remove(n);
945 8922 : _constraint_rows.erase(n);
946 :
947 : // But not yet from the container; we might invalidate
948 : // an iterator that way!
949 :
950 : //_nodes.erase(n->id());
951 :
952 : // Instead, we set it to nullptr for now
953 :
954 50479443 : _nodes[n->id()] = nullptr;
955 :
956 : // delete the node
957 50479443 : delete n;
958 50479443 : }
959 :
960 :
961 :
962 6897011 : void DistributedMesh::renumber_node(const dof_id_type old_id,
963 : const dof_id_type new_id)
964 : {
965 : // This could be a no-op
966 6897011 : if (old_id == new_id)
967 0 : return;
968 :
969 6896419 : Node * nd = _nodes[old_id];
970 3234 : libmesh_assert (nd);
971 3234 : libmesh_assert_equal_to (nd->id(), old_id);
972 :
973 : // If we have nodes shipped to this processor for NodeConstraints
974 : // use, then those nodes will exist in _nodes, but may not be
975 : // locatable via a TopologyMap due to the insufficiency of elements
976 : // connecting to them. If local refinement then wants to create a
977 : // *new* node in the same location, it will initially get a temporary
978 : // id, and then make_node_ids_parallel_consistent() will try to move
979 : // it to the canonical id. We need to account for this case to
980 : // avoid false positives and memory leaks.
981 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
982 6468 : if (_nodes[new_id])
983 : {
984 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
985 : *(Point *)_nodes[old_id]);
986 0 : _nodes.erase(new_id);
987 : }
988 : #else
989 : // If we aren't shipping nodes for NodeConstraints, there should be
990 : // no reason for renumbering one node onto another.
991 : libmesh_assert (!_nodes[new_id]);
992 : #endif
993 6896419 : _nodes[new_id] = nd;
994 6896419 : nd->set_id(new_id);
995 :
996 6896419 : _nodes.erase(old_id);
997 : }
998 :
999 :
1000 :
1001 554311 : void DistributedMesh::clear ()
1002 : {
1003 : // Call parent clear function
1004 554311 : MeshBase::clear();
1005 :
1006 : // Clear our elements and nodes
1007 : // There is no need to remove them from
1008 : // the BoundaryInfo data structure since we
1009 : // already cleared it.
1010 554311 : this->DistributedMesh::clear_elems();
1011 :
1012 23248916 : for (auto & node : _nodes)
1013 41549634 : delete node;
1014 :
1015 564 : _nodes.clear();
1016 :
1017 : // We're no longer distributed if we were before
1018 554311 : _is_serial = true;
1019 554311 : _is_serial_on_proc_0 = true;
1020 :
1021 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1022 : // all is copacetic.
1023 554311 : _deleted_coarse_elements = false;
1024 :
1025 : // Correct our caches
1026 554311 : _n_nodes = 0;
1027 554311 : _max_node_id = 0;
1028 554875 : _next_free_local_node_id = this->processor_id();
1029 554311 : _next_free_unpartitioned_node_id = this->n_processors();
1030 554311 : }
1031 :
1032 :
1033 :
1034 560327 : void DistributedMesh::clear_elems ()
1035 : {
1036 9340987 : for (auto & elem : _elements)
1037 8780660 : delete elem;
1038 :
1039 564 : _elements.clear();
1040 :
1041 : // Correct our caches
1042 560327 : _n_elem = 0;
1043 560327 : _max_elem_id = 0;
1044 560891 : _next_free_local_elem_id = this->processor_id();
1045 560327 : _next_free_unpartitioned_elem_id = this->n_processors();
1046 560327 : }
1047 :
1048 :
1049 :
1050 553573 : void DistributedMesh::redistribute ()
1051 : {
1052 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1053 553573 : if (!this->is_serial())
1054 : {
1055 : // Construct a MeshCommunication object to actually redistribute the nodes
1056 : // and elements according to the partitioner, and then to re-gather the neighbors.
1057 : MeshCommunication mc;
1058 68133 : mc.redistribute(*this);
1059 :
1060 68133 : this->update_parallel_id_counts();
1061 :
1062 : // We ought to still have valid neighbor links; we communicate
1063 : // them for newly-redistributed elements
1064 : // this->find_neighbors();
1065 :
1066 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1067 : // anyway... but users can always call partition directly, in which case we do need
1068 : // to call delete_remote_elements()...
1069 : //
1070 : // Regardless of whether it's necessary, it isn't safe. We
1071 : // haven't communicated new node processor_ids yet, and we can't
1072 : // delete nodes until we do.
1073 : // this->delete_remote_elements();
1074 : }
1075 : else
1076 : // The base class can handle non-distributed things, like
1077 : // notifying any GhostingFunctors of changes
1078 485440 : MeshBase::redistribute();
1079 553573 : }
1080 :
1081 :
1082 :
1083 475630 : void DistributedMesh::update_post_partitioning ()
1084 : {
1085 : // this->recalculate_n_partitions();
1086 :
1087 : // Let's do the base class cache clearing first, just in case our
1088 : // later computations are ever changed to make use of a local
1089 : // elements range cache
1090 475630 : this->MeshBase::update_post_partitioning();
1091 :
1092 : // Partitioning changes our numbers of unpartitioned objects
1093 475630 : this->update_parallel_id_counts();
1094 475630 : }
1095 :
1096 :
1097 :
1098 : template <typename T>
1099 3300 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1100 : {
1101 : // This function must be run on all processors at once
1102 3300 : parallel_object_only();
1103 :
1104 3300 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1105 3300 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1106 3300 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1107 :
1108 906732 : for (dof_id_type i=0; i != pmax_id; ++i)
1109 : {
1110 903432 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1111 :
1112 : // Local lookups by id should return the requested object
1113 903432 : libmesh_assert(!obj || obj->id() == i);
1114 :
1115 : // All processors with an object should agree on id
1116 : #ifndef NDEBUG
1117 903432 : const dof_id_type dofid = obj && obj->valid_id() ?
1118 : obj->id() : DofObject::invalid_id;
1119 903432 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1120 : #endif
1121 :
1122 : // All processors with an object should agree on processor id
1123 903432 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1124 : obj->processor_id() : DofObject::invalid_processor_id;
1125 903432 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1126 :
1127 903432 : dof_id_type min_procid = procid;
1128 903432 : this->comm().min(min_procid);
1129 :
1130 : // Either:
1131 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1132 : // or
1133 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1134 :
1135 : // Original assert logic
1136 : // libmesh_assert (min_procid != this->processor_id() || obj);
1137 :
1138 : // More human-understandable logic...
1139 903432 : libmesh_assert (
1140 : ((min_procid == this->processor_id()) && obj)
1141 : ||
1142 : (min_procid != this->processor_id())
1143 : );
1144 :
1145 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1146 : // All processors with an object should agree on unique id
1147 903432 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1148 903432 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1149 : #endif
1150 : }
1151 3300 : }
1152 :
1153 :
1154 :
1155 1650 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1156 : {
1157 1650 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1158 1650 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1159 1650 : }
1160 :
1161 :
1162 :
1163 394 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1164 : {
1165 : #ifndef NDEBUG
1166 : // This function must be run on all processors at once
1167 394 : parallel_object_only();
1168 :
1169 394 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1170 :
1171 21872 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1172 : {
1173 21478 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1174 :
1175 21478 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1176 :
1177 : // All processors with an active element should agree on p level
1178 21478 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1179 : }
1180 : #endif
1181 394 : }
1182 :
1183 :
1184 :
1185 :
1186 1598 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1187 : {
1188 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1189 : // This function must be run on all processors at once
1190 1598 : parallel_object_only();
1191 :
1192 1598 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1193 :
1194 148394 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1195 : {
1196 146796 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1197 :
1198 146796 : unsigned int refinement_flag = el ?
1199 65517 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1200 146796 : unsigned int p_refinement_flag = el ?
1201 65517 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1202 :
1203 146796 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1204 :
1205 : // p refinement flags aren't always kept correct on inactive
1206 : // ghost elements
1207 146796 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1208 : }
1209 : #endif // LIBMESH_ENABLE_AMR
1210 1598 : }
1211 :
1212 :
1213 :
1214 : template <typename T>
1215 : dof_id_type
1216 1353094 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1217 : {
1218 : // This function must be run on all processors at once
1219 784 : parallel_object_only();
1220 :
1221 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1222 :
1223 : // In parallel we may not know what objects other processors have.
1224 : // Start by figuring out how many
1225 784 : dof_id_type unpartitioned_objects = 0;
1226 :
1227 : std::unordered_map<processor_id_type, dof_id_type>
1228 1568 : ghost_objects_from_proc;
1229 :
1230 1353094 : object_iterator it = objects.begin();
1231 784 : object_iterator end = objects.end();
1232 :
1233 103993340 : while (it != end)
1234 : {
1235 102640246 : T * obj = *it;
1236 :
1237 : // Remove any nullptr container entries while we're here.
1238 102640246 : if (!obj)
1239 1288570 : it = objects.erase(it);
1240 : else
1241 : {
1242 101351676 : processor_id_type obj_procid = obj->processor_id();
1243 101351676 : if (obj_procid == DofObject::invalid_processor_id)
1244 42383115 : unpartitioned_objects++;
1245 : else
1246 58968561 : ghost_objects_from_proc[obj_procid]++;
1247 :
1248 : // Finally, increment the iterator
1249 56228 : ++it;
1250 : }
1251 : }
1252 :
1253 1354662 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1254 1353878 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1255 784 : this->comm().allgather
1256 1353577 : ((this_it == ghost_objects_from_proc.end()) ?
1257 483 : dof_id_type(0) : this_it->second, objects_on_proc);
1258 :
1259 : #ifndef NDEBUG
1260 784 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1261 2352 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1262 1568 : if (ghost_objects_from_proc.count(p))
1263 938 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1264 : else
1265 630 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1266 : #endif
1267 :
1268 : // We'll renumber objects in blocks by processor id
1269 1354662 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1270 14353318 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1271 13003360 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1272 784 : objects_on_proc[i-1];
1273 1353878 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1274 1353094 : dof_id_type first_free_id =
1275 1354662 : first_object_on_proc[this->n_processors()-1] +
1276 784 : objects_on_proc[this->n_processors()-1] +
1277 : unpartitioned_objects;
1278 :
1279 : // First set new local object ids and build request sets
1280 : // for non-local object ids
1281 :
1282 : // Request sets to send to each processor
1283 : std::map<processor_id_type, std::vector<dof_id_type>>
1284 784 : requested_ids;
1285 :
1286 : // We know how many objects live on each processor, so reserve() space for
1287 : // each.
1288 784 : auto ghost_end = ghost_objects_from_proc.end();
1289 15706412 : for (auto p : make_range(this->n_processors()))
1290 14354886 : if (p != this->processor_id())
1291 : {
1292 13001008 : if (const auto p_it = ghost_objects_from_proc.find(p);
1293 784 : p_it != ghost_end)
1294 2815126 : requested_ids[p].reserve(p_it->second);
1295 : }
1296 :
1297 784 : end = objects.end();
1298 102704770 : for (it = objects.begin(); it != end; ++it)
1299 : {
1300 101351676 : T * obj = *it;
1301 101351676 : if (!obj)
1302 0 : continue;
1303 101407904 : if (obj->processor_id() == this->processor_id())
1304 18233770 : obj->set_id(next_id++);
1305 83117906 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1306 40734791 : requested_ids[obj->processor_id()].push_back(obj->id());
1307 : }
1308 :
1309 : // Next set ghost object ids from other processors
1310 :
1311 1370237 : auto gather_functor =
1312 43516541 : [
1313 : #ifndef NDEBUG
1314 : this,
1315 : &first_object_on_proc,
1316 : &objects_on_proc,
1317 : #endif
1318 : &objects]
1319 : (processor_id_type, const std::vector<dof_id_type> & ids,
1320 910 : std::vector<dof_id_type> & new_ids)
1321 : {
1322 910 : std::size_t ids_size = ids.size();
1323 2815126 : new_ids.resize(ids_size);
1324 :
1325 43549917 : for (std::size_t i=0; i != ids_size; ++i)
1326 : {
1327 40734791 : T * obj = objects[ids[i]];
1328 16233 : libmesh_assert(obj);
1329 16233 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1330 40751024 : new_ids[i] = obj->id();
1331 :
1332 16233 : libmesh_assert_greater_equal (new_ids[i],
1333 : first_object_on_proc[this->processor_id()]);
1334 16233 : libmesh_assert_less (new_ids[i],
1335 : first_object_on_proc[this->processor_id()] +
1336 : objects_on_proc[this->processor_id()]);
1337 : }
1338 : };
1339 :
1340 1370237 : auto action_functor =
1341 43516541 : [
1342 : #ifndef NDEBUG
1343 : &first_object_on_proc,
1344 : &objects_on_proc,
1345 : #endif
1346 : &objects]
1347 : (processor_id_type libmesh_dbg_var(pid),
1348 : const std::vector<dof_id_type> & ids,
1349 910 : const std::vector<dof_id_type> & data)
1350 : {
1351 : // Copy the id changes we've now been informed of
1352 43549917 : for (auto i : index_range(ids))
1353 : {
1354 40734791 : T * obj = objects[ids[i]];
1355 16233 : libmesh_assert (obj);
1356 16233 : libmesh_assert_equal_to (obj->processor_id(), pid);
1357 16233 : libmesh_assert_greater_equal (data[i],
1358 : first_object_on_proc[pid]);
1359 16233 : libmesh_assert_less (data[i],
1360 : first_object_on_proc[pid] +
1361 : objects_on_proc[pid]);
1362 40751024 : obj->set_id(data[i]);
1363 : }
1364 : };
1365 :
1366 784 : const dof_id_type * ex = nullptr;
1367 : Parallel::pull_parallel_vector_data
1368 1353094 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1369 :
1370 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1371 1370237 : auto unique_gather_functor =
1372 43516541 : [
1373 : #ifndef NDEBUG
1374 : this,
1375 : #endif
1376 : &objects]
1377 : (processor_id_type, const std::vector<dof_id_type> & ids,
1378 910 : std::vector<unique_id_type> & data)
1379 : {
1380 910 : std::size_t ids_size = ids.size();
1381 2815126 : data.resize(ids_size);
1382 :
1383 43549917 : for (std::size_t i=0; i != ids_size; ++i)
1384 : {
1385 40734791 : T * obj = objects[ids[i]];
1386 16233 : libmesh_assert(obj);
1387 16233 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1388 81469582 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1389 : }
1390 : };
1391 :
1392 1370237 : auto unique_action_functor =
1393 43516541 : [&objects]
1394 : (processor_id_type libmesh_dbg_var(pid),
1395 : const std::vector<dof_id_type> & ids,
1396 910 : const std::vector<unique_id_type> & data)
1397 : {
1398 43549917 : for (auto i : index_range(ids))
1399 : {
1400 40734791 : T * obj = objects[ids[i]];
1401 16233 : libmesh_assert (obj);
1402 16233 : libmesh_assert_equal_to (obj->processor_id(), pid);
1403 40734791 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1404 0 : obj->set_unique_id(data[i]);
1405 : }
1406 : };
1407 :
1408 784 : const unique_id_type * unique_ex = nullptr;
1409 : Parallel::pull_parallel_vector_data
1410 1353094 : (this->comm(), requested_ids, unique_gather_functor,
1411 : unique_action_functor, unique_ex);
1412 : #endif
1413 :
1414 : // Next set unpartitioned object ids
1415 784 : next_id = 0;
1416 15706412 : for (auto i : make_range(this->n_processors()))
1417 14354886 : next_id += objects_on_proc[i];
1418 102704770 : for (it = objects.begin(); it != end; ++it)
1419 : {
1420 101351676 : T * obj = *it;
1421 101351676 : if (!obj)
1422 0 : continue;
1423 101351676 : if (obj->processor_id() == DofObject::invalid_processor_id)
1424 42383115 : obj->set_id(next_id++);
1425 : }
1426 :
1427 : // Finally shuffle around objects so that container indices
1428 : // match ids
1429 1353094 : it = objects.begin();
1430 784 : end = objects.end();
1431 108944249 : while (it != end)
1432 : {
1433 107591155 : T * obj = *it;
1434 107591155 : if (obj) // don't try shuffling already-nullptr entries
1435 : {
1436 107591155 : T * next = objects[obj->id()];
1437 : // If we have to move this object
1438 107591155 : if (next != obj)
1439 : {
1440 : // nullptr out its original position for now
1441 : // (our shuffling may put another object there shortly)
1442 39667916 : *it = nullptr;
1443 :
1444 : // There may already be another object with this id that
1445 : // needs to be moved itself
1446 50995593 : while (next)
1447 : {
1448 : // We shouldn't be trying to give two objects the
1449 : // same id
1450 9326 : libmesh_assert_not_equal_to (next->id(), obj->id());
1451 11327677 : objects[obj->id()] = obj;
1452 9326 : obj = next;
1453 11327677 : next = objects[obj->id()];
1454 : }
1455 39667916 : objects[obj->id()] = obj;
1456 : }
1457 : }
1458 :
1459 : // Remove any container entries that were left as nullptr.
1460 114904 : if (!obj)
1461 0 : it = objects.erase(it);
1462 : else
1463 57452 : ++it;
1464 : }
1465 :
1466 1353878 : return first_free_id;
1467 : }
1468 :
1469 :
1470 678217 : void DistributedMesh::renumber_nodes_and_elements ()
1471 : {
1472 394 : parallel_object_only();
1473 :
1474 : #ifdef DEBUG
1475 : // Make sure our ids and flags are consistent
1476 394 : this->libmesh_assert_valid_parallel_ids();
1477 394 : this->libmesh_assert_valid_parallel_flags();
1478 394 : this->libmesh_assert_valid_parallel_p_levels();
1479 : #endif
1480 :
1481 394 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1482 :
1483 : // Nodes not connected to any elements, and nullptr node entries
1484 : // in our container, should be deleted. But wait! If we've deleted coarse
1485 : // local elements on some processor, other processors might have ghosted
1486 : // nodes from it that are now no longer connected to any elements on it, but
1487 : // that are connected to their own semilocal elements. We'll have to
1488 : // communicate to ascertain if that's the case.
1489 678217 : this->comm().max(_deleted_coarse_elements);
1490 :
1491 : // What used nodes do we see on our proc?
1492 394 : std::set<dof_id_type> used_nodes;
1493 :
1494 : // What used node info should we send from our proc? Could we take ownership
1495 : // of each node if we needed to?
1496 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1497 394 : used_nodes_on_proc;
1498 :
1499 : // flag the nodes we need
1500 54558016 : for (auto & elem : this->element_ptr_range())
1501 283841613 : for (const Node & node : elem->node_ref_range())
1502 : {
1503 257211323 : const dof_id_type n = node.id();
1504 257067209 : used_nodes.insert(n);
1505 257211323 : if (_deleted_coarse_elements)
1506 : {
1507 3123685 : const processor_id_type p = node.processor_id();
1508 3123969 : if (p != this->processor_id())
1509 : {
1510 2237336 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1511 2237478 : if (elem->processor_id() == this->processor_id())
1512 215584 : used_nodes_on_p[n] = true;
1513 : else
1514 105 : if (!used_nodes_on_p.count(n))
1515 516282 : used_nodes_on_p[n] = false;
1516 : }
1517 : }
1518 677429 : }
1519 :
1520 678217 : if (_deleted_coarse_elements)
1521 : {
1522 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1523 : typedef unsigned char boolish;
1524 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1525 28 : used_nodes_on_proc_vecs;
1526 70355 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1527 54485 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1528 :
1529 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1530 : std::map<processor_id_type, std::set<dof_id_type>>
1531 28 : repartitioned_node_sets_to_push;
1532 :
1533 : auto ids_action_functor =
1534 54459 : [&used_nodes, &repartitioned_node_pids,
1535 : &repartitioned_node_sets_to_push]
1536 : (processor_id_type pid,
1537 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1538 : {
1539 580910 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1540 : {
1541 : // If we don't see a use for our own node, but someone
1542 : // else does, better figure out who should own it next.
1543 97 : if (!used_nodes.count(n))
1544 : {
1545 2 : if (auto it = repartitioned_node_pids.find(n);
1546 2 : sender_could_become_owner)
1547 : {
1548 1 : if (it != repartitioned_node_pids.end() &&
1549 0 : pid < it->second)
1550 0 : it->second = pid;
1551 : else
1552 1 : repartitioned_node_pids[n] = pid;
1553 : }
1554 : else
1555 1 : if (it == repartitioned_node_pids.end())
1556 0 : repartitioned_node_pids[n] =
1557 : DofObject::invalid_processor_id;
1558 :
1559 2 : repartitioned_node_sets_to_push[pid].insert(n);
1560 : }
1561 : }
1562 15882 : };
1563 :
1564 : // We need two pushes instead of a pull here because we need to
1565 : // know *all* the queries for a particular node before we can
1566 : // respond to *any* of them.
1567 : Parallel::push_parallel_vector_data
1568 15870 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1569 :
1570 : // Repartition (what used to be) our own nodes first
1571 15871 : for (auto & [n, p] : repartitioned_node_pids)
1572 : {
1573 1 : Node & node = this->node_ref(n);
1574 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1575 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1576 1 : node.processor_id() = p;
1577 : }
1578 :
1579 : // Then push to repartition others' ghosted copies.
1580 :
1581 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1582 28 : repartitioned_node_vecs;
1583 :
1584 15872 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1585 : {
1586 2 : auto & rn_vec = repartitioned_node_vecs[p];
1587 4 : for (auto n : nodeset)
1588 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1589 : }
1590 :
1591 : auto repartition_node_functor =
1592 2 : [this]
1593 : (processor_id_type libmesh_dbg_var(pid),
1594 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1595 : {
1596 4 : for (auto [n, p] : ids_and_pids)
1597 : {
1598 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1599 2 : Node & node = this->node_ref(n);
1600 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1601 2 : node.processor_id() = p;
1602 : }
1603 15842 : };
1604 :
1605 : Parallel::push_parallel_vector_data
1606 15870 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1607 : }
1608 :
1609 678217 : _deleted_coarse_elements = false;
1610 :
1611 : // Nodes not connected to any local elements, and nullptr node entries
1612 : // in our container, are deleted
1613 : {
1614 678217 : node_iterator_imp it = _nodes.begin();
1615 394 : node_iterator_imp end = _nodes.end();
1616 :
1617 81504562 : while (it != end)
1618 : {
1619 80826345 : Node * nd = *it;
1620 80826345 : if (!nd)
1621 2743685 : it = _nodes.erase(it);
1622 78082660 : else if (!used_nodes.count(nd->id()))
1623 : {
1624 : // remove any boundary information associated with
1625 : // this node
1626 2327679 : this->get_boundary_info().remove (nd);
1627 2828 : _constraint_rows.erase(nd);
1628 :
1629 : // delete the node
1630 2327679 : delete nd;
1631 :
1632 2327679 : it = _nodes.erase(it);
1633 : }
1634 : else
1635 45139 : ++it;
1636 : }
1637 : }
1638 :
1639 678217 : this->_preparation.has_removed_orphaned_nodes = true;
1640 :
1641 678217 : if (_skip_renumber_nodes_and_elements)
1642 : {
1643 1670 : this->update_parallel_id_counts();
1644 2 : return;
1645 : }
1646 :
1647 : // Finally renumber all the elements
1648 676547 : _n_elem = this->renumber_dof_objects (this->_elements);
1649 :
1650 : // and all the remaining nodes
1651 676547 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1652 :
1653 : // And figure out what IDs we should use when adding new nodes and
1654 : // new elements
1655 676547 : this->update_parallel_id_counts();
1656 :
1657 : // Make sure our caches are up to date and our
1658 : // DofObjects are well packed
1659 : #ifdef DEBUG
1660 392 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1661 392 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1662 392 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1663 392 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1664 392 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1665 392 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1666 392 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1667 392 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1668 :
1669 : // Make sure our ids and flags are consistent
1670 392 : this->libmesh_assert_valid_parallel_ids();
1671 392 : this->libmesh_assert_valid_parallel_flags();
1672 :
1673 : // And make sure we've made our numbering monotonic
1674 392 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1675 : #endif
1676 : }
1677 :
1678 :
1679 :
1680 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1681 : {
1682 : // We can't use range-for here because we need access to the special
1683 : // iterators' methods, not just to their dereferenced values.
1684 :
1685 : // Nodes first
1686 0 : for (auto pr = this->_nodes.begin(),
1687 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1688 : {
1689 5427360 : Node * n = *pr;
1690 5427360 : if (n != nullptr)
1691 : {
1692 0 : const dof_id_type id = pr.index();
1693 5204550 : n->set_id() = id;
1694 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1695 : }
1696 : }
1697 :
1698 : // Elements next
1699 0 : for (auto pr = this->_elements.begin(),
1700 5671231 : end = this->_elements.end(); pr != end; ++pr)
1701 : {
1702 5653821 : Elem * e = *pr;
1703 5653821 : if (e != nullptr)
1704 : {
1705 0 : const dof_id_type id = pr.index();
1706 5590229 : e->set_id() = id;
1707 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1708 : }
1709 : }
1710 17410 : }
1711 :
1712 :
1713 :
1714 631970 : dof_id_type DistributedMesh::n_active_elem () const
1715 : {
1716 540 : parallel_object_only();
1717 :
1718 : // Get local active elements first
1719 : dof_id_type active_elements =
1720 1263400 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1721 1894830 : this->active_local_elements_end()));
1722 631970 : this->comm().sum(active_elements);
1723 :
1724 : // Then add unpartitioned active elements, which should exist on
1725 : // every processor
1726 631970 : active_elements +=
1727 631970 : static_cast<dof_id_type>(std::distance
1728 1263400 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1729 632510 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1730 631970 : return active_elements;
1731 : }
1732 :
1733 :
1734 :
1735 403244 : void DistributedMesh::delete_remote_elements()
1736 : {
1737 : #ifdef DEBUG
1738 : // Make sure our neighbor links are all fine
1739 382 : MeshTools::libmesh_assert_valid_neighbors(*this);
1740 :
1741 : // And our child/parent links, and our flags
1742 382 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1743 :
1744 : // Make sure our ids and flags are consistent
1745 382 : this->libmesh_assert_valid_parallel_ids();
1746 382 : this->libmesh_assert_valid_parallel_flags();
1747 :
1748 382 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1749 382 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1750 382 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1751 382 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1752 382 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1753 382 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1754 : #endif
1755 :
1756 403244 : _is_serial = false;
1757 403244 : _is_serial_on_proc_0 = false;
1758 :
1759 403244 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1760 :
1761 382 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1762 :
1763 : // Now make sure the containers actually shrink - strip
1764 : // any newly-created nullptr voids out of the element array
1765 403244 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1766 382 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1767 53520534 : while (e_it != e_end)
1768 53117290 : if (!*e_it)
1769 39588284 : e_it = _elements.erase(e_it);
1770 : else
1771 17361 : ++e_it;
1772 :
1773 403244 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1774 382 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1775 100184839 : while (n_it != n_end)
1776 99781595 : if (!*n_it)
1777 66632226 : n_it = _nodes.erase(n_it);
1778 : else
1779 48503 : ++n_it;
1780 :
1781 : // We may have deleted no-longer-connected nodes or coarsened-away
1782 : // elements; let's update our caches.
1783 403244 : this->update_parallel_id_counts();
1784 :
1785 : // We may have deleted nodes or elements that were the only local
1786 : // representatives of some particular boundary id(s); let's update
1787 : // those caches.
1788 403244 : this->get_boundary_info().regenerate_id_sets();
1789 :
1790 : #ifdef DEBUG
1791 : // We might not have well-packed objects if the user didn't allow us
1792 : // to renumber
1793 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1794 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1795 :
1796 : // Make sure our neighbor links are all fine
1797 382 : MeshTools::libmesh_assert_valid_neighbors(*this);
1798 :
1799 : // And our child/parent links, and our flags
1800 382 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1801 :
1802 : // Make sure our ids and flags are consistent
1803 382 : this->libmesh_assert_valid_parallel_ids();
1804 382 : this->libmesh_assert_valid_parallel_flags();
1805 : #endif
1806 :
1807 403244 : this->_preparation.has_removed_remote_elements = true;
1808 403244 : }
1809 :
1810 :
1811 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1812 : {
1813 : // First add the elem like normal
1814 0 : add_elem(e);
1815 :
1816 : // Now add it to the set that won't be deleted when we call
1817 : // delete_remote_elements()
1818 0 : _extra_ghost_elems.insert(e);
1819 0 : }
1820 :
1821 : void
1822 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1823 : {
1824 0 : std::set<Elem *> tmp;
1825 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1826 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1827 0 : std::inserter(tmp, tmp.begin()));
1828 0 : _extra_ghost_elems = tmp;
1829 0 : }
1830 :
1831 96828 : void DistributedMesh::allgather()
1832 : {
1833 96828 : if (_is_serial)
1834 0 : return;
1835 96826 : MeshCommunication().allgather(*this);
1836 96826 : _is_serial = true;
1837 96826 : _is_serial_on_proc_0 = true;
1838 :
1839 : // Make sure our caches are up to date and our
1840 : // DofObjects are well packed
1841 : #ifdef DEBUG
1842 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1843 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1844 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1845 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1846 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1847 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1848 :
1849 : // If we've disabled renumbering we can't be sure we're contiguous
1850 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1851 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1852 :
1853 : // Make sure our neighbor links are all fine
1854 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1855 :
1856 : // Make sure our ids and flags are consistent
1857 48 : this->libmesh_assert_valid_parallel_ids();
1858 48 : this->libmesh_assert_valid_parallel_flags();
1859 : #endif
1860 : }
1861 :
1862 18096 : void DistributedMesh::gather_to_zero()
1863 : {
1864 18096 : if (_is_serial_on_proc_0)
1865 4 : return;
1866 :
1867 13858 : _is_serial_on_proc_0 = true;
1868 13858 : MeshCommunication().gather(0, *this);
1869 : }
1870 :
1871 :
1872 : } // namespace libMesh
|
