Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/distributed_mesh.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/boundary_info.h"
25 : #include "libmesh/elem.h"
26 : #include "libmesh/libmesh_logging.h"
27 : #include "libmesh/mesh_communication.h"
28 : #include "libmesh/mesh_tools.h"
29 : #include "libmesh/partitioner.h"
30 : #include "libmesh/string_to_enum.h"
31 :
32 : // TIMPI includes
33 : #include "timpi/parallel_implementation.h"
34 : #include "timpi/parallel_sync.h"
35 :
36 :
37 : namespace libMesh
38 : {
39 :
40 : // ------------------------------------------------------------
41 : // DistributedMesh class member functions
42 277019 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
43 277019 : unsigned char d) :
44 276535 : UnstructuredMesh (comm_in,d), _is_serial(true),
45 276535 : _is_serial_on_proc_0(true),
46 276535 : _deleted_coarse_elements(false),
47 276535 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48 277261 : _next_free_local_node_id(this->processor_id()),
49 277019 : _next_free_local_elem_id(this->processor_id()),
50 277019 : _next_free_unpartitioned_node_id(this->n_processors()),
51 277019 : _next_free_unpartitioned_elem_id(this->n_processors())
52 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
53 277019 : , _next_unpartitioned_unique_id(this->n_processors())
54 : #endif
55 : {
56 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
57 277019 : _next_unique_id = this->processor_id();
58 : #endif
59 :
60 277261 : const std::string default_partitioner = "parmetis";
61 : const std::string my_partitioner =
62 : libMesh::command_line_value("--default-partitioner",
63 554038 : default_partitioner);
64 : _partitioner = Partitioner::build
65 553796 : (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 277019 : }
67 :
68 270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
69 : {
70 4 : LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 :
72 : // Move assign as an UnstructuredMesh.
73 4 : this->UnstructuredMesh::operator=(std::move(other_mesh));
74 :
75 : // Nodes and elements belong to DistributedMesh and have to be
76 : // moved before we can move arbitrary GhostingFunctor, Partitioner,
77 : // etc. subclasses.
78 270 : this->move_nodes_and_elements(std::move(other_mesh));
79 :
80 : // But move_nodes_and_elems misses (or guesses about) some of our
81 : // subclass values, and we want more precision than a guess.
82 270 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83 4 : _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 :
85 : // Handle remaining MeshBase moves.
86 270 : this->post_dofobject_moves(std::move(other_mesh));
87 :
88 274 : return *this;
89 : }
90 :
91 270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
92 : {
93 270 : *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 :
95 270 : return *this;
96 : }
97 :
98 13026 : std::string_view DistributedMesh::subclass_first_difference_from (const MeshBase & other_mesh_base) const
99 : {
100 708 : const DistributedMesh * dist_mesh_ptr =
101 13026 : dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102 13026 : if (!dist_mesh_ptr)
103 0 : return "DistributedMesh class";
104 708 : const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 :
106 : #define CHECK_MEMBER(member_name) \
107 : if (member_name != other_mesh.member_name) \
108 : return #member_name;
109 :
110 13026 : CHECK_MEMBER(_is_serial);
111 13026 : CHECK_MEMBER(_is_serial_on_proc_0);
112 13026 : CHECK_MEMBER(_deleted_coarse_elements);
113 13026 : CHECK_MEMBER(_n_nodes);
114 13026 : CHECK_MEMBER(_n_elem);
115 13026 : CHECK_MEMBER(_max_node_id);
116 13026 : CHECK_MEMBER(_max_elem_id);
117 : // We expect these things to change in a prepare_for_use();
118 : // they're conceptually "mutable"...
119 : /*
120 : CHECK_MEMBER(_next_free_local_node_id);
121 : CHECK_MEMBER(_next_free_local_elem_id);
122 : CHECK_MEMBER(_next_free_unpartitioned_node_id);
123 : CHECK_MEMBER(_next_free_unpartitioned_elem_id);
124 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
125 : CHECK_MEMBER(_next_unpartitioned_unique_id);
126 : #endif
127 : */
128 13026 : if (!this->nodes_and_elements_equal(other_mesh))
129 63 : return "nodes and/or elements";
130 :
131 13669 : if (_extra_ghost_elems.size() !=
132 706 : other_mesh._extra_ghost_elems.size())
133 0 : return "_extra_ghost_elems size";
134 12963 : for (auto & elem : _extra_ghost_elems)
135 : {
136 0 : libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
137 0 : const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
138 0 : if (!other_elem ||
139 0 : !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
140 0 : return "_extra_ghost_elems entry";
141 : }
142 :
143 12963 : return "";
144 : }
145 :
146 319596 : DistributedMesh::~DistributedMesh ()
147 : {
148 297507 : this->DistributedMesh::clear(); // Free nodes and elements
149 319596 : }
150 :
151 :
152 : // This might be specialized later, but right now it's just here to
153 : // make sure the compiler doesn't give us a default (non-deep) copy
154 : // constructor instead.
155 18003 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
156 18003 : DistributedMesh(static_cast<const MeshBase &>(other_mesh))
157 : {
158 18003 : _is_serial = other_mesh._is_serial;
159 18003 : _is_serial_on_proc_0 = other_mesh._is_serial_on_proc_0;
160 18003 : _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
161 :
162 18003 : _n_nodes = other_mesh.n_nodes();
163 18003 : _n_elem = other_mesh.n_elem();
164 18003 : _max_node_id = other_mesh.max_node_id();
165 18003 : _max_elem_id = other_mesh.max_elem_id();
166 18003 : _next_free_local_node_id =
167 18003 : other_mesh._next_free_local_node_id;
168 18003 : _next_free_local_elem_id =
169 18003 : other_mesh._next_free_local_elem_id;
170 18003 : _next_free_unpartitioned_node_id =
171 18003 : other_mesh._next_free_unpartitioned_node_id;
172 18003 : _next_free_unpartitioned_elem_id =
173 18003 : other_mesh._next_free_unpartitioned_elem_id;
174 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
175 18003 : _next_unique_id =
176 18003 : other_mesh._next_unique_id;
177 18003 : _next_unpartitioned_unique_id =
178 18003 : other_mesh._next_unpartitioned_unique_id;
179 : #endif
180 :
181 : // Need to copy extra_ghost_elems
182 18003 : for (auto & elem : other_mesh._extra_ghost_elems)
183 0 : _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
184 18003 : }
185 :
186 :
187 :
188 20488 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh) :
189 20488 : UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
190 20916 : _is_serial_on_proc_0(other_mesh.is_serial()),
191 19972 : _deleted_coarse_elements(true), // better safe than sorry...
192 19972 : _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
193 20916 : _next_free_local_node_id(this->processor_id()),
194 20828 : _next_free_local_elem_id(this->processor_id()),
195 20828 : _next_free_unpartitioned_node_id(this->n_processors()),
196 40888 : _next_free_unpartitioned_elem_id(this->n_processors())
197 : {
198 : // Just copy, skipping preparation
199 20488 : this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
200 :
201 516 : this->allow_find_neighbors(other_mesh.allow_find_neighbors());
202 516 : this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
203 516 : this->allow_renumbering(other_mesh.allow_renumbering());
204 516 : this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
205 516 : this->skip_partitioning(other_mesh.skip_partitioning());
206 :
207 20488 : this->copy_constraint_rows(other_mesh);
208 :
209 428 : auto & this_boundary_info = this->get_boundary_info();
210 428 : const auto & other_boundary_info = other_mesh.get_boundary_info();
211 :
212 20488 : this_boundary_info = other_boundary_info;
213 :
214 428 : this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
215 :
216 20488 : this->_preparation = other_mesh.preparation();
217 :
218 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
219 20488 : _next_unique_id = other_mesh.parallel_max_unique_id() +
220 20576 : this->processor_id();
221 20916 : _next_unpartitioned_unique_id = _next_unique_id +
222 20488 : (this->n_processors() - this->processor_id());
223 : #endif
224 20488 : this->update_parallel_id_counts();
225 20488 : }
226 :
227 270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
228 : {
229 4 : DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
230 :
231 4 : this->_nodes = std::move(other_mesh._nodes);
232 270 : this->_n_nodes = other_mesh.n_nodes();
233 :
234 4 : this->_elements = std::move(other_mesh._elements);
235 270 : this->_n_elem = other_mesh.n_elem();
236 :
237 270 : _is_serial = other_mesh.is_serial();
238 270 : _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
239 270 : _deleted_coarse_elements = true; // Better safe than sorry
240 :
241 270 : _max_node_id = other_mesh.max_node_id();
242 270 : _max_elem_id = other_mesh.max_elem_id();
243 :
244 270 : _next_free_local_node_id = other_mesh._next_free_local_node_id;
245 270 : _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
246 270 : _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
247 270 : _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
248 :
249 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
250 270 : _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
251 : #endif
252 270 : }
253 :
254 : // We use cached values for these so they can be called
255 : // from one processor without bothering the rest, but
256 : // we may need to update those caches before doing a full
257 : // renumbering
258 1888059 : void DistributedMesh::update_parallel_id_counts()
259 : {
260 : // This function must be run on all processors at once
261 2402 : parallel_object_only();
262 :
263 1888059 : _n_elem = this->parallel_n_elem();
264 1888059 : _n_nodes = this->parallel_n_nodes();
265 1888059 : _max_node_id = this->parallel_max_node_id();
266 1888059 : _max_elem_id = this->parallel_max_elem_id();
267 :
268 1888059 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
269 21653 : _next_free_unpartitioned_elem_id =
270 21653 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
271 21653 : (this->n_processors() + 1) + this->n_processors();
272 1888059 : if (_next_free_local_elem_id < _max_elem_id)
273 21135 : _next_free_local_elem_id =
274 21157 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
275 21135 : (this->n_processors() + 1) + this->processor_id();
276 :
277 1888059 : if (_next_free_unpartitioned_node_id < _max_node_id)
278 28280 : _next_free_unpartitioned_node_id =
279 28280 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
280 28280 : (this->n_processors() + 1) + this->n_processors();
281 1888059 : if (_next_free_local_node_id < _max_node_id)
282 28925 : _next_free_local_node_id =
283 28950 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
284 28925 : (this->n_processors() + 1) + this->processor_id();
285 :
286 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
287 1888059 : _next_unique_id = this->parallel_max_unique_id();
288 1888059 : _next_unpartitioned_unique_id =
289 1889781 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
290 1888059 : (this->n_processors() + 1) + this->n_processors();
291 1888059 : _next_unique_id =
292 1888059 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
293 1888059 : (this->n_processors() + 1) + this->processor_id();
294 : #endif
295 :
296 1888059 : this->_preparation.has_synched_id_counts = true;
297 1888059 : }
298 :
299 :
300 : // Or in debug mode we may want to test the uncached values without
301 : // changing the cache
302 1895770 : dof_id_type DistributedMesh::parallel_n_elem() const
303 : {
304 : // This function must be run on all processors at once
305 3236 : parallel_object_only();
306 :
307 1895770 : dof_id_type n_local = this->n_local_elem();
308 1895770 : this->comm().sum(n_local);
309 1895770 : n_local += this->n_unpartitioned_elem();
310 1895770 : return n_local;
311 : }
312 :
313 :
314 :
315 1894937 : dof_id_type DistributedMesh::parallel_max_elem_id() const
316 : {
317 : // This function must be run on all processors at once
318 9280 : parallel_object_only();
319 :
320 1894937 : dof_id_type max_local = 0;
321 :
322 : dofobject_container<Elem>::const_reverse_veclike_iterator
323 9280 : rit = _elements.rbegin();
324 :
325 : const dofobject_container<Elem>::const_reverse_veclike_iterator
326 9280 : rend = _elements.rend();
327 :
328 : // Look for the maximum element id. Search backwards through
329 : // elements so we can break out early. Beware of nullptr entries that
330 : // haven't yet been cleared from _elements.
331 11315348 : for (; rit != rend; ++rit)
332 : {
333 10791322 : const DofObject *d = *rit;
334 19535 : if (d)
335 : {
336 9043 : libmesh_assert(_elements[d->id()] == d);
337 1370911 : max_local = d->id() + 1;
338 1370911 : break;
339 : }
340 : }
341 :
342 1894937 : this->comm().max(max_local);
343 1894937 : return max_local;
344 : }
345 :
346 :
347 :
348 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
349 1990911 : unique_id_type DistributedMesh::parallel_max_unique_id() const
350 : {
351 : // This function must be run on all processors at once
352 3200 : parallel_object_only();
353 :
354 3981822 : unique_id_type max_local = std::max(_next_unique_id,
355 2302489 : _next_unpartitioned_unique_id);
356 1990911 : this->comm().max(max_local);
357 1990911 : return max_local;
358 : }
359 :
360 :
361 :
362 78212 : void DistributedMesh::set_next_unique_id(unique_id_type id)
363 : {
364 482 : _next_unique_id = id;
365 78212 : _next_unpartitioned_unique_id =
366 78354 : ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
367 78212 : (this->n_processors() + 1) + this->n_processors();
368 78212 : _next_unique_id =
369 78212 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
370 78212 : (this->n_processors() + 1) + this->processor_id();
371 78212 : }
372 : #endif
373 :
374 :
375 :
376 1896218 : dof_id_type DistributedMesh::parallel_n_nodes() const
377 : {
378 : // This function must be run on all processors at once
379 3236 : parallel_object_only();
380 :
381 1896218 : dof_id_type n_local = this->n_local_nodes();
382 1896218 : this->comm().sum(n_local);
383 1896218 : n_local += this->n_unpartitioned_nodes();
384 1896218 : return n_local;
385 : }
386 :
387 :
388 :
389 1892563 : dof_id_type DistributedMesh::parallel_max_node_id() const
390 : {
391 : // This function must be run on all processors at once
392 6906 : parallel_object_only();
393 :
394 1892563 : dof_id_type max_local = 0;
395 :
396 : dofobject_container<Node>::const_reverse_veclike_iterator
397 6906 : rit = _nodes.rbegin();
398 :
399 : const dofobject_container<Node>::const_reverse_veclike_iterator
400 6906 : rend = _nodes.rend();
401 :
402 : // Look for the maximum node id. Search backwards through
403 : // nodes so we can break out early. Beware of nullptr entries that
404 : // haven't yet been cleared from _nodes
405 39582183 : for (; rit != rend; ++rit)
406 : {
407 39058227 : const DofObject *d = *rit;
408 63819 : if (d)
409 : {
410 6739 : libmesh_assert(_nodes[d->id()] == d);
411 1368607 : max_local = d->id() + 1;
412 1368607 : break;
413 : }
414 : }
415 :
416 1892563 : this->comm().max(max_local);
417 1892563 : return max_local;
418 : }
419 :
420 :
421 :
422 42399244 : const Point & DistributedMesh::point (const dof_id_type i) const
423 : {
424 42399244 : return this->node_ref(i);
425 : }
426 :
427 :
428 :
429 43504633 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
430 : {
431 136653 : libmesh_assert(_nodes[i]);
432 136653 : libmesh_assert_equal_to (_nodes[i]->id(), i);
433 :
434 43504633 : return _nodes[i];
435 : }
436 :
437 :
438 :
439 :
440 734336431 : Node * DistributedMesh::node_ptr (const dof_id_type i)
441 : {
442 704117 : libmesh_assert(_nodes[i]);
443 704117 : libmesh_assert_equal_to (_nodes[i]->id(), i);
444 :
445 734336431 : return _nodes[i];
446 : }
447 :
448 :
449 :
450 :
451 3539316 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
452 : {
453 3539316 : if (const auto it = _nodes.find(i);
454 1087593 : it != _nodes.end())
455 : {
456 3102772 : const Node * n = *it;
457 651049 : libmesh_assert (!n || n->id() == i);
458 651049 : return n;
459 : }
460 :
461 436544 : return nullptr;
462 : }
463 :
464 :
465 :
466 :
467 367375651 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
468 : {
469 367375651 : if (auto it = _nodes.find(i);
470 244321 : it != _nodes.end())
471 : {
472 284553977 : Node * n = *it;
473 156798 : libmesh_assert (!n || n->id() == i);
474 156798 : return n;
475 : }
476 :
477 87523 : return nullptr;
478 : }
479 :
480 :
481 :
482 :
483 3049923 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
484 : {
485 10085 : libmesh_assert(_elements[i]);
486 10085 : libmesh_assert_equal_to (_elements[i]->id(), i);
487 :
488 3049923 : return _elements[i];
489 : }
490 :
491 :
492 :
493 :
494 286884998 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
495 : {
496 177789 : libmesh_assert(_elements[i]);
497 177789 : libmesh_assert_equal_to (_elements[i]->id(), i);
498 :
499 286884998 : return _elements[i];
500 : }
501 :
502 :
503 :
504 :
505 1792857 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
506 : {
507 1792857 : if (const auto it = _elements.find(i);
508 953887 : it != _elements.end())
509 : {
510 1246870 : const Elem * e = *it;
511 416781 : libmesh_assert (!e || e->id() == i);
512 416781 : return e;
513 : }
514 :
515 537106 : return nullptr;
516 : }
517 :
518 :
519 :
520 :
521 603582363 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
522 : {
523 603582363 : if (auto it = _elements.find(i);
524 217082 : it != _elements.end())
525 : {
526 518035954 : Elem * e = *it;
527 213403 : libmesh_assert (!e || e->id() == i);
528 213403 : return e;
529 : }
530 :
531 3679 : return nullptr;
532 : }
533 :
534 :
535 :
536 :
537 44055870 : Elem * DistributedMesh::add_elem (Elem * e)
538 : {
539 : // Don't try to add nullptrs!
540 35269 : libmesh_assert(e);
541 :
542 : // Trying to add an existing element is a no-op
543 44055870 : if (e->valid_id() && _elements[e->id()] == e)
544 0 : return e;
545 :
546 44055870 : const processor_id_type elem_procid = e->processor_id();
547 :
548 44055870 : if (!e->valid_id())
549 : {
550 : // We should only be creating new ids past the end of the range
551 : // of existing ids
552 3568 : libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
553 : _max_elem_id);
554 3568 : libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
555 :
556 : // Use the unpartitioned ids for unpartitioned elems, and
557 : // temporarily for ghost elems
558 3857693 : dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
559 3861261 : if (elem_procid == this->processor_id())
560 1133556 : next_id = &_next_free_local_elem_id;
561 3857693 : e->set_id (*next_id);
562 : }
563 :
564 : {
565 : // Advance next_ids up high enough that each is pointing to an
566 : // unused id and any subsequent increments will still point us
567 : // to unused ids
568 88111740 : _max_elem_id = std::max(_max_elem_id,
569 44055870 : static_cast<dof_id_type>(e->id()+1));
570 :
571 44055870 : if (_next_free_unpartitioned_elem_id < _max_elem_id)
572 5146930 : _next_free_unpartitioned_elem_id =
573 5146930 : ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
574 5146930 : (this->n_processors() + 1) + this->n_processors();
575 44055870 : if (_next_free_local_elem_id < _max_elem_id)
576 3454280 : _next_free_local_elem_id =
577 3460520 : ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
578 3454280 : (this->n_processors() + 1) + this->processor_id();
579 :
580 : #ifndef NDEBUG
581 : // We need a const dofobject_container so we don't inadvertently create
582 : // nullptr entries when testing for non-nullptr ones
583 35269 : const dofobject_container<Elem> & const_elements = _elements;
584 : #endif
585 35269 : libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
586 35269 : libmesh_assert(!const_elements[_next_free_local_elem_id]);
587 : }
588 :
589 : // Don't try to overwrite existing elems
590 35269 : libmesh_assert (!_elements[e->id()]);
591 :
592 44055870 : _elements[e->id()] = e;
593 :
594 : // We actually added a new element. Some of our caches might still
595 : // be valid, but we should clear the ones which definitely are not.
596 44055870 : this->clear_point_locator();
597 44055870 : this->clear_stored_ranges();
598 :
599 : // Try to make the cached elem data more accurate
600 44074751 : if (elem_procid == this->processor_id() ||
601 : elem_procid == DofObject::invalid_processor_id)
602 12916533 : _n_elem++;
603 :
604 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
605 44055870 : if (!e->valid_unique_id())
606 : {
607 13502719 : if (processor_id() == e->processor_id())
608 : {
609 1133556 : e->set_unique_id(_next_unique_id);
610 1133556 : _next_unique_id += this->n_processors() + 1;
611 : }
612 : else
613 : {
614 12369163 : e->set_unique_id(_next_unpartitioned_unique_id);
615 12369163 : _next_unpartitioned_unique_id += this->n_processors() + 1;
616 : }
617 : }
618 : else
619 : {
620 30577496 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
621 30553151 : _next_unique_id =
622 30553151 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
623 30553151 : (this->n_processors() + 1) + this->processor_id();
624 : }
625 : #endif
626 :
627 : // Unpartitioned elems should be added on every processor
628 : // And shouldn't be added in the same batch as ghost elems
629 : // But we might be just adding on processor 0 to
630 : // broadcast later
631 : // #ifdef DEBUG
632 : // if (elem_procid == DofObject::invalid_processor_id)
633 : // {
634 : // dof_id_type elem_id = e->id();
635 : // this->comm().max(elem_id);
636 : // libmesh_assert_equal_to (elem_id, e->id());
637 : // }
638 : // #endif
639 :
640 : // Make sure any new element is given space for any extra integers
641 : // we've requested
642 44055870 : e->add_extra_integers(_elem_integer_names.size(),
643 44055870 : _elem_integer_default_values);
644 :
645 : // And set mapping type and data on any new element
646 54150 : e->set_mapping_type(this->default_mapping_type());
647 54150 : e->set_mapping_data(this->default_mapping_data());
648 :
649 44055870 : return e;
650 : }
651 :
652 :
653 :
654 14755106 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
655 : {
656 : // The mesh now takes ownership of the Elem. Eventually the guts of
657 : // add_elem() will get moved to a private helper function, and
658 : // calling add_elem() directly will be deprecated.
659 14755106 : return add_elem(e.release());
660 : }
661 :
662 :
663 :
664 3348652 : Elem * DistributedMesh::insert_elem (Elem * e)
665 : {
666 3348652 : if (_elements[e->id()])
667 3348652 : this->delete_elem(_elements[e->id()]);
668 :
669 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
670 3348652 : if (!e->valid_unique_id())
671 : {
672 0 : if (processor_id() == e->processor_id())
673 : {
674 0 : e->set_unique_id(_next_unique_id);
675 0 : _next_unique_id += this->n_processors() + 1;
676 : }
677 : else
678 : {
679 0 : e->set_unique_id(_next_unpartitioned_unique_id);
680 0 : _next_unpartitioned_unique_id += this->n_processors() + 1;
681 : }
682 : }
683 : else
684 : {
685 3374032 : _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
686 3348652 : _next_unique_id =
687 3352348 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
688 3348652 : (this->n_processors() + 1) + this->processor_id();
689 : }
690 : #endif
691 :
692 : // Try to make the cached elem data more accurate
693 3348652 : processor_id_type elem_procid = e->processor_id();
694 3352348 : if (elem_procid == this->processor_id() ||
695 : elem_procid == DofObject::invalid_processor_id)
696 3219576 : _n_elem++;
697 :
698 3348652 : _elements[e->id()] = e;
699 :
700 : // We actually added a new element. Some of our caches might still
701 : // be valid, but we should clear the ones which definitely are not.
702 3348652 : this->clear_point_locator();
703 3348652 : this->clear_stored_ranges();
704 :
705 : // Make sure any new element is given space for any extra integers
706 : // we've requested
707 3348652 : e->add_extra_integers(_elem_integer_names.size(),
708 3348652 : _elem_integer_default_values);
709 :
710 : // And set mapping type and data on any new element
711 7392 : e->set_mapping_type(this->default_mapping_type());
712 7392 : e->set_mapping_data(this->default_mapping_data());
713 :
714 3348652 : return e;
715 : }
716 :
717 3348652 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
718 : {
719 : // The mesh now takes ownership of the Elem. Eventually the guts of
720 : // insert_elem(Elem*) will get moved to a private helper function, and
721 : // calling insert_elem(Elem*) directly will be deprecated.
722 3348652 : return insert_elem(e.release());
723 : }
724 :
725 :
726 40089690 : void DistributedMesh::delete_elem(Elem * e)
727 : {
728 7586 : libmesh_assert (e);
729 :
730 : // Try to make the cached elem data more accurate
731 40089690 : _n_elem--;
732 :
733 : // Was this a coarse element, not just a coarsening where we still
734 : // have some ancestor structure? Was it a *local* element, that we
735 : // might have been depending on as an owner of local nodes? We'll
736 : // have to be more careful with our nodes in contract() later; no
737 : // telling if we just locally orphaned a node that should be
738 : // globally retained.
739 40099096 : if (e->processor_id() == this->processor_id() &&
740 3640 : !e->parent())
741 80473 : _deleted_coarse_elements = true;
742 :
743 : // Delete the element from the BoundaryInfo object
744 40089690 : this->get_boundary_info().remove(e);
745 :
746 : // But not yet from the container; we might invalidate
747 : // an iterator that way!
748 :
749 : //_elements.erase(e->id());
750 :
751 : // Instead, we set it to nullptr for now
752 :
753 40089690 : _elements[e->id()] = nullptr;
754 :
755 : // delete the element
756 40089690 : delete e;
757 :
758 : // Some of our caches might still be valid, but we should clear the
759 : // ones which definitely are not.
760 40089690 : this->clear_point_locator();
761 40089690 : this->clear_stored_ranges();
762 40089690 : }
763 :
764 :
765 :
766 3207784 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
767 : const dof_id_type new_id)
768 : {
769 : // This could be a no-op
770 3207784 : if (old_id == new_id)
771 0 : return;
772 :
773 3207784 : Elem * el = _elements[old_id];
774 1160 : libmesh_assert (el);
775 1160 : libmesh_assert_equal_to (el->id(), old_id);
776 :
777 3207784 : el->set_id(new_id);
778 1160 : libmesh_assert (!_elements[new_id]);
779 3207784 : _elements[new_id] = el;
780 3207784 : _elements.erase(old_id);
781 :
782 : // Should we delete any caches here? Our point locator indexes by
783 : // element pointer and should be fine with an id change. Our stored
784 : // ranges are no longer sorted, which is *probably* fine, but let's
785 : // just be safe.
786 3207784 : this->clear_stored_ranges();
787 : }
788 :
789 :
790 :
791 40974734 : Node * DistributedMesh::add_point (const Point & p,
792 : const dof_id_type id,
793 : const processor_id_type proc_id)
794 : {
795 40974734 : Node * old_n = this->query_node_ptr(id);
796 :
797 40974734 : if (old_n)
798 : {
799 0 : *old_n = p;
800 0 : old_n->processor_id() = proc_id;
801 :
802 0 : return old_n;
803 : }
804 :
805 40934979 : Node * n = Node::build(p, id).release();
806 40974734 : n->processor_id() = proc_id;
807 :
808 40974734 : return DistributedMesh::add_node(n);
809 : }
810 :
811 :
812 4130 : void DistributedMesh::own_node (Node & n)
813 : {
814 : // This had better be a node in our mesh
815 0 : libmesh_assert(_nodes[n.id()] == &n);
816 :
817 4130 : _nodes[n.id()] = nullptr;
818 4130 : _n_nodes--;
819 :
820 0 : n.set_id(DofObject::invalid_id);
821 4130 : n.processor_id() = this->processor_id();
822 :
823 4130 : this->add_node(&n);
824 4130 : }
825 :
826 :
827 71747743 : Node * DistributedMesh::add_node (Node * n)
828 : {
829 : // Don't try to add nullptrs!
830 81737 : libmesh_assert(n);
831 :
832 : // Trying to add an existing node is a no-op
833 71747743 : if (n->valid_id() && _nodes[n->id()] == n)
834 0 : return n;
835 :
836 71747743 : const processor_id_type node_procid = n->processor_id();
837 :
838 71747743 : if (!n->valid_id())
839 : {
840 : // We should only be creating new ids past the end of the range
841 : // of existing ids
842 17829 : libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
843 : _max_node_id);
844 17829 : libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
845 :
846 : // Use the unpartitioned ids for unpartitioned nodes,
847 : // and temporarily for ghost nodes
848 17183774 : dof_id_type * next_id = &_next_free_unpartitioned_node_id;
849 17201603 : if (node_procid == this->processor_id())
850 2128321 : next_id = &_next_free_local_node_id;
851 17183774 : n->set_id (*next_id);
852 : }
853 :
854 : {
855 : // Advance next_ids up high enough that each is pointing to an
856 : // unused id and any subsequent increments will still point us
857 : // to unused ids
858 143495486 : _max_node_id = std::max(_max_node_id,
859 71747743 : static_cast<dof_id_type>(n->id()+1));
860 :
861 71747743 : if (_next_free_unpartitioned_node_id < _max_node_id)
862 20123386 : _next_free_unpartitioned_node_id =
863 20123386 : ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
864 20123386 : (this->n_processors() + 1) + this->n_processors();
865 71747743 : if (_next_free_local_node_id < _max_node_id)
866 11730384 : _next_free_local_node_id =
867 11748176 : ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
868 11730384 : (this->n_processors() + 1) + this->processor_id();
869 :
870 : #ifndef NDEBUG
871 : // We need a const dofobject_container so we don't inadvertently create
872 : // nullptr entries when testing for non-nullptr ones
873 81737 : const dofobject_container<Node> & const_nodes = _nodes;
874 : #endif
875 81737 : libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
876 81737 : libmesh_assert(!const_nodes[_next_free_local_node_id]);
877 : }
878 :
879 : // Don't try to overwrite existing nodes
880 81737 : libmesh_assert (!_nodes[n->id()]);
881 :
882 71747743 : _nodes[n->id()] = n;
883 :
884 : // Try to make the cached node data more accurate
885 71789210 : if (node_procid == this->processor_id() ||
886 : node_procid == DofObject::invalid_processor_id)
887 38545498 : _n_nodes++;
888 :
889 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
890 71747743 : if (!n->valid_unique_id())
891 : {
892 40976462 : if (processor_id() == n->processor_id())
893 : {
894 2569114 : n->set_unique_id(_next_unique_id);
895 2569114 : _next_unique_id += this->n_processors() + 1;
896 : }
897 : else
898 : {
899 38407348 : n->set_unique_id(_next_unpartitioned_unique_id);
900 38407348 : _next_unpartitioned_unique_id += this->n_processors() + 1;
901 : }
902 : }
903 : else
904 : {
905 30901702 : _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
906 30771281 : _next_unique_id =
907 30771281 : ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
908 30771281 : (this->n_processors() + 1) + this->processor_id();
909 : }
910 : #endif
911 :
912 71747743 : n->add_extra_integers(_node_integer_names.size(),
913 71747743 : _node_integer_default_values);
914 :
915 : // Unpartitioned nodes should be added on every processor
916 : // And shouldn't be added in the same batch as ghost nodes
917 : // But we might be just adding on processor 0 to
918 : // broadcast later
919 : // #ifdef DEBUG
920 : // if (node_procid == DofObject::invalid_processor_id)
921 : // {
922 : // dof_id_type node_id = n->id();
923 : // this->comm().max(node_id);
924 : // libmesh_assert_equal_to (node_id, n->id());
925 : // }
926 : // #endif
927 :
928 71747743 : return n;
929 : }
930 :
931 30768879 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
932 : {
933 : // The mesh now takes ownership of the Node. Eventually the guts of
934 : // add_node() will get moved to a private helper function, and
935 : // calling add_node() directly will be deprecated.
936 30768879 : return add_node(n.release());
937 : }
938 :
939 50483394 : void DistributedMesh::delete_node(Node * n)
940 : {
941 4461 : libmesh_assert(n);
942 4461 : libmesh_assert(_nodes[n->id()]);
943 :
944 : // Try to make the cached elem data more accurate
945 50483394 : _n_nodes--;
946 :
947 : // Delete the node from the BoundaryInfo object
948 50483394 : this->get_boundary_info().remove(n);
949 8922 : _constraint_rows.erase(n);
950 :
951 : // But not yet from the container; we might invalidate
952 : // an iterator that way!
953 :
954 : //_nodes.erase(n->id());
955 :
956 : // Instead, we set it to nullptr for now
957 :
958 50483394 : _nodes[n->id()] = nullptr;
959 :
960 : // delete the node
961 50483394 : delete n;
962 50483394 : }
963 :
964 :
965 :
966 6896924 : void DistributedMesh::renumber_node(const dof_id_type old_id,
967 : const dof_id_type new_id)
968 : {
969 : // This could be a no-op
970 6896924 : if (old_id == new_id)
971 0 : return;
972 :
973 6896326 : Node * nd = _nodes[old_id];
974 3234 : libmesh_assert (nd);
975 3234 : libmesh_assert_equal_to (nd->id(), old_id);
976 :
977 : // If we have nodes shipped to this processor for NodeConstraints
978 : // use, then those nodes will exist in _nodes, but may not be
979 : // locatable via a TopologyMap due to the insufficiency of elements
980 : // connecting to them. If local refinement then wants to create a
981 : // *new* node in the same location, it will initially get a temporary
982 : // id, and then make_node_ids_parallel_consistent() will try to move
983 : // it to the canonical id. We need to account for this case to
984 : // avoid false positives and memory leaks.
985 : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
986 6468 : if (_nodes[new_id])
987 : {
988 0 : libmesh_assert_equal_to (*(Point *)_nodes[new_id],
989 : *(Point *)_nodes[old_id]);
990 0 : _nodes.erase(new_id);
991 : }
992 : #else
993 : // If we aren't shipping nodes for NodeConstraints, there should be
994 : // no reason for renumbering one node onto another.
995 : libmesh_assert (!_nodes[new_id]);
996 : #endif
997 6896326 : _nodes[new_id] = nd;
998 6896326 : nd->set_id(new_id);
999 :
1000 6896326 : _nodes.erase(old_id);
1001 : }
1002 :
1003 :
1004 :
1005 555291 : void DistributedMesh::clear ()
1006 : {
1007 : // Call parent clear function
1008 555291 : MeshBase::clear();
1009 :
1010 : // Clear our elements and nodes
1011 : // There is no need to remove them from
1012 : // the BoundaryInfo data structure since we
1013 : // already cleared it.
1014 555291 : this->DistributedMesh::clear_elems();
1015 :
1016 22619057 : for (auto & node : _nodes)
1017 40920444 : delete node;
1018 :
1019 904 : _nodes.clear();
1020 :
1021 : // We're no longer distributed if we were before
1022 555291 : _is_serial = true;
1023 555291 : _is_serial_on_proc_0 = true;
1024 :
1025 : // We deleted a ton of coarse elements, but their nodes got deleted too so
1026 : // all is copacetic.
1027 555291 : _deleted_coarse_elements = false;
1028 :
1029 : // Correct our caches
1030 555291 : _n_nodes = 0;
1031 555291 : _max_node_id = 0;
1032 555855 : _next_free_local_node_id = this->processor_id();
1033 555291 : _next_free_unpartitioned_node_id = this->n_processors();
1034 555291 : }
1035 :
1036 :
1037 :
1038 561563 : void DistributedMesh::clear_elems ()
1039 : {
1040 9222184 : for (auto & elem : _elements)
1041 8660621 : delete elem;
1042 :
1043 904 : _elements.clear();
1044 :
1045 : // Correct our caches
1046 561563 : _n_elem = 0;
1047 561563 : _max_elem_id = 0;
1048 562127 : _next_free_local_elem_id = this->processor_id();
1049 561563 : _next_free_unpartitioned_elem_id = this->n_processors();
1050 561563 : }
1051 :
1052 :
1053 :
1054 554589 : void DistributedMesh::redistribute ()
1055 : {
1056 : // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1057 554589 : if (!this->is_serial())
1058 : {
1059 : // Construct a MeshCommunication object to actually redistribute the nodes
1060 : // and elements according to the partitioner, and then to re-gather the neighbors.
1061 : MeshCommunication mc;
1062 68133 : mc.redistribute(*this);
1063 :
1064 68133 : this->update_parallel_id_counts();
1065 :
1066 : // We communicate valid neighbor links for newly-redistributed
1067 : // elements, but we may still have remote_elem links that should
1068 : // be corrected but whose correction wasn't communicated.
1069 68405 : this->find_neighbors(/*reset_remote_elements*/ false,
1070 : /*reset_current_list*/ false /*non-default*/,
1071 : /*assert_valid*/ false,
1072 272 : /*check_non_remote*/ false /*non-default!*/);
1073 :
1074 : // Is this necessary? If we are called from prepare_for_use(), this will be called
1075 : // anyway... but users can always call partition directly, in which case we do need
1076 : // to call delete_remote_elements()...
1077 : //
1078 : // Regardless of whether it's necessary, it isn't safe. We
1079 : // haven't communicated new node processor_ids yet, and we can't
1080 : // delete nodes until we do.
1081 : // this->delete_remote_elements();
1082 : }
1083 : else
1084 : // The base class can handle non-distributed things, like
1085 : // notifying any GhostingFunctors of changes
1086 486456 : MeshBase::redistribute();
1087 554589 : }
1088 :
1089 :
1090 :
1091 476512 : void DistributedMesh::update_post_partitioning ()
1092 : {
1093 : // this->recalculate_n_partitions();
1094 :
1095 : // Let's do the base class cache clearing first, just in case our
1096 : // later computations are ever changed to make use of a local
1097 : // elements range cache
1098 476512 : this->MeshBase::update_post_partitioning();
1099 :
1100 : // Partitioning changes our numbers of unpartitioned objects
1101 476512 : this->update_parallel_id_counts();
1102 476512 : }
1103 :
1104 :
1105 :
1106 : template <typename T>
1107 3300 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
1108 : {
1109 : // This function must be run on all processors at once
1110 3300 : parallel_object_only();
1111 :
1112 3300 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1113 3300 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1114 3300 : const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1115 :
1116 906732 : for (dof_id_type i=0; i != pmax_id; ++i)
1117 : {
1118 903432 : T * obj = objects[i]; // Returns nullptr if there's no map entry
1119 :
1120 : // Local lookups by id should return the requested object
1121 903432 : libmesh_assert(!obj || obj->id() == i);
1122 :
1123 : // All processors with an object should agree on id
1124 : #ifndef NDEBUG
1125 903432 : const dof_id_type dofid = obj && obj->valid_id() ?
1126 : obj->id() : DofObject::invalid_id;
1127 903432 : libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1128 : #endif
1129 :
1130 : // All processors with an object should agree on processor id
1131 903432 : const dof_id_type procid = obj && obj->valid_processor_id() ?
1132 : obj->processor_id() : DofObject::invalid_processor_id;
1133 903432 : libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1134 :
1135 903432 : dof_id_type min_procid = procid;
1136 903432 : this->comm().min(min_procid);
1137 :
1138 : // Either:
1139 : // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1140 : // or
1141 : // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1142 :
1143 : // Original assert logic
1144 : // libmesh_assert (min_procid != this->processor_id() || obj);
1145 :
1146 : // More human-understandable logic...
1147 903432 : libmesh_assert (
1148 : ((min_procid == this->processor_id()) && obj)
1149 : ||
1150 : (min_procid != this->processor_id())
1151 : );
1152 :
1153 : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1154 : // All processors with an object should agree on unique id
1155 903432 : const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1156 903432 : libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1157 : #endif
1158 : }
1159 3300 : }
1160 :
1161 :
1162 :
1163 1650 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
1164 : {
1165 1650 : this->libmesh_assert_valid_parallel_object_ids (this->_elements);
1166 1650 : this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1167 1650 : }
1168 :
1169 :
1170 :
1171 394 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
1172 : {
1173 : #ifndef NDEBUG
1174 : // This function must be run on all processors at once
1175 394 : parallel_object_only();
1176 :
1177 394 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1178 :
1179 21872 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1180 : {
1181 21478 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1182 :
1183 21478 : unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1184 :
1185 : // All processors with an active element should agree on p level
1186 21478 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1187 : }
1188 : #endif
1189 394 : }
1190 :
1191 :
1192 :
1193 :
1194 1598 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
1195 : {
1196 : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1197 : // This function must be run on all processors at once
1198 1598 : parallel_object_only();
1199 :
1200 1598 : dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1201 :
1202 148394 : for (dof_id_type i=0; i != pmax_elem_id; ++i)
1203 : {
1204 146796 : Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1205 :
1206 146796 : unsigned int refinement_flag = el ?
1207 65517 : static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1208 146796 : unsigned int p_refinement_flag = el ?
1209 65517 : static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1210 :
1211 146796 : libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1212 :
1213 : // p refinement flags aren't always kept correct on inactive
1214 : // ghost elements
1215 146796 : libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1216 : }
1217 : #endif // LIBMESH_ENABLE_AMR
1218 1598 : }
1219 :
1220 :
1221 :
1222 : template <typename T>
1223 : dof_id_type
1224 1706118 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
1225 : {
1226 : // This function must be run on all processors at once
1227 784 : parallel_object_only();
1228 :
1229 : typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1230 :
1231 : // In parallel we may not know what objects other processors have.
1232 : // Start by figuring out how many
1233 784 : dof_id_type unpartitioned_objects = 0;
1234 :
1235 : std::unordered_map<processor_id_type, dof_id_type>
1236 1568 : ghost_objects_from_proc;
1237 :
1238 1706118 : object_iterator it = objects.begin();
1239 784 : object_iterator end = objects.end();
1240 :
1241 109665725 : while (it != end)
1242 : {
1243 107959607 : T * obj = *it;
1244 :
1245 : // Remove any nullptr container entries while we're here.
1246 107959607 : if (!obj)
1247 1437488 : it = objects.erase(it);
1248 : else
1249 : {
1250 106522119 : processor_id_type obj_procid = obj->processor_id();
1251 106522119 : if (obj_procid == DofObject::invalid_processor_id)
1252 42390731 : unpartitioned_objects++;
1253 : else
1254 64131388 : ghost_objects_from_proc[obj_procid]++;
1255 :
1256 : // Finally, increment the iterator
1257 56228 : ++it;
1258 : }
1259 : }
1260 :
1261 1707686 : std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1262 1706902 : auto this_it = ghost_objects_from_proc.find(this->processor_id());
1263 784 : this->comm().allgather
1264 1706601 : ((this_it == ghost_objects_from_proc.end()) ?
1265 483 : dof_id_type(0) : this_it->second, objects_on_proc);
1266 :
1267 : #ifndef NDEBUG
1268 784 : libmesh_assert(this->comm().verify(unpartitioned_objects));
1269 2352 : for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1270 1568 : if (ghost_objects_from_proc.count(p))
1271 938 : libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1272 : else
1273 630 : libmesh_assert_less_equal (0, objects_on_proc[p]);
1274 : #endif
1275 :
1276 : // We'll renumber objects in blocks by processor id
1277 1707686 : std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1278 18104198 : for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1279 16401216 : first_object_on_proc[i] = first_object_on_proc[i-1] +
1280 784 : objects_on_proc[i-1];
1281 1706902 : dof_id_type next_id = first_object_on_proc[this->processor_id()];
1282 1706118 : dof_id_type first_free_id =
1283 1707686 : first_object_on_proc[this->n_processors()-1] +
1284 784 : objects_on_proc[this->n_processors()-1] +
1285 : unpartitioned_objects;
1286 :
1287 : // First set new local object ids and build request sets
1288 : // for non-local object ids
1289 :
1290 : // Request sets to send to each processor
1291 : std::map<processor_id_type, std::vector<dof_id_type>>
1292 784 : requested_ids;
1293 :
1294 : // We know how many objects live on each processor, so reserve() space for
1295 : // each.
1296 784 : auto ghost_end = ghost_objects_from_proc.end();
1297 19810316 : for (auto p : make_range(this->n_processors()))
1298 18105766 : if (p != this->processor_id())
1299 : {
1300 16398864 : if (const auto p_it = ghost_objects_from_proc.find(p);
1301 784 : p_it != ghost_end)
1302 3283527 : requested_ids[p].reserve(p_it->second);
1303 : }
1304 :
1305 784 : end = objects.end();
1306 108228237 : for (it = objects.begin(); it != end; ++it)
1307 : {
1308 106522119 : T * obj = *it;
1309 106522119 : if (!obj)
1310 0 : continue;
1311 106578347 : if (obj->processor_id() == this->processor_id())
1312 19759196 : obj->set_id(next_id++);
1313 86762923 : else if (obj->processor_id() != DofObject::invalid_processor_id)
1314 44372192 : requested_ids[obj->processor_id()].push_back(obj->id());
1315 : }
1316 :
1317 : // Next set ghost object ids from other processors
1318 :
1319 1723261 : auto gather_functor =
1320 47622343 : [
1321 : #ifndef NDEBUG
1322 : this,
1323 : &first_object_on_proc,
1324 : &objects_on_proc,
1325 : #endif
1326 : &objects]
1327 : (processor_id_type, const std::vector<dof_id_type> & ids,
1328 910 : std::vector<dof_id_type> & new_ids)
1329 : {
1330 910 : std::size_t ids_size = ids.size();
1331 3283527 : new_ids.resize(ids_size);
1332 :
1333 47655719 : for (std::size_t i=0; i != ids_size; ++i)
1334 : {
1335 44372192 : T * obj = objects[ids[i]];
1336 16233 : libmesh_assert(obj);
1337 16233 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1338 44388425 : new_ids[i] = obj->id();
1339 :
1340 16233 : libmesh_assert_greater_equal (new_ids[i],
1341 : first_object_on_proc[this->processor_id()]);
1342 16233 : libmesh_assert_less (new_ids[i],
1343 : first_object_on_proc[this->processor_id()] +
1344 : objects_on_proc[this->processor_id()]);
1345 : }
1346 : };
1347 :
1348 1723261 : auto action_functor =
1349 47622343 : [
1350 : #ifndef NDEBUG
1351 : &first_object_on_proc,
1352 : &objects_on_proc,
1353 : #endif
1354 : &objects]
1355 : (processor_id_type libmesh_dbg_var(pid),
1356 : const std::vector<dof_id_type> & ids,
1357 910 : const std::vector<dof_id_type> & data)
1358 : {
1359 : // Copy the id changes we've now been informed of
1360 47655719 : for (auto i : index_range(ids))
1361 : {
1362 44372192 : T * obj = objects[ids[i]];
1363 16233 : libmesh_assert (obj);
1364 16233 : libmesh_assert_equal_to (obj->processor_id(), pid);
1365 16233 : libmesh_assert_greater_equal (data[i],
1366 : first_object_on_proc[pid]);
1367 16233 : libmesh_assert_less (data[i],
1368 : first_object_on_proc[pid] +
1369 : objects_on_proc[pid]);
1370 44388425 : obj->set_id(data[i]);
1371 : }
1372 : };
1373 :
1374 784 : const dof_id_type * ex = nullptr;
1375 : Parallel::pull_parallel_vector_data
1376 1706118 : (this->comm(), requested_ids, gather_functor, action_functor, ex);
1377 :
1378 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
1379 1723261 : auto unique_gather_functor =
1380 47622343 : [
1381 : #ifndef NDEBUG
1382 : this,
1383 : #endif
1384 : &objects]
1385 : (processor_id_type, const std::vector<dof_id_type> & ids,
1386 910 : std::vector<unique_id_type> & data)
1387 : {
1388 910 : std::size_t ids_size = ids.size();
1389 3283527 : data.resize(ids_size);
1390 :
1391 47655719 : for (std::size_t i=0; i != ids_size; ++i)
1392 : {
1393 44372192 : T * obj = objects[ids[i]];
1394 16233 : libmesh_assert(obj);
1395 16233 : libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1396 88744384 : data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1397 : }
1398 : };
1399 :
1400 1723261 : auto unique_action_functor =
1401 47622343 : [&objects]
1402 : (processor_id_type libmesh_dbg_var(pid),
1403 : const std::vector<dof_id_type> & ids,
1404 910 : const std::vector<unique_id_type> & data)
1405 : {
1406 47655719 : for (auto i : index_range(ids))
1407 : {
1408 44372192 : T * obj = objects[ids[i]];
1409 16233 : libmesh_assert (obj);
1410 16233 : libmesh_assert_equal_to (obj->processor_id(), pid);
1411 44372192 : if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1412 0 : obj->set_unique_id(data[i]);
1413 : }
1414 : };
1415 :
1416 784 : const unique_id_type * unique_ex = nullptr;
1417 : Parallel::pull_parallel_vector_data
1418 1706118 : (this->comm(), requested_ids, unique_gather_functor,
1419 : unique_action_functor, unique_ex);
1420 : #endif
1421 :
1422 : // Next set unpartitioned object ids
1423 784 : next_id = 0;
1424 19810316 : for (auto i : make_range(this->n_processors()))
1425 18105766 : next_id += objects_on_proc[i];
1426 108228237 : for (it = objects.begin(); it != end; ++it)
1427 : {
1428 106522119 : T * obj = *it;
1429 106522119 : if (!obj)
1430 0 : continue;
1431 106522119 : if (obj->processor_id() == DofObject::invalid_processor_id)
1432 42390731 : obj->set_id(next_id++);
1433 : }
1434 :
1435 : // Finally shuffle around objects so that container indices
1436 : // match ids
1437 1706118 : it = objects.begin();
1438 784 : end = objects.end();
1439 114468041 : while (it != end)
1440 : {
1441 112761923 : T * obj = *it;
1442 112761923 : if (obj) // don't try shuffling already-nullptr entries
1443 : {
1444 112761923 : T * next = objects[obj->id()];
1445 : // If we have to move this object
1446 112761923 : if (next != obj)
1447 : {
1448 : // nullptr out its original position for now
1449 : // (our shuffling may put another object there shortly)
1450 39670898 : *it = nullptr;
1451 :
1452 : // There may already be another object with this id that
1453 : // needs to be moved itself
1454 51000278 : while (next)
1455 : {
1456 : // We shouldn't be trying to give two objects the
1457 : // same id
1458 9326 : libmesh_assert_not_equal_to (next->id(), obj->id());
1459 11329380 : objects[obj->id()] = obj;
1460 9326 : obj = next;
1461 11329380 : next = objects[obj->id()];
1462 : }
1463 39670898 : objects[obj->id()] = obj;
1464 : }
1465 : }
1466 :
1467 : // Remove any container entries that were left as nullptr.
1468 114904 : if (!obj)
1469 0 : it = objects.erase(it);
1470 : else
1471 57452 : ++it;
1472 : }
1473 :
1474 1706902 : return first_free_id;
1475 : }
1476 :
1477 :
1478 854729 : void DistributedMesh::renumber_nodes_and_elements ()
1479 : {
1480 394 : parallel_object_only();
1481 :
1482 : #ifdef DEBUG
1483 : // Make sure our ids and flags are consistent
1484 394 : this->libmesh_assert_valid_parallel_ids();
1485 394 : this->libmesh_assert_valid_parallel_flags();
1486 394 : this->libmesh_assert_valid_parallel_p_levels();
1487 : #endif
1488 :
1489 394 : LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1490 :
1491 : // Nodes not connected to any elements, and nullptr node entries
1492 : // in our container, should be deleted. But wait! If we've deleted coarse
1493 : // local elements on some processor, other processors might have ghosted
1494 : // nodes from it that are now no longer connected to any elements on it, but
1495 : // that are connected to their own semilocal elements. We'll have to
1496 : // communicate to ascertain if that's the case.
1497 854729 : this->comm().max(_deleted_coarse_elements);
1498 :
1499 : // What used nodes do we see on our proc?
1500 394 : std::set<dof_id_type> used_nodes;
1501 :
1502 : // What used node info should we send from our proc? Could we take ownership
1503 : // of each node if we needed to?
1504 : std::map<processor_id_type, std::map<dof_id_type, bool>>
1505 394 : used_nodes_on_proc;
1506 :
1507 : // flag the nodes we need
1508 56811522 : for (auto & elem : this->element_ptr_range())
1509 296201218 : for (const Node & node : elem->node_ref_range())
1510 : {
1511 268620687 : const dof_id_type n = node.id();
1512 268476573 : used_nodes.insert(n);
1513 268620687 : if (_deleted_coarse_elements)
1514 : {
1515 3123685 : const processor_id_type p = node.processor_id();
1516 3123969 : if (p != this->processor_id())
1517 : {
1518 2237336 : auto & used_nodes_on_p = used_nodes_on_proc[p];
1519 2237478 : if (elem->processor_id() == this->processor_id())
1520 215584 : used_nodes_on_p[n] = true;
1521 : else
1522 105 : if (!used_nodes_on_p.count(n))
1523 516282 : used_nodes_on_p[n] = false;
1524 : }
1525 : }
1526 853941 : }
1527 :
1528 854729 : if (_deleted_coarse_elements)
1529 : {
1530 : // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1531 : typedef unsigned char boolish;
1532 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1533 28 : used_nodes_on_proc_vecs;
1534 70355 : for (auto & [pid, nodemap] : used_nodes_on_proc)
1535 54485 : used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1536 :
1537 28 : std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1538 : std::map<processor_id_type, std::set<dof_id_type>>
1539 28 : repartitioned_node_sets_to_push;
1540 :
1541 : auto ids_action_functor =
1542 54459 : [&used_nodes, &repartitioned_node_pids,
1543 : &repartitioned_node_sets_to_push]
1544 : (processor_id_type pid,
1545 112 : const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1546 : {
1547 580910 : for (auto [n, sender_could_become_owner] : ids_and_bools)
1548 : {
1549 : // If we don't see a use for our own node, but someone
1550 : // else does, better figure out who should own it next.
1551 97 : if (!used_nodes.count(n))
1552 : {
1553 2 : if (auto it = repartitioned_node_pids.find(n);
1554 2 : sender_could_become_owner)
1555 : {
1556 1 : if (it != repartitioned_node_pids.end() &&
1557 1 : pid < it->second)
1558 1 : it->second = pid;
1559 : else
1560 0 : repartitioned_node_pids[n] = pid;
1561 : }
1562 : else
1563 1 : if (it == repartitioned_node_pids.end())
1564 1 : repartitioned_node_pids[n] =
1565 : DofObject::invalid_processor_id;
1566 :
1567 2 : repartitioned_node_sets_to_push[pid].insert(n);
1568 : }
1569 : }
1570 15882 : };
1571 :
1572 : // We need two pushes instead of a pull here because we need to
1573 : // know *all* the queries for a particular node before we can
1574 : // respond to *any* of them.
1575 : Parallel::push_parallel_vector_data
1576 15870 : (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1577 :
1578 : // Repartition (what used to be) our own nodes first
1579 15871 : for (auto & [n, p] : repartitioned_node_pids)
1580 : {
1581 1 : Node & node = this->node_ref(n);
1582 0 : libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1583 0 : libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1584 1 : node.processor_id() = p;
1585 : }
1586 :
1587 : // Then push to repartition others' ghosted copies.
1588 :
1589 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1590 28 : repartitioned_node_vecs;
1591 :
1592 15872 : for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1593 : {
1594 2 : auto & rn_vec = repartitioned_node_vecs[p];
1595 4 : for (auto n : nodeset)
1596 2 : rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1597 : }
1598 :
1599 : auto repartition_node_functor =
1600 2 : [this]
1601 : (processor_id_type libmesh_dbg_var(pid),
1602 2 : const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1603 : {
1604 4 : for (auto [n, p] : ids_and_pids)
1605 : {
1606 0 : libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1607 2 : Node & node = this->node_ref(n);
1608 0 : libmesh_assert_equal_to(node.processor_id(), pid);
1609 2 : node.processor_id() = p;
1610 : }
1611 15842 : };
1612 :
1613 : Parallel::push_parallel_vector_data
1614 15870 : (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1615 : }
1616 :
1617 854729 : _deleted_coarse_elements = false;
1618 :
1619 : // Nodes not connected to any local elements, and nullptr node entries
1620 : // in our container, are deleted
1621 : {
1622 854729 : node_iterator_imp it = _nodes.begin();
1623 394 : node_iterator_imp end = _nodes.end();
1624 :
1625 86628783 : while (it != end)
1626 : {
1627 85774054 : Node * nd = *it;
1628 85774054 : if (!nd)
1629 3471192 : it = _nodes.erase(it);
1630 82302862 : else if (!used_nodes.count(nd->id()))
1631 : {
1632 : // remove any boundary information associated with
1633 : // this node
1634 2327679 : this->get_boundary_info().remove (nd);
1635 2828 : _constraint_rows.erase(nd);
1636 :
1637 : // delete the node
1638 2327679 : delete nd;
1639 :
1640 2327679 : it = _nodes.erase(it);
1641 : }
1642 : else
1643 45139 : ++it;
1644 : }
1645 : }
1646 :
1647 854729 : this->_preparation.has_removed_orphaned_nodes = true;
1648 :
1649 854729 : if (_skip_renumber_nodes_and_elements)
1650 : {
1651 1670 : this->update_parallel_id_counts();
1652 2 : return;
1653 : }
1654 :
1655 : // Finally renumber all the elements
1656 853059 : _n_elem = this->renumber_dof_objects (this->_elements);
1657 :
1658 : // and all the remaining nodes
1659 853059 : _n_nodes = this->renumber_dof_objects (this->_nodes);
1660 :
1661 : // And figure out what IDs we should use when adding new nodes and
1662 : // new elements
1663 853059 : this->update_parallel_id_counts();
1664 :
1665 : // Make sure our caches are up to date and our
1666 : // DofObjects are well packed
1667 : #ifdef DEBUG
1668 392 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1669 392 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1670 392 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1671 392 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1672 392 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1673 392 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1674 392 : libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1675 392 : libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1676 :
1677 : // Make sure our ids and flags are consistent
1678 392 : this->libmesh_assert_valid_parallel_ids();
1679 392 : this->libmesh_assert_valid_parallel_flags();
1680 :
1681 : // And make sure we've made our numbering monotonic
1682 392 : MeshTools::libmesh_assert_valid_elem_ids(*this);
1683 : #endif
1684 : }
1685 :
1686 :
1687 :
1688 17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
1689 : {
1690 : // We can't use range-for here because we need access to the special
1691 : // iterators' methods, not just to their dereferenced values.
1692 :
1693 : // Nodes first
1694 0 : for (auto pr = this->_nodes.begin(),
1695 5444770 : end = this->_nodes.end(); pr != end; ++pr)
1696 : {
1697 5427360 : Node * n = *pr;
1698 5427360 : if (n != nullptr)
1699 : {
1700 0 : const dof_id_type id = pr.index();
1701 5204550 : n->set_id() = id;
1702 0 : libmesh_assert_equal_to(this->node_ptr(id), n);
1703 : }
1704 : }
1705 :
1706 : // Elements next
1707 0 : for (auto pr = this->_elements.begin(),
1708 5671231 : end = this->_elements.end(); pr != end; ++pr)
1709 : {
1710 5653821 : Elem * e = *pr;
1711 5653821 : if (e != nullptr)
1712 : {
1713 0 : const dof_id_type id = pr.index();
1714 5590229 : e->set_id() = id;
1715 0 : libmesh_assert_equal_to(this->elem_ptr(id), e);
1716 : }
1717 : }
1718 17410 : }
1719 :
1720 :
1721 :
1722 632987 : dof_id_type DistributedMesh::n_active_elem () const
1723 : {
1724 540 : parallel_object_only();
1725 :
1726 : // Get local active elements first
1727 : dof_id_type active_elements =
1728 1265434 : static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1729 1897881 : this->active_local_elements_end()));
1730 632987 : this->comm().sum(active_elements);
1731 :
1732 : // Then add unpartitioned active elements, which should exist on
1733 : // every processor
1734 632987 : active_elements +=
1735 632987 : static_cast<dof_id_type>(std::distance
1736 1265434 : (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1737 633527 : this->active_pid_elements_end(DofObject::invalid_processor_id)));
1738 632987 : return active_elements;
1739 : }
1740 :
1741 :
1742 :
1743 403756 : void DistributedMesh::delete_remote_elements()
1744 : {
1745 : #ifdef DEBUG
1746 : // Make sure our neighbor links are all fine
1747 382 : MeshTools::libmesh_assert_valid_neighbors(*this);
1748 :
1749 : // And our child/parent links, and our flags
1750 382 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1751 :
1752 : // Make sure our ids and flags are consistent
1753 382 : this->libmesh_assert_valid_parallel_ids();
1754 382 : this->libmesh_assert_valid_parallel_flags();
1755 :
1756 382 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1757 382 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1758 382 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1759 382 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1760 382 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1761 382 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1762 : #endif
1763 :
1764 403756 : _is_serial = false;
1765 403756 : _is_serial_on_proc_0 = false;
1766 :
1767 403756 : MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1768 :
1769 382 : libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1770 :
1771 : // Now make sure the containers actually shrink - strip
1772 : // any newly-created nullptr voids out of the element array
1773 403756 : dofobject_container<Elem>::veclike_iterator e_it = _elements.begin();
1774 382 : const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
1775 53523786 : while (e_it != e_end)
1776 53120030 : if (!*e_it)
1777 39590070 : e_it = _elements.erase(e_it);
1778 : else
1779 17361 : ++e_it;
1780 :
1781 403756 : dofobject_container<Node>::veclike_iterator n_it = _nodes.begin();
1782 382 : const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
1783 100190784 : while (n_it != n_end)
1784 99787028 : if (!*n_it)
1785 66635675 : n_it = _nodes.erase(n_it);
1786 : else
1787 48503 : ++n_it;
1788 :
1789 : // We may have deleted no-longer-connected nodes or coarsened-away
1790 : // elements; let's update our caches.
1791 403756 : this->update_parallel_id_counts();
1792 :
1793 : // We may have deleted nodes or elements that were the only local
1794 : // representatives of some particular boundary id(s); let's update
1795 : // those caches.
1796 403756 : this->get_boundary_info().regenerate_id_sets();
1797 :
1798 : #ifdef DEBUG
1799 : // We might not have well-packed objects if the user didn't allow us
1800 : // to renumber
1801 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1802 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1803 :
1804 : // Make sure our neighbor links are all fine
1805 382 : MeshTools::libmesh_assert_valid_neighbors(*this);
1806 :
1807 : // And our child/parent links, and our flags
1808 382 : MeshTools::libmesh_assert_valid_refinement_tree(*this);
1809 :
1810 : // Make sure our ids and flags are consistent
1811 382 : this->libmesh_assert_valid_parallel_ids();
1812 382 : this->libmesh_assert_valid_parallel_flags();
1813 : #endif
1814 :
1815 403756 : this->_preparation.has_removed_remote_elements = true;
1816 403756 : }
1817 :
1818 :
1819 0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
1820 : {
1821 : // First add the elem like normal
1822 0 : add_elem(e);
1823 :
1824 : // Now add it to the set that won't be deleted when we call
1825 : // delete_remote_elements()
1826 0 : _extra_ghost_elems.insert(e);
1827 0 : }
1828 :
1829 : void
1830 0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1831 : {
1832 0 : std::set<Elem *> tmp;
1833 0 : std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1834 : extra_ghost_elems.begin(), extra_ghost_elems.end(),
1835 0 : std::inserter(tmp, tmp.begin()));
1836 0 : _extra_ghost_elems = tmp;
1837 0 : }
1838 :
1839 96956 : void DistributedMesh::allgather()
1840 : {
1841 96956 : if (_is_serial)
1842 0 : return;
1843 96954 : MeshCommunication().allgather(*this);
1844 96954 : _is_serial = true;
1845 96954 : _is_serial_on_proc_0 = true;
1846 :
1847 : // Make sure our caches are up to date and our
1848 : // DofObjects are well packed
1849 : #ifdef DEBUG
1850 48 : libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1851 48 : libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1852 48 : const dof_id_type pmax_node_id = this->parallel_max_node_id();
1853 48 : const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1854 48 : libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1855 48 : libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1856 :
1857 : // If we've disabled renumbering we can't be sure we're contiguous
1858 : // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1859 : // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1860 :
1861 : // Make sure our neighbor links are all fine
1862 48 : MeshTools::libmesh_assert_valid_neighbors(*this);
1863 :
1864 : // Make sure our ids and flags are consistent
1865 48 : this->libmesh_assert_valid_parallel_ids();
1866 48 : this->libmesh_assert_valid_parallel_flags();
1867 : #endif
1868 : }
1869 :
1870 18096 : void DistributedMesh::gather_to_zero()
1871 : {
1872 18096 : if (_is_serial_on_proc_0)
1873 4 : return;
1874 :
1875 13858 : _is_serial_on_proc_0 = true;
1876 13858 : MeshCommunication().gather(0, *this);
1877 : }
1878 :
1879 :
1880 : } // namespace libMesh
|
