libMesh
distributed_mesh.C
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2025 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 
20 // Local includes
21 #include "libmesh/distributed_mesh.h"
22 
23 // libMesh includes
24 #include "libmesh/boundary_info.h"
25 #include "libmesh/elem.h"
26 #include "libmesh/libmesh_logging.h"
27 #include "libmesh/mesh_communication.h"
28 #include "libmesh/mesh_tools.h"
29 #include "libmesh/partitioner.h"
30 #include "libmesh/string_to_enum.h"
31 
32 // TIMPI includes
34 #include "timpi/parallel_sync.h"
35 
36 
37 namespace libMesh
38 {
39 
40 // ------------------------------------------------------------
41 // DistributedMesh class member functions
43  unsigned char d) :
44  UnstructuredMesh (comm_in,d), _is_serial(true),
45  _is_serial_on_proc_0(true),
46  _deleted_coarse_elements(false),
47  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
48  _next_free_local_node_id(this->processor_id()),
49  _next_free_local_elem_id(this->processor_id()),
50  _next_free_unpartitioned_node_id(this->n_processors()),
51  _next_free_unpartitioned_elem_id(this->n_processors())
52 #ifdef LIBMESH_ENABLE_UNIQUE_ID
53  , _next_unpartitioned_unique_id(this->n_processors())
54 #endif
55 {
56 #ifdef LIBMESH_ENABLE_UNIQUE_ID
57  _next_unique_id = this->processor_id();
58 #endif
59 
60  const std::string default_partitioner = "parmetis";
61  const std::string my_partitioner =
62  libMesh::command_line_value("--default-partitioner",
63  default_partitioner);
65  (Utility::string_to_enum<PartitionerType>(my_partitioner));
66 }
67 
69 {
70  LOG_SCOPE("operator=(&&)", "DistributedMesh");
71 
72  // Move assign as an UnstructuredMesh.
73  this->UnstructuredMesh::operator=(std::move(other_mesh));
74 
75  // Nodes and elements belong to DistributedMesh and have to be
76  // moved before we can move arbitrary GhostingFunctor, Partitioner,
77  // etc. subclasses.
78  this->move_nodes_and_elements(std::move(other_mesh));
79 
80  // But move_nodes_and_elems misses (or guesses about) some of our
81  // subclass values, and we want more precision than a guess.
82  _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
83  _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
84 
85  // Handle remaining MeshBase moves.
86  this->post_dofobject_moves(std::move(other_mesh));
87 
88  return *this;
89 }
90 
92 {
93  *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
94 
95  return *this;
96 }
97 
98 bool DistributedMesh::subclass_locally_equals(const MeshBase & other_mesh_base) const
99 {
100  const DistributedMesh * dist_mesh_ptr =
101  dynamic_cast<const DistributedMesh *>(&other_mesh_base);
102  if (!dist_mesh_ptr)
103  return false;
104  const DistributedMesh & other_mesh = *dist_mesh_ptr;
105 
106  if (_is_serial != other_mesh._is_serial ||
109  _n_nodes != other_mesh._n_nodes ||
110  _n_elem != other_mesh._n_elem ||
111  _max_node_id != other_mesh._max_node_id ||
112  _max_elem_id != other_mesh._max_elem_id ||
113  // We expect these things to change in a prepare_for_use();
114  // they're conceptually "mutable"...
115 /*
116  _next_free_local_node_id != other_mesh._next_free_local_node_id ||
117  _next_free_local_elem_id != other_mesh._next_free_local_elem_id ||
118  _next_free_unpartitioned_node_id != other_mesh._next_free_unpartitioned_node_id ||
119  _next_free_unpartitioned_elem_id != other_mesh._next_free_unpartitioned_elem_id ||
120 #ifdef LIBMESH_ENABLE_UNIQUE_ID
121  _next_unpartitioned_unique_id != other_mesh._next_unpartitioned_unique_id ||
122 #endif
123 */
124  !this->nodes_and_elements_equal(other_mesh))
125  return false;
126 
127  if (_extra_ghost_elems.size() !=
128  other_mesh._extra_ghost_elems.size())
129  return false;
130  for (auto & elem : _extra_ghost_elems)
131  {
132  libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
133  const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
134  if (!other_elem ||
135  !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
136  return false;
137  }
138 
139  return true;
140 }
141 
143 {
144  this->DistributedMesh::clear(); // Free nodes and elements
145 }
146 
147 
148 // This might be specialized later, but right now it's just here to
149 // make sure the compiler doesn't give us a default (non-deep) copy
150 // constructor instead.
152  DistributedMesh(static_cast<const MeshBase &>(other_mesh))
153 {
154  _is_serial = other_mesh._is_serial;
157 
158  _n_nodes = other_mesh.n_nodes();
159  _n_elem = other_mesh.n_elem();
160  _max_node_id = other_mesh.max_node_id();
161  _max_elem_id = other_mesh.max_elem_id();
163  other_mesh._next_free_local_node_id;
165  other_mesh._next_free_local_elem_id;
170 #ifdef LIBMESH_ENABLE_UNIQUE_ID
172  other_mesh._next_unique_id;
175 #endif
176 
177  // Need to copy extra_ghost_elems
178  for (auto & elem : other_mesh._extra_ghost_elems)
179  _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
180 }
181 
182 
183 
185  UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
186  _is_serial_on_proc_0(other_mesh.is_serial()),
187  _deleted_coarse_elements(true), // better safe than sorry...
188  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
189  _next_free_local_node_id(this->processor_id()),
190  _next_free_local_elem_id(this->processor_id()),
191  _next_free_unpartitioned_node_id(this->n_processors()),
192  _next_free_unpartitioned_elem_id(this->n_processors())
193 {
194  this->copy_nodes_and_elements(other_mesh, true);
195 
196  this->allow_find_neighbors(other_mesh.allow_find_neighbors());
197  this->allow_renumbering(other_mesh.allow_renumbering());
199  this->skip_partitioning(other_mesh.skip_partitioning());
200 
201  // The prepare_for_use() in copy_nodes_and_elements() is going to be
202  // tricky to remove without breaking backwards compatibility, but it
203  // updates some things we want to just copy.
204  this->copy_cached_data(other_mesh);
205 
206  this->copy_constraint_rows(other_mesh);
207 
208  this->_is_prepared = other_mesh.is_prepared();
209 
210  auto & this_boundary_info = this->get_boundary_info();
211  const auto & other_boundary_info = other_mesh.get_boundary_info();
212 
213  this_boundary_info = other_boundary_info;
214 
215  this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
216 
217 #ifdef LIBMESH_ENABLE_UNIQUE_ID
218  _next_unique_id = other_mesh.parallel_max_unique_id() +
219  this->processor_id();
221  (this->n_processors() - this->processor_id());
222 #endif
224 }
225 
227 {
228  DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
229 
230  this->_nodes = std::move(other_mesh._nodes);
231  this->_n_nodes = other_mesh.n_nodes();
232 
233  this->_elements = std::move(other_mesh._elements);
234  this->_n_elem = other_mesh.n_elem();
235 
236  _is_serial = other_mesh.is_serial();
238  _deleted_coarse_elements = true; // Better safe than sorry
239 
240  _max_node_id = other_mesh.max_node_id();
241  _max_elem_id = other_mesh.max_elem_id();
242 
247 
248  #ifdef LIBMESH_ENABLE_UNIQUE_ID
250  #endif
251 }
252 
253 // We use cached values for these so they can be called
254 // from one processor without bothering the rest, but
255 // we may need to update those caches before doing a full
256 // renumbering
258 {
259  // This function must be run on all processors at once
260  parallel_object_only();
261 
262  _n_elem = this->parallel_n_elem();
263  _n_nodes = this->parallel_n_nodes();
266 
269  ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
270  (this->n_processors() + 1) + this->n_processors();
273  ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
274  (this->n_processors() + 1) + this->processor_id();
275 
278  ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
279  (this->n_processors() + 1) + this->n_processors();
282  ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
283  (this->n_processors() + 1) + this->processor_id();
284 
285 #ifdef LIBMESH_ENABLE_UNIQUE_ID
288  ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
289  (this->n_processors() + 1) + this->n_processors();
291  ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
292  (this->n_processors() + 1) + this->processor_id();
293 #endif
294 }
295 
296 
297 // Or in debug mode we may want to test the uncached values without
298 // changing the cache
300 {
301  // This function must be run on all processors at once
302  parallel_object_only();
303 
304  dof_id_type n_local = this->n_local_elem();
305  this->comm().sum(n_local);
306  n_local += this->n_unpartitioned_elem();
307  return n_local;
308 }
309 
310 
311 
313 {
314  // This function must be run on all processors at once
315  parallel_object_only();
316 
317  dof_id_type max_local = 0;
318 
320  rit = _elements.rbegin();
321 
323  rend = _elements.rend();
324 
325  // Look for the maximum element id. Search backwards through
326  // elements so we can break out early. Beware of nullptr entries that
327  // haven't yet been cleared from _elements.
328  for (; rit != rend; ++rit)
329  {
330  const DofObject *d = *rit;
331  if (d)
332  {
333  libmesh_assert(_elements[d->id()] == d);
334  max_local = d->id() + 1;
335  break;
336  }
337  }
338 
339  this->comm().max(max_local);
340  return max_local;
341 }
342 
343 
344 
345 #ifdef LIBMESH_ENABLE_UNIQUE_ID
347 {
348  // This function must be run on all processors at once
349  parallel_object_only();
350 
351  unique_id_type max_local = std::max(_next_unique_id,
353  this->comm().max(max_local);
354  return max_local;
355 }
356 
357 
358 
360 {
361  _next_unique_id = id;
363  ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
364  (this->n_processors() + 1) + this->n_processors();
366  ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
367  (this->n_processors() + 1) + this->processor_id();
368 }
369 #endif
370 
371 
372 
374 {
375  // This function must be run on all processors at once
376  parallel_object_only();
377 
378  dof_id_type n_local = this->n_local_nodes();
379  this->comm().sum(n_local);
380  n_local += this->n_unpartitioned_nodes();
381  return n_local;
382 }
383 
384 
385 
387 {
388  // This function must be run on all processors at once
389  parallel_object_only();
390 
391  dof_id_type max_local = 0;
392 
394  rit = _nodes.rbegin();
395 
397  rend = _nodes.rend();
398 
399  // Look for the maximum node id. Search backwards through
400  // nodes so we can break out early. Beware of nullptr entries that
401  // haven't yet been cleared from _nodes
402  for (; rit != rend; ++rit)
403  {
404  const DofObject *d = *rit;
405  if (d)
406  {
407  libmesh_assert(_nodes[d->id()] == d);
408  max_local = d->id() + 1;
409  break;
410  }
411  }
412 
413  this->comm().max(max_local);
414  return max_local;
415 }
416 
417 
418 
419 const Point & DistributedMesh::point (const dof_id_type i) const
420 {
421  return this->node_ref(i);
422 }
423 
424 
425 
427 {
429  libmesh_assert_equal_to (_nodes[i]->id(), i);
430 
431  return _nodes[i];
432 }
433 
434 
435 
436 
438 {
440  libmesh_assert_equal_to (_nodes[i]->id(), i);
441 
442  return _nodes[i];
443 }
444 
445 
446 
447 
449 {
450  if (const auto it = _nodes.find(i);
451  it != _nodes.end())
452  {
453  const Node * n = *it;
454  libmesh_assert (!n || n->id() == i);
455  return n;
456  }
457 
458  return nullptr;
459 }
460 
461 
462 
463 
465 {
466  if (auto it = _nodes.find(i);
467  it != _nodes.end())
468  {
469  Node * n = *it;
470  libmesh_assert (!n || n->id() == i);
471  return n;
472  }
473 
474  return nullptr;
475 }
476 
477 
478 
479 
481 {
483  libmesh_assert_equal_to (_elements[i]->id(), i);
484 
485  return _elements[i];
486 }
487 
488 
489 
490 
492 {
494  libmesh_assert_equal_to (_elements[i]->id(), i);
495 
496  return _elements[i];
497 }
498 
499 
500 
501 
503 {
504  if (const auto it = _elements.find(i);
505  it != _elements.end())
506  {
507  const Elem * e = *it;
508  libmesh_assert (!e || e->id() == i);
509  return e;
510  }
511 
512  return nullptr;
513 }
514 
515 
516 
517 
519 {
520  if (auto it = _elements.find(i);
521  it != _elements.end())
522  {
523  Elem * e = *it;
524  libmesh_assert (!e || e->id() == i);
525  return e;
526  }
527 
528  return nullptr;
529 }
530 
531 
532 
533 
535 {
536  // Don't try to add nullptrs!
537  libmesh_assert(e);
538 
539  // Trying to add an existing element is a no-op
540  if (e->valid_id() && _elements[e->id()] == e)
541  return e;
542 
543  const processor_id_type elem_procid = e->processor_id();
544 
545  if (!e->valid_id())
546  {
547  // We should only be creating new ids past the end of the range
548  // of existing ids
549  libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
550  _max_elem_id);
551  libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
552 
553  // Use the unpartitioned ids for unpartitioned elems, and
554  // temporarily for ghost elems
556  if (elem_procid == this->processor_id())
557  next_id = &_next_free_local_elem_id;
558  e->set_id (*next_id);
559  }
560 
561  {
562  // Advance next_ids up high enough that each is pointing to an
563  // unused id and any subsequent increments will still point us
564  // to unused ids
565  _max_elem_id = std::max(_max_elem_id,
566  static_cast<dof_id_type>(e->id()+1));
567 
570  ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
571  (this->n_processors() + 1) + this->n_processors();
574  ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
575  (this->n_processors() + 1) + this->processor_id();
576 
577 #ifndef NDEBUG
578  // We need a const dofobject_container so we don't inadvertently create
579  // nullptr entries when testing for non-nullptr ones
580  const dofobject_container<Elem> & const_elements = _elements;
581 #endif
583  libmesh_assert(!const_elements[_next_free_local_elem_id]);
584  }
585 
586  // Don't try to overwrite existing elems
587  libmesh_assert (!_elements[e->id()]);
588 
589  _elements[e->id()] = e;
590 
591  // Try to make the cached elem data more accurate
592  if (elem_procid == this->processor_id() ||
593  elem_procid == DofObject::invalid_processor_id)
594  _n_elem++;
595 
596 #ifdef LIBMESH_ENABLE_UNIQUE_ID
597  if (!e->valid_unique_id())
598  {
599  if (processor_id() == e->processor_id())
600  {
602  _next_unique_id += this->n_processors() + 1;
603  }
604  else
605  {
608  }
609  }
610  else
611  {
612  _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
614  ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
615  (this->n_processors() + 1) + this->processor_id();
616  }
617 #endif
618 
619  // Unpartitioned elems should be added on every processor
620  // And shouldn't be added in the same batch as ghost elems
621  // But we might be just adding on processor 0 to
622  // broadcast later
623  // #ifdef DEBUG
624  // if (elem_procid == DofObject::invalid_processor_id)
625  // {
626  // dof_id_type elem_id = e->id();
627  // this->comm().max(elem_id);
628  // libmesh_assert_equal_to (elem_id, e->id());
629  // }
630  // #endif
631 
632  // Make sure any new element is given space for any extra integers
633  // we've requested
636 
637  // And set mapping type and data on any new element
640 
641  return e;
642 }
643 
644 
645 
646 Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
647 {
648  // The mesh now takes ownership of the Elem. Eventually the guts of
649  // add_elem() will get moved to a private helper function, and
650  // calling add_elem() directly will be deprecated.
651  return add_elem(e.release());
652 }
653 
654 
655 
657 {
658  if (_elements[e->id()])
659  this->delete_elem(_elements[e->id()]);
660 
661 #ifdef LIBMESH_ENABLE_UNIQUE_ID
662  if (!e->valid_unique_id())
663  {
664  if (processor_id() == e->processor_id())
665  {
667  _next_unique_id += this->n_processors() + 1;
668  }
669  else
670  {
673  }
674  }
675  else
676  {
677  _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
679  ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
680  (this->n_processors() + 1) + this->processor_id();
681  }
682 #endif
683 
684  // Try to make the cached elem data more accurate
685  processor_id_type elem_procid = e->processor_id();
686  if (elem_procid == this->processor_id() ||
687  elem_procid == DofObject::invalid_processor_id)
688  _n_elem++;
689 
690  _elements[e->id()] = e;
691 
692  // Make sure any new element is given space for any extra integers
693  // we've requested
696 
697  // And set mapping type and data on any new element
700 
701  return e;
702 }
703 
704 Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
705 {
706  // The mesh now takes ownership of the Elem. Eventually the guts of
707  // insert_elem(Elem*) will get moved to a private helper function, and
708  // calling insert_elem(Elem*) directly will be deprecated.
709  return insert_elem(e.release());
710 }
711 
712 
714 {
715  libmesh_assert (e);
716 
717  // Try to make the cached elem data more accurate
718  _n_elem--;
719 
720  // Was this a coarse element, not just a coarsening where we still
721  // have some ancestor structure? Was it a *local* element, that we
722  // might have been depending on as an owner of local nodes? We'll
723  // have to be more careful with our nodes in contract() later; no
724  // telling if we just locally orphaned a node that should be
725  // globally retained.
726  if (e->processor_id() == this->processor_id() &&
727  !e->parent())
729 
730  // Delete the element from the BoundaryInfo object
731  this->get_boundary_info().remove(e);
732 
733  // But not yet from the container; we might invalidate
734  // an iterator that way!
735 
736  //_elements.erase(e->id());
737 
738  // Instead, we set it to nullptr for now
739 
740  _elements[e->id()] = nullptr;
741 
742  // delete the element
743  delete e;
744 }
745 
746 
747 
749  const dof_id_type new_id)
750 {
751  // This could be a no-op
752  if (old_id == new_id)
753  return;
754 
755  Elem * el = _elements[old_id];
756  libmesh_assert (el);
757  libmesh_assert_equal_to (el->id(), old_id);
758 
759  el->set_id(new_id);
760  libmesh_assert (!_elements[new_id]);
761  _elements[new_id] = el;
762  _elements.erase(old_id);
763 }
764 
765 
766 
768  const dof_id_type id,
769  const processor_id_type proc_id)
770 {
771  Node * old_n = this->query_node_ptr(id);
772 
773  if (old_n)
774  {
775  *old_n = p;
776  old_n->processor_id() = proc_id;
777 
778  return old_n;
779  }
780 
781  Node * n = Node::build(p, id).release();
782  n->processor_id() = proc_id;
783 
784  return DistributedMesh::add_node(n);
785 }
786 
787 
789 {
790  // This had better be a node in our mesh
791  libmesh_assert(_nodes[n.id()] == &n);
792 
793  _nodes[n.id()] = nullptr;
794  _n_nodes--;
795 
797  n.processor_id() = this->processor_id();
798 
799  this->add_node(&n);
800 }
801 
802 
804 {
805  // Don't try to add nullptrs!
806  libmesh_assert(n);
807 
808  // Trying to add an existing node is a no-op
809  if (n->valid_id() && _nodes[n->id()] == n)
810  return n;
811 
812  const processor_id_type node_procid = n->processor_id();
813 
814  if (!n->valid_id())
815  {
816  // We should only be creating new ids past the end of the range
817  // of existing ids
818  libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
819  _max_node_id);
820  libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
821 
822  // Use the unpartitioned ids for unpartitioned nodes,
823  // and temporarily for ghost nodes
825  if (node_procid == this->processor_id())
826  next_id = &_next_free_local_node_id;
827  n->set_id (*next_id);
828  }
829 
830  {
831  // Advance next_ids up high enough that each is pointing to an
832  // unused id and any subsequent increments will still point us
833  // to unused ids
834  _max_node_id = std::max(_max_node_id,
835  static_cast<dof_id_type>(n->id()+1));
836 
839  ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
840  (this->n_processors() + 1) + this->n_processors();
843  ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
844  (this->n_processors() + 1) + this->processor_id();
845 
846 #ifndef NDEBUG
847  // We need a const dofobject_container so we don't inadvertently create
848  // nullptr entries when testing for non-nullptr ones
849  const dofobject_container<Node> & const_nodes = _nodes;
850 #endif
853  }
854 
855  // Don't try to overwrite existing nodes
856  libmesh_assert (!_nodes[n->id()]);
857 
858  _nodes[n->id()] = n;
859 
860  // Try to make the cached node data more accurate
861  if (node_procid == this->processor_id() ||
862  node_procid == DofObject::invalid_processor_id)
863  _n_nodes++;
864 
865 #ifdef LIBMESH_ENABLE_UNIQUE_ID
866  if (!n->valid_unique_id())
867  {
868  if (processor_id() == n->processor_id())
869  {
871  _next_unique_id += this->n_processors() + 1;
872  }
873  else
874  {
877  }
878  }
879  else
880  {
881  _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
883  ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
884  (this->n_processors() + 1) + this->processor_id();
885  }
886 #endif
887 
890 
891  // Unpartitioned nodes should be added on every processor
892  // And shouldn't be added in the same batch as ghost nodes
893  // But we might be just adding on processor 0 to
894  // broadcast later
895  // #ifdef DEBUG
896  // if (node_procid == DofObject::invalid_processor_id)
897  // {
898  // dof_id_type node_id = n->id();
899  // this->comm().max(node_id);
900  // libmesh_assert_equal_to (node_id, n->id());
901  // }
902  // #endif
903 
904  return n;
905 }
906 
907 Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
908 {
909  // The mesh now takes ownership of the Node. Eventually the guts of
910  // add_node() will get moved to a private helper function, and
911  // calling add_node() directly will be deprecated.
912  return add_node(n.release());
913 }
914 
915 #ifdef LIBMESH_ENABLE_DEPRECATED
916 
918 {
919  libmesh_deprecated();
920  return DistributedMesh::add_node(n);
921 }
922 
923 Node * DistributedMesh::insert_node(std::unique_ptr<Node> n)
924 {
925  libmesh_deprecated();
926  return insert_node(n.release());
927 }
928 
929 #endif
930 
932 {
933  libmesh_assert(n);
934  libmesh_assert(_nodes[n->id()]);
935 
936  // Try to make the cached elem data more accurate
937  _n_nodes--;
938 
939  // Delete the node from the BoundaryInfo object
940  this->get_boundary_info().remove(n);
941  _constraint_rows.erase(n);
942 
943  // But not yet from the container; we might invalidate
944  // an iterator that way!
945 
946  //_nodes.erase(n->id());
947 
948  // Instead, we set it to nullptr for now
949 
950  _nodes[n->id()] = nullptr;
951 
952  // delete the node
953  delete n;
954 }
955 
956 
957 
959  const dof_id_type new_id)
960 {
961  // This could be a no-op
962  if (old_id == new_id)
963  return;
964 
965  Node * nd = _nodes[old_id];
966  libmesh_assert (nd);
967  libmesh_assert_equal_to (nd->id(), old_id);
968 
969  // If we have nodes shipped to this processor for NodeConstraints
970  // use, then those nodes will exist in _nodes, but may not be
971  // locatable via a TopologyMap due to the insufficiency of elements
972  // connecting to them. If local refinement then wants to create a
973  // *new* node in the same location, it will initially get a temporary
974  // id, and then make_node_ids_parallel_consistent() will try to move
975  // it to the canonical id. We need to account for this case to
976  // avoid false positives and memory leaks.
977 #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
978  if (_nodes[new_id])
979  {
980  libmesh_assert_equal_to (*(Point *)_nodes[new_id],
981  *(Point *)_nodes[old_id]);
982  _nodes.erase(new_id);
983  }
984 #else
985  // If we aren't shipping nodes for NodeConstraints, there should be
986  // no reason for renumbering one node onto another.
987  libmesh_assert (!_nodes[new_id]);
988 #endif
989  _nodes[new_id] = nd;
990  nd->set_id(new_id);
991 
992  _nodes.erase(old_id);
993 }
994 
995 
996 
998 {
999  // Call parent clear function
1000  MeshBase::clear();
1001 
1002  // Clear our elements and nodes
1003  // There is no need to remove them from
1004  // the BoundaryInfo data structure since we
1005  // already cleared it.
1007 
1008  for (auto & node : _nodes)
1009  delete node;
1010 
1011  _nodes.clear();
1012 
1013  // We're no longer distributed if we were before
1014  _is_serial = true;
1015  _is_serial_on_proc_0 = true;
1016 
1017  // We deleted a ton of coarse elements, but their nodes got deleted too so
1018  // all is copacetic.
1019  _deleted_coarse_elements = false;
1020 
1021  // Correct our caches
1022  _n_nodes = 0;
1023  _max_node_id = 0;
1026 }
1027 
1028 
1029 
1031 {
1032  for (auto & elem : _elements)
1033  delete elem;
1034 
1035  _elements.clear();
1036 
1037  // Correct our caches
1038  _n_elem = 0;
1039  _max_elem_id = 0;
1042 }
1043 
1044 
1045 
1047 {
1048  // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
1049  if (!this->is_serial())
1050  {
1051  // Construct a MeshCommunication object to actually redistribute the nodes
1052  // and elements according to the partitioner, and then to re-gather the neighbors.
1053  MeshCommunication mc;
1054  mc.redistribute(*this);
1055 
1056  this->update_parallel_id_counts();
1057 
1058  // We ought to still have valid neighbor links; we communicate
1059  // them for newly-redistributed elements
1060  // this->find_neighbors();
1061 
1062  // Is this necessary? If we are called from prepare_for_use(), this will be called
1063  // anyway... but users can always call partition directly, in which case we do need
1064  // to call delete_remote_elements()...
1065  //
1066  // Regardless of whether it's necessary, it isn't safe. We
1067  // haven't communicated new node processor_ids yet, and we can't
1068  // delete nodes until we do.
1069  // this->delete_remote_elements();
1070  }
1071  else
1072  // The base class can handle non-distributed things, like
1073  // notifying any GhostingFunctors of changes
1075 }
1076 
1077 
1078 
1080 {
1081  // this->recalculate_n_partitions();
1082 
1083  // Partitioning changes our numbers of unpartitioned objects
1084  this->update_parallel_id_counts();
1085 }
1086 
1087 
1088 
1089 template <typename T>
1091 {
1092  // This function must be run on all processors at once
1093  parallel_object_only();
1094 
1095  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1096  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1097  const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
1098 
1099  for (dof_id_type i=0; i != pmax_id; ++i)
1100  {
1101  T * obj = objects[i]; // Returns nullptr if there's no map entry
1102 
1103  // Local lookups by id should return the requested object
1104  libmesh_assert(!obj || obj->id() == i);
1105 
1106  // All processors with an object should agree on id
1107 #ifndef NDEBUG
1108  const dof_id_type dofid = obj && obj->valid_id() ?
1109  obj->id() : DofObject::invalid_id;
1110  libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
1111 #endif
1112 
1113  // All processors with an object should agree on processor id
1114  const dof_id_type procid = obj && obj->valid_processor_id() ?
1115  obj->processor_id() : DofObject::invalid_processor_id;
1116  libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
1117 
1118  dof_id_type min_procid = procid;
1119  this->comm().min(min_procid);
1120 
1121  // Either:
1122  // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
1123  // or
1124  // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
1125 
1126  // Original assert logic
1127  // libmesh_assert (min_procid != this->processor_id() || obj);
1128 
1129  // More human-understandable logic...
1130  libmesh_assert (
1131  ((min_procid == this->processor_id()) && obj)
1132  ||
1133  (min_procid != this->processor_id())
1134  );
1135 
1136 #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
1137  // All processors with an object should agree on unique id
1138  const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
1139  libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
1140 #endif
1141  }
1142 }
1143 
1144 
1145 
1147 {
1149  this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
1150 }
1151 
1152 
1153 
1155 {
1156 #ifndef NDEBUG
1157  // This function must be run on all processors at once
1158  parallel_object_only();
1159 
1160  dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1161 
1162  for (dof_id_type i=0; i != pmax_elem_id; ++i)
1163  {
1164  Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1165 
1166  unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
1167 
1168  // All processors with an active element should agree on p level
1169  libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
1170  }
1171 #endif
1172 }
1173 
1174 
1175 
1176 
1178 {
1179 #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
1180  // This function must be run on all processors at once
1181  parallel_object_only();
1182 
1183  dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1184 
1185  for (dof_id_type i=0; i != pmax_elem_id; ++i)
1186  {
1187  Elem * el = _elements[i]; // Returns nullptr if there's no map entry
1188 
1189  unsigned int refinement_flag = el ?
1190  static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
1191  unsigned int p_refinement_flag = el ?
1192  static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
1193 
1194  libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
1195 
1196  // p refinement flags aren't always kept correct on inactive
1197  // ghost elements
1198  libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
1199  }
1200 #endif // LIBMESH_ENABLE_AMR
1201 }
1202 
1203 
1204 
1205 template <typename T>
1208 {
1209  // This function must be run on all processors at once
1210  parallel_object_only();
1211 
1212  typedef typename dofobject_container<T>::veclike_iterator object_iterator;
1213 
1214  // In parallel we may not know what objects other processors have.
1215  // Start by figuring out how many
1216  dof_id_type unpartitioned_objects = 0;
1217 
1218  std::unordered_map<processor_id_type, dof_id_type>
1219  ghost_objects_from_proc;
1220 
1221  object_iterator it = objects.begin();
1222  object_iterator end = objects.end();
1223 
1224  while (it != end)
1225  {
1226  T * obj = *it;
1227 
1228  // Remove any nullptr container entries while we're here.
1229  if (!obj)
1230  it = objects.erase(it);
1231  else
1232  {
1233  processor_id_type obj_procid = obj->processor_id();
1234  if (obj_procid == DofObject::invalid_processor_id)
1235  unpartitioned_objects++;
1236  else
1237  ghost_objects_from_proc[obj_procid]++;
1238 
1239  // Finally, increment the iterator
1240  ++it;
1241  }
1242  }
1243 
1244  std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
1245  auto this_it = ghost_objects_from_proc.find(this->processor_id());
1246  this->comm().allgather
1247  ((this_it == ghost_objects_from_proc.end()) ?
1248  dof_id_type(0) : this_it->second, objects_on_proc);
1249 
1250 #ifndef NDEBUG
1251  libmesh_assert(this->comm().verify(unpartitioned_objects));
1252  for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1253  if (ghost_objects_from_proc.count(p))
1254  libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1255  else
1256  libmesh_assert_less_equal (0, objects_on_proc[p]);
1257 #endif
1258 
1259  // We'll renumber objects in blocks by processor id
1260  std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1261  for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
1262  first_object_on_proc[i] = first_object_on_proc[i-1] +
1263  objects_on_proc[i-1];
1264  dof_id_type next_id = first_object_on_proc[this->processor_id()];
1265  dof_id_type first_free_id =
1266  first_object_on_proc[this->n_processors()-1] +
1267  objects_on_proc[this->n_processors()-1] +
1268  unpartitioned_objects;
1269 
1270  // First set new local object ids and build request sets
1271  // for non-local object ids
1272 
1273  // Request sets to send to each processor
1274  std::map<processor_id_type, std::vector<dof_id_type>>
1275  requested_ids;
1276 
1277  // We know how many objects live on each processor, so reserve() space for
1278  // each.
1279  auto ghost_end = ghost_objects_from_proc.end();
1280  for (auto p : make_range(this->n_processors()))
1281  if (p != this->processor_id())
1282  {
1283  if (const auto p_it = ghost_objects_from_proc.find(p);
1284  p_it != ghost_end)
1285  requested_ids[p].reserve(p_it->second);
1286  }
1287 
1288  end = objects.end();
1289  for (it = objects.begin(); it != end; ++it)
1290  {
1291  T * obj = *it;
1292  if (!obj)
1293  continue;
1294  if (obj->processor_id() == this->processor_id())
1295  obj->set_id(next_id++);
1296  else if (obj->processor_id() != DofObject::invalid_processor_id)
1297  requested_ids[obj->processor_id()].push_back(obj->id());
1298  }
1299 
1300  // Next set ghost object ids from other processors
1301 
1302  auto gather_functor =
1303  [
1304 #ifndef NDEBUG
1305  this,
1306  &first_object_on_proc,
1307  &objects_on_proc,
1308 #endif
1309  &objects]
1310  (processor_id_type, const std::vector<dof_id_type> & ids,
1311  std::vector<dof_id_type> & new_ids)
1312  {
1313  std::size_t ids_size = ids.size();
1314  new_ids.resize(ids_size);
1315 
1316  for (std::size_t i=0; i != ids_size; ++i)
1317  {
1318  T * obj = objects[ids[i]];
1319  libmesh_assert(obj);
1320  libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1321  new_ids[i] = obj->id();
1322 
1323  libmesh_assert_greater_equal (new_ids[i],
1324  first_object_on_proc[this->processor_id()]);
1325  libmesh_assert_less (new_ids[i],
1326  first_object_on_proc[this->processor_id()] +
1327  objects_on_proc[this->processor_id()]);
1328  }
1329  };
1330 
1331  auto action_functor =
1332  [
1333 #ifndef NDEBUG
1334  &first_object_on_proc,
1335  &objects_on_proc,
1336 #endif
1337  &objects]
1338  (processor_id_type libmesh_dbg_var(pid),
1339  const std::vector<dof_id_type> & ids,
1340  const std::vector<dof_id_type> & data)
1341  {
1342  // Copy the id changes we've now been informed of
1343  for (auto i : index_range(ids))
1344  {
1345  T * obj = objects[ids[i]];
1346  libmesh_assert (obj);
1347  libmesh_assert_equal_to (obj->processor_id(), pid);
1348  libmesh_assert_greater_equal (data[i],
1349  first_object_on_proc[pid]);
1350  libmesh_assert_less (data[i],
1351  first_object_on_proc[pid] +
1352  objects_on_proc[pid]);
1353  obj->set_id(data[i]);
1354  }
1355  };
1356 
1357  const dof_id_type * ex = nullptr;
1358  Parallel::pull_parallel_vector_data
1359  (this->comm(), requested_ids, gather_functor, action_functor, ex);
1360 
1361 #ifdef LIBMESH_ENABLE_UNIQUE_ID
1362  auto unique_gather_functor =
1363  [
1364 #ifndef NDEBUG
1365  this,
1366 #endif
1367  &objects]
1368  (processor_id_type, const std::vector<dof_id_type> & ids,
1369  std::vector<unique_id_type> & data)
1370  {
1371  std::size_t ids_size = ids.size();
1372  data.resize(ids_size);
1373 
1374  for (std::size_t i=0; i != ids_size; ++i)
1375  {
1376  T * obj = objects[ids[i]];
1377  libmesh_assert(obj);
1378  libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1379  data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1380  }
1381  };
1382 
1383  auto unique_action_functor =
1384  [&objects]
1385  (processor_id_type libmesh_dbg_var(pid),
1386  const std::vector<dof_id_type> & ids,
1387  const std::vector<unique_id_type> & data)
1388  {
1389  for (auto i : index_range(ids))
1390  {
1391  T * obj = objects[ids[i]];
1392  libmesh_assert (obj);
1393  libmesh_assert_equal_to (obj->processor_id(), pid);
1394  if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1395  obj->set_unique_id(data[i]);
1396  }
1397  };
1398 
1399  const unique_id_type * unique_ex = nullptr;
1400  Parallel::pull_parallel_vector_data
1401  (this->comm(), requested_ids, unique_gather_functor,
1402  unique_action_functor, unique_ex);
1403 #endif
1404 
1405  // Next set unpartitioned object ids
1406  next_id = 0;
1407  for (auto i : make_range(this->n_processors()))
1408  next_id += objects_on_proc[i];
1409  for (it = objects.begin(); it != end; ++it)
1410  {
1411  T * obj = *it;
1412  if (!obj)
1413  continue;
1414  if (obj->processor_id() == DofObject::invalid_processor_id)
1415  obj->set_id(next_id++);
1416  }
1417 
1418  // Finally shuffle around objects so that container indices
1419  // match ids
1420  it = objects.begin();
1421  end = objects.end();
1422  while (it != end)
1423  {
1424  T * obj = *it;
1425  if (obj) // don't try shuffling already-nullptr entries
1426  {
1427  T * next = objects[obj->id()];
1428  // If we have to move this object
1429  if (next != obj)
1430  {
1431  // nullptr out its original position for now
1432  // (our shuffling may put another object there shortly)
1433  *it = nullptr;
1434 
1435  // There may already be another object with this id that
1436  // needs to be moved itself
1437  while (next)
1438  {
1439  // We shouldn't be trying to give two objects the
1440  // same id
1441  libmesh_assert_not_equal_to (next->id(), obj->id());
1442  objects[obj->id()] = obj;
1443  obj = next;
1444  next = objects[obj->id()];
1445  }
1446  objects[obj->id()] = obj;
1447  }
1448  }
1449 
1450  // Remove any container entries that were left as nullptr.
1451  if (!obj)
1452  it = objects.erase(it);
1453  else
1454  ++it;
1455  }
1456 
1457  return first_free_id;
1458 }
1459 
1460 
1462 {
1463  parallel_object_only();
1464 
1465 #ifdef DEBUG
1466  // Make sure our ids and flags are consistent
1470 #endif
1471 
1472  LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1473 
1474  // Nodes not connected to any elements, and nullptr node entries
1475  // in our container, should be deleted. But wait! If we've deleted coarse
1476  // local elements on some processor, other processors might have ghosted
1477  // nodes from it that are now no longer connected to any elements on it, but
1478  // that are connected to their own semilocal elements. We'll have to
1479  // communicate to ascertain if that's the case.
1481 
1482  // What used nodes do we see on our proc?
1483  std::set<dof_id_type> used_nodes;
1484 
1485  // What used node info should we send from our proc? Could we take ownership
1486  // of each node if we needed to?
1487  std::map<processor_id_type, std::map<dof_id_type, bool>>
1488  used_nodes_on_proc;
1489 
1490  // flag the nodes we need
1491  for (auto & elem : this->element_ptr_range())
1492  for (const Node & node : elem->node_ref_range())
1493  {
1494  const dof_id_type n = node.id();
1495  used_nodes.insert(n);
1497  {
1498  const processor_id_type p = node.processor_id();
1499  if (p != this->processor_id())
1500  {
1501  auto & used_nodes_on_p = used_nodes_on_proc[p];
1502  if (elem->processor_id() == this->processor_id())
1503  used_nodes_on_p[n] = true;
1504  else
1505  if (!used_nodes_on_p.count(n))
1506  used_nodes_on_p[n] = false;
1507  }
1508  }
1509  }
1510 
1512  {
1513  // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
1514  typedef unsigned char boolish;
1515  std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
1516  used_nodes_on_proc_vecs;
1517  for (auto & [pid, nodemap] : used_nodes_on_proc)
1518  used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
1519 
1520  std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
1521  std::map<processor_id_type, std::set<dof_id_type>>
1522  repartitioned_node_sets_to_push;
1523 
1524  auto ids_action_functor =
1525  [&used_nodes, &repartitioned_node_pids,
1526  &repartitioned_node_sets_to_push]
1528  const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
1529  {
1530  for (auto [n, sender_could_become_owner] : ids_and_bools)
1531  {
1532  // If we don't see a use for our own node, but someone
1533  // else does, better figure out who should own it next.
1534  if (!used_nodes.count(n))
1535  {
1536  if (auto it = repartitioned_node_pids.find(n);
1537  sender_could_become_owner)
1538  {
1539  if (it != repartitioned_node_pids.end() &&
1540  pid < it->second)
1541  it->second = pid;
1542  else
1543  repartitioned_node_pids[n] = pid;
1544  }
1545  else
1546  if (it == repartitioned_node_pids.end())
1547  repartitioned_node_pids[n] =
1549 
1550  repartitioned_node_sets_to_push[pid].insert(n);
1551  }
1552  }
1553  };
1554 
1555  // We need two pushes instead of a pull here because we need to
1556  // know *all* the queries for a particular node before we can
1557  // respond to *any* of them.
1558  Parallel::push_parallel_vector_data
1559  (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
1560 
1561  // Repartition (what used to be) our own nodes first
1562  for (auto & [n, p] : repartitioned_node_pids)
1563  {
1564  Node & node = this->node_ref(n);
1565  libmesh_assert_equal_to(node.processor_id(), this->processor_id());
1566  libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
1567  node.processor_id() = p;
1568  }
1569 
1570  // Then push to repartition others' ghosted copies.
1571 
1572  std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
1573  repartitioned_node_vecs;
1574 
1575  for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
1576  {
1577  auto & rn_vec = repartitioned_node_vecs[p];
1578  for (auto n : nodeset)
1579  rn_vec.emplace_back(n, repartitioned_node_pids[n]);
1580  }
1581 
1582  auto repartition_node_functor =
1583  [this]
1584  (processor_id_type libmesh_dbg_var(pid),
1585  const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
1586  {
1587  for (auto [n, p] : ids_and_pids)
1588  {
1589  libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
1590  Node & node = this->node_ref(n);
1591  libmesh_assert_equal_to(node.processor_id(), pid);
1592  node.processor_id() = p;
1593  }
1594  };
1595 
1596  Parallel::push_parallel_vector_data
1597  (this->comm(), repartitioned_node_vecs, repartition_node_functor);
1598  }
1599 
1600  _deleted_coarse_elements = false;
1601 
1602  // Nodes not connected to any local elements, and nullptr node entries
1603  // in our container, are deleted
1604  {
1605  node_iterator_imp it = _nodes.begin();
1606  node_iterator_imp end = _nodes.end();
1607 
1608  while (it != end)
1609  {
1610  Node * nd = *it;
1611  if (!nd)
1612  it = _nodes.erase(it);
1613  else if (!used_nodes.count(nd->id()))
1614  {
1615  // remove any boundary information associated with
1616  // this node
1617  this->get_boundary_info().remove (nd);
1618  _constraint_rows.erase(nd);
1619 
1620  // delete the node
1621  delete nd;
1622 
1623  it = _nodes.erase(it);
1624  }
1625  else
1626  ++it;
1627  }
1628  }
1629 
1631  {
1632  this->update_parallel_id_counts();
1633  return;
1634  }
1635 
1636  // Finally renumber all the elements
1637  _n_elem = this->renumber_dof_objects (this->_elements);
1638 
1639  // and all the remaining nodes
1640  _n_nodes = this->renumber_dof_objects (this->_nodes);
1641 
1642  // And figure out what IDs we should use when adding new nodes and
1643  // new elements
1644  this->update_parallel_id_counts();
1645 
1646  // Make sure our caches are up to date and our
1647  // DofObjects are well packed
1648 #ifdef DEBUG
1649  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1650  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1651  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1652  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1653  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1654  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1655  libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1656  libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1657 
1658  // Make sure our ids and flags are consistent
1661 
1662  // And make sure we've made our numbering monotonic
1664 #endif
1665 }
1666 
1667 
1668 
1670 {
1671  // We can't use range-for here because we need access to the special
1672  // iterators' methods, not just to their dereferenced values.
1673 
1674  // Nodes first
1675  for (auto pr = this->_nodes.begin(),
1676  end = this->_nodes.end(); pr != end; ++pr)
1677  {
1678  Node * n = *pr;
1679  if (n != nullptr)
1680  {
1681  const dof_id_type id = pr.index();
1682  n->set_id() = id;
1683  libmesh_assert_equal_to(this->node_ptr(id), n);
1684  }
1685  }
1686 
1687  // Elements next
1688  for (auto pr = this->_elements.begin(),
1689  end = this->_elements.end(); pr != end; ++pr)
1690  {
1691  Elem * e = *pr;
1692  if (e != nullptr)
1693  {
1694  const dof_id_type id = pr.index();
1695  e->set_id() = id;
1696  libmesh_assert_equal_to(this->elem_ptr(id), e);
1697  }
1698  }
1699 }
1700 
1701 
1702 
1704 {
1705  parallel_object_only();
1706 
1707  // Get local active elements first
1708  dof_id_type active_elements =
1709  static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1710  this->active_local_elements_end()));
1711  this->comm().sum(active_elements);
1712 
1713  // Then add unpartitioned active elements, which should exist on
1714  // every processor
1715  active_elements +=
1716  static_cast<dof_id_type>(std::distance
1717  (this->active_pid_elements_begin(DofObject::invalid_processor_id),
1718  this->active_pid_elements_end(DofObject::invalid_processor_id)));
1719  return active_elements;
1720 }
1721 
1722 
1723 
1725 {
1726 #ifdef DEBUG
1727  // Make sure our neighbor links are all fine
1729 
1730  // And our child/parent links, and our flags
1732 
1733  // Make sure our ids and flags are consistent
1736 
1737  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1738  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1739  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1740  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1741  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1742  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1743 #endif
1744 
1745  _is_serial = false;
1746  _is_serial_on_proc_0 = false;
1747 
1748  MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
1749 
1750  libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1751 
1752  // Now make sure the containers actually shrink - strip
1753  // any newly-created nullptr voids out of the element array
1756  while (e_it != e_end)
1757  if (!*e_it)
1758  e_it = _elements.erase(e_it);
1759  else
1760  ++e_it;
1761 
1764  while (n_it != n_end)
1765  if (!*n_it)
1766  n_it = _nodes.erase(n_it);
1767  else
1768  ++n_it;
1769 
1770  // We may have deleted no-longer-connected nodes or coarsened-away
1771  // elements; let's update our caches.
1772  this->update_parallel_id_counts();
1773 
1774  // We may have deleted nodes or elements that were the only local
1775  // representatives of some particular boundary id(s); let's update
1776  // those caches.
1778 
1779 #ifdef DEBUG
1780  // We might not have well-packed objects if the user didn't allow us
1781  // to renumber
1782  // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1783  // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1784 
1785  // Make sure our neighbor links are all fine
1787 
1788  // And our child/parent links, and our flags
1790 
1791  // Make sure our ids and flags are consistent
1794 #endif
1795 }
1796 
1797 
1799 {
1800  // First add the elem like normal
1801  add_elem(e);
1802 
1803  // Now add it to the set that won't be deleted when we call
1804  // delete_remote_elements()
1805  _extra_ghost_elems.insert(e);
1806 }
1807 
1808 void
1809 DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
1810 {
1811  std::set<Elem *> tmp;
1812  std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
1813  extra_ghost_elems.begin(), extra_ghost_elems.end(),
1814  std::inserter(tmp, tmp.begin()));
1815  _extra_ghost_elems = tmp;
1816 }
1817 
1819 {
1820  if (_is_serial)
1821  return;
1822  MeshCommunication().allgather(*this);
1823  _is_serial = true;
1824  _is_serial_on_proc_0 = true;
1825 
1826  // Make sure our caches are up to date and our
1827  // DofObjects are well packed
1828 #ifdef DEBUG
1829  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1830  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1831  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1832  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1833  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1834  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1835 
1836  // If we've disabled renumbering we can't be sure we're contiguous
1837  // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1838  // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1839 
1840  // Make sure our neighbor links are all fine
1842 
1843  // Make sure our ids and flags are consistent
1846 #endif
1847 }
1848 
1850 {
1852  return;
1853 
1854  _is_serial_on_proc_0 = true;
1855  MeshCommunication().gather(0, *this);
1856 }
1857 
1858 
1859 } // namespace libMesh
UnstructuredMesh & operator=(const UnstructuredMesh &)=delete
Copy assignment is not allowed.
This mapvector templated class is intended to provide the performance characteristics of a std::map w...
Definition: mapvector.h:42
RefinementState refinement_flag() const
Definition: elem.h:3210
virtual Node * add_point(const Point &p, const dof_id_type id=DofObject::invalid_id, const processor_id_type proc_id=DofObject::invalid_processor_id) override final
functions for adding /deleting nodes elements.
virtual void update_parallel_id_counts() override
Updates parallel caches so that methods like n_elem() accurately reflect changes on other processors...
void allgather(const T &send_data, std::vector< T, A > &recv_data) const
unique_id_type & set_unique_id()
Definition: dof_object.h:858
virtual dof_id_type parallel_n_nodes() const override
const Elem * parent() const
Definition: elem.h:3030
bool _skip_renumber_nodes_and_elements
If this is true then renumbering will be kept to a minimum.
Definition: mesh_base.h:1950
virtual Elem * add_elem(Elem *e) override final
Add elem e to the end of the element array.
bool is_prepared() const
Definition: mesh_base.h:198
virtual void copy_nodes_and_elements(const MeshBase &other_mesh, const bool skip_find_neighbors=false, dof_id_type element_id_offset=0, dof_id_type node_id_offset=0, unique_id_type unique_id_offset=0, std::unordered_map< subdomain_id_type, subdomain_id_type > *id_remapping=nullptr)
Deep copy of nodes and elements from another mesh object (used by subclass copy constructors and by m...
A Node is like a Point, but with more information.
Definition: node.h:52
virtual unique_id_type parallel_max_unique_id() const =0
dof_id_type n_unpartitioned_nodes() const
Definition: mesh_base.h:448
const unsigned int invalid_uint
A number which is used quite often to represent an invalid or uninitialized value for an unsigned int...
Definition: libmesh.h:310
std::vector< std::string > _elem_integer_names
The array of names for integer data associated with each element in the mesh.
Definition: mesh_base.h:2033
virtual void libmesh_assert_valid_parallel_ids() const override
Verify id and processor_id consistency of our elements and nodes containers.
virtual void fix_broken_node_and_element_numbering() override
There is no reason for a user to ever call this function.
void set_mapping_type(const ElemMappingType type)
Sets the value of the mapping type for the element.
Definition: elem.h:3128
void allow_renumbering(bool allow)
If false is passed in then this mesh will no longer be renumbered when being prepared for use...
Definition: mesh_base.h:1196
virtual void renumber_elem(dof_id_type old_id, dof_id_type new_id) override final
Changes the id of element old_id, both by changing elem(old_id)->id() and by moving elem(old_id) in t...
void skip_partitioning(bool skip)
If true is passed in then nothing on this mesh will be (re)partitioned.
Definition: mesh_base.h:1254
bool _is_serial_on_proc_0
A boolean remembering whether we&#39;re serialized to proc 0 or not.
void copy_cached_data(const MeshBase &other_mesh)
Helper class to copy cached data, to synchronize with a possibly unprepared other_mesh.
Definition: mesh_base.C:2004
dof_id_type _n_nodes
Cached data from the last renumber_nodes_and_elements call.
virtual void clear() override
Clear all internal data.
virtual void renumber_nodes_and_elements() override
Remove nullptr elements from arrays.
RefinementState p_refinement_flag() const
Definition: elem.h:3226
void libmesh_assert_valid_refinement_tree(const MeshBase &mesh)
A function for verifying that elements on this processor have valid descendants and consistent active...
Definition: mesh_tools.C:1547
virtual dof_id_type n_nodes() const override final
dof_id_type _next_free_local_node_id
Guaranteed globally unused IDs for use when adding new nodes or elements.
void remove(const Node *node)
Removes the boundary conditions associated with node node, if any exist.
void sum(T &r) const
virtual void allgather() override
Gathers all elements and nodes of the mesh onto every processor.
This is the base class from which all geometric element types are derived.
Definition: elem.h:94
dof_id_type n_local_nodes() const
Definition: mesh_base.h:442
constraint_rows_type _constraint_rows
Definition: mesh_base.h:2104
unique_id_type unique_id() const
Definition: dof_object.h:844
void copy_constraint_rows(const MeshBase &other_mesh)
Copy the constraints from the other mesh to this mesh.
Definition: mesh_base.C:2063
const Parallel::Communicator & comm() const
bool _is_serial
A boolean remembering whether we&#39;re serialized or not.
unsigned int p_level() const
Definition: elem.h:3108
void libmesh_assert_valid_parallel_flags() const
Verify refinement_flag and p_refinement_flag consistency of our elements containers.
The libMesh namespace provides an interface to certain functionality in the library.
virtual void delete_elem(Elem *e) override final
Removes element e from the mesh.
const BoundaryInfo & get_boundary_info() const
The information about boundary ids on the mesh.
Definition: mesh_base.h:165
virtual void add_extra_ghost_elem(Elem *e)
Inserts the element and adds it to a list of elements that should not get deleted or have their desce...
dof_id_type n_unpartitioned_elem() const
Definition: mesh_base.h:554
std::vector< std::string > _node_integer_names
The array of names for integer data associated with each node in the mesh.
Definition: mesh_base.h:2045
virtual Node * add_node(Node *n) override final
Add Node n to the end of the vertex array.
Real distance(const Point &p)
virtual const Point & point(const dof_id_type i) const override final
virtual unique_id_type parallel_max_unique_id() const override
dof_id_type n_local_elem() const
Definition: mesh_base.h:548
virtual void delete_remote_elements() override
Deletes all nonlocal elements of the mesh except for "ghosts" which touch a local element...
void libmesh_assert_valid_parallel_object_ids(const dofobject_container< T > &) const
Verify id, processor_id, and if applicable unique_id consistency of a parallel objects container...
uint8_t processor_id_type
Definition: id_types.h:104
This is the MeshBase class.
Definition: mesh_base.h:75
void libmesh_assert_valid_parallel_p_levels() const
Verify p_level consistency of our elements containers.
dof_id_type & set_id()
Definition: dof_object.h:836
ElemMappingType default_mapping_type() const
Returns the default master space to physical space mapping basis functions to be used on newly added ...
Definition: mesh_base.h:812
unique_id_type _next_unique_id
The next available unique id for assigning ids to DOF objects.
Definition: mesh_base.h:1925
virtual void clear() override
Free all new memory associated with the object, but restore its original state, with the mesh pointer...
Definition: dof_map.C:901
processor_id_type pid unsigned int level std::set< subdomain_id_type > ss const DofMap &dof_map LIBMESH_COMMA unsigned int dof_map LIBMESH_COMMA var_num unsigned char rflag processor_id_type pid const DofMap &dof_map LIBMESH_COMMA unsigned int dof_map LIBMESH_COMMA var_num DECLARE_NODE_ITERATORS(multi_evaluable_, std::vector< const DofMap * > dof_maps, dof_maps) protected dofobject_container< Node > _nodes
Move node and elements from a DistributedMesh.
void allow_remote_element_removal(bool allow)
If false is passed in then this mesh will no longer have remote elements deleted when being prepared ...
Definition: mesh_base.h:1212
processor_id_type n_processors() const
virtual const Elem * elem_ptr(const dof_id_type i) const override final
friend class MeshCommunication
Make the MeshCommunication class a friend so that it can directly broadcast *_integer_names.
Definition: mesh_base.h:2134
virtual dof_id_type parallel_n_elem() const override
T command_line_value(const std::string &, T)
Definition: libmesh.C:1024
This is the MeshCommunication class.
static std::unique_ptr< Partitioner > build(const PartitionerType solver_package)
Builds a Partitioner of the type specified by partitioner_type.
Definition: partitioner.C:159
dof_id_type id() const
Definition: dof_object.h:828
void min(const T &r, T &o, Request &req) const
const std::map< subdomain_id_type, std::string > & get_subdomain_name_map() const
Definition: mesh_base.h:1694
static const unique_id_type invalid_unique_id
An invalid unique_id to distinguish an uninitialized DofObject.
Definition: dof_object.h:487
virtual void move_nodes_and_elements(MeshBase &&other_mesh)=0
Move node and elements from other_mesh to this mesh.
static const processor_id_type invalid_processor_id
An invalid processor_id to distinguish DoFs that have not been assigned to a processor.
Definition: dof_object.h:493
std::set< Elem * > _extra_ghost_elems
These are extra ghost elements that we want to make sure not to delete when we call delete_remote_ele...
The UnstructuredMesh class is derived from the MeshBase class.
unsigned char default_mapping_data() const
Returns any default data value used by the master space to physical space mapping.
Definition: mesh_base.h:830
void erase(index_t i)
Definition: mapvector.h:183
virtual bool is_serial_on_zero() const override final
std::unique_ptr< Partitioner > _partitioner
A partitioner to use at each prepare_for_use().
Definition: mesh_base.h:1919
void set_mapping_data(const unsigned char data)
Sets the value of the mapping data for the element.
Definition: elem.h:3144
virtual void set_next_unique_id(unique_id_type id) override
Sets the next available unique id to be used.
virtual void clear_elems() override
Clear internal Elem data.
bool allow_find_neighbors() const
Definition: mesh_base.h:1204
libmesh_assert(ctx)
static const dof_id_type invalid_id
An invalid id to distinguish an uninitialized DofObject.
Definition: dof_object.h:482
virtual void own_node(Node &n) override final
Takes ownership of node n on this partition of a distributed mesh, by setting n.processor_id() to thi...
virtual void clear_extra_ghost_elems()
Clears extra ghost elements.
const std::set< Elem * > & extra_ghost_elems() const
Const accessor to the ghosted elements.
bool valid_unique_id() const
Definition: dof_object.h:893
virtual const Node * query_node_ptr(const dof_id_type i) const override final
void allow_find_neighbors(bool allow)
If false is passed then this mesh will no longer work to find element neighbors when being prepared f...
Definition: mesh_base.h:1203
virtual bool subclass_locally_equals(const MeshBase &other_mesh) const override
Shim to allow operator == (&) to behave like a virtual function without having to be one...
dof_id_type parallel_max_node_id() const
veclike_iterator end()
Definition: mapvector.h:203
bool allow_remote_element_removal() const
Definition: mesh_base.h:1213
The DistributedMesh class is derived from the MeshBase class, and is intended to provide identical fu...
void regenerate_id_sets()
Clears and regenerates the cached sets of ids.
virtual void clear()
Deletes all the element and node data that is currently stored.
Definition: mesh_base.C:920
dof_id_type _next_free_unpartitioned_node_id
dofobject_container< Elem > _elements
The elements in the mesh.
virtual void delete_node(Node *n) override final
Removes the Node n from the mesh.
bool skip_partitioning() const
Definition: mesh_base.h:1256
virtual void renumber_node(dof_id_type old_id, dof_id_type new_id) override final
Changes the id of node old_id, both by changing node(old_id)->id() and by moving node(old_id) in the ...
std::vector< dof_id_type > _node_integer_default_values
The array of default initialization values for integer data associated with each node in the mesh...
Definition: mesh_base.h:2051
static std::unique_ptr< Node > build(const Node &n)
Definition: node.h:315
static const unsigned int next[3]
A lookup table for the increment modulo 3 operation, for iterating through the three nodes per elemen...
virtual void redistribute() override
Redistribute elements between processors.
std::vector< dof_id_type > _elem_integer_default_values
The array of default initialization values for integer data associated with each element in the mesh...
Definition: mesh_base.h:2039
DistributedMesh & operator=(const DistributedMesh &)=delete
Copy assignment is not allowed.
virtual dof_id_type n_active_elem() const override final
dof_id_type _next_free_unpartitioned_elem_id
bool valid_id() const
Definition: dof_object.h:885
void max(const T &r, T &o, Request &req) const
DistributedMesh(const Parallel::Communicator &comm_in, unsigned char dim=1)
Constructor.
veclike_iterator begin()
Definition: mapvector.h:193
void post_dofobject_moves(MeshBase &&other_mesh)
Moves any superclass data (e.g.
Definition: mesh_base.C:1969
virtual void gather_to_zero() override
Gathers all elements and nodes of the mesh onto processor zero.
virtual Elem * insert_elem(Elem *e) override final
Insert elem e to the element array, preserving its id and replacing/deleting any existing element wit...
virtual dof_id_type max_node_id() const override final
bool _is_prepared
Flag indicating if the mesh has been prepared for use.
Definition: mesh_base.h:1896
IntRange< T > make_range(T beg, T end)
The 2-parameter make_range() helper function returns an IntRange<T> when both input parameters are of...
Definition: int_range.h:140
The DofObject defines an abstract base class for objects that have degrees of freedom associated with...
Definition: dof_object.h:54
void add_extra_integers(const unsigned int n_integers)
Assigns a set of extra integers to this DofObject.
Definition: dof_object.C:490
virtual const Node & node_ref(const dof_id_type i) const
Definition: mesh_base.h:596
virtual void update_post_partitioning() override
Recalculate cached data after elements and nodes have been repartitioned.
bool allow_renumbering() const
Definition: mesh_base.h:1197
unique_id_type _next_unpartitioned_unique_id
The next available unique id for assigning ids to unpartitioned DOF objects.
virtual MeshBase & assign(MeshBase &&other_mesh) override
Shim to call the move assignment operator for this class.
virtual dof_id_type n_elem() const override final
void libmesh_assert_valid_neighbors(const MeshBase &mesh, bool assert_valid_remote_elems=true)
A function for verifying that neighbor connectivity is correct (each element is a neighbor of or desc...
Definition: mesh_tools.C:2205
virtual ~DistributedMesh()
Destructor.
virtual Node * insert_node(Node *n) override final
These methods are deprecated.
processor_id_type processor_id() const
virtual bool is_serial() const override final
const DofMap &dof_map LIBMESH_COMMA unsigned int std::string & set_subdomain_name_map()
Definition: mesh_base.h:1692
virtual dof_id_type max_elem_id() const override final
bool active() const
Definition: elem.h:2941
dof_id_type _next_free_local_elem_id
virtual void redistribute()
Redistribute elements between processors.
Definition: mesh_base.C:985
processor_id_type processor_id() const
Definition: dof_object.h:905
bool _deleted_coarse_elements
A boolean remembering whether we&#39;ve recently deleted top-level elements or not.
A Point defines a location in LIBMESH_DIM dimensional Real space.
Definition: point.h:39
dof_id_type renumber_dof_objects(dofobject_container< T > &)
Renumber a parallel objects container.
uint8_t unique_id_type
Definition: id_types.h:86
void redistribute(DistributedMesh &mesh, bool newly_coarsened_only=false) const
This method takes a parallel distributed mesh and redistributes the elements.
auto index_range(const T &sizable)
Helper function that returns an IntRange<std::size_t> representing all the indices of the passed-in v...
Definition: int_range.h:117
dof_id_type parallel_max_elem_id() const
uint8_t dof_id_type
Definition: id_types.h:67
void libmesh_assert_valid_elem_ids(const MeshBase &mesh)
A function for verifying that ids and processor assignment of elements are correctly sorted (monotone...
Definition: mesh_tools.C:1318
virtual const Node * node_ptr(const dof_id_type i) const override final
virtual const Elem * query_elem_ptr(const dof_id_type i) const override final