LCOV - code coverage report
Current view: top level - src/mesh - distributed_mesh.C (source / functions) Hit Total Coverage
Test: libMesh/libmesh: #4573 (924896) with base 3f0912 Lines: 738 819 90.1 %
Date: 2026-11-18 21:52:26 Functions: 88 96 91.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : // The libMesh Finite Element Library.
       2             : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
       3             : 
       4             : // This library is free software; you can redistribute it and/or
       5             : // modify it under the terms of the GNU Lesser General Public
       6             : // License as published by the Free Software Foundation; either
       7             : // version 2.1 of the License, or (at your option) any later version.
       8             : 
       9             : // This library is distributed in the hope that it will be useful,
      10             : // but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      12             : // Lesser General Public License for more details.
      13             : 
      14             : // You should have received a copy of the GNU Lesser General Public
      15             : // License along with this library; if not, write to the Free Software
      16             : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
      17             : 
      18             : 
      19             : 
      20             : // Local includes
      21             : #include "libmesh/distributed_mesh.h"
      22             : 
      23             : // libMesh includes
      24             : #include "libmesh/boundary_info.h"
      25             : #include "libmesh/elem.h"
      26             : #include "libmesh/libmesh_logging.h"
      27             : #include "libmesh/mesh_communication.h"
      28             : #include "libmesh/mesh_tools.h"
      29             : #include "libmesh/partitioner.h"
      30             : #include "libmesh/string_to_enum.h"
      31             : 
      32             : // TIMPI includes
      33             : #include "timpi/parallel_implementation.h"
      34             : #include "timpi/parallel_sync.h"
      35             : 
      36             : 
      37             : namespace libMesh
      38             : {
      39             : 
      40             : // ------------------------------------------------------------
      41             : // DistributedMesh class member functions
      42         911 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
      43         911 :                                   unsigned char d) :
      44         387 :   UnstructuredMesh (comm_in,d), _is_serial(true),
      45         387 :   _is_serial_on_proc_0(true),
      46         387 :   _deleted_coarse_elements(false),
      47         387 :   _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
      48        1173 :   _next_free_local_node_id(this->processor_id()),
      49         911 :   _next_free_local_elem_id(this->processor_id()),
      50         911 :   _next_free_unpartitioned_node_id(this->n_processors()),
      51         911 :   _next_free_unpartitioned_elem_id(this->n_processors())
      52             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
      53         911 :   , _next_unpartitioned_unique_id(this->n_processors())
      54             : #endif
      55             : {
      56             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
      57         911 :   _next_unique_id = this->processor_id();
      58             : #endif
      59             : 
      60        1173 :   const std::string default_partitioner = "parmetis";
      61             :   const std::string my_partitioner =
      62             :     libMesh::command_line_value("--default-partitioner",
      63        1822 :                                 default_partitioner);
      64             :   _partitioner = Partitioner::build
      65        1560 :     (Utility::string_to_enum<PartitionerType>(my_partitioner));
      66         911 : }
      67             : 
      68          14 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
      69             : {
      70           8 :   LOG_SCOPE("operator=(&&)", "DistributedMesh");
      71             : 
      72             :   // Move assign as an UnstructuredMesh.
      73           4 :   this->UnstructuredMesh::operator=(std::move(other_mesh));
      74             : 
      75             :   // Nodes and elements belong to DistributedMesh and have to be
      76             :   // moved before we can move arbitrary GhostingFunctor, Partitioner,
      77             :   // etc. subclasses.
      78          14 :   this->move_nodes_and_elements(std::move(other_mesh));
      79             : 
      80             :   // But move_nodes_and_elems misses (or guesses about) some of our
      81             :   // subclass values, and we want more precision than a guess.
      82          14 :   _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
      83           4 :   _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
      84             : 
      85             :   // Handle remaining MeshBase moves.
      86          14 :   this->post_dofobject_moves(std::move(other_mesh));
      87             : 
      88          18 :   return *this;
      89             : }
      90             : 
      91          14 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
      92             : {
      93          14 :   *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
      94             : 
      95          14 :   return *this;
      96             : }
      97             : 
      98         778 : std::string_view DistributedMesh::subclass_first_difference_from (const MeshBase & other_mesh_base) const
      99             : {
     100         748 :   const DistributedMesh * dist_mesh_ptr =
     101         778 :     dynamic_cast<const DistributedMesh *>(&other_mesh_base);
     102         778 :   if (!dist_mesh_ptr)
     103           0 :     return "DistributedMesh class";
     104         748 :   const DistributedMesh & other_mesh = *dist_mesh_ptr;
     105             : 
     106             : #define CHECK_MEMBER(member_name) \
     107             :   if (member_name != other_mesh.member_name) \
     108             :     return #member_name;
     109             : 
     110         778 :   CHECK_MEMBER(_is_serial);
     111         778 :   CHECK_MEMBER(_is_serial_on_proc_0);
     112         778 :   CHECK_MEMBER(_deleted_coarse_elements);
     113         778 :   CHECK_MEMBER(_n_nodes);
     114         778 :   CHECK_MEMBER(_n_elem);
     115         778 :   CHECK_MEMBER(_max_node_id);
     116         778 :   CHECK_MEMBER(_max_elem_id);
     117             :   // We expect these things to change in a prepare_for_use();
     118             :   // they're conceptually "mutable"...
     119             : /*
     120             :   CHECK_MEMBER(_next_free_local_node_id);
     121             :   CHECK_MEMBER(_next_free_local_elem_id);
     122             :   CHECK_MEMBER(_next_free_unpartitioned_node_id);
     123             :   CHECK_MEMBER(_next_free_unpartitioned_elem_id);
     124             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     125             :   CHECK_MEMBER(_next_unpartitioned_unique_id);
     126             : #endif
     127             : */
     128         778 :   if (!this->nodes_and_elements_equal(other_mesh))
     129           7 :     return "nodes and/or elements";
     130             : 
     131        1517 :   if (_extra_ghost_elems.size() !=
     132         746 :       other_mesh._extra_ghost_elems.size())
     133           0 :     return "_extra_ghost_elems size";
     134         771 :   for (auto & elem : _extra_ghost_elems)
     135             :     {
     136           0 :       libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
     137           0 :       const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
     138           0 :       if (!other_elem ||
     139           0 :           !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
     140           0 :         return "_extra_ghost_elems entry";
     141             :     }
     142             : 
     143         771 :   return "";
     144             : }
     145             : 
     146        2688 : DistributedMesh::~DistributedMesh ()
     147             : {
     148        1600 :   this->DistributedMesh::clear();  // Free nodes and elements
     149        2688 : }
     150             : 
     151             : 
     152             : // This might be specialized later, but right now it's just here to
     153             : // make sure the compiler doesn't give us a default (non-deep) copy
     154             : // constructor instead.
     155         430 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
     156             :   DistributedMesh(static_cast<const MeshBase &>(other_mesh),
     157         430 :                   /*bool other_is_distributed_type=*/true)
     158             : {
     159         430 :   _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
     160         430 :   _next_free_local_node_id =
     161         430 :     other_mesh._next_free_local_node_id;
     162         430 :   _next_free_local_elem_id =
     163         430 :     other_mesh._next_free_local_elem_id;
     164         430 :   _next_free_unpartitioned_node_id =
     165         430 :     other_mesh._next_free_unpartitioned_node_id;
     166         430 :   _next_free_unpartitioned_elem_id =
     167         430 :     other_mesh._next_free_unpartitioned_elem_id;
     168             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     169         430 :   _next_unique_id =
     170         430 :     other_mesh._next_unique_id;
     171         430 :   _next_unpartitioned_unique_id =
     172         430 :     other_mesh._next_unpartitioned_unique_id;
     173             : #endif
     174             : 
     175             :   // Need to copy extra_ghost_elems
     176         430 :   for (auto & elem : other_mesh._extra_ghost_elems)
     177           0 :     _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
     178         430 : }
     179             : 
     180             : 
     181             : 
     182         689 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
     183         689 :   UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
     184        1143 :   _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
     185         141 :   _deleted_coarse_elements(true), // better safe than sorry...
     186         141 :   _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
     187        1143 :   _next_free_local_node_id(this->processor_id()),
     188        1049 :   _next_free_local_elem_id(this->processor_id()),
     189        1049 :   _next_free_unpartitioned_node_id(this->n_processors()),
     190        1284 :   _next_free_unpartitioned_elem_id(this->n_processors())
     191             : {
     192             :   // Just copy, skipping preparation
     193         689 :   this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
     194             : 
     195         548 :   this->allow_find_neighbors(other_mesh.allow_find_neighbors());
     196         548 :   this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
     197         548 :   this->allow_renumbering(other_mesh.allow_renumbering());
     198         548 :   this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
     199         548 :   this->skip_partitioning(other_mesh.skip_partitioning());
     200             : 
     201         689 :   this->copy_constraint_rows(other_mesh);
     202             : 
     203         454 :   auto & this_boundary_info = this->get_boundary_info();
     204         454 :   const auto & other_boundary_info = other_mesh.get_boundary_info();
     205             : 
     206         689 :   this_boundary_info = other_boundary_info;
     207             : 
     208         454 :   this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
     209             : 
     210             :   // add_node()/add_elem() only increment _n_nodes/_n_elem for
     211             :   // nodes/elements they consider locally owned (or unpartitioned), so after
     212             :   // copying only the objects visible to this processor, those counts
     213             :   // reflect an incomplete local view rather than other_mesh's totals,
     214             :   // and need to be copied afresh.
     215         689 :   _n_nodes = other_mesh.n_nodes();
     216         689 :   _n_elem = other_mesh.n_elem();
     217         689 :   _max_node_id = other_mesh.max_node_id();
     218         689 :   _max_elem_id = other_mesh.max_elem_id();
     219             : 
     220             :   // If other_mesh is actually a DistributedMesh, we can just copy its raw
     221             :   // counters directly, mirroring other_mesh's own bookkeeping exactly. This
     222             :   // is done in the DistributedMesh copy constructor.
     223             :   // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
     224             :   // raw counterpart fields to copy, so derive the free-id-range
     225             :   // counters from _max_elem_id/_max_node_id the same way
     226             :   // update_parallel_id_counts() does via the set_next_ids() method. For
     227             :   // the unique id counters we copy other_mesh's own local next_unique_id()
     228             :   // and derive the next available unique ids
     229             :   // (partitioned and unpartitioned) via set_next_unique_ids().
     230         689 :   if (!other_is_distributed_type)
     231             :     {
     232         259 :       this->set_next_ids();
     233             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     234         259 :       this->set_next_unique_ids(other_mesh.next_unique_id());
     235             : #endif
     236             :     }
     237             : 
     238             :   // Copy other_mesh's actual preparation state, including whatever it
     239             :   // reports for has_synched_id_counts: the counts above are a faithful copy
     240             :   // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
     241             :   // verified computation, so our synced-ness should match other_mesh's.
     242         689 :   this->_preparation = other_mesh.preparation();
     243         689 : }
     244             : 
     245          14 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
     246             : {
     247           4 :   DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
     248             : 
     249           4 :   this->_nodes = std::move(other_mesh._nodes);
     250          14 :   this->_n_nodes = other_mesh.n_nodes();
     251             : 
     252           4 :   this->_elements = std::move(other_mesh._elements);
     253          14 :   this->_n_elem = other_mesh.n_elem();
     254             : 
     255          14 :   _is_serial = other_mesh.is_serial();
     256          14 :   _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
     257          14 :   _deleted_coarse_elements = true; // Better safe than sorry
     258             : 
     259          14 :   _max_node_id = other_mesh.max_node_id();
     260          14 :   _max_elem_id = other_mesh.max_elem_id();
     261             : 
     262          14 :   _next_free_local_node_id = other_mesh._next_free_local_node_id;
     263          14 :   _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
     264          14 :   _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
     265          14 :   _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
     266             : 
     267             :   #ifdef LIBMESH_ENABLE_UNIQUE_ID
     268          14 :   _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
     269             :   #endif
     270          14 : }
     271             : 
     272        9021 : void DistributedMesh::set_next_ids()
     273             : {
     274        9021 :   _next_free_unpartitioned_elem_id =
     275       11643 :     ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
     276        9021 :     (this->n_processors() + 1) + this->n_processors();
     277        9021 :   _next_free_local_elem_id =
     278        9021 :     ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     279        9021 :     (this->n_processors() + 1) + this->processor_id();
     280             : 
     281        9021 :   _next_free_unpartitioned_node_id =
     282       11643 :     ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
     283        9021 :     (this->n_processors() + 1) + this->n_processors();
     284        9021 :   _next_free_local_node_id =
     285        9021 :     ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     286        9021 :     (this->n_processors() + 1) + this->processor_id();
     287        9021 : }
     288             : 
     289             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     290        9021 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
     291             : {
     292             :   // Unique ids are laid out in repeating groups of (n_processors()+1)
     293             :   // consecutive integers ("alignment" here means a value's position, i.e.
     294             :   // its residue mod (n_processors()+1), within one of these groups). In
     295             :   // every group, the first n_processors() slots (residues 0..n_processors()-1)
     296             :   // are each permanently earmarked for one processor's locally-owned
     297             :   // objects, and the last slot (residue n_processors()) is permanently
     298             :   // earmarked for unpartitioned objects.
     299             :   //
     300             :   // parallel_max_unique_id carries no guarantee of landing on any
     301             :   // particular residue, so rather than simply offsetting it by
     302             :   // processor_id()/n_processors(), we have to explicitly round up to the
     303             :   // start of the next full, entirely-unused group before adding this
     304             :   // processor's (or the unpartitioned pool's) fixed offset into that group.
     305             :   //
     306             :   // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
     307             :   // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
     308             :   // the number of complete groups needed to reach at least past the
     309             :   // current max id. Multiplying back by (n_processors()+1) gives the first
     310             :   // id of the next group that is guaranteed to be entirely unused; adding
     311             :   // n_processors(), the unpartitioned pool's fixed offset within a group,
     312             :   // lands on that pool's reserved slot in that group.
     313        9021 :   _next_unpartitioned_unique_id =
     314       11643 :       ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
     315        9021 :           (this->n_processors() + 1) +
     316        9021 :       this->n_processors();
     317             : 
     318             :   // Same idiom, but shifted by (n_processors()-1) before dividing so that
     319             :   // the rounding accounts for processor_id() possibly being less than the
     320             :   // residue the raw max id already occupies within its group; this can
     321             :   // land this processor's slot in the same group as the unpartitioned
     322             :   // pool's above, or push it into the following group, depending on where
     323             :   // the max id and this processor's offset happen to fall.
     324        9021 :   _next_unique_id =
     325        9021 :       ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     326        9021 :           (this->n_processors() + 1) +
     327        9021 :       this->processor_id();
     328        9021 : }
     329             : #endif
     330             : 
     331             : // We use cached values for these so they can be called
     332             : // from one processor without bothering the rest, but
     333             : // we may need to update those caches before doing a full
     334             : // renumbering
     335        8762 : void DistributedMesh::update_parallel_id_counts()
     336             : {
     337             :   // This function must be run on all processors at once
     338        2908 :   parallel_object_only();
     339             : 
     340        8762 :   _n_elem  = this->parallel_n_elem();
     341        8762 :   _n_nodes = this->parallel_n_nodes();
     342        8762 :   _max_node_id = this->parallel_max_node_id();
     343        8762 :   _max_elem_id = this->parallel_max_elem_id();
     344             : 
     345        8762 :   this->set_next_ids();
     346             : 
     347             :   // set_next_unique_ids() needs the true parallel max unique id (not just
     348             :   // this processor's possibly-stale cached _next_unique_id) since the whole
     349             :   // point of this function is to recompute counters that may have drifted
     350             :   // out of sync.
     351             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     352        8762 :   this->set_next_unique_ids(this->parallel_max_unique_id());
     353             : #endif
     354             : 
     355        8762 :   _n_constraint_rows = this->count_constraint_rows();
     356             : 
     357        8762 :   this->_preparation.has_synched_id_counts = true;
     358        8762 : }
     359             : 
     360             : 
     361             : // Or in debug mode we may want to test the uncached values without
     362             : // changing the cache
     363        9681 : dof_id_type DistributedMesh::parallel_n_elem() const
     364             : {
     365             :   // This function must be run on all processors at once
     366        3798 :   parallel_object_only();
     367             : 
     368        9681 :   dof_id_type n_local = this->n_local_elem();
     369        9681 :   this->comm().sum(n_local);
     370        9681 :   n_local += this->n_unpartitioned_elem();
     371        9681 :   return n_local;
     372             : }
     373             : 
     374             : 
     375             : 
     376       16108 : dof_id_type DistributedMesh::parallel_max_elem_id() const
     377             : {
     378             :   // This function must be run on all processors at once
     379       10254 :   parallel_object_only();
     380             : 
     381       16108 :   dof_id_type max_local = 0;
     382             : 
     383             :   dofobject_container<Elem>::const_reverse_veclike_iterator
     384       10254 :     rit = _elements.rbegin();
     385             : 
     386             :   const dofobject_container<Elem>::const_reverse_veclike_iterator
     387       10254 :     rend = _elements.rend();
     388             : 
     389             :   // Look for the maximum element id.  Search backwards through
     390             :   // elements so we can break out early.  Beware of nullptr entries that
     391             :   // haven't yet been cleared from _elements.
     392       34149 :   for (; rit != rend; ++rit)
     393             :     {
     394       33845 :       const DofObject *d = *rit;
     395       21497 :       if (d)
     396             :         {
     397       10026 :           libmesh_assert(_elements[d->id()] == d);
     398       15804 :           max_local = d->id() + 1;
     399       15804 :           break;
     400             :         }
     401             :     }
     402             : 
     403       16108 :   this->comm().max(max_local);
     404       16108 :   return max_local;
     405             : }
     406             : 
     407             : 
     408             : 
     409             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     410        9478 : unique_id_type DistributedMesh::parallel_max_unique_id() const
     411             : {
     412             :   // This function must be run on all processors at once
     413        3370 :   parallel_object_only();
     414             : 
     415       18956 :   unique_id_type max_local = std::max(_next_unique_id,
     416       11469 :                                       _next_unpartitioned_unique_id);
     417        9478 :   this->comm().max(max_local);
     418        9478 :   return max_local;
     419             : }
     420             : 
     421             : 
     422             : 
     423         877 : void DistributedMesh::set_next_unique_id(unique_id_type id)
     424             : {
     425         508 :   _next_unique_id = id;
     426         877 :   _next_unpartitioned_unique_id =
     427        1025 :     ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
     428         877 :     (this->n_processors() + 1) + this->n_processors();
     429         877 :   _next_unique_id =
     430         877 :     ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     431         877 :     (this->n_processors() + 1) + this->processor_id();
     432         877 : }
     433             : #endif
     434             : 
     435             : 
     436             : 
     437        9681 : dof_id_type DistributedMesh::parallel_n_nodes() const
     438             : {
     439             :   // This function must be run on all processors at once
     440        3798 :   parallel_object_only();
     441             : 
     442        9681 :   dof_id_type n_local = this->n_local_nodes();
     443        9681 :   this->comm().sum(n_local);
     444        9681 :   n_local += this->n_unpartitioned_nodes();
     445        9681 :   return n_local;
     446             : }
     447             : 
     448             : 
     449             : 
     450       13566 : dof_id_type DistributedMesh::parallel_max_node_id() const
     451             : {
     452             :   // This function must be run on all processors at once
     453        7712 :   parallel_object_only();
     454             : 
     455       13566 :   dof_id_type max_local = 0;
     456             : 
     457             :   dofobject_container<Node>::const_reverse_veclike_iterator
     458        7712 :     rit = _nodes.rbegin();
     459             : 
     460             :   const dofobject_container<Node>::const_reverse_veclike_iterator
     461        7712 :     rend = _nodes.rend();
     462             : 
     463             :   // Look for the maximum node id.  Search backwards through
     464             :   // nodes so we can break out early.  Beware of nullptr entries that
     465             :   // haven't yet been cleared from _nodes
     466      134470 :   for (; rit != rend; ++rit)
     467             :     {
     468      134236 :       const DofObject *d = *rit;
     469       74783 :       if (d)
     470             :         {
     471        7554 :           libmesh_assert(_nodes[d->id()] == d);
     472       13332 :           max_local = d->id() + 1;
     473       13332 :           break;
     474             :         }
     475             :     }
     476             : 
     477       13566 :   this->comm().max(max_local);
     478       13566 :   return max_local;
     479             : }
     480             : 
     481             : 
     482             : 
     483      421297 : const Point & DistributedMesh::point (const dof_id_type i) const
     484             : {
     485      421297 :   return this->node_ref(i);
     486             : }
     487             : 
     488             : 
     489             : 
     490      474847 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
     491             : {
     492      137685 :   libmesh_assert(_nodes[i]);
     493      137685 :   libmesh_assert_equal_to (_nodes[i]->id(), i);
     494             : 
     495      474847 :   return _nodes[i];
     496             : }
     497             : 
     498             : 
     499             : 
     500             : 
     501     1654383 : Node * DistributedMesh::node_ptr (const dof_id_type i)
     502             : {
     503      656010 :   libmesh_assert(_nodes[i]);
     504      656010 :   libmesh_assert_equal_to (_nodes[i]->id(), i);
     505             : 
     506     1654383 :   return _nodes[i];
     507             : }
     508             : 
     509             : 
     510             : 
     511             : 
     512     1085839 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
     513             : {
     514     1085839 :   if (const auto it = _nodes.find(i);
     515     1083543 :       it != _nodes.end())
     516             :     {
     517      657952 :       const Node * n = *it;
     518      655656 :       libmesh_assert (!n || n->id() == i);
     519      655656 :       return n;
     520             :     }
     521             : 
     522      427887 :   return nullptr;
     523             : }
     524             : 
     525             : 
     526             : 
     527             : 
     528      586408 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
     529             : {
     530      586408 :   if (auto it = _nodes.find(i);
     531      246421 :       it != _nodes.end())
     532             :     {
     533      382326 :       Node * n = *it;
     534      157965 :       libmesh_assert (!n || n->id() == i);
     535      157965 :       return n;
     536             :     }
     537             : 
     538       88456 :   return nullptr;
     539             : }
     540             : 
     541             : 
     542             : 
     543             : 
     544       13537 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
     545             : {
     546       10249 :   libmesh_assert(_elements[i]);
     547       10249 :   libmesh_assert_equal_to (_elements[i]->id(), i);
     548             : 
     549       13537 :   return _elements[i];
     550             : }
     551             : 
     552             : 
     553             : 
     554             : 
     555      516431 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
     556             : {
     557      178609 :   libmesh_assert(_elements[i]);
     558      178609 :   libmesh_assert_equal_to (_elements[i]->id(), i);
     559             : 
     560      516431 :   return _elements[i];
     561             : }
     562             : 
     563             : 
     564             : 
     565             : 
     566      931827 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
     567             : {
     568      931827 :   if (const auto it = _elements.find(i);
     569      930607 :       it != _elements.end())
     570             :     {
     571      421928 :       const Elem * e = *it;
     572      420925 :       libmesh_assert (!e || e->id() == i);
     573      420925 :       return e;
     574             :     }
     575             : 
     576      509682 :   return nullptr;
     577             : }
     578             : 
     579             : 
     580             : 
     581             : 
     582      607375 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
     583             : {
     584      607375 :   if (auto it = _elements.find(i);
     585      218068 :       it != _elements.end())
     586             :     {
     587      596196 :       Elem * e = *it;
     588      214366 :       libmesh_assert (!e || e->id() == i);
     589      214366 :       return e;
     590             :     }
     591             : 
     592        3702 :   return nullptr;
     593             : }
     594             : 
     595             : 
     596             : 
     597             : 
     598       83732 : Elem * DistributedMesh::add_elem (Elem * e)
     599             : {
     600             :   // Don't try to add nullptrs!
     601       35765 :   libmesh_assert(e);
     602             : 
     603             :   // Trying to add an existing element is a no-op
     604       83732 :   if (e->valid_id() && _elements[e->id()] == e)
     605           0 :     return e;
     606             : 
     607       83732 :   const processor_id_type elem_procid = e->processor_id();
     608             : 
     609       83732 :   if (!e->valid_id())
     610             :     {
     611             :       // We should only be creating new ids past the end of the range
     612             :       // of existing ids
     613        3568 :       libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
     614             :                                    _max_elem_id);
     615        3568 :       libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
     616             : 
     617             :       // Use the unpartitioned ids for unpartitioned elems, and
     618             :       // temporarily for ghost elems
     619       12186 :       dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
     620       15754 :       if (elem_procid == this->processor_id())
     621        7264 :         next_id = &_next_free_local_elem_id;
     622       12186 :       e->set_id (*next_id);
     623             :     }
     624             : 
     625             :   {
     626             :     // Advance next_ids up high enough that each is pointing to an
     627             :     // unused id and any subsequent increments will still point us
     628             :     // to unused ids
     629      167464 :     _max_elem_id = std::max(_max_elem_id,
     630       83732 :                             static_cast<dof_id_type>(e->id()+1));
     631             : 
     632       83732 :     if (_next_free_unpartitioned_elem_id < _max_elem_id)
     633       34334 :       _next_free_unpartitioned_elem_id =
     634       34334 :         ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
     635       34334 :         (this->n_processors() + 1) + this->n_processors();
     636       83732 :     if (_next_free_local_elem_id < _max_elem_id)
     637       31920 :       _next_free_local_elem_id =
     638       38249 :         ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     639       31920 :         (this->n_processors() + 1) + this->processor_id();
     640             : 
     641             : #ifndef NDEBUG
     642             :     // We need a const dofobject_container so we don't inadvertently create
     643             :     // nullptr entries when testing for non-nullptr ones
     644       35765 :     const dofobject_container<Elem> & const_elements = _elements;
     645             : #endif
     646       35765 :     libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
     647       35765 :     libmesh_assert(!const_elements[_next_free_local_elem_id]);
     648             :   }
     649             : 
     650             :   // Don't try to overwrite existing elems
     651       35765 :   libmesh_assert (!_elements[e->id()]);
     652             : 
     653       83732 :   _elements[e->id()] = e;
     654             : 
     655             :   // We actually added a new element.  Some of our caches might still
     656             :   // be valid, but we should clear the ones which definitely are not.
     657       83732 :   this->clear_point_locator();
     658       83732 :   this->clear_stored_ranges();
     659             : 
     660             :   // Try to make the cached elem data more accurate
     661      102959 :   if (elem_procid == this->processor_id() ||
     662             :       elem_procid == DofObject::invalid_processor_id)
     663       61714 :     _n_elem++;
     664             : 
     665             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     666       83732 :   if (!e->valid_unique_id())
     667             :     {
     668       34181 :       if (processor_id() == e->processor_id())
     669             :         {
     670        7264 :           e->set_unique_id(_next_unique_id);
     671        7264 :           _next_unique_id += this->n_processors() + 1;
     672             :         }
     673             :       else
     674             :         {
     675       26917 :           e->set_unique_id(_next_unpartitioned_unique_id);
     676       26917 :           _next_unpartitioned_unique_id += this->n_processors() + 1;
     677             :         }
     678             :     }
     679             :   else
     680             :     {
     681       49911 :       _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
     682       49551 :       _next_unique_id =
     683       49551 :         ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     684       49551 :         (this->n_processors() + 1) + this->processor_id();
     685             :     }
     686             : #endif
     687             : 
     688             :   // Unpartitioned elems should be added on every processor
     689             :   // And shouldn't be added in the same batch as ghost elems
     690             :   // But we might be just adding on processor 0 to
     691             :   // broadcast later
     692             :   // #ifdef DEBUG
     693             :   //   if (elem_procid == DofObject::invalid_processor_id)
     694             :   //     {
     695             :   //       dof_id_type elem_id = e->id();
     696             :   //       this->comm().max(elem_id);
     697             :   //       libmesh_assert_equal_to (elem_id, e->id());
     698             :   //     }
     699             :   // #endif
     700             : 
     701             :   // Make sure any new element is given space for any extra integers
     702             :   // we've requested
     703       83732 :   e->add_extra_integers(_elem_integer_names.size(),
     704       83732 :                         _elem_integer_default_values);
     705             : 
     706             :   // And set mapping type and data on any new element
     707       54992 :   e->set_mapping_type(this->default_mapping_type());
     708       54992 :   e->set_mapping_data(this->default_mapping_data());
     709             : 
     710       83732 :   return e;
     711             : }
     712             : 
     713             : 
     714             : 
     715       82280 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
     716             : {
     717             :   // The mesh now takes ownership of the Elem. Eventually the guts of
     718             :   // add_elem() will get moved to a private helper function, and
     719             :   // calling add_elem() directly will be deprecated.
     720       82280 :   return add_elem(e.release());
     721             : }
     722             : 
     723             : 
     724             : 
     725       13080 : Elem * DistributedMesh::insert_elem (Elem * e)
     726             : {
     727       13080 :   if (_elements[e->id()])
     728       13080 :     this->delete_elem(_elements[e->id()]);
     729             : 
     730             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     731       13080 :   if (!e->valid_unique_id())
     732             :     {
     733           0 :       if (processor_id() == e->processor_id())
     734             :         {
     735           0 :           e->set_unique_id(_next_unique_id);
     736           0 :           _next_unique_id += this->n_processors() + 1;
     737             :         }
     738             :       else
     739             :         {
     740           0 :           e->set_unique_id(_next_unpartitioned_unique_id);
     741           0 :           _next_unpartitioned_unique_id += this->n_processors() + 1;
     742             :         }
     743             :     }
     744             :   else
     745             :     {
     746       13080 :       _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
     747       13080 :       _next_unique_id =
     748       16776 :         ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     749       13080 :         (this->n_processors() + 1) + this->processor_id();
     750             :     }
     751             : #endif
     752             : 
     753             :   // Try to make the cached elem data more accurate
     754       13080 :   processor_id_type elem_procid = e->processor_id();
     755       16776 :   if (elem_procid == this->processor_id() ||
     756             :       elem_procid == DofObject::invalid_processor_id)
     757       12576 :     _n_elem++;
     758             : 
     759       13080 :   _elements[e->id()] = e;
     760             : 
     761             :   // We actually added a new element.  Some of our caches might still
     762             :   // be valid, but we should clear the ones which definitely are not.
     763       13080 :   this->clear_point_locator();
     764       13080 :   this->clear_stored_ranges();
     765             : 
     766             :   // Make sure any new element is given space for any extra integers
     767             :   // we've requested
     768       13080 :   e->add_extra_integers(_elem_integer_names.size(),
     769       13080 :                         _elem_integer_default_values);
     770             : 
     771             :   // And set mapping type and data on any new element
     772        7392 :   e->set_mapping_type(this->default_mapping_type());
     773        7392 :   e->set_mapping_data(this->default_mapping_data());
     774             : 
     775       13080 :   return e;
     776             : }
     777             : 
     778       13080 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
     779             : {
     780             :   // The mesh now takes ownership of the Elem. Eventually the guts of
     781             :   // insert_elem(Elem*) will get moved to a private helper function, and
     782             :   // calling insert_elem(Elem*) directly will be deprecated.
     783       13080 :   return insert_elem(e.release());
     784             : }
     785             : 
     786             : 
     787       26443 : void DistributedMesh::delete_elem(Elem * e)
     788             : {
     789        7652 :   libmesh_assert (e);
     790             : 
     791             :   // Try to make the cached elem data more accurate
     792       26443 :   _n_elem--;
     793             : 
     794             :   // Was this a coarse element, not just a coarsening where we still
     795             :   // have some ancestor structure?  Was it a *local* element, that we
     796             :   // might have been depending on as an owner of local nodes?  We'll
     797             :   // have to be more careful with our nodes in contract() later; no
     798             :   // telling if we just locally orphaned a node that should be
     799             :   // globally retained.
     800       35915 :   if (e->processor_id() == this->processor_id() &&
     801        3640 :       !e->parent())
     802        2304 :     _deleted_coarse_elements = true;
     803             : 
     804             :   // Delete the element from the BoundaryInfo object
     805       26443 :   this->get_boundary_info().remove(e);
     806             : 
     807             :   // But not yet from the container; we might invalidate
     808             :   // an iterator that way!
     809             : 
     810             :   //_elements.erase(e->id());
     811             : 
     812             :   // Instead, we set it to nullptr for now
     813             : 
     814       26443 :   _elements[e->id()] = nullptr;
     815             : 
     816             :   // delete the element
     817       26443 :   delete e;
     818             : 
     819             :   // Some of our caches might still be valid, but we should clear the
     820             :   // ones which definitely are not.
     821       26443 :   this->clear_point_locator();
     822       26443 :   this->clear_stored_ranges();
     823       26443 : }
     824             : 
     825             : 
     826             : 
     827        3480 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
     828             :                                     const dof_id_type new_id)
     829             : {
     830             :   // This could be a no-op
     831        3480 :   if (old_id == new_id)
     832           0 :     return;
     833             : 
     834        3480 :   Elem * el = _elements[old_id];
     835        1160 :   libmesh_assert (el);
     836        1160 :   libmesh_assert_equal_to (el->id(), old_id);
     837             : 
     838        3480 :   el->set_id(new_id);
     839        1160 :   libmesh_assert (!_elements[new_id]);
     840        3480 :   _elements[new_id] = el;
     841        3480 :   _elements.erase(old_id);
     842             : 
     843             :   // Should we delete any caches here?  Our point locator indexes by
     844             :   // element pointer and should be fine with an id change.  Our stored
     845             :   // ranges are no longer sorted, which is *probably* fine, but let's
     846             :   // just be safe.
     847        3480 :   this->clear_stored_ranges();
     848             : }
     849             : 
     850             : 
     851             : 
     852      181173 : Node * DistributedMesh::add_point (const Point & p,
     853             :                                    const dof_id_type id,
     854             :                                    const processor_id_type proc_id)
     855             : {
     856      181173 :   Node * old_n = this->query_node_ptr(id);
     857             : 
     858      181173 :   if (old_n)
     859             :     {
     860           2 :       *old_n = p;
     861           7 :       old_n->processor_id() = proc_id;
     862             : 
     863           7 :       return old_n;
     864             :     }
     865             : 
     866      140851 :   Node * n = Node::build(p, id).release();
     867      181166 :   n->processor_id() = proc_id;
     868             : 
     869      181166 :   return DistributedMesh::add_node(n);
     870             : }
     871             : 
     872             : 
     873           0 : void DistributedMesh::own_node (Node & n)
     874             : {
     875             :   // This had better be a node in our mesh
     876           0 :   libmesh_assert(_nodes[n.id()] == &n);
     877             : 
     878           0 :   _nodes[n.id()] = nullptr;
     879           0 :   _n_nodes--;
     880             : 
     881           0 :   n.set_id(DofObject::invalid_id);
     882           0 :   n.processor_id() = this->processor_id();
     883             : 
     884           0 :   this->add_node(&n);
     885           0 : }
     886             : 
     887             : 
     888      186671 : Node * DistributedMesh::add_node (Node * n)
     889             : {
     890             :   // Don't try to add nullptrs!
     891       82685 :   libmesh_assert(n);
     892             : 
     893             :   // Trying to add an existing node is a no-op
     894      186671 :   if (n->valid_id() && _nodes[n->id()] == n)
     895           0 :     return n;
     896             : 
     897      186671 :   const processor_id_type node_procid = n->processor_id();
     898             : 
     899      186671 :   if (!n->valid_id())
     900             :     {
     901             :       // We should only be creating new ids past the end of the range
     902             :       // of existing ids
     903       17832 :       libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
     904             :                                    _max_node_id);
     905       17832 :       libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
     906             : 
     907             :       // Use the unpartitioned ids for unpartitioned nodes,
     908             :       // and temporarily for ghost nodes
     909       61957 :       dof_id_type * next_id = &_next_free_unpartitioned_node_id;
     910       79789 :       if (node_procid == this->processor_id())
     911       25654 :         next_id = &_next_free_local_node_id;
     912       61957 :       n->set_id (*next_id);
     913             :     }
     914             : 
     915             :   {
     916             :     // Advance next_ids up high enough that each is pointing to an
     917             :     // unused id and any subsequent increments will still point us
     918             :     // to unused ids
     919      373342 :     _max_node_id = std::max(_max_node_id,
     920      186671 :                             static_cast<dof_id_type>(n->id()+1));
     921             : 
     922      186671 :     if (_next_free_unpartitioned_node_id < _max_node_id)
     923      102308 :       _next_free_unpartitioned_node_id =
     924      102308 :         ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
     925      102308 :         (this->n_processors() + 1) + this->n_processors();
     926      186671 :     if (_next_free_local_node_id < _max_node_id)
     927       85514 :       _next_free_local_node_id =
     928      103798 :         ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     929       85514 :         (this->n_processors() + 1) + this->processor_id();
     930             : 
     931             : #ifndef NDEBUG
     932             :     // We need a const dofobject_container so we don't inadvertently create
     933             :     // nullptr entries when testing for non-nullptr ones
     934       82685 :     const dofobject_container<Node> & const_nodes = _nodes;
     935             : #endif
     936       82685 :     libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
     937       82685 :     libmesh_assert(!const_nodes[_next_free_local_node_id]);
     938             :   }
     939             : 
     940             :   // Don't try to overwrite existing nodes
     941       82685 :   libmesh_assert (!_nodes[n->id()]);
     942             : 
     943      186671 :   _nodes[n->id()] = n;
     944             : 
     945             :   // Try to make the cached node data more accurate
     946      228769 :   if (node_procid == this->processor_id() ||
     947             :       node_procid == DofObject::invalid_processor_id)
     948      143146 :     _n_nodes++;
     949             : 
     950             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     951      186671 :   if (!n->valid_unique_id())
     952             :     {
     953      181177 :       if (processor_id() == n->processor_id())
     954             :         {
     955       71182 :           n->set_unique_id(_next_unique_id);
     956       71182 :           _next_unique_id += this->n_processors() + 1;
     957             :         }
     958             :       else
     959             :         {
     960      109995 :           n->set_unique_id(_next_unpartitioned_unique_id);
     961      109995 :           _next_unpartitioned_unique_id += this->n_processors() + 1;
     962             :         }
     963             :     }
     964             :   else
     965             :     {
     966        5511 :       _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
     967        5494 :       _next_unique_id =
     968        5494 :         ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     969        5494 :         (this->n_processors() + 1) + this->processor_id();
     970             :     }
     971             : #endif
     972             : 
     973             :   // A new node invalidates any cached node ranges
     974      186671 :   this->clear_stored_ranges();
     975             : 
     976      186671 :   n->add_extra_integers(_node_integer_names.size(),
     977      186671 :                         _node_integer_default_values);
     978             : 
     979             :   // Unpartitioned nodes should be added on every processor
     980             :   // And shouldn't be added in the same batch as ghost nodes
     981             :   // But we might be just adding on processor 0 to
     982             :   // broadcast later
     983             :   // #ifdef DEBUG
     984             :   //   if (node_procid == DofObject::invalid_processor_id)
     985             :   //     {
     986             :   //       dof_id_type node_id = n->id();
     987             :   //       this->comm().max(node_id);
     988             :   //       libmesh_assert_equal_to (node_id, n->id());
     989             :   //     }
     990             :   // #endif
     991             : 
     992      186671 :   return n;
     993             : }
     994             : 
     995        5505 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
     996             : {
     997             :   // The mesh now takes ownership of the Node. Eventually the guts of
     998             :   // add_node() will get moved to a private helper function, and
     999             :   // calling add_node() directly will be deprecated.
    1000        5505 :   return add_node(n.release());
    1001             : }
    1002             : 
    1003       14407 : void DistributedMesh::delete_node(Node * n)
    1004             : {
    1005        4586 :   libmesh_assert(n);
    1006        4586 :   libmesh_assert(_nodes[n->id()]);
    1007             : 
    1008             :   // Try to make the cached elem data more accurate
    1009       14407 :   _n_nodes--;
    1010             : 
    1011             :   // Delete the node from the BoundaryInfo object
    1012       14407 :   this->get_boundary_info().remove(n);
    1013        9172 :   _constraint_rows.erase(n);
    1014             : 
    1015             :   // But not yet from the container; we might invalidate
    1016             :   // an iterator that way!
    1017             : 
    1018             :   //_nodes.erase(n->id());
    1019             : 
    1020             :   // Instead, we set it to nullptr for now
    1021             : 
    1022       14407 :   _nodes[n->id()] = nullptr;
    1023             : 
    1024             :   // delete the node
    1025       14407 :   delete n;
    1026             : 
    1027             :   // We've invalidated any ranges that contained this node
    1028       14407 :   this->clear_stored_ranges();
    1029       14407 : }
    1030             : 
    1031             : 
    1032             : 
    1033        9702 : void DistributedMesh::renumber_node(const dof_id_type old_id,
    1034             :                                     const dof_id_type new_id)
    1035             : {
    1036             :   // This could be a no-op
    1037        9702 :   if (old_id == new_id)
    1038           0 :     return;
    1039             : 
    1040        9702 :   Node * nd = _nodes[old_id];
    1041        3234 :   libmesh_assert (nd);
    1042        3234 :   libmesh_assert_equal_to (nd->id(), old_id);
    1043             : 
    1044             :   // If we have nodes shipped to this processor for NodeConstraints
    1045             :   // use, then those nodes will exist in _nodes, but may not be
    1046             :   // locatable via a TopologyMap due to the insufficiency of elements
    1047             :   // connecting to them.  If local refinement then wants to create a
    1048             :   // *new* node in the same location, it will initially get a temporary
    1049             :   // id, and then make_node_ids_parallel_consistent() will try to move
    1050             :   // it to the canonical id.  We need to account for this case to
    1051             :   // avoid false positives and memory leaks.
    1052             : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
    1053        6468 :   if (_nodes[new_id])
    1054             :     {
    1055           0 :       libmesh_assert_equal_to (*(Point *)_nodes[new_id],
    1056             :                                *(Point *)_nodes[old_id]);
    1057           0 :       _nodes.erase(new_id);
    1058             :     }
    1059             : #else
    1060             :   // If we aren't shipping nodes for NodeConstraints, there should be
    1061             :   // no reason for renumbering one node onto another.
    1062             :   libmesh_assert (!_nodes[new_id]);
    1063             : #endif
    1064        9702 :   _nodes[new_id] = nd;
    1065        9702 :   nd->set_id(new_id);
    1066             : 
    1067        9702 :   _nodes.erase(old_id);
    1068             : }
    1069             : 
    1070             : 
    1071             : 
    1072        2463 : void DistributedMesh::clear ()
    1073             : {
    1074             :   // Call parent clear function
    1075        2463 :   MeshBase::clear();
    1076             : 
    1077             :   // Clear our elements and nodes
    1078             :   // There is no need to remove them from
    1079             :   // the BoundaryInfo data structure since we
    1080             :   // already cleared it.
    1081        2463 :   this->DistributedMesh::clear_elems();
    1082             : 
    1083      170641 :   for (auto & node : _nodes)
    1084      258807 :     delete node;
    1085             : 
    1086         964 :   _nodes.clear();
    1087             : 
    1088             :   // We're no longer distributed if we were before
    1089        2463 :   _is_serial = true;
    1090        2463 :   _is_serial_on_proc_0 = true;
    1091             : 
    1092             :   // We deleted a ton of coarse elements, but their nodes got deleted too so
    1093             :   // all is copacetic.
    1094        2463 :   _deleted_coarse_elements = false;
    1095             : 
    1096             :   // Correct our caches
    1097        2463 :   _n_nodes = 0;
    1098        2463 :   _max_node_id = 0;
    1099        3067 :   _next_free_local_node_id = this->processor_id();
    1100        2463 :   _next_free_unpartitioned_node_id = this->n_processors();
    1101        2463 : }
    1102             : 
    1103             : 
    1104             : 
    1105        2463 : void DistributedMesh::clear_elems ()
    1106             : {
    1107       73252 :   for (auto & elem : _elements)
    1108       70789 :     delete elem;
    1109             : 
    1110         964 :   _elements.clear();
    1111             : 
    1112             :   // Correct our caches
    1113        2463 :   _n_elem = 0;
    1114        2463 :   _max_elem_id = 0;
    1115        3067 :   _next_free_local_elem_id = this->processor_id();
    1116        2463 :   _next_free_unpartitioned_elem_id = this->n_processors();
    1117        2463 : }
    1118             : 
    1119             : 
    1120             : 
    1121        1507 : void DistributedMesh::redistribute ()
    1122             : {
    1123             :   // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
    1124        1507 :   if (!this->is_serial())
    1125             :     {
    1126             :       // Construct a MeshCommunication object to actually redistribute the nodes
    1127             :       // and elements according to the partitioner, and then to re-gather the neighbors.
    1128             :       MeshCommunication mc;
    1129         414 :       mc.redistribute(*this);
    1130             : 
    1131         414 :       this->update_parallel_id_counts();
    1132             : 
    1133             :       // We communicate valid neighbor links for newly-redistributed
    1134             :       // elements, but we may still have remote_elem links that should
    1135             :       // be corrected but whose correction wasn't communicated.
    1136         690 :       this->find_neighbors(/*reset_remote_elements*/ false,
    1137             :                            /*reset_current_list*/ false /*non-default*/,
    1138             :                            /*assert_valid*/ false,
    1139         276 :                            /*check_non_remote*/ false /*non-default!*/);
    1140             : 
    1141             :       // Is this necessary?  If we are called from prepare_for_use(), this will be called
    1142             :       // anyway... but users can always call partition directly, in which case we do need
    1143             :       // to call delete_remote_elements()...
    1144             :       //
    1145             :       // Regardless of whether it's necessary, it isn't safe.  We
    1146             :       // haven't communicated new node processor_ids yet, and we can't
    1147             :       // delete nodes until we do.
    1148             :       // this->delete_remote_elements();
    1149             :     }
    1150             :   else
    1151             :     // The base class can handle non-distributed things, like
    1152             :     // notifying any GhostingFunctors of changes
    1153        1093 :     MeshBase::redistribute();
    1154        1507 : }
    1155             : 
    1156             : 
    1157             : 
    1158        4093 : void DistributedMesh::update_post_partitioning ()
    1159             : {
    1160             :   // this->recalculate_n_partitions();
    1161             : 
    1162             :   // Let's do the base class cache clearing first, just in case our
    1163             :   // later computations are ever changed to make use of a local
    1164             :   // elements range cache
    1165        4093 :   this->MeshBase::update_post_partitioning();
    1166             : 
    1167             :   // Partitioning changes our numbers of unpartitioned objects
    1168        4093 :   this->update_parallel_id_counts();
    1169        4093 : }
    1170             : 
    1171             : 
    1172             : 
    1173             : template <typename T>
    1174        3524 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
    1175             : {
    1176             :   // This function must be run on all processors at once
    1177        3524 :   parallel_object_only();
    1178             : 
    1179        3524 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1180        3524 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1181        3524 :   const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
    1182             : 
    1183      898180 :   for (dof_id_type i=0; i != pmax_id; ++i)
    1184             :     {
    1185      894656 :       T * obj = objects[i]; // Returns nullptr if there's no map entry
    1186             : 
    1187             :       // Local lookups by id should return the requested object
    1188      894656 :       libmesh_assert(!obj || obj->id() == i);
    1189             : 
    1190             :       // All processors with an object should agree on id
    1191             : #ifndef NDEBUG
    1192      894656 :       const dof_id_type dofid = obj && obj->valid_id() ?
    1193             :         obj->id() : DofObject::invalid_id;
    1194      894656 :       libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
    1195             : #endif
    1196             : 
    1197             :       // All processors with an object should agree on processor id
    1198      894656 :       const dof_id_type procid = obj && obj->valid_processor_id() ?
    1199             :         obj->processor_id() : DofObject::invalid_processor_id;
    1200      894656 :       libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
    1201             : 
    1202      894656 :       dof_id_type min_procid = procid;
    1203      894656 :       this->comm().min(min_procid);
    1204             : 
    1205             :       // Either:
    1206             :       // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
    1207             :       // or
    1208             :       // 2.) I don't own this elem (min_procid != this->processor_id()).  (In this case I may or may not have a valid pointer to it.)
    1209             : 
    1210             :       // Original assert logic
    1211             :       // libmesh_assert (min_procid != this->processor_id() || obj);
    1212             : 
    1213             :       // More human-understandable logic...
    1214      894656 :       libmesh_assert (
    1215             :                       ((min_procid == this->processor_id()) && obj)
    1216             :                       ||
    1217             :                       (min_procid != this->processor_id())
    1218             :                       );
    1219             : 
    1220             : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
    1221             :       // All processors with an object should agree on unique id
    1222      894656 :       const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
    1223      894656 :       libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
    1224             : #endif
    1225             :     }
    1226        3524 : }
    1227             : 
    1228             : 
    1229             : 
    1230        1762 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
    1231             : {
    1232        1762 :   this->libmesh_assert_valid_parallel_object_ids (this->_elements);
    1233        1762 :   this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
    1234        1762 : }
    1235             : 
    1236             : 
    1237             : 
    1238         430 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
    1239             : {
    1240             : #ifndef NDEBUG
    1241             :   // This function must be run on all processors at once
    1242         430 :   parallel_object_only();
    1243             : 
    1244         430 :   dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1245             : 
    1246       22314 :   for (dof_id_type i=0; i != pmax_elem_id; ++i)
    1247             :     {
    1248       21884 :       Elem * el = _elements[i]; // Returns nullptr if there's no map entry
    1249             : 
    1250       21884 :       unsigned int p_level = el ?  (el->p_level()) : libMesh::invalid_uint;
    1251             : 
    1252             :       // All processors with an active element should agree on p level
    1253       21884 :       libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
    1254             :     }
    1255             : #endif
    1256         430 : }
    1257             : 
    1258             : 
    1259             : 
    1260             : 
    1261        1710 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
    1262             : {
    1263             : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
    1264             :   // This function must be run on all processors at once
    1265        1710 :   parallel_object_only();
    1266             : 
    1267        1710 :   dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1268             : 
    1269      145736 :   for (dof_id_type i=0; i != pmax_elem_id; ++i)
    1270             :     {
    1271      144026 :       Elem * el = _elements[i]; // Returns nullptr if there's no map entry
    1272             : 
    1273      144026 :       unsigned int refinement_flag   = el ?
    1274       66551 :         static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
    1275      144026 :       unsigned int p_refinement_flag = el ?
    1276       66551 :         static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
    1277             : 
    1278      144026 :       libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
    1279             : 
    1280             :       // p refinement flags aren't always kept correct on inactive
    1281             :       // ghost elements
    1282      144026 :       libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
    1283             :     }
    1284             : #endif // LIBMESH_ENABLE_AMR
    1285        1710 : }
    1286             : 
    1287             : 
    1288             : 
    1289             : template <typename T>
    1290             : dof_id_type
    1291        2976 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
    1292             : {
    1293             :   // This function must be run on all processors at once
    1294         856 :   parallel_object_only();
    1295             : 
    1296             :   typedef typename dofobject_container<T>::veclike_iterator object_iterator;
    1297             : 
    1298             :   // In parallel we may not know what objects other processors have.
    1299             :   // Start by figuring out how many
    1300         856 :   dof_id_type unpartitioned_objects = 0;
    1301             : 
    1302             :   std::unordered_map<processor_id_type, dof_id_type>
    1303        1712 :     ghost_objects_from_proc;
    1304             : 
    1305        2976 :   object_iterator it  = objects.begin();
    1306         856 :   object_iterator end = objects.end();
    1307             : 
    1308      201236 :   while (it != end)
    1309             :     {
    1310      198260 :       T * obj = *it;
    1311             : 
    1312             :       // Remove any nullptr container entries while we're here.
    1313      198260 :       if (!obj)
    1314         641 :         it = objects.erase(it);
    1315             :       else
    1316             :         {
    1317      197619 :           processor_id_type obj_procid = obj->processor_id();
    1318      197619 :           if (obj_procid == DofObject::invalid_processor_id)
    1319       55017 :             unpartitioned_objects++;
    1320             :           else
    1321      142602 :             ghost_objects_from_proc[obj_procid]++;
    1322             : 
    1323             :           // Finally, increment the iterator
    1324       57213 :           ++it;
    1325             :         }
    1326             :     }
    1327             : 
    1328        4688 :   std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
    1329        3832 :   auto this_it = ghost_objects_from_proc.find(this->processor_id());
    1330         856 :   this->comm().allgather
    1331        3494 :     ((this_it == ghost_objects_from_proc.end()) ?
    1332         518 :      dof_id_type(0) : this_it->second, objects_on_proc);
    1333             : 
    1334             : #ifndef NDEBUG
    1335         856 :   libmesh_assert(this->comm().verify(unpartitioned_objects));
    1336        2568 :   for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
    1337        1712 :     if (ghost_objects_from_proc.count(p))
    1338         993 :       libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
    1339             :     else
    1340         719 :       libmesh_assert_less_equal (0, objects_on_proc[p]);
    1341             : #endif
    1342             : 
    1343             :   // We'll renumber objects in blocks by processor id
    1344        4688 :   std::vector<dof_id_type> first_object_on_proc(this->n_processors());
    1345        5544 :   for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
    1346        5992 :     first_object_on_proc[i] = first_object_on_proc[i-1] +
    1347         856 :       objects_on_proc[i-1];
    1348        3832 :   dof_id_type next_id = first_object_on_proc[this->processor_id()];
    1349        2976 :   dof_id_type first_free_id =
    1350        4688 :     first_object_on_proc[this->n_processors()-1] +
    1351         856 :     objects_on_proc[this->n_processors()-1] +
    1352             :     unpartitioned_objects;
    1353             : 
    1354             :   // First set new local object ids and build request sets
    1355             :   // for non-local object ids
    1356             : 
    1357             :   // Request sets to send to each processor
    1358             :   std::map<processor_id_type, std::vector<dof_id_type>>
    1359         856 :     requested_ids;
    1360             : 
    1361             :   // We know how many objects live on each processor, so reserve() space for
    1362             :   // each.
    1363         856 :   auto ghost_end = ghost_objects_from_proc.end();
    1364        8520 :   for (auto p : make_range(this->n_processors()))
    1365        7256 :     if (p != this->processor_id())
    1366             :       {
    1367        3424 :         if (const auto p_it = ghost_objects_from_proc.find(p);
    1368         856 :             p_it != ghost_end)
    1369        1425 :           requested_ids[p].reserve(p_it->second);
    1370             :       }
    1371             : 
    1372         856 :   end = objects.end();
    1373      200595 :   for (it = objects.begin(); it != end; ++it)
    1374             :     {
    1375      197619 :       T * obj = *it;
    1376      197619 :       if (!obj)
    1377           0 :         continue;
    1378      254832 :       if (obj->processor_id() == this->processor_id())
    1379       93612 :         obj->set_id(next_id++);
    1380      104007 :       else if (obj->processor_id() != DofObject::invalid_processor_id)
    1381       48990 :         requested_ids[obj->processor_id()].push_back(obj->id());
    1382             :     }
    1383             : 
    1384             :   // Next set ghost object ids from other processors
    1385             : 
    1386       20262 :   auto gather_functor =
    1387       16793 :     [
    1388             : #ifndef NDEBUG
    1389             :      this,
    1390             :      &first_object_on_proc,
    1391             :      &objects_on_proc,
    1392             : #endif
    1393             :      &objects]
    1394             :     (processor_id_type, const std::vector<dof_id_type> & ids,
    1395         950 :      std::vector<dof_id_type> & new_ids)
    1396             :     {
    1397         950 :       std::size_t ids_size = ids.size();
    1398        1425 :       new_ids.resize(ids_size);
    1399             : 
    1400       50415 :       for (std::size_t i=0; i != ids_size; ++i)
    1401             :         {
    1402       48990 :           T * obj = objects[ids[i]];
    1403       16336 :           libmesh_assert(obj);
    1404       16336 :           libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
    1405       65326 :           new_ids[i] = obj->id();
    1406             : 
    1407       16336 :           libmesh_assert_greater_equal (new_ids[i],
    1408             :                                         first_object_on_proc[this->processor_id()]);
    1409       16336 :           libmesh_assert_less (new_ids[i],
    1410             :                                first_object_on_proc[this->processor_id()] +
    1411             :                                objects_on_proc[this->processor_id()]);
    1412             :         }
    1413             :     };
    1414             : 
    1415       20262 :   auto action_functor =
    1416       16793 :     [
    1417             : #ifndef NDEBUG
    1418             :      &first_object_on_proc,
    1419             :      &objects_on_proc,
    1420             : #endif
    1421             :      &objects]
    1422             :     (processor_id_type libmesh_dbg_var(pid),
    1423             :      const std::vector<dof_id_type> & ids,
    1424         950 :      const std::vector<dof_id_type> & data)
    1425             :     {
    1426             :       // Copy the id changes we've now been informed of
    1427       50415 :       for (auto i : index_range(ids))
    1428             :         {
    1429       48990 :           T * obj = objects[ids[i]];
    1430       16336 :           libmesh_assert (obj);
    1431       16336 :           libmesh_assert_equal_to (obj->processor_id(), pid);
    1432       16336 :           libmesh_assert_greater_equal (data[i],
    1433             :                                         first_object_on_proc[pid]);
    1434       16336 :           libmesh_assert_less (data[i],
    1435             :                                first_object_on_proc[pid] +
    1436             :                                objects_on_proc[pid]);
    1437       65326 :           obj->set_id(data[i]);
    1438             :         }
    1439             :     };
    1440             : 
    1441         856 :   const dof_id_type * ex = nullptr;
    1442             :   Parallel::pull_parallel_vector_data
    1443        2976 :     (this->comm(), requested_ids, gather_functor, action_functor, ex);
    1444             : 
    1445             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
    1446       20262 :   auto unique_gather_functor =
    1447       16793 :     [
    1448             : #ifndef NDEBUG
    1449             :      this,
    1450             : #endif
    1451             :      &objects]
    1452             :     (processor_id_type, const std::vector<dof_id_type> & ids,
    1453         950 :      std::vector<unique_id_type> & data)
    1454             :     {
    1455         950 :       std::size_t ids_size = ids.size();
    1456        1425 :       data.resize(ids_size);
    1457             : 
    1458       50415 :       for (std::size_t i=0; i != ids_size; ++i)
    1459             :         {
    1460       48990 :           T * obj = objects[ids[i]];
    1461       16336 :           libmesh_assert(obj);
    1462       16336 :           libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
    1463       97980 :           data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
    1464             :         }
    1465             :     };
    1466             : 
    1467       20262 :   auto unique_action_functor =
    1468       16793 :     [&objects]
    1469             :     (processor_id_type libmesh_dbg_var(pid),
    1470             :      const std::vector<dof_id_type> & ids,
    1471         950 :      const std::vector<unique_id_type> & data)
    1472             :     {
    1473       50415 :       for (auto i : index_range(ids))
    1474             :         {
    1475       48990 :           T * obj = objects[ids[i]];
    1476       16336 :           libmesh_assert (obj);
    1477       16336 :           libmesh_assert_equal_to (obj->processor_id(), pid);
    1478       48990 :           if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
    1479           0 :             obj->set_unique_id(data[i]);
    1480             :         }
    1481             :     };
    1482             : 
    1483         856 :   const unique_id_type * unique_ex = nullptr;
    1484             :   Parallel::pull_parallel_vector_data
    1485        2976 :     (this->comm(), requested_ids, unique_gather_functor,
    1486             :      unique_action_functor, unique_ex);
    1487             : #endif
    1488             : 
    1489             :   // Next set unpartitioned object ids
    1490         856 :   next_id = 0;
    1491        8520 :   for (auto i : make_range(this->n_processors()))
    1492        7256 :     next_id += objects_on_proc[i];
    1493      200595 :   for (it = objects.begin(); it != end; ++it)
    1494             :     {
    1495      197619 :       T * obj = *it;
    1496      197619 :       if (!obj)
    1497           0 :         continue;
    1498      197619 :       if (obj->processor_id() == DofObject::invalid_processor_id)
    1499       55017 :         obj->set_id(next_id++);
    1500             :     }
    1501             : 
    1502             :   // Finally shuffle around objects so that container indices
    1503             :   // match ids
    1504        2976 :   it = objects.begin();
    1505         856 :   end = objects.end();
    1506      204326 :   while (it != end)
    1507             :     {
    1508      201350 :       T * obj = *it;
    1509      201350 :       if (obj) // don't try shuffling already-nullptr entries
    1510             :         {
    1511      201350 :           T * next = objects[obj->id()];
    1512             :           // If we have to move this object
    1513      201350 :           if (next != obj)
    1514             :             {
    1515             :               // nullptr out its original position for now
    1516             :               // (our shuffling may put another object there shortly)
    1517       50885 :               *it = nullptr;
    1518             : 
    1519             :               // There may already be another object with this id that
    1520             :               // needs to be moved itself
    1521       79121 :               while (next)
    1522             :                 {
    1523             :                   // We shouldn't be trying to give two objects the
    1524             :                   // same id
    1525        9414 :                   libmesh_assert_not_equal_to (next->id(), obj->id());
    1526       28236 :                   objects[obj->id()] = obj;
    1527        9414 :                   obj = next;
    1528       28236 :                   next = objects[obj->id()];
    1529             :                 }
    1530       50885 :               objects[obj->id()] = obj;
    1531             :             }
    1532             :         }
    1533             : 
    1534             :       // Remove any container entries that were left as nullptr.
    1535      116912 :       if (!obj)
    1536           0 :         it = objects.erase(it);
    1537             :       else
    1538       58456 :         ++it;
    1539             :     }
    1540             : 
    1541        3832 :   return first_free_id;
    1542             : }
    1543             : 
    1544             : 
    1545        1494 : void DistributedMesh::renumber_nodes_and_elements ()
    1546             : {
    1547         430 :   parallel_object_only();
    1548             : 
    1549             : #ifdef DEBUG
    1550             :   // Make sure our ids and flags are consistent
    1551         430 :   this->libmesh_assert_valid_parallel_ids();
    1552         430 :   this->libmesh_assert_valid_parallel_flags();
    1553         430 :   this->libmesh_assert_valid_parallel_p_levels();
    1554             : #endif
    1555             : 
    1556         858 :   LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
    1557             : 
    1558             :   // Nodes not connected to any elements, and nullptr node entries
    1559             :   // in our container, should be deleted.  But wait!  If we've deleted coarse
    1560             :   // local elements on some processor, other processors might have ghosted
    1561             :   // nodes from it that are now no longer connected to any elements on it, but
    1562             :   // that are connected to their own semilocal elements.  We'll have to
    1563             :   // communicate to ascertain if that's the case.
    1564        1494 :   this->comm().max(_deleted_coarse_elements);
    1565             : 
    1566             :   // What used nodes do we see on our proc?
    1567         430 :   std::set<dof_id_type> used_nodes;
    1568             : 
    1569             :   // What used node info should we send from our proc?  Could we take ownership
    1570             :   // of each node if we needed to?
    1571             :   std::map<processor_id_type, std::map<dof_id_type, bool>>
    1572         430 :     used_nodes_on_proc;
    1573             : 
    1574             :   // flag the nodes we need
    1575       75956 :   for (auto & elem : this->element_ptr_range())
    1576      569043 :     for (const Node & node : elem->node_ref_range())
    1577             :       {
    1578      502374 :         const dof_id_type n = node.id();
    1579      357002 :         used_nodes.insert(n);
    1580      502374 :         if (_deleted_coarse_elements)
    1581             :           {
    1582         994 :             const processor_id_type p = node.processor_id();
    1583        1278 :             if (p != this->processor_id())
    1584             :               {
    1585         426 :                 auto & used_nodes_on_p = used_nodes_on_proc[p];
    1586         568 :                 if (elem->processor_id() == this->processor_id())
    1587         111 :                   used_nodes_on_p[n] = true;
    1588             :                 else
    1589         105 :                   if (!used_nodes_on_p.count(n))
    1590         261 :                     used_nodes_on_p[n] = false;
    1591             :               }
    1592             :           }
    1593         634 :       }
    1594             : 
    1595        1494 :   if (_deleted_coarse_elements)
    1596             :     {
    1597             :       // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
    1598             :       typedef unsigned char boolish;
    1599             :       std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
    1600          28 :         used_nodes_on_proc_vecs;
    1601          88 :       for (auto & [pid, nodemap] : used_nodes_on_proc)
    1602          39 :         used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
    1603             : 
    1604          28 :       std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
    1605             :       std::map<processor_id_type, std::set<dof_id_type>>
    1606          28 :         repartitioned_node_sets_to_push;
    1607             : 
    1608             :       auto ids_action_functor =
    1609          13 :         [&used_nodes, &repartitioned_node_pids,
    1610             :          &repartitioned_node_sets_to_push]
    1611             :         (processor_id_type pid,
    1612         110 :          const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
    1613             :         {
    1614         330 :           for (auto [n, sender_could_become_owner] : ids_and_bools)
    1615             :             {
    1616             :               // If we don't see a use for our own node, but someone
    1617             :               // else does, better figure out who should own it next.
    1618          97 :               if (!used_nodes.count(n))
    1619             :                 {
    1620           0 :                   if (auto it = repartitioned_node_pids.find(n);
    1621           0 :                       sender_could_become_owner)
    1622             :                     {
    1623           0 :                       if (it != repartitioned_node_pids.end() &&
    1624           0 :                           pid < it->second)
    1625           0 :                         it->second = pid;
    1626             :                       else
    1627           0 :                         repartitioned_node_pids[n] = pid;
    1628             :                     }
    1629             :                   else
    1630           0 :                     if (it == repartitioned_node_pids.end())
    1631           0 :                       repartitioned_node_pids[n] =
    1632             :                         DofObject::invalid_processor_id;
    1633             : 
    1634           0 :                   repartitioned_node_sets_to_push[pid].insert(n);
    1635             :                 }
    1636             :             }
    1637          61 :         };
    1638             : 
    1639             :       // We need two pushes instead of a pull here because we need to
    1640             :       // know *all* the queries for a particular node before we can
    1641             :       // respond to *any* of them.
    1642             :       Parallel::push_parallel_vector_data
    1643          49 :       (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
    1644             : 
    1645             :       // Repartition (what used to be) our own nodes first
    1646          49 :       for (auto & [n, p] : repartitioned_node_pids)
    1647             :         {
    1648           0 :           Node & node = this->node_ref(n);
    1649           0 :           libmesh_assert_equal_to(node.processor_id(), this->processor_id());
    1650           0 :           libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
    1651           0 :           node.processor_id() = p;
    1652             :         }
    1653             : 
    1654             :       // Then push to repartition others' ghosted copies.
    1655             : 
    1656             :       std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
    1657          28 :         repartitioned_node_vecs;
    1658             : 
    1659          49 :       for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
    1660             :         {
    1661           0 :           auto & rn_vec = repartitioned_node_vecs[p];
    1662           0 :           for (auto n : nodeset)
    1663           0 :             rn_vec.emplace_back(n, repartitioned_node_pids[n]);
    1664             :         }
    1665             : 
    1666             :       auto repartition_node_functor =
    1667           0 :         [this]
    1668             :         (processor_id_type libmesh_dbg_var(pid),
    1669           0 :          const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
    1670             :         {
    1671           0 :           for (auto [n, p] : ids_and_pids)
    1672             :             {
    1673           0 :               libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
    1674           0 :               Node & node = this->node_ref(n);
    1675           0 :               libmesh_assert_equal_to(node.processor_id(), pid);
    1676           0 :               node.processor_id() = p;
    1677             :             }
    1678          21 :         };
    1679             : 
    1680             :       Parallel::push_parallel_vector_data
    1681          49 :       (this->comm(), repartitioned_node_vecs, repartition_node_functor);
    1682             :     }
    1683             : 
    1684        1494 :   _deleted_coarse_elements = false;
    1685             : 
    1686             :   // Nodes not connected to any local elements, and nullptr node entries
    1687             :   // in our container, are deleted
    1688             :   {
    1689        1494 :     node_iterator_imp  it = _nodes.begin();
    1690         430 :     node_iterator_imp end = _nodes.end();
    1691             : 
    1692      165991 :     while (it != end)
    1693             :       {
    1694      164497 :         Node * nd = *it;
    1695      164497 :         if (!nd)
    1696        2926 :           it = _nodes.erase(it);
    1697      161571 :         else if (!used_nodes.count(nd->id()))
    1698             :           {
    1699             :             // remove any boundary information associated with
    1700             :             // this node
    1701        4950 :             this->get_boundary_info().remove (nd);
    1702        2828 :             _constraint_rows.erase(nd);
    1703             : 
    1704             :             // delete the node
    1705        4950 :             delete nd;
    1706             : 
    1707        4950 :             it = _nodes.erase(it);
    1708             :           }
    1709             :         else
    1710       45790 :           ++it;
    1711             :       }
    1712             :   }
    1713             : 
    1714        1494 :   this->_preparation.has_removed_orphaned_nodes = true;
    1715             : 
    1716        1494 :   if (_skip_renumber_nodes_and_elements)
    1717             :     {
    1718           6 :       this->update_parallel_id_counts();
    1719           2 :       return;
    1720             :     }
    1721             : 
    1722             :   // Finally renumber all the elements
    1723        1488 :   _n_elem = this->renumber_dof_objects (this->_elements);
    1724             : 
    1725             :   // and all the remaining nodes
    1726        1488 :   _n_nodes = this->renumber_dof_objects (this->_nodes);
    1727             : 
    1728             :   // And figure out what IDs we should use when adding new nodes and
    1729             :   // new elements
    1730        1488 :   this->update_parallel_id_counts();
    1731             : 
    1732             :   // We've probably done *some* node removal and/or renumbering; let's
    1733             :   // clear caches to be safe.
    1734        1488 :   this->clear_stored_ranges();
    1735             : 
    1736             :   // Make sure our caches are up to date and our
    1737             :   // DofObjects are well packed
    1738             : #ifdef DEBUG
    1739         428 :   libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
    1740         428 :   libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
    1741         428 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1742         428 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1743         428 :   libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
    1744         428 :   libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
    1745         428 :   libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
    1746         428 :   libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
    1747             : 
    1748             :   // Make sure our ids and flags are consistent
    1749         428 :   this->libmesh_assert_valid_parallel_ids();
    1750         428 :   this->libmesh_assert_valid_parallel_flags();
    1751             : 
    1752             :   // And make sure we've made our numbering monotonic
    1753         428 :   MeshTools::libmesh_assert_valid_elem_ids(*this);
    1754             : #endif
    1755             : }
    1756             : 
    1757             : 
    1758             : 
    1759           0 : void DistributedMesh::fix_broken_node_and_element_numbering ()
    1760             : {
    1761             :   // We can't use range-for here because we need access to the special
    1762             :   // iterators' methods, not just to their dereferenced values.
    1763             : 
    1764             :   // Nodes first
    1765           0 :   for (auto pr = this->_nodes.begin(),
    1766           0 :            end = this->_nodes.end(); pr != end; ++pr)
    1767             :     {
    1768           0 :       Node * n = *pr;
    1769           0 :       if (n != nullptr)
    1770             :         {
    1771           0 :           const dof_id_type id = pr.index();
    1772           0 :           n->set_id() = id;
    1773           0 :           libmesh_assert_equal_to(this->node_ptr(id), n);
    1774             :         }
    1775             :     }
    1776             : 
    1777             :   // Elements next
    1778           0 :   for (auto pr = this->_elements.begin(),
    1779           0 :            end = this->_elements.end(); pr != end; ++pr)
    1780             :     {
    1781           0 :       Elem * e = *pr;
    1782           0 :       if (e != nullptr)
    1783             :         {
    1784           0 :           const dof_id_type id = pr.index();
    1785           0 :           e->set_id() = id;
    1786           0 :           libmesh_assert_equal_to(this->elem_ptr(id), e);
    1787             :         }
    1788             :     }
    1789           0 : }
    1790             : 
    1791             : 
    1792             : 
    1793        1983 : dof_id_type DistributedMesh::n_active_elem () const
    1794             : {
    1795         582 :   parallel_object_only();
    1796             : 
    1797             :   // Get local active elements first
    1798             :   dof_id_type active_elements =
    1799        3384 :     static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
    1800        4785 :                                             this->active_local_elements_end()));
    1801        1983 :   this->comm().sum(active_elements);
    1802             : 
    1803             :   // Then add unpartitioned active elements, which should exist on
    1804             :   // every processor
    1805        1983 :   active_elements +=
    1806        1983 :     static_cast<dof_id_type>(std::distance
    1807        3384 :                              (this->active_pid_elements_begin(DofObject::invalid_processor_id),
    1808        2565 :                               this->active_pid_elements_end(DofObject::invalid_processor_id)));
    1809        1983 :   return active_elements;
    1810             : }
    1811             : 
    1812             : 
    1813             : 
    1814        1396 : void DistributedMesh::delete_remote_elements()
    1815             : {
    1816             : #ifdef DEBUG
    1817             :   // Make sure our neighbor links are all fine
    1818         402 :   MeshTools::libmesh_assert_valid_neighbors(*this);
    1819             : 
    1820             :   // And our child/parent links, and our flags
    1821         402 :   MeshTools::libmesh_assert_valid_refinement_tree(*this);
    1822             : 
    1823             :   // Make sure our ids and flags are consistent
    1824         402 :   this->libmesh_assert_valid_parallel_ids();
    1825         402 :   this->libmesh_assert_valid_parallel_flags();
    1826             : 
    1827         402 :   libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
    1828         402 :   libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
    1829         402 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1830         402 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1831         402 :   libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
    1832         402 :   libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
    1833             : #endif
    1834             : 
    1835        1396 :   _is_serial = false;
    1836        1396 :   _is_serial_on_proc_0 = false;
    1837             : 
    1838        1396 :   MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
    1839             : 
    1840         402 :   libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
    1841             : 
    1842             :   // Now make sure the containers actually shrink - strip
    1843             :   // any newly-created nullptr voids out of the element array
    1844        1396 :   dofobject_container<Elem>::veclike_iterator e_it        = _elements.begin();
    1845         402 :   const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
    1846       74597 :   while (e_it != e_end)
    1847       73201 :     if (!*e_it)
    1848        9863 :       e_it = _elements.erase(e_it);
    1849             :     else
    1850       17511 :       ++e_it;
    1851             : 
    1852        1396 :   dofobject_container<Node>::veclike_iterator n_it        = _nodes.begin();
    1853         402 :   const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
    1854      209360 :   while (n_it != n_end)
    1855      207964 :     if (!*n_it)
    1856       34123 :       n_it = _nodes.erase(n_it);
    1857             :     else
    1858       48820 :       ++n_it;
    1859             : 
    1860             :   // We may have deleted no-longer-connected nodes or coarsened-away
    1861             :   // elements; let's update our caches if they're expected to be up to
    1862             :   // date.
    1863        1396 :   if (_preparation.has_synched_id_counts)
    1864        1396 :     this->update_parallel_id_counts();
    1865             : 
    1866             :   // We may have deleted nodes or elements that were the only local
    1867             :   // representatives of some particular boundary id(s); let's update
    1868             :   // those caches if they're expected to be up to date.
    1869        1396 :   if (_preparation.has_boundary_id_sets)
    1870          89 :     this->get_boundary_info().regenerate_id_sets();
    1871             : 
    1872             : #ifdef DEBUG
    1873             :   // We might not have well-packed objects if the user didn't allow us
    1874             :   // to renumber
    1875             :   // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
    1876             :   // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
    1877             : 
    1878             :   // Make sure our neighbor links are all fine
    1879         402 :   MeshTools::libmesh_assert_valid_neighbors(*this);
    1880             : 
    1881             :   // And our child/parent links, and our flags
    1882         402 :   MeshTools::libmesh_assert_valid_refinement_tree(*this);
    1883             : 
    1884             :   // Make sure our ids and flags are consistent
    1885         402 :   this->libmesh_assert_valid_parallel_ids();
    1886         402 :   this->libmesh_assert_valid_parallel_flags();
    1887             : #endif
    1888             : 
    1889        1396 :   this->_preparation.has_removed_remote_elements = true;
    1890        1396 : }
    1891             : 
    1892             : 
    1893           0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
    1894             : {
    1895             :   // First add the elem like normal
    1896           0 :   add_elem(e);
    1897             : 
    1898             :   // Now add it to the set that won't be deleted when we call
    1899             :   // delete_remote_elements()
    1900           0 :   _extra_ghost_elems.insert(e);
    1901           0 : }
    1902             : 
    1903             : void
    1904           0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
    1905             : {
    1906           0 :   std::set<Elem *> tmp;
    1907           0 :   std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
    1908             :                       extra_ghost_elems.begin(), extra_ghost_elems.end(),
    1909           0 :                       std::inserter(tmp, tmp.begin()));
    1910           0 :   _extra_ghost_elems = tmp;
    1911           0 : }
    1912             : 
    1913         159 : void DistributedMesh::allgather()
    1914             : {
    1915         159 :   if (_is_serial)
    1916           0 :     return;
    1917         158 :   MeshCommunication().allgather(*this);
    1918         158 :   _is_serial = true;
    1919         158 :   _is_serial_on_proc_0 = true;
    1920             : 
    1921             :   // Make sure our caches are up to date and our
    1922             :   // DofObjects are well packed
    1923             : #ifdef DEBUG
    1924          48 :   libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
    1925          48 :   libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
    1926          48 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1927          48 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1928          48 :   libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
    1929          48 :   libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
    1930             : 
    1931             :   // If we've disabled renumbering we can't be sure we're contiguous
    1932             :   // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
    1933             :   // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
    1934             : 
    1935             :   // Make sure our neighbor links are all fine
    1936          48 :   MeshTools::libmesh_assert_valid_neighbors(*this);
    1937             : 
    1938             :   // Make sure our ids and flags are consistent
    1939          48 :   this->libmesh_assert_valid_parallel_ids();
    1940          48 :   this->libmesh_assert_valid_parallel_flags();
    1941             : #endif
    1942             : }
    1943             : 
    1944          67 : void DistributedMesh::gather_to_zero()
    1945             : {
    1946          67 :   if (_is_serial_on_proc_0)
    1947           6 :     return;
    1948             : 
    1949          47 :   _is_serial_on_proc_0 = true;
    1950          47 :   MeshCommunication().gather(0, *this);
    1951             : }
    1952             : 
    1953             : 
    1954             : } // namespace libMesh

Generated by: LCOV version 1.14