LCOV - code coverage report
Current view: top level - src/mesh - distributed_mesh.C (source / functions) Hit Total Coverage
Test: libMesh/libmesh: #4667 (c50533) with base aa4ea8 Lines: 770 819 94.0 %
Date: 2026-11-17 18:48:15 Functions: 91 96 94.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : // The libMesh Finite Element Library.
       2             : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
       3             : 
       4             : // This library is free software; you can redistribute it and/or
       5             : // modify it under the terms of the GNU Lesser General Public
       6             : // License as published by the Free Software Foundation; either
       7             : // version 2.1 of the License, or (at your option) any later version.
       8             : 
       9             : // This library is distributed in the hope that it will be useful,
      10             : // but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      12             : // Lesser General Public License for more details.
      13             : 
      14             : // You should have received a copy of the GNU Lesser General Public
      15             : // License along with this library; if not, write to the Free Software
      16             : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
      17             : 
      18             : 
      19             : 
      20             : // Local includes
      21             : #include "libmesh/distributed_mesh.h"
      22             : 
      23             : // libMesh includes
      24             : #include "libmesh/boundary_info.h"
      25             : #include "libmesh/elem.h"
      26             : #include "libmesh/libmesh_logging.h"
      27             : #include "libmesh/mesh_communication.h"
      28             : #include "libmesh/mesh_tools.h"
      29             : #include "libmesh/partitioner.h"
      30             : #include "libmesh/string_to_enum.h"
      31             : 
      32             : // TIMPI includes
      33             : #include "timpi/parallel_implementation.h"
      34             : #include "timpi/parallel_sync.h"
      35             : 
      36             : 
      37             : namespace libMesh
      38             : {
      39             : 
      40             : // ------------------------------------------------------------
      41             : // DistributedMesh class member functions
      42      419167 : DistributedMesh::DistributedMesh (const Parallel::Communicator & comm_in,
      43      419167 :                                   unsigned char d) :
      44      418643 :   UnstructuredMesh (comm_in,d), _is_serial(true),
      45      418643 :   _is_serial_on_proc_0(true),
      46      418643 :   _deleted_coarse_elements(false),
      47      418643 :   _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
      48      419429 :   _next_free_local_node_id(this->processor_id()),
      49      419167 :   _next_free_local_elem_id(this->processor_id()),
      50      419167 :   _next_free_unpartitioned_node_id(this->n_processors()),
      51      419167 :   _next_free_unpartitioned_elem_id(this->n_processors())
      52             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
      53      419167 :   , _next_unpartitioned_unique_id(this->n_processors())
      54             : #endif
      55             : {
      56             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
      57      419167 :   _next_unique_id = this->processor_id();
      58             : #endif
      59             : 
      60      419429 :   const std::string default_partitioner = "parmetis";
      61             :   const std::string my_partitioner =
      62             :     libMesh::command_line_value("--default-partitioner",
      63      838334 :                                 default_partitioner);
      64             :   _partitioner = Partitioner::build
      65      838072 :     (Utility::string_to_enum<PartitionerType>(my_partitioner));
      66      419167 : }
      67             : 
      68         270 : DistributedMesh & DistributedMesh::operator= (DistributedMesh && other_mesh)
      69             : {
      70           8 :   LOG_SCOPE("operator=(&&)", "DistributedMesh");
      71             : 
      72             :   // Move assign as an UnstructuredMesh.
      73           4 :   this->UnstructuredMesh::operator=(std::move(other_mesh));
      74             : 
      75             :   // Nodes and elements belong to DistributedMesh and have to be
      76             :   // moved before we can move arbitrary GhostingFunctor, Partitioner,
      77             :   // etc. subclasses.
      78         270 :   this->move_nodes_and_elements(std::move(other_mesh));
      79             : 
      80             :   // But move_nodes_and_elems misses (or guesses about) some of our
      81             :   // subclass values, and we want more precision than a guess.
      82         270 :   _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
      83           4 :   _extra_ghost_elems = std::move(other_mesh._extra_ghost_elems);
      84             : 
      85             :   // Handle remaining MeshBase moves.
      86         270 :   this->post_dofobject_moves(std::move(other_mesh));
      87             : 
      88         274 :   return *this;
      89             : }
      90             : 
      91         270 : MeshBase & DistributedMesh::assign(MeshBase && other_mesh)
      92             : {
      93         270 :   *this = std::move(cast_ref<DistributedMesh&>(other_mesh));
      94             : 
      95         270 :   return *this;
      96             : }
      97             : 
      98       13066 : std::string_view DistributedMesh::subclass_first_difference_from (const MeshBase & other_mesh_base) const
      99             : {
     100         748 :   const DistributedMesh * dist_mesh_ptr =
     101       13066 :     dynamic_cast<const DistributedMesh *>(&other_mesh_base);
     102       13066 :   if (!dist_mesh_ptr)
     103           0 :     return "DistributedMesh class";
     104         748 :   const DistributedMesh & other_mesh = *dist_mesh_ptr;
     105             : 
     106             : #define CHECK_MEMBER(member_name) \
     107             :   if (member_name != other_mesh.member_name) \
     108             :     return #member_name;
     109             : 
     110       13066 :   CHECK_MEMBER(_is_serial);
     111       13066 :   CHECK_MEMBER(_is_serial_on_proc_0);
     112       13066 :   CHECK_MEMBER(_deleted_coarse_elements);
     113       13066 :   CHECK_MEMBER(_n_nodes);
     114       13066 :   CHECK_MEMBER(_n_elem);
     115       13066 :   CHECK_MEMBER(_max_node_id);
     116       13066 :   CHECK_MEMBER(_max_elem_id);
     117             :   // We expect these things to change in a prepare_for_use();
     118             :   // they're conceptually "mutable"...
     119             : /*
     120             :   CHECK_MEMBER(_next_free_local_node_id);
     121             :   CHECK_MEMBER(_next_free_local_elem_id);
     122             :   CHECK_MEMBER(_next_free_unpartitioned_node_id);
     123             :   CHECK_MEMBER(_next_free_unpartitioned_elem_id);
     124             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     125             :   CHECK_MEMBER(_next_unpartitioned_unique_id);
     126             : #endif
     127             : */
     128       13066 :   if (!this->nodes_and_elements_equal(other_mesh))
     129          63 :     return "nodes and/or elements";
     130             : 
     131       13749 :   if (_extra_ghost_elems.size() !=
     132         746 :       other_mesh._extra_ghost_elems.size())
     133           0 :     return "_extra_ghost_elems size";
     134       13003 :   for (auto & elem : _extra_ghost_elems)
     135             :     {
     136           0 :       libmesh_assert(this->query_elem_ptr(elem->id()) == elem);
     137           0 :       const Elem * other_elem = other_mesh.query_elem_ptr(elem->id());
     138           0 :       if (!other_elem ||
     139           0 :           !other_mesh._extra_ghost_elems.count(const_cast<Elem *>(other_elem)))
     140           0 :         return "_extra_ghost_elems entry";
     141             :     }
     142             : 
     143       13003 :   return "";
     144             : }
     145             : 
     146      462352 : DistributedMesh::~DistributedMesh ()
     147             : {
     148      439888 :   this->DistributedMesh::clear();  // Free nodes and elements
     149      462352 : }
     150             : 
     151             : 
     152             : // This might be specialized later, but right now it's just here to
     153             : // make sure the compiler doesn't give us a default (non-deep) copy
     154             : // constructor instead.
     155       18094 : DistributedMesh::DistributedMesh (const DistributedMesh & other_mesh) :
     156             :   DistributedMesh(static_cast<const MeshBase &>(other_mesh),
     157       18094 :                   /*bool other_is_distributed_type=*/true)
     158             : {
     159       18094 :   _deleted_coarse_elements = other_mesh._deleted_coarse_elements;
     160       18094 :   _next_free_local_node_id =
     161       18094 :     other_mesh._next_free_local_node_id;
     162       18094 :   _next_free_local_elem_id =
     163       18094 :     other_mesh._next_free_local_elem_id;
     164       18094 :   _next_free_unpartitioned_node_id =
     165       18094 :     other_mesh._next_free_unpartitioned_node_id;
     166       18094 :   _next_free_unpartitioned_elem_id =
     167       18094 :     other_mesh._next_free_unpartitioned_elem_id;
     168             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     169       18094 :   _next_unique_id =
     170       18094 :     other_mesh._next_unique_id;
     171       18094 :   _next_unpartitioned_unique_id =
     172       18094 :     other_mesh._next_unpartitioned_unique_id;
     173             : #endif
     174             : 
     175             :   // Need to copy extra_ghost_elems
     176       18094 :   for (auto & elem : other_mesh._extra_ghost_elems)
     177           0 :     _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
     178       18094 : }
     179             : 
     180             : 
     181             : 
     182       20721 : DistributedMesh::DistributedMesh (const MeshBase & other_mesh, bool other_is_distributed_type) :
     183       20721 :   UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
     184       21175 :   _is_serial_on_proc_0(other_mesh.is_serial_on_zero()),
     185       20173 :   _deleted_coarse_elements(true), // better safe than sorry...
     186       20173 :   _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0), // recomputed below
     187       21175 :   _next_free_local_node_id(this->processor_id()),
     188       21081 :   _next_free_local_elem_id(this->processor_id()),
     189       21081 :   _next_free_unpartitioned_node_id(this->n_processors()),
     190       41348 :   _next_free_unpartitioned_elem_id(this->n_processors())
     191             : {
     192             :   // Just copy, skipping preparation
     193       20721 :   this->copy_nodes_and_elements(other_mesh, true, 0, 0, 0, nullptr, true);
     194             : 
     195         548 :   this->allow_find_neighbors(other_mesh.allow_find_neighbors());
     196         548 :   this->allow_detect_interior_parents(other_mesh.allow_detect_interior_parents());
     197         548 :   this->allow_renumbering(other_mesh.allow_renumbering());
     198         548 :   this->allow_remote_element_removal(other_mesh.allow_remote_element_removal());
     199         548 :   this->skip_partitioning(other_mesh.skip_partitioning());
     200             : 
     201       20721 :   this->copy_constraint_rows(other_mesh);
     202             : 
     203         454 :   auto & this_boundary_info = this->get_boundary_info();
     204         454 :   const auto & other_boundary_info = other_mesh.get_boundary_info();
     205             : 
     206       20721 :   this_boundary_info = other_boundary_info;
     207             : 
     208         454 :   this->set_subdomain_name_map() = other_mesh.get_subdomain_name_map();
     209             : 
     210             :   // add_node()/add_elem() only increment _n_nodes/_n_elem for
     211             :   // nodes/elements they consider locally owned (or unpartitioned), so after
     212             :   // copying only the objects visible to this processor, those counts
     213             :   // reflect an incomplete local view rather than other_mesh's totals,
     214             :   // and need to be copied afresh.
     215       20721 :   _n_nodes = other_mesh.n_nodes();
     216       20721 :   _n_elem = other_mesh.n_elem();
     217       20721 :   _max_node_id = other_mesh.max_node_id();
     218       20721 :   _max_elem_id = other_mesh.max_elem_id();
     219             : 
     220             :   // If other_mesh is actually a DistributedMesh, we can just copy its raw
     221             :   // counters directly, mirroring other_mesh's own bookkeeping exactly. This
     222             :   // is done in the DistributedMesh copy constructor.
     223             :   // For any other MeshBase subclass (e.g. ReplicatedMesh) we have no
     224             :   // raw counterpart fields to copy, so derive the free-id-range
     225             :   // counters from _max_elem_id/_max_node_id the same way
     226             :   // update_parallel_id_counts() does via the set_next_ids() method. For
     227             :   // the unique id counters we copy other_mesh's own local next_unique_id()
     228             :   // and derive the next available unique ids
     229             :   // (partitioned and unpartitioned) via set_next_unique_ids().
     230       20721 :   if (!other_is_distributed_type)
     231             :     {
     232        2627 :       this->set_next_ids();
     233             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     234        2627 :       this->set_next_unique_ids(other_mesh.next_unique_id());
     235             : #endif
     236             :     }
     237             : 
     238             :   // Copy other_mesh's actual preparation state, including whatever it
     239             :   // reports for has_synched_id_counts: the counts above are a faithful copy
     240             :   // of other_mesh's own (possibly not-yet-synced) counts, not a freshly
     241             :   // verified computation, so our synced-ness should match other_mesh's.
     242       20721 :   this->_preparation = other_mesh.preparation();
     243       20721 : }
     244             : 
     245         270 : void DistributedMesh::move_nodes_and_elements(MeshBase && other_meshbase)
     246             : {
     247           4 :   DistributedMesh & other_mesh = cast_ref<DistributedMesh&>(other_meshbase);
     248             : 
     249           4 :   this->_nodes = std::move(other_mesh._nodes);
     250         270 :   this->_n_nodes = other_mesh.n_nodes();
     251             : 
     252           4 :   this->_elements = std::move(other_mesh._elements);
     253         270 :   this->_n_elem = other_mesh.n_elem();
     254             : 
     255         270 :   _is_serial = other_mesh.is_serial();
     256         270 :   _is_serial_on_proc_0 = other_mesh.is_serial_on_zero();
     257         270 :   _deleted_coarse_elements = true; // Better safe than sorry
     258             : 
     259         270 :   _max_node_id = other_mesh.max_node_id();
     260         270 :   _max_elem_id = other_mesh.max_elem_id();
     261             : 
     262         270 :   _next_free_local_node_id = other_mesh._next_free_local_node_id;
     263         270 :   _next_free_local_elem_id = other_mesh._next_free_local_elem_id;
     264         270 :   _next_free_unpartitioned_node_id = other_mesh._next_free_unpartitioned_node_id;
     265         270 :   _next_free_unpartitioned_elem_id = other_mesh._next_free_unpartitioned_elem_id;
     266             : 
     267             :   #ifdef LIBMESH_ENABLE_UNIQUE_ID
     268         270 :   _next_unpartitioned_unique_id = other_mesh._next_unpartitioned_unique_id;
     269             :   #endif
     270         270 : }
     271             : 
     272     4122757 : void DistributedMesh::set_next_ids()
     273             : {
     274     4122757 :   _next_free_unpartitioned_elem_id =
     275     4125379 :     ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
     276     4122757 :     (this->n_processors() + 1) + this->n_processors();
     277     4122757 :   _next_free_local_elem_id =
     278     4122757 :     ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     279     4122757 :     (this->n_processors() + 1) + this->processor_id();
     280             : 
     281     4122757 :   _next_free_unpartitioned_node_id =
     282     4125379 :     ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
     283     4122757 :     (this->n_processors() + 1) + this->n_processors();
     284     4122757 :   _next_free_local_node_id =
     285     4122757 :     ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     286     4122757 :     (this->n_processors() + 1) + this->processor_id();
     287     4122757 : }
     288             : 
     289             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     290     4122757 : void DistributedMesh::set_next_unique_ids(unique_id_type parallel_max_unique_id)
     291             : {
     292             :   // Unique ids are laid out in repeating groups of (n_processors()+1)
     293             :   // consecutive integers ("alignment" here means a value's position, i.e.
     294             :   // its residue mod (n_processors()+1), within one of these groups). In
     295             :   // every group, the first n_processors() slots (residues 0..n_processors()-1)
     296             :   // are each permanently earmarked for one processor's locally-owned
     297             :   // objects, and the last slot (residue n_processors()) is permanently
     298             :   // earmarked for unpartitioned objects.
     299             :   //
     300             :   // parallel_max_unique_id carries no guarantee of landing on any
     301             :   // particular residue, so rather than simply offsetting it by
     302             :   // processor_id()/n_processors(), we have to explicitly round up to the
     303             :   // start of the next full, entirely-unused group before adding this
     304             :   // processor's (or the unpartitioned pool's) fixed offset into that group.
     305             :   //
     306             :   // (parallel_max_unique_id - 1) / (n_processors()+1) + 1 is the standard
     307             :   // integer idiom for ceil(parallel_max_unique_id / (n_processors()+1)):
     308             :   // the number of complete groups needed to reach at least past the
     309             :   // current max id. Multiplying back by (n_processors()+1) gives the first
     310             :   // id of the next group that is guaranteed to be entirely unused; adding
     311             :   // n_processors(), the unpartitioned pool's fixed offset within a group,
     312             :   // lands on that pool's reserved slot in that group.
     313     4122757 :   _next_unpartitioned_unique_id =
     314     4125379 :       ((parallel_max_unique_id - 1) / (this->n_processors() + 1) + 1) *
     315     4122757 :           (this->n_processors() + 1) +
     316     4122757 :       this->n_processors();
     317             : 
     318             :   // Same idiom, but shifted by (n_processors()-1) before dividing so that
     319             :   // the rounding accounts for processor_id() possibly being less than the
     320             :   // residue the raw max id already occupies within its group; this can
     321             :   // land this processor's slot in the same group as the unpartitioned
     322             :   // pool's above, or push it into the following group, depending on where
     323             :   // the max id and this processor's offset happen to fall.
     324     4122757 :   _next_unique_id =
     325     4122757 :       ((parallel_max_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     326     4122757 :           (this->n_processors() + 1) +
     327     4122757 :       this->processor_id();
     328     4122757 : }
     329             : #endif
     330             : 
     331             : // We use cached values for these so they can be called
     332             : // from one processor without bothering the rest, but
     333             : // we may need to update those caches before doing a full
     334             : // renumbering
     335     4120130 : void DistributedMesh::update_parallel_id_counts()
     336             : {
     337             :   // This function must be run on all processors at once
     338        2908 :   parallel_object_only();
     339             : 
     340     4120130 :   _n_elem  = this->parallel_n_elem();
     341     4120130 :   _n_nodes = this->parallel_n_nodes();
     342     4120130 :   _max_node_id = this->parallel_max_node_id();
     343     4120130 :   _max_elem_id = this->parallel_max_elem_id();
     344             : 
     345     4120130 :   this->set_next_ids();
     346             : 
     347             :   // set_next_unique_ids() needs the true parallel max unique id (not just
     348             :   // this processor's possibly-stale cached _next_unique_id) since the whole
     349             :   // point of this function is to recompute counters that may have drifted
     350             :   // out of sync.
     351             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     352     4120130 :   this->set_next_unique_ids(this->parallel_max_unique_id());
     353             : #endif
     354             : 
     355     4120130 :   _n_constraint_rows = this->count_constraint_rows();
     356             : 
     357     4120130 :   this->_preparation.has_synched_id_counts = true;
     358     4120130 : }
     359             : 
     360             : 
     361             : // Or in debug mode we may want to test the uncached values without
     362             : // changing the cache
     363     4127897 : dof_id_type DistributedMesh::parallel_n_elem() const
     364             : {
     365             :   // This function must be run on all processors at once
     366        3798 :   parallel_object_only();
     367             : 
     368     4127897 :   dof_id_type n_local = this->n_local_elem();
     369     4127897 :   this->comm().sum(n_local);
     370     4127897 :   n_local += this->n_unpartitioned_elem();
     371     4127897 :   return n_local;
     372             : }
     373             : 
     374             : 
     375             : 
     376     4127476 : dof_id_type DistributedMesh::parallel_max_elem_id() const
     377             : {
     378             :   // This function must be run on all processors at once
     379       10254 :   parallel_object_only();
     380             : 
     381     4127476 :   dof_id_type max_local = 0;
     382             : 
     383             :   dofobject_container<Elem>::const_reverse_veclike_iterator
     384       10254 :     rit = _elements.rbegin();
     385             : 
     386             :   const dofobject_container<Elem>::const_reverse_veclike_iterator
     387       10254 :     rend = _elements.rend();
     388             : 
     389             :   // Look for the maximum element id.  Search backwards through
     390             :   // elements so we can break out early.  Beware of nullptr entries that
     391             :   // haven't yet been cleared from _elements.
     392    16468912 :   for (; rit != rend; ++rit)
     393             :     {
     394    15639909 :       const DofObject *d = *rit;
     395       21497 :       if (d)
     396             :         {
     397       10026 :           libmesh_assert(_elements[d->id()] == d);
     398     3298473 :           max_local = d->id() + 1;
     399     3298473 :           break;
     400             :         }
     401             :     }
     402             : 
     403     4127476 :   this->comm().max(max_local);
     404     4127476 :   return max_local;
     405             : }
     406             : 
     407             : 
     408             : 
     409             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     410     4215182 : unique_id_type DistributedMesh::parallel_max_unique_id() const
     411             : {
     412             :   // This function must be run on all processors at once
     413        3370 :   parallel_object_only();
     414             : 
     415     8430364 :   unique_id_type max_local = std::max(_next_unique_id,
     416     4666181 :                                       _next_unpartitioned_unique_id);
     417     4215182 :   this->comm().max(max_local);
     418     4215182 :   return max_local;
     419             : }
     420             : 
     421             : 
     422             : 
     423       90093 : void DistributedMesh::set_next_unique_id(unique_id_type id)
     424             : {
     425         508 :   _next_unique_id = id;
     426       90093 :   _next_unpartitioned_unique_id =
     427       90241 :     ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
     428       90093 :     (this->n_processors() + 1) + this->n_processors();
     429       90093 :   _next_unique_id =
     430       90093 :     ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     431       90093 :     (this->n_processors() + 1) + this->processor_id();
     432       90093 : }
     433             : #endif
     434             : 
     435             : 
     436             : 
     437     4128345 : dof_id_type DistributedMesh::parallel_n_nodes() const
     438             : {
     439             :   // This function must be run on all processors at once
     440        3798 :   parallel_object_only();
     441             : 
     442     4128345 :   dof_id_type n_local = this->n_local_nodes();
     443     4128345 :   this->comm().sum(n_local);
     444     4128345 :   n_local += this->n_unpartitioned_nodes();
     445     4128345 :   return n_local;
     446             : }
     447             : 
     448             : 
     449             : 
     450     4124934 : dof_id_type DistributedMesh::parallel_max_node_id() const
     451             : {
     452             :   // This function must be run on all processors at once
     453        7712 :   parallel_object_only();
     454             : 
     455     4124934 :   dof_id_type max_local = 0;
     456             : 
     457             :   dofobject_container<Node>::const_reverse_veclike_iterator
     458        7712 :     rit = _nodes.rbegin();
     459             : 
     460             :   const dofobject_container<Node>::const_reverse_veclike_iterator
     461        7712 :     rend = _nodes.rend();
     462             : 
     463             :   // Look for the maximum node id.  Search backwards through
     464             :   // nodes so we can break out early.  Beware of nullptr entries that
     465             :   // haven't yet been cleared from _nodes
     466    77839300 :   for (; rit != rend; ++rit)
     467             :     {
     468    77010367 :       const DofObject *d = *rit;
     469       74783 :       if (d)
     470             :         {
     471        7554 :           libmesh_assert(_nodes[d->id()] == d);
     472     3296001 :           max_local = d->id() + 1;
     473     3296001 :           break;
     474             :         }
     475             :     }
     476             : 
     477     4124934 :   this->comm().max(max_local);
     478     4124934 :   return max_local;
     479             : }
     480             : 
     481             : 
     482             : 
     483    46397113 : const Point & DistributedMesh::point (const dof_id_type i) const
     484             : {
     485    46397113 :   return this->node_ref(i);
     486             : }
     487             : 
     488             : 
     489             : 
     490    47514190 : const Node * DistributedMesh::node_ptr (const dof_id_type i) const
     491             : {
     492      137685 :   libmesh_assert(_nodes[i]);
     493      137685 :   libmesh_assert_equal_to (_nodes[i]->id(), i);
     494             : 
     495    47514190 :   return _nodes[i];
     496             : }
     497             : 
     498             : 
     499             : 
     500             : 
     501   675570237 : Node * DistributedMesh::node_ptr (const dof_id_type i)
     502             : {
     503      656010 :   libmesh_assert(_nodes[i]);
     504      656010 :   libmesh_assert_equal_to (_nodes[i]->id(), i);
     505             : 
     506   675570237 :   return _nodes[i];
     507             : }
     508             : 
     509             : 
     510             : 
     511             : 
     512     3535266 : const Node * DistributedMesh::query_node_ptr (const dof_id_type i) const
     513             : {
     514     3535266 :   if (const auto it = _nodes.find(i);
     515     1083543 :       it != _nodes.end())
     516             :     {
     517     3107379 :       const Node * n = *it;
     518      655656 :       libmesh_assert (!n || n->id() == i);
     519      655656 :       return n;
     520             :     }
     521             : 
     522      427887 :   return nullptr;
     523             : }
     524             : 
     525             : 
     526             : 
     527             : 
     528   394254893 : Node * DistributedMesh::query_node_ptr (const dof_id_type i)
     529             : {
     530   394254893 :   if (auto it = _nodes.find(i);
     531      246425 :       it != _nodes.end())
     532             :     {
     533   306401485 :       Node * n = *it;
     534      157969 :       libmesh_assert (!n || n->id() == i);
     535      157969 :       return n;
     536             :     }
     537             : 
     538       88456 :   return nullptr;
     539             : }
     540             : 
     541             : 
     542             : 
     543             : 
     544     3145478 : const Elem * DistributedMesh::elem_ptr (const dof_id_type i) const
     545             : {
     546       10249 :   libmesh_assert(_elements[i]);
     547       10249 :   libmesh_assert_equal_to (_elements[i]->id(), i);
     548             : 
     549     3145478 :   return _elements[i];
     550             : }
     551             : 
     552             : 
     553             : 
     554             : 
     555   298651292 : Elem * DistributedMesh::elem_ptr (const dof_id_type i)
     556             : {
     557      178609 :   libmesh_assert(_elements[i]);
     558      178609 :   libmesh_assert_equal_to (_elements[i]->id(), i);
     559             : 
     560   298651292 :   return _elements[i];
     561             : }
     562             : 
     563             : 
     564             : 
     565             : 
     566     1769577 : const Elem * DistributedMesh::query_elem_ptr (const dof_id_type i) const
     567             : {
     568     1769577 :   if (const auto it = _elements.find(i);
     569      930607 :       it != _elements.end())
     570             :     {
     571     1250980 :       const Elem * e = *it;
     572      420925 :       libmesh_assert (!e || e->id() == i);
     573      420925 :       return e;
     574             :     }
     575             : 
     576      509682 :   return nullptr;
     577             : }
     578             : 
     579             : 
     580             : 
     581             : 
     582   634052291 : Elem * DistributedMesh::query_elem_ptr (const dof_id_type i)
     583             : {
     584   634052291 :   if (auto it = _elements.find(i);
     585      218068 :       it != _elements.end())
     586             :     {
     587   547792203 :       Elem * e = *it;
     588      214366 :       libmesh_assert (!e || e->id() == i);
     589      214366 :       return e;
     590             :     }
     591             : 
     592        3702 :   return nullptr;
     593             : }
     594             : 
     595             : 
     596             : 
     597             : 
     598    45201381 : Elem * DistributedMesh::add_elem (Elem * e)
     599             : {
     600             :   // Don't try to add nullptrs!
     601       35765 :   libmesh_assert(e);
     602             : 
     603             :   // Trying to add an existing element is a no-op
     604    45201381 :   if (e->valid_id() && _elements[e->id()] == e)
     605           0 :     return e;
     606             : 
     607    45201381 :   const processor_id_type elem_procid = e->processor_id();
     608             : 
     609    45201381 :   if (!e->valid_id())
     610             :     {
     611             :       // We should only be creating new ids past the end of the range
     612             :       // of existing ids
     613        3568 :       libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
     614             :                                    _max_elem_id);
     615        3568 :       libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
     616             : 
     617             :       // Use the unpartitioned ids for unpartitioned elems, and
     618             :       // temporarily for ghost elems
     619     3891621 :       dof_id_type * next_id = &_next_free_unpartitioned_elem_id;
     620     3895189 :       if (elem_procid == this->processor_id())
     621     1135136 :         next_id = &_next_free_local_elem_id;
     622     3891621 :       e->set_id (*next_id);
     623             :     }
     624             : 
     625             :   {
     626             :     // Advance next_ids up high enough that each is pointing to an
     627             :     // unused id and any subsequent increments will still point us
     628             :     // to unused ids
     629    90402762 :     _max_elem_id = std::max(_max_elem_id,
     630    45201381 :                             static_cast<dof_id_type>(e->id()+1));
     631             : 
     632    45201381 :     if (_next_free_unpartitioned_elem_id < _max_elem_id)
     633     5302238 :       _next_free_unpartitioned_elem_id =
     634     5302238 :         ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
     635     5302238 :         (this->n_processors() + 1) + this->n_processors();
     636    45201381 :     if (_next_free_local_elem_id < _max_elem_id)
     637     3577053 :       _next_free_local_elem_id =
     638     3583382 :         ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     639     3577053 :         (this->n_processors() + 1) + this->processor_id();
     640             : 
     641             : #ifndef NDEBUG
     642             :     // We need a const dofobject_container so we don't inadvertently create
     643             :     // nullptr entries when testing for non-nullptr ones
     644       35765 :     const dofobject_container<Elem> & const_elements = _elements;
     645             : #endif
     646       35765 :     libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
     647       35765 :     libmesh_assert(!const_elements[_next_free_local_elem_id]);
     648             :   }
     649             : 
     650             :   // Don't try to overwrite existing elems
     651       35765 :   libmesh_assert (!_elements[e->id()]);
     652             : 
     653    45201381 :   _elements[e->id()] = e;
     654             : 
     655             :   // We actually added a new element.  Some of our caches might still
     656             :   // be valid, but we should clear the ones which definitely are not.
     657    45201381 :   this->clear_point_locator();
     658    45201381 :   this->clear_stored_ranges();
     659             : 
     660             :   // Try to make the cached elem data more accurate
     661    45220608 :   if (elem_procid == this->processor_id() ||
     662             :       elem_procid == DofObject::invalid_processor_id)
     663    13994444 :     _n_elem++;
     664             : 
     665             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     666    45201381 :   if (!e->valid_unique_id())
     667             :     {
     668    14556543 :       if (processor_id() == e->processor_id())
     669             :         {
     670     1135136 :           e->set_unique_id(_next_unique_id);
     671     1135136 :           _next_unique_id += this->n_processors() + 1;
     672             :         }
     673             :       else
     674             :         {
     675    13421407 :           e->set_unique_id(_next_unpartitioned_unique_id);
     676    13421407 :           _next_unpartitioned_unique_id += this->n_processors() + 1;
     677             :         }
     678             :     }
     679             :   else
     680             :     {
     681    30669278 :       _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
     682    30644838 :       _next_unique_id =
     683    30644838 :         ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     684    30644838 :         (this->n_processors() + 1) + this->processor_id();
     685             :     }
     686             : #endif
     687             : 
     688             :   // Unpartitioned elems should be added on every processor
     689             :   // And shouldn't be added in the same batch as ghost elems
     690             :   // But we might be just adding on processor 0 to
     691             :   // broadcast later
     692             :   // #ifdef DEBUG
     693             :   //   if (elem_procid == DofObject::invalid_processor_id)
     694             :   //     {
     695             :   //       dof_id_type elem_id = e->id();
     696             :   //       this->comm().max(elem_id);
     697             :   //       libmesh_assert_equal_to (elem_id, e->id());
     698             :   //     }
     699             :   // #endif
     700             : 
     701             :   // Make sure any new element is given space for any extra integers
     702             :   // we've requested
     703    45201381 :   e->add_extra_integers(_elem_integer_names.size(),
     704    45201381 :                         _elem_integer_default_values);
     705             : 
     706             :   // And set mapping type and data on any new element
     707       54992 :   e->set_mapping_type(this->default_mapping_type());
     708       54992 :   e->set_mapping_data(this->default_mapping_data());
     709             : 
     710    45201381 :   return e;
     711             : }
     712             : 
     713             : 
     714             : 
     715    15813062 : Elem * DistributedMesh::add_elem (std::unique_ptr<Elem> e)
     716             : {
     717             :   // The mesh now takes ownership of the Elem. Eventually the guts of
     718             :   // add_elem() will get moved to a private helper function, and
     719             :   // calling add_elem() directly will be deprecated.
     720    15813062 :   return add_elem(e.release());
     721             : }
     722             : 
     723             : 
     724             : 
     725     3838118 : Elem * DistributedMesh::insert_elem (Elem * e)
     726             : {
     727     3838118 :   if (_elements[e->id()])
     728     3838118 :     this->delete_elem(_elements[e->id()]);
     729             : 
     730             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     731     3838118 :   if (!e->valid_unique_id())
     732             :     {
     733           0 :       if (processor_id() == e->processor_id())
     734             :         {
     735           0 :           e->set_unique_id(_next_unique_id);
     736           0 :           _next_unique_id += this->n_processors() + 1;
     737             :         }
     738             :       else
     739             :         {
     740           0 :           e->set_unique_id(_next_unpartitioned_unique_id);
     741           0 :           _next_unpartitioned_unique_id += this->n_processors() + 1;
     742             :         }
     743             :     }
     744             :   else
     745             :     {
     746     3838118 :       _next_unique_id = std::max(_next_unique_id, e->unique_id()+1);
     747     3838118 :       _next_unique_id =
     748     3841814 :         ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     749     3838118 :         (this->n_processors() + 1) + this->processor_id();
     750             :     }
     751             : #endif
     752             : 
     753             :   // Try to make the cached elem data more accurate
     754     3838118 :   processor_id_type elem_procid = e->processor_id();
     755     3841814 :   if (elem_procid == this->processor_id() ||
     756             :       elem_procid == DofObject::invalid_processor_id)
     757     3704344 :     _n_elem++;
     758             : 
     759     3838118 :   _elements[e->id()] = e;
     760             : 
     761             :   // We actually added a new element.  Some of our caches might still
     762             :   // be valid, but we should clear the ones which definitely are not.
     763     3838118 :   this->clear_point_locator();
     764     3838118 :   this->clear_stored_ranges();
     765             : 
     766             :   // Make sure any new element is given space for any extra integers
     767             :   // we've requested
     768     3838118 :   e->add_extra_integers(_elem_integer_names.size(),
     769     3838118 :                         _elem_integer_default_values);
     770             : 
     771             :   // And set mapping type and data on any new element
     772        7392 :   e->set_mapping_type(this->default_mapping_type());
     773        7392 :   e->set_mapping_data(this->default_mapping_data());
     774             : 
     775     3838118 :   return e;
     776             : }
     777             : 
     778     3838118 : Elem * DistributedMesh::insert_elem (std::unique_ptr<Elem> e)
     779             : {
     780             :   // The mesh now takes ownership of the Elem. Eventually the guts of
     781             :   // insert_elem(Elem*) will get moved to a private helper function, and
     782             :   // calling insert_elem(Elem*) directly will be deprecated.
     783     3838118 :   return insert_elem(e.release());
     784             : }
     785             : 
     786             : 
     787    41044102 : void DistributedMesh::delete_elem(Elem * e)
     788             : {
     789        7652 :   libmesh_assert (e);
     790             : 
     791             :   // Try to make the cached elem data more accurate
     792    41044102 :   _n_elem--;
     793             : 
     794             :   // Was this a coarse element, not just a coarsening where we still
     795             :   // have some ancestor structure?  Was it a *local* element, that we
     796             :   // might have been depending on as an owner of local nodes?  We'll
     797             :   // have to be more careful with our nodes in contract() later; no
     798             :   // telling if we just locally orphaned a node that should be
     799             :   // globally retained.
     800    41053574 :   if (e->processor_id() == this->processor_id() &&
     801        3640 :       !e->parent())
     802       81593 :     _deleted_coarse_elements = true;
     803             : 
     804             :   // Delete the element from the BoundaryInfo object
     805    41044102 :   this->get_boundary_info().remove(e);
     806             : 
     807             :   // But not yet from the container; we might invalidate
     808             :   // an iterator that way!
     809             : 
     810             :   //_elements.erase(e->id());
     811             : 
     812             :   // Instead, we set it to nullptr for now
     813             : 
     814    41044102 :   _elements[e->id()] = nullptr;
     815             : 
     816             :   // delete the element
     817    41044102 :   delete e;
     818             : 
     819             :   // Some of our caches might still be valid, but we should clear the
     820             :   // ones which definitely are not.
     821    41044102 :   this->clear_point_locator();
     822    41044102 :   this->clear_stored_ranges();
     823    41044102 : }
     824             : 
     825             : 
     826             : 
     827     3212724 : void DistributedMesh::renumber_elem(const dof_id_type old_id,
     828             :                                     const dof_id_type new_id)
     829             : {
     830             :   // This could be a no-op
     831     3212724 :   if (old_id == new_id)
     832           0 :     return;
     833             : 
     834     3212724 :   Elem * el = _elements[old_id];
     835        1160 :   libmesh_assert (el);
     836        1160 :   libmesh_assert_equal_to (el->id(), old_id);
     837             : 
     838     3212724 :   el->set_id(new_id);
     839        1160 :   libmesh_assert (!_elements[new_id]);
     840     3212724 :   _elements[new_id] = el;
     841     3212724 :   _elements.erase(old_id);
     842             : 
     843             :   // Should we delete any caches here?  Our point locator indexes by
     844             :   // element pointer and should be fine with an id change.  Our stored
     845             :   // ranges are no longer sorted, which is *probably* fine, but let's
     846             :   // just be safe.
     847     3212724 :   this->clear_stored_ranges();
     848             : }
     849             : 
     850             : 
     851             : 
     852    45766867 : Node * DistributedMesh::add_point (const Point & p,
     853             :                                    const dof_id_type id,
     854             :                                    const processor_id_type proc_id)
     855             : {
     856    45766867 :   Node * old_n = this->query_node_ptr(id);
     857             : 
     858    45766867 :   if (old_n)
     859             :     {
     860           2 :       *old_n = p;
     861          60 :       old_n->processor_id() = proc_id;
     862             : 
     863          60 :       return old_n;
     864             :     }
     865             : 
     866    45726492 :   Node * n = Node::build(p, id).release();
     867    45766807 :   n->processor_id() = proc_id;
     868             : 
     869    45766807 :   return DistributedMesh::add_node(n);
     870             : }
     871             : 
     872             : 
     873        4130 : void DistributedMesh::own_node (Node & n)
     874             : {
     875             :   // This had better be a node in our mesh
     876           0 :   libmesh_assert(_nodes[n.id()] == &n);
     877             : 
     878        4130 :   _nodes[n.id()] = nullptr;
     879        4130 :   _n_nodes--;
     880             : 
     881           0 :   n.set_id(DofObject::invalid_id);
     882        4130 :   n.processor_id() = this->processor_id();
     883             : 
     884        4130 :   this->add_node(&n);
     885        4130 : }
     886             : 
     887             : 
     888    76763493 : Node * DistributedMesh::add_node (Node * n)
     889             : {
     890             :   // Don't try to add nullptrs!
     891       82685 :   libmesh_assert(n);
     892             : 
     893             :   // Trying to add an existing node is a no-op
     894    76763493 :   if (n->valid_id() && _nodes[n->id()] == n)
     895           0 :     return n;
     896             : 
     897    76763493 :   const processor_id_type node_procid = n->processor_id();
     898             : 
     899    76763493 :   if (!n->valid_id())
     900             :     {
     901             :       // We should only be creating new ids past the end of the range
     902             :       // of existing ids
     903       17832 :       libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
     904             :                                    _max_node_id);
     905       17832 :       libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
     906             : 
     907             :       // Use the unpartitioned ids for unpartitioned nodes,
     908             :       // and temporarily for ghost nodes
     909    18903241 :       dof_id_type * next_id = &_next_free_unpartitioned_node_id;
     910    18921073 :       if (node_procid == this->processor_id())
     911     2149042 :         next_id = &_next_free_local_node_id;
     912    18903241 :       n->set_id (*next_id);
     913             :     }
     914             : 
     915             :   {
     916             :     // Advance next_ids up high enough that each is pointing to an
     917             :     // unused id and any subsequent increments will still point us
     918             :     // to unused ids
     919   153526986 :     _max_node_id = std::max(_max_node_id,
     920    76763493 :                             static_cast<dof_id_type>(n->id()+1));
     921             : 
     922    76763493 :     if (_next_free_unpartitioned_node_id < _max_node_id)
     923    22098875 :       _next_free_unpartitioned_node_id =
     924    22098875 :         ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
     925    22098875 :         (this->n_processors() + 1) + this->n_processors();
     926    76763493 :     if (_next_free_local_node_id < _max_node_id)
     927    12844556 :       _next_free_local_node_id =
     928    12862837 :         ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     929    12844556 :         (this->n_processors() + 1) + this->processor_id();
     930             : 
     931             : #ifndef NDEBUG
     932             :     // We need a const dofobject_container so we don't inadvertently create
     933             :     // nullptr entries when testing for non-nullptr ones
     934       82685 :     const dofobject_container<Node> & const_nodes = _nodes;
     935             : #endif
     936       82685 :     libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
     937       82685 :     libmesh_assert(!const_nodes[_next_free_local_node_id]);
     938             :   }
     939             : 
     940             :   // Don't try to overwrite existing nodes
     941       82685 :   libmesh_assert (!_nodes[n->id()]);
     942             : 
     943    76763493 :   _nodes[n->id()] = n;
     944             : 
     945             :   // Try to make the cached node data more accurate
     946    76805591 :   if (node_procid == this->processor_id() ||
     947             :       node_procid == DofObject::invalid_processor_id)
     948    43266059 :     _n_nodes++;
     949             : 
     950             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
     951    76763493 :   if (!n->valid_unique_id())
     952             :     {
     953    45769538 :       if (processor_id() == n->processor_id())
     954             :         {
     955     2591444 :           n->set_unique_id(_next_unique_id);
     956     2591444 :           _next_unique_id += this->n_processors() + 1;
     957             :         }
     958             :       else
     959             :         {
     960    43178094 :           n->set_unique_id(_next_unpartitioned_unique_id);
     961    43178094 :           _next_unpartitioned_unique_id += this->n_processors() + 1;
     962             :         }
     963             :     }
     964             :   else
     965             :     {
     966    31126328 :       _next_unique_id = std::max(_next_unique_id, n->unique_id()+1);
     967    30993955 :       _next_unique_id =
     968    30993955 :         ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
     969    30993955 :         (this->n_processors() + 1) + this->processor_id();
     970             :     }
     971             : #endif
     972             : 
     973             :   // A new node invalidates any cached node ranges
     974    76763493 :   this->clear_stored_ranges();
     975             : 
     976    76763493 :   n->add_extra_integers(_node_integer_names.size(),
     977    76763493 :                         _node_integer_default_values);
     978             : 
     979             :   // Unpartitioned nodes should be added on every processor
     980             :   // And shouldn't be added in the same batch as ghost nodes
     981             :   // But we might be just adding on processor 0 to
     982             :   // broadcast later
     983             :   // #ifdef DEBUG
     984             :   //   if (node_procid == DofObject::invalid_processor_id)
     985             :   //     {
     986             :   //       dof_id_type node_id = n->id();
     987             :   //       this->comm().max(node_id);
     988             :   //       libmesh_assert_equal_to (node_id, n->id());
     989             :   //     }
     990             :   // #endif
     991             : 
     992    76763493 :   return n;
     993             : }
     994             : 
     995    30992556 : Node * DistributedMesh::add_node (std::unique_ptr<Node> n)
     996             : {
     997             :   // The mesh now takes ownership of the Node. Eventually the guts of
     998             :   // add_node() will get moved to a private helper function, and
     999             :   // calling add_node() directly will be deprecated.
    1000    30992556 :   return add_node(n.release());
    1001             : }
    1002             : 
    1003    52427661 : void DistributedMesh::delete_node(Node * n)
    1004             : {
    1005        4586 :   libmesh_assert(n);
    1006        4586 :   libmesh_assert(_nodes[n->id()]);
    1007             : 
    1008             :   // Try to make the cached elem data more accurate
    1009    52427661 :   _n_nodes--;
    1010             : 
    1011             :   // Delete the node from the BoundaryInfo object
    1012    52427661 :   this->get_boundary_info().remove(n);
    1013        9172 :   _constraint_rows.erase(n);
    1014             : 
    1015             :   // But not yet from the container; we might invalidate
    1016             :   // an iterator that way!
    1017             : 
    1018             :   //_nodes.erase(n->id());
    1019             : 
    1020             :   // Instead, we set it to nullptr for now
    1021             : 
    1022    52427661 :   _nodes[n->id()] = nullptr;
    1023             : 
    1024             :   // delete the node
    1025    52427661 :   delete n;
    1026             : 
    1027             :   // We've invalidated any ranges that contained this node
    1028    52427661 :   this->clear_stored_ranges();
    1029    52427661 : }
    1030             : 
    1031             : 
    1032             : 
    1033     6984483 : void DistributedMesh::renumber_node(const dof_id_type old_id,
    1034             :                                     const dof_id_type new_id)
    1035             : {
    1036             :   // This could be a no-op
    1037     6984483 :   if (old_id == new_id)
    1038           0 :     return;
    1039             : 
    1040     6983896 :   Node * nd = _nodes[old_id];
    1041        3234 :   libmesh_assert (nd);
    1042        3234 :   libmesh_assert_equal_to (nd->id(), old_id);
    1043             : 
    1044             :   // If we have nodes shipped to this processor for NodeConstraints
    1045             :   // use, then those nodes will exist in _nodes, but may not be
    1046             :   // locatable via a TopologyMap due to the insufficiency of elements
    1047             :   // connecting to them.  If local refinement then wants to create a
    1048             :   // *new* node in the same location, it will initially get a temporary
    1049             :   // id, and then make_node_ids_parallel_consistent() will try to move
    1050             :   // it to the canonical id.  We need to account for this case to
    1051             :   // avoid false positives and memory leaks.
    1052             : #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
    1053        6468 :   if (_nodes[new_id])
    1054             :     {
    1055           0 :       libmesh_assert_equal_to (*(Point *)_nodes[new_id],
    1056             :                                *(Point *)_nodes[old_id]);
    1057           0 :       _nodes.erase(new_id);
    1058             :     }
    1059             : #else
    1060             :   // If we aren't shipping nodes for NodeConstraints, there should be
    1061             :   // no reason for renumbering one node onto another.
    1062             :   libmesh_assert (!_nodes[new_id]);
    1063             : #endif
    1064     6983896 :   _nodes[new_id] = nd;
    1065     6983896 :   nd->set_id(new_id);
    1066             : 
    1067     6983896 :   _nodes.erase(old_id);
    1068             : }
    1069             : 
    1070             : 
    1071             : 
    1072      834775 : void DistributedMesh::clear ()
    1073             : {
    1074             :   // Call parent clear function
    1075      834775 :   MeshBase::clear();
    1076             : 
    1077             :   // Clear our elements and nodes
    1078             :   // There is no need to remove them from
    1079             :   // the BoundaryInfo data structure since we
    1080             :   // already cleared it.
    1081      834775 :   this->DistributedMesh::clear_elems();
    1082             : 
    1083    25629896 :   for (auto & node : _nodes)
    1084    46215504 :     delete node;
    1085             : 
    1086         964 :   _nodes.clear();
    1087             : 
    1088             :   // We're no longer distributed if we were before
    1089      834775 :   _is_serial = true;
    1090      834775 :   _is_serial_on_proc_0 = true;
    1091             : 
    1092             :   // We deleted a ton of coarse elements, but their nodes got deleted too so
    1093             :   // all is copacetic.
    1094      834775 :   _deleted_coarse_elements = false;
    1095             : 
    1096             :   // Correct our caches
    1097      834775 :   _n_nodes = 0;
    1098      834775 :   _max_node_id = 0;
    1099      835379 :   _next_free_local_node_id = this->processor_id();
    1100      834775 :   _next_free_unpartitioned_node_id = this->n_processors();
    1101      834775 : }
    1102             : 
    1103             : 
    1104             : 
    1105      841239 : void DistributedMesh::clear_elems ()
    1106             : {
    1107    10250103 :   for (auto & elem : _elements)
    1108     9408864 :     delete elem;
    1109             : 
    1110         964 :   _elements.clear();
    1111             : 
    1112             :   // Correct our caches
    1113      841239 :   _n_elem = 0;
    1114      841239 :   _max_elem_id = 0;
    1115      841843 :   _next_free_local_elem_id = this->processor_id();
    1116      841239 :   _next_free_unpartitioned_elem_id = this->n_processors();
    1117      841239 : }
    1118             : 
    1119             : 
    1120             : 
    1121      779457 : void DistributedMesh::redistribute ()
    1122             : {
    1123             :   // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
    1124      779457 :   if (!this->is_serial())
    1125             :     {
    1126             :       // Construct a MeshCommunication object to actually redistribute the nodes
    1127             :       // and elements according to the partitioner, and then to re-gather the neighbors.
    1128             :       MeshCommunication mc;
    1129       80676 :       mc.redistribute(*this);
    1130             : 
    1131       80676 :       this->update_parallel_id_counts();
    1132             : 
    1133             :       // We communicate valid neighbor links for newly-redistributed
    1134             :       // elements, but we may still have remote_elem links that should
    1135             :       // be corrected but whose correction wasn't communicated.
    1136       80952 :       this->find_neighbors(/*reset_remote_elements*/ false,
    1137             :                            /*reset_current_list*/ false /*non-default*/,
    1138             :                            /*assert_valid*/ false,
    1139         276 :                            /*check_non_remote*/ false /*non-default!*/);
    1140             : 
    1141             :       // Is this necessary?  If we are called from prepare_for_use(), this will be called
    1142             :       // anyway... but users can always call partition directly, in which case we do need
    1143             :       // to call delete_remote_elements()...
    1144             :       //
    1145             :       // Regardless of whether it's necessary, it isn't safe.  We
    1146             :       // haven't communicated new node processor_ids yet, and we can't
    1147             :       // delete nodes until we do.
    1148             :       // this->delete_remote_elements();
    1149             :     }
    1150             :   else
    1151             :     // The base class can handle non-distributed things, like
    1152             :     // notifying any GhostingFunctors of changes
    1153      698781 :     MeshBase::redistribute();
    1154      779457 : }
    1155             : 
    1156             : 
    1157             : 
    1158     2085790 : void DistributedMesh::update_post_partitioning ()
    1159             : {
    1160             :   // this->recalculate_n_partitions();
    1161             : 
    1162             :   // Let's do the base class cache clearing first, just in case our
    1163             :   // later computations are ever changed to make use of a local
    1164             :   // elements range cache
    1165     2085790 :   this->MeshBase::update_post_partitioning();
    1166             : 
    1167             :   // Partitioning changes our numbers of unpartitioned objects
    1168     2085790 :   this->update_parallel_id_counts();
    1169     2085790 : }
    1170             : 
    1171             : 
    1172             : 
    1173             : template <typename T>
    1174        3524 : void DistributedMesh::libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> & objects) const
    1175             : {
    1176             :   // This function must be run on all processors at once
    1177        3524 :   parallel_object_only();
    1178             : 
    1179        3524 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1180        3524 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1181        3524 :   const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
    1182             : 
    1183      898180 :   for (dof_id_type i=0; i != pmax_id; ++i)
    1184             :     {
    1185      894656 :       T * obj = objects[i]; // Returns nullptr if there's no map entry
    1186             : 
    1187             :       // Local lookups by id should return the requested object
    1188      894656 :       libmesh_assert(!obj || obj->id() == i);
    1189             : 
    1190             :       // All processors with an object should agree on id
    1191             : #ifndef NDEBUG
    1192      894656 :       const dof_id_type dofid = obj && obj->valid_id() ?
    1193             :         obj->id() : DofObject::invalid_id;
    1194      894656 :       libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
    1195             : #endif
    1196             : 
    1197             :       // All processors with an object should agree on processor id
    1198      894656 :       const dof_id_type procid = obj && obj->valid_processor_id() ?
    1199             :         obj->processor_id() : DofObject::invalid_processor_id;
    1200      894656 :       libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
    1201             : 
    1202      894656 :       dof_id_type min_procid = procid;
    1203      894656 :       this->comm().min(min_procid);
    1204             : 
    1205             :       // Either:
    1206             :       // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
    1207             :       // or
    1208             :       // 2.) I don't own this elem (min_procid != this->processor_id()).  (In this case I may or may not have a valid pointer to it.)
    1209             : 
    1210             :       // Original assert logic
    1211             :       // libmesh_assert (min_procid != this->processor_id() || obj);
    1212             : 
    1213             :       // More human-understandable logic...
    1214      894656 :       libmesh_assert (
    1215             :                       ((min_procid == this->processor_id()) && obj)
    1216             :                       ||
    1217             :                       (min_procid != this->processor_id())
    1218             :                       );
    1219             : 
    1220             : #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
    1221             :       // All processors with an object should agree on unique id
    1222      894656 :       const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
    1223      894656 :       libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
    1224             : #endif
    1225             :     }
    1226        3524 : }
    1227             : 
    1228             : 
    1229             : 
    1230        1762 : void DistributedMesh::libmesh_assert_valid_parallel_ids () const
    1231             : {
    1232        1762 :   this->libmesh_assert_valid_parallel_object_ids (this->_elements);
    1233        1762 :   this->libmesh_assert_valid_parallel_object_ids (this->_nodes);
    1234        1762 : }
    1235             : 
    1236             : 
    1237             : 
    1238         430 : void DistributedMesh::libmesh_assert_valid_parallel_p_levels () const
    1239             : {
    1240             : #ifndef NDEBUG
    1241             :   // This function must be run on all processors at once
    1242         430 :   parallel_object_only();
    1243             : 
    1244         430 :   dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1245             : 
    1246       22314 :   for (dof_id_type i=0; i != pmax_elem_id; ++i)
    1247             :     {
    1248       21884 :       Elem * el = _elements[i]; // Returns nullptr if there's no map entry
    1249             : 
    1250       21884 :       unsigned int p_level = el ?  (el->p_level()) : libMesh::invalid_uint;
    1251             : 
    1252             :       // All processors with an active element should agree on p level
    1253       21884 :       libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
    1254             :     }
    1255             : #endif
    1256         430 : }
    1257             : 
    1258             : 
    1259             : 
    1260             : 
    1261        1710 : void DistributedMesh::libmesh_assert_valid_parallel_flags () const
    1262             : {
    1263             : #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
    1264             :   // This function must be run on all processors at once
    1265        1710 :   parallel_object_only();
    1266             : 
    1267        1710 :   dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1268             : 
    1269      145736 :   for (dof_id_type i=0; i != pmax_elem_id; ++i)
    1270             :     {
    1271      144026 :       Elem * el = _elements[i]; // Returns nullptr if there's no map entry
    1272             : 
    1273      144026 :       unsigned int refinement_flag   = el ?
    1274       66551 :         static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
    1275      144026 :       unsigned int p_refinement_flag = el ?
    1276       66551 :         static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
    1277             : 
    1278      144026 :       libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
    1279             : 
    1280             :       // p refinement flags aren't always kept correct on inactive
    1281             :       // ghost elements
    1282      144026 :       libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
    1283             :     }
    1284             : #endif // LIBMESH_ENABLE_AMR
    1285        1710 : }
    1286             : 
    1287             : 
    1288             : 
    1289             : template <typename T>
    1290             : dof_id_type
    1291     2563322 : DistributedMesh::renumber_dof_objects(dofobject_container<T> & objects)
    1292             : {
    1293             :   // This function must be run on all processors at once
    1294         856 :   parallel_object_only();
    1295             : 
    1296             :   typedef typename dofobject_container<T>::veclike_iterator object_iterator;
    1297             : 
    1298             :   // In parallel we may not know what objects other processors have.
    1299             :   // Start by figuring out how many
    1300         856 :   dof_id_type unpartitioned_objects = 0;
    1301             : 
    1302             :   std::unordered_map<processor_id_type, dof_id_type>
    1303        1712 :     ghost_objects_from_proc;
    1304             : 
    1305     2563322 :   object_iterator it  = objects.begin();
    1306         856 :   object_iterator end = objects.end();
    1307             : 
    1308   124110281 :   while (it != end)
    1309             :     {
    1310   121546959 :       T * obj = *it;
    1311             : 
    1312             :       // Remove any nullptr container entries while we're here.
    1313   121546959 :       if (!obj)
    1314     1491382 :         it = objects.erase(it);
    1315             :       else
    1316             :         {
    1317   120055577 :           processor_id_type obj_procid = obj->processor_id();
    1318   120055577 :           if (obj_procid == DofObject::invalid_processor_id)
    1319    47563654 :             unpartitioned_objects++;
    1320             :           else
    1321    72491923 :             ghost_objects_from_proc[obj_procid]++;
    1322             : 
    1323             :           // Finally, increment the iterator
    1324       57213 :           ++it;
    1325             :         }
    1326             :     }
    1327             : 
    1328     2565034 :   std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
    1329     2564178 :   auto this_it = ghost_objects_from_proc.find(this->processor_id());
    1330         856 :   this->comm().allgather
    1331     2563840 :     ((this_it == ghost_objects_from_proc.end()) ?
    1332         518 :      dof_id_type(0) : this_it->second, objects_on_proc);
    1333             : 
    1334             : #ifndef NDEBUG
    1335         856 :   libmesh_assert(this->comm().verify(unpartitioned_objects));
    1336        2568 :   for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
    1337        1712 :     if (ghost_objects_from_proc.count(p))
    1338         993 :       libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
    1339             :     else
    1340         719 :       libmesh_assert_less_equal (0, objects_on_proc[p]);
    1341             : #endif
    1342             : 
    1343             :   // We'll renumber objects in blocks by processor id
    1344     2565034 :   std::vector<dof_id_type> first_object_on_proc(this->n_processors());
    1345    27209858 :   for (processor_id_type i=1, np=this->n_processors(); i != np; ++i)
    1346    24649960 :     first_object_on_proc[i] = first_object_on_proc[i-1] +
    1347         856 :       objects_on_proc[i-1];
    1348     2564178 :   dof_id_type next_id = first_object_on_proc[this->processor_id()];
    1349     2563322 :   dof_id_type first_free_id =
    1350     2565034 :     first_object_on_proc[this->n_processors()-1] +
    1351         856 :     objects_on_proc[this->n_processors()-1] +
    1352             :     unpartitioned_objects;
    1353             : 
    1354             :   // First set new local object ids and build request sets
    1355             :   // for non-local object ids
    1356             : 
    1357             :   // Request sets to send to each processor
    1358             :   std::map<processor_id_type, std::vector<dof_id_type>>
    1359         856 :     requested_ids;
    1360             : 
    1361             :   // We know how many objects live on each processor, so reserve() space for
    1362             :   // each.
    1363         856 :   auto ghost_end = ghost_objects_from_proc.end();
    1364    29773180 :   for (auto p : make_range(this->n_processors()))
    1365    27211570 :     if (p != this->processor_id())
    1366             :       {
    1367    24647392 :         if (const auto p_it = ghost_objects_from_proc.find(p);
    1368         856 :             p_it != ghost_end)
    1369     4177680 :           requested_ids[p].reserve(p_it->second);
    1370             :       }
    1371             : 
    1372         856 :   end = objects.end();
    1373   122618899 :   for (it = objects.begin(); it != end; ++it)
    1374             :     {
    1375   120055577 :       T * obj = *it;
    1376   120055577 :       if (!obj)
    1377           0 :         continue;
    1378   120112790 :       if (obj->processor_id() == this->processor_id())
    1379    21396492 :         obj->set_id(next_id++);
    1380    98659085 :       else if (obj->processor_id() != DofObject::invalid_processor_id)
    1381    51095431 :         requested_ids[obj->processor_id()].push_back(obj->id());
    1382             :     }
    1383             : 
    1384             :   // Next set ghost object ids from other processors
    1385             : 
    1386     2580608 :   auto gather_functor =
    1387    55239489 :     [
    1388             : #ifndef NDEBUG
    1389             :      this,
    1390             :      &first_object_on_proc,
    1391             :      &objects_on_proc,
    1392             : #endif
    1393             :      &objects]
    1394             :     (processor_id_type, const std::vector<dof_id_type> & ids,
    1395         950 :      std::vector<dof_id_type> & new_ids)
    1396             :     {
    1397         950 :       std::size_t ids_size = ids.size();
    1398     4177680 :       new_ids.resize(ids_size);
    1399             : 
    1400    55273111 :       for (std::size_t i=0; i != ids_size; ++i)
    1401             :         {
    1402    51095431 :           T * obj = objects[ids[i]];
    1403       16336 :           libmesh_assert(obj);
    1404       16336 :           libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
    1405    51111767 :           new_ids[i] = obj->id();
    1406             : 
    1407       16336 :           libmesh_assert_greater_equal (new_ids[i],
    1408             :                                         first_object_on_proc[this->processor_id()]);
    1409       16336 :           libmesh_assert_less (new_ids[i],
    1410             :                                first_object_on_proc[this->processor_id()] +
    1411             :                                objects_on_proc[this->processor_id()]);
    1412             :         }
    1413             :     };
    1414             : 
    1415     2580608 :   auto action_functor =
    1416    55239489 :     [
    1417             : #ifndef NDEBUG
    1418             :      &first_object_on_proc,
    1419             :      &objects_on_proc,
    1420             : #endif
    1421             :      &objects]
    1422             :     (processor_id_type libmesh_dbg_var(pid),
    1423             :      const std::vector<dof_id_type> & ids,
    1424         950 :      const std::vector<dof_id_type> & data)
    1425             :     {
    1426             :       // Copy the id changes we've now been informed of
    1427    55273111 :       for (auto i : index_range(ids))
    1428             :         {
    1429    51095431 :           T * obj = objects[ids[i]];
    1430       16336 :           libmesh_assert (obj);
    1431       16336 :           libmesh_assert_equal_to (obj->processor_id(), pid);
    1432       16336 :           libmesh_assert_greater_equal (data[i],
    1433             :                                         first_object_on_proc[pid]);
    1434       16336 :           libmesh_assert_less (data[i],
    1435             :                                first_object_on_proc[pid] +
    1436             :                                objects_on_proc[pid]);
    1437    51111767 :           obj->set_id(data[i]);
    1438             :         }
    1439             :     };
    1440             : 
    1441         856 :   const dof_id_type * ex = nullptr;
    1442             :   Parallel::pull_parallel_vector_data
    1443     2563322 :     (this->comm(), requested_ids, gather_functor, action_functor, ex);
    1444             : 
    1445             : #ifdef LIBMESH_ENABLE_UNIQUE_ID
    1446     2580608 :   auto unique_gather_functor =
    1447    55239489 :     [
    1448             : #ifndef NDEBUG
    1449             :      this,
    1450             : #endif
    1451             :      &objects]
    1452             :     (processor_id_type, const std::vector<dof_id_type> & ids,
    1453         950 :      std::vector<unique_id_type> & data)
    1454             :     {
    1455         950 :       std::size_t ids_size = ids.size();
    1456     4177680 :       data.resize(ids_size);
    1457             : 
    1458    55273111 :       for (std::size_t i=0; i != ids_size; ++i)
    1459             :         {
    1460    51095431 :           T * obj = objects[ids[i]];
    1461       16336 :           libmesh_assert(obj);
    1462       16336 :           libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
    1463   102190862 :           data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
    1464             :         }
    1465             :     };
    1466             : 
    1467     2580608 :   auto unique_action_functor =
    1468    55239489 :     [&objects]
    1469             :     (processor_id_type libmesh_dbg_var(pid),
    1470             :      const std::vector<dof_id_type> & ids,
    1471         950 :      const std::vector<unique_id_type> & data)
    1472             :     {
    1473    55273111 :       for (auto i : index_range(ids))
    1474             :         {
    1475    51095431 :           T * obj = objects[ids[i]];
    1476       16336 :           libmesh_assert (obj);
    1477       16336 :           libmesh_assert_equal_to (obj->processor_id(), pid);
    1478    51095431 :           if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
    1479           0 :             obj->set_unique_id(data[i]);
    1480             :         }
    1481             :     };
    1482             : 
    1483         856 :   const unique_id_type * unique_ex = nullptr;
    1484             :   Parallel::pull_parallel_vector_data
    1485     2563322 :     (this->comm(), requested_ids, unique_gather_functor,
    1486             :      unique_action_functor, unique_ex);
    1487             : #endif
    1488             : 
    1489             :   // Next set unpartitioned object ids
    1490         856 :   next_id = 0;
    1491    29773180 :   for (auto i : make_range(this->n_processors()))
    1492    27211570 :     next_id += objects_on_proc[i];
    1493   122618899 :   for (it = objects.begin(); it != end; ++it)
    1494             :     {
    1495   120055577 :       T * obj = *it;
    1496   120055577 :       if (!obj)
    1497           0 :         continue;
    1498   120055577 :       if (obj->processor_id() == DofObject::invalid_processor_id)
    1499    47563654 :         obj->set_id(next_id++);
    1500             :     }
    1501             : 
    1502             :   // Finally shuffle around objects so that container indices
    1503             :   // match ids
    1504     2563322 :   it = objects.begin();
    1505         856 :   end = objects.end();
    1506   129462455 :   while (it != end)
    1507             :     {
    1508   126899133 :       T * obj = *it;
    1509   126899133 :       if (obj) // don't try shuffling already-nullptr entries
    1510             :         {
    1511   126899133 :           T * next = objects[obj->id()];
    1512             :           // If we have to move this object
    1513   126899133 :           if (next != obj)
    1514             :             {
    1515             :               // nullptr out its original position for now
    1516             :               // (our shuffling may put another object there shortly)
    1517    43073816 :               *it = nullptr;
    1518             : 
    1519             :               // There may already be another object with this id that
    1520             :               // needs to be moved itself
    1521    56741475 :               while (next)
    1522             :                 {
    1523             :                   // We shouldn't be trying to give two objects the
    1524             :                   // same id
    1525        9414 :                   libmesh_assert_not_equal_to (next->id(), obj->id());
    1526    13667659 :                   objects[obj->id()] = obj;
    1527        9414 :                   obj = next;
    1528    13667659 :                   next = objects[obj->id()];
    1529             :                 }
    1530    43073816 :               objects[obj->id()] = obj;
    1531             :             }
    1532             :         }
    1533             : 
    1534             :       // Remove any container entries that were left as nullptr.
    1535      116912 :       if (!obj)
    1536           0 :         it = objects.erase(it);
    1537             :       else
    1538       58456 :         ++it;
    1539             :     }
    1540             : 
    1541     2564178 :   return first_free_id;
    1542             : }
    1543             : 
    1544             : 
    1545     1283331 : void DistributedMesh::renumber_nodes_and_elements ()
    1546             : {
    1547         430 :   parallel_object_only();
    1548             : 
    1549             : #ifdef DEBUG
    1550             :   // Make sure our ids and flags are consistent
    1551         430 :   this->libmesh_assert_valid_parallel_ids();
    1552         430 :   this->libmesh_assert_valid_parallel_flags();
    1553         430 :   this->libmesh_assert_valid_parallel_p_levels();
    1554             : #endif
    1555             : 
    1556         858 :   LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
    1557             : 
    1558             :   // Nodes not connected to any elements, and nullptr node entries
    1559             :   // in our container, should be deleted.  But wait!  If we've deleted coarse
    1560             :   // local elements on some processor, other processors might have ghosted
    1561             :   // nodes from it that are now no longer connected to any elements on it, but
    1562             :   // that are connected to their own semilocal elements.  We'll have to
    1563             :   // communicate to ascertain if that's the case.
    1564     1283331 :   this->comm().max(_deleted_coarse_elements);
    1565             : 
    1566             :   // What used nodes do we see on our proc?
    1567         430 :   std::set<dof_id_type> used_nodes;
    1568             : 
    1569             :   // What used node info should we send from our proc?  Could we take ownership
    1570             :   // of each node if we needed to?
    1571             :   std::map<processor_id_type, std::map<dof_id_type, bool>>
    1572         430 :     used_nodes_on_proc;
    1573             : 
    1574             :   // flag the nodes we need
    1575    62913586 :   for (auto & elem : this->element_ptr_range())
    1576   328273897 :     for (const Node & node : elem->node_ref_range())
    1577             :       {
    1578   298070250 :         const dof_id_type n = node.id();
    1579   297924878 :         used_nodes.insert(n);
    1580   298070250 :         if (_deleted_coarse_elements)
    1581             :           {
    1582     3232367 :             const processor_id_type p = node.processor_id();
    1583     3232651 :             if (p != this->processor_id())
    1584             :               {
    1585     2327522 :                 auto & used_nodes_on_p = used_nodes_on_proc[p];
    1586     2327664 :                 if (elem->processor_id() == this->processor_id())
    1587      221409 :                   used_nodes_on_p[n] = true;
    1588             :                 else
    1589         105 :                   if (!used_nodes_on_p.count(n))
    1590      567261 :                     used_nodes_on_p[n] = false;
    1591             :               }
    1592             :           }
    1593     1282471 :       }
    1594             : 
    1595     1283331 :   if (_deleted_coarse_elements)
    1596             :     {
    1597             :       // "unsigned char" == "bool, but MPI::BOOL is iffy to use"
    1598             :       typedef unsigned char boolish;
    1599             :       std::map<processor_id_type, std::vector<std::pair<dof_id_type, boolish>>>
    1600          28 :         used_nodes_on_proc_vecs;
    1601       77110 :       for (auto & [pid, nodemap] : used_nodes_on_proc)
    1602       59576 :         used_nodes_on_proc_vecs[pid].assign(nodemap.begin(), nodemap.end());
    1603             : 
    1604          28 :       std::map<dof_id_type,processor_id_type> repartitioned_node_pids;
    1605             :       std::map<processor_id_type, std::set<dof_id_type>>
    1606          28 :         repartitioned_node_sets_to_push;
    1607             : 
    1608             :       auto ids_action_functor =
    1609       59550 :         [&used_nodes, &repartitioned_node_pids,
    1610             :          &repartitioned_node_sets_to_push]
    1611             :         (processor_id_type pid,
    1612         112 :          const std::vector<std::pair<dof_id_type, boolish>> & ids_and_bools)
    1613             :         {
    1614      637024 :           for (auto [n, sender_could_become_owner] : ids_and_bools)
    1615             :             {
    1616             :               // If we don't see a use for our own node, but someone
    1617             :               // else does, better figure out who should own it next.
    1618          97 :               if (!used_nodes.count(n))
    1619             :                 {
    1620           2 :                   if (auto it = repartitioned_node_pids.find(n);
    1621           2 :                       sender_could_become_owner)
    1622             :                     {
    1623           1 :                       if (it != repartitioned_node_pids.end() &&
    1624           0 :                           pid < it->second)
    1625           0 :                         it->second = pid;
    1626             :                       else
    1627           1 :                         repartitioned_node_pids[n] = pid;
    1628             :                     }
    1629             :                   else
    1630           1 :                     if (it == repartitioned_node_pids.end())
    1631           0 :                       repartitioned_node_pids[n] =
    1632             :                         DofObject::invalid_processor_id;
    1633             : 
    1634           2 :                   repartitioned_node_sets_to_push[pid].insert(n);
    1635             :                 }
    1636             :             }
    1637       17546 :         };
    1638             : 
    1639             :       // We need two pushes instead of a pull here because we need to
    1640             :       // know *all* the queries for a particular node before we can
    1641             :       // respond to *any* of them.
    1642             :       Parallel::push_parallel_vector_data
    1643       17534 :       (this->comm(), used_nodes_on_proc_vecs, ids_action_functor);
    1644             : 
    1645             :       // Repartition (what used to be) our own nodes first
    1646       17535 :       for (auto & [n, p] : repartitioned_node_pids)
    1647             :         {
    1648           1 :           Node & node = this->node_ref(n);
    1649           0 :           libmesh_assert_equal_to(node.processor_id(), this->processor_id());
    1650           0 :           libmesh_assert_not_equal_to_msg(p, DofObject::invalid_processor_id, "Node " << n << " is lost?");
    1651           1 :           node.processor_id() = p;
    1652             :         }
    1653             : 
    1654             :       // Then push to repartition others' ghosted copies.
    1655             : 
    1656             :       std::map<processor_id_type, std::vector<std::pair<dof_id_type,processor_id_type>>>
    1657          28 :         repartitioned_node_vecs;
    1658             : 
    1659       17536 :       for (auto & [p, nodeset] : repartitioned_node_sets_to_push)
    1660             :         {
    1661           2 :           auto & rn_vec = repartitioned_node_vecs[p];
    1662           4 :           for (auto n : nodeset)
    1663           2 :             rn_vec.emplace_back(n, repartitioned_node_pids[n]);
    1664             :         }
    1665             : 
    1666             :       auto repartition_node_functor =
    1667           2 :         [this]
    1668             :         (processor_id_type libmesh_dbg_var(pid),
    1669           2 :          const std::vector<std::pair<dof_id_type, processor_id_type>> & ids_and_pids)
    1670             :         {
    1671           4 :           for (auto [n, p] : ids_and_pids)
    1672             :             {
    1673           0 :               libmesh_assert_not_equal_to(p, DofObject::invalid_processor_id);
    1674           2 :               Node & node = this->node_ref(n);
    1675           0 :               libmesh_assert_equal_to(node.processor_id(), pid);
    1676           2 :               node.processor_id() = p;
    1677             :             }
    1678       17506 :         };
    1679             : 
    1680             :       Parallel::push_parallel_vector_data
    1681       17534 :       (this->comm(), repartitioned_node_vecs, repartition_node_functor);
    1682             :     }
    1683             : 
    1684     1283331 :   _deleted_coarse_elements = false;
    1685             : 
    1686             :   // Nodes not connected to any local elements, and nullptr node entries
    1687             :   // in our container, are deleted
    1688             :   {
    1689     1283331 :     node_iterator_imp  it = _nodes.begin();
    1690         430 :     node_iterator_imp end = _nodes.end();
    1691             : 
    1692    98712540 :     while (it != end)
    1693             :       {
    1694    97429209 :         Node * nd = *it;
    1695    97429209 :         if (!nd)
    1696     3708716 :           it = _nodes.erase(it);
    1697    93720493 :         else if (!used_nodes.count(nd->id()))
    1698             :           {
    1699             :             // remove any boundary information associated with
    1700             :             // this node
    1701     2834634 :             this->get_boundary_info().remove (nd);
    1702        2828 :             _constraint_rows.erase(nd);
    1703             : 
    1704             :             // delete the node
    1705     2834634 :             delete nd;
    1706             : 
    1707     2834634 :             it = _nodes.erase(it);
    1708             :           }
    1709             :         else
    1710       45790 :           ++it;
    1711             :       }
    1712             :   }
    1713             : 
    1714     1283331 :   this->_preparation.has_removed_orphaned_nodes = true;
    1715             : 
    1716     1283331 :   if (_skip_renumber_nodes_and_elements)
    1717             :     {
    1718        1670 :       this->update_parallel_id_counts();
    1719           2 :       return;
    1720             :     }
    1721             : 
    1722             :   // Finally renumber all the elements
    1723     1281661 :   _n_elem = this->renumber_dof_objects (this->_elements);
    1724             : 
    1725             :   // and all the remaining nodes
    1726     1281661 :   _n_nodes = this->renumber_dof_objects (this->_nodes);
    1727             : 
    1728             :   // And figure out what IDs we should use when adding new nodes and
    1729             :   // new elements
    1730     1281661 :   this->update_parallel_id_counts();
    1731             : 
    1732             :   // We've probably done *some* node removal and/or renumbering; let's
    1733             :   // clear caches to be safe.
    1734     1281661 :   this->clear_stored_ranges();
    1735             : 
    1736             :   // Make sure our caches are up to date and our
    1737             :   // DofObjects are well packed
    1738             : #ifdef DEBUG
    1739         428 :   libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
    1740         428 :   libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
    1741         428 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1742         428 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1743         428 :   libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
    1744         428 :   libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
    1745         428 :   libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
    1746         428 :   libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
    1747             : 
    1748             :   // Make sure our ids and flags are consistent
    1749         428 :   this->libmesh_assert_valid_parallel_ids();
    1750         428 :   this->libmesh_assert_valid_parallel_flags();
    1751             : 
    1752             :   // And make sure we've made our numbering monotonic
    1753         428 :   MeshTools::libmesh_assert_valid_elem_ids(*this);
    1754             : #endif
    1755             : }
    1756             : 
    1757             : 
    1758             : 
    1759       17410 : void DistributedMesh::fix_broken_node_and_element_numbering ()
    1760             : {
    1761             :   // We can't use range-for here because we need access to the special
    1762             :   // iterators' methods, not just to their dereferenced values.
    1763             : 
    1764             :   // Nodes first
    1765           0 :   for (auto pr = this->_nodes.begin(),
    1766     5444770 :            end = this->_nodes.end(); pr != end; ++pr)
    1767             :     {
    1768     5427360 :       Node * n = *pr;
    1769     5427360 :       if (n != nullptr)
    1770             :         {
    1771           0 :           const dof_id_type id = pr.index();
    1772     5204550 :           n->set_id() = id;
    1773           0 :           libmesh_assert_equal_to(this->node_ptr(id), n);
    1774             :         }
    1775             :     }
    1776             : 
    1777             :   // Elements next
    1778           0 :   for (auto pr = this->_elements.begin(),
    1779     5671231 :            end = this->_elements.end(); pr != end; ++pr)
    1780             :     {
    1781     5653821 :       Elem * e = *pr;
    1782     5653821 :       if (e != nullptr)
    1783             :         {
    1784           0 :           const dof_id_type id = pr.index();
    1785     5590229 :           e->set_id() = id;
    1786           0 :           libmesh_assert_equal_to(this->elem_ptr(id), e);
    1787             :         }
    1788             :     }
    1789       17410 : }
    1790             : 
    1791             : 
    1792             : 
    1793      859071 : dof_id_type DistributedMesh::n_active_elem () const
    1794             : {
    1795         582 :   parallel_object_only();
    1796             : 
    1797             :   // Get local active elements first
    1798             :   dof_id_type active_elements =
    1799     1717560 :     static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
    1800     2576049 :                                             this->active_local_elements_end()));
    1801      859071 :   this->comm().sum(active_elements);
    1802             : 
    1803             :   // Then add unpartitioned active elements, which should exist on
    1804             :   // every processor
    1805      859071 :   active_elements +=
    1806      859071 :     static_cast<dof_id_type>(std::distance
    1807     1717560 :                              (this->active_pid_elements_begin(DofObject::invalid_processor_id),
    1808      859653 :                               this->active_pid_elements_end(DofObject::invalid_processor_id)));
    1809      859071 :   return active_elements;
    1810             : }
    1811             : 
    1812             : 
    1813             : 
    1814      557776 : void DistributedMesh::delete_remote_elements()
    1815             : {
    1816             : #ifdef DEBUG
    1817             :   // Make sure our neighbor links are all fine
    1818         402 :   MeshTools::libmesh_assert_valid_neighbors(*this);
    1819             : 
    1820             :   // And our child/parent links, and our flags
    1821         402 :   MeshTools::libmesh_assert_valid_refinement_tree(*this);
    1822             : 
    1823             :   // Make sure our ids and flags are consistent
    1824         402 :   this->libmesh_assert_valid_parallel_ids();
    1825         402 :   this->libmesh_assert_valid_parallel_flags();
    1826             : 
    1827         402 :   libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
    1828         402 :   libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
    1829         402 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1830         402 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1831         402 :   libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
    1832         402 :   libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
    1833             : #endif
    1834             : 
    1835      557776 :   _is_serial = false;
    1836      557776 :   _is_serial_on_proc_0 = false;
    1837             : 
    1838      557776 :   MeshCommunication().delete_remote_elements(*this, _extra_ghost_elems);
    1839             : 
    1840         402 :   libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
    1841             : 
    1842             :   // Now make sure the containers actually shrink - strip
    1843             :   // any newly-created nullptr voids out of the element array
    1844      557776 :   dofobject_container<Elem>::veclike_iterator e_it        = _elements.begin();
    1845         402 :   const dofobject_container<Elem>::veclike_iterator e_end = _elements.end();
    1846    55009024 :   while (e_it != e_end)
    1847    54451248 :     if (!*e_it)
    1848    39946807 :       e_it = _elements.erase(e_it);
    1849             :     else
    1850       17511 :       ++e_it;
    1851             : 
    1852      557776 :   dofobject_container<Node>::veclike_iterator n_it        = _nodes.begin();
    1853         402 :   const dofobject_container<Node>::veclike_iterator n_end = _nodes.end();
    1854   107731182 :   while (n_it != n_end)
    1855   107173406 :     if (!*n_it)
    1856    70023989 :       n_it = _nodes.erase(n_it);
    1857             :     else
    1858       48820 :       ++n_it;
    1859             : 
    1860             :   // We may have deleted no-longer-connected nodes or coarsened-away
    1861             :   // elements; let's update our caches if they're expected to be up to
    1862             :   // date.
    1863      557776 :   if (_preparation.has_synched_id_counts)
    1864      555536 :     this->update_parallel_id_counts();
    1865             : 
    1866             :   // We may have deleted nodes or elements that were the only local
    1867             :   // representatives of some particular boundary id(s); let's update
    1868             :   // those caches if they're expected to be up to date.
    1869      557776 :   if (_preparation.has_boundary_id_sets)
    1870       54599 :     this->get_boundary_info().regenerate_id_sets();
    1871             : 
    1872             : #ifdef DEBUG
    1873             :   // We might not have well-packed objects if the user didn't allow us
    1874             :   // to renumber
    1875             :   // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
    1876             :   // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
    1877             : 
    1878             :   // Make sure our neighbor links are all fine
    1879         402 :   MeshTools::libmesh_assert_valid_neighbors(*this);
    1880             : 
    1881             :   // And our child/parent links, and our flags
    1882         402 :   MeshTools::libmesh_assert_valid_refinement_tree(*this);
    1883             : 
    1884             :   // Make sure our ids and flags are consistent
    1885         402 :   this->libmesh_assert_valid_parallel_ids();
    1886         402 :   this->libmesh_assert_valid_parallel_flags();
    1887             : #endif
    1888             : 
    1889      557776 :   this->_preparation.has_removed_remote_elements = true;
    1890      557776 : }
    1891             : 
    1892             : 
    1893           0 : void DistributedMesh::add_extra_ghost_elem(Elem * e)
    1894             : {
    1895             :   // First add the elem like normal
    1896           0 :   add_elem(e);
    1897             : 
    1898             :   // Now add it to the set that won't be deleted when we call
    1899             :   // delete_remote_elements()
    1900           0 :   _extra_ghost_elems.insert(e);
    1901           0 : }
    1902             : 
    1903             : void
    1904           0 : DistributedMesh::clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems)
    1905             : {
    1906           0 :   std::set<Elem *> tmp;
    1907           0 :   std::set_difference(_extra_ghost_elems.begin(), _extra_ghost_elems.end(),
    1908             :                       extra_ghost_elems.begin(), extra_ghost_elems.end(),
    1909           0 :                       std::inserter(tmp, tmp.begin()));
    1910           0 :   _extra_ghost_elems = tmp;
    1911           0 : }
    1912             : 
    1913      106609 : void DistributedMesh::allgather()
    1914             : {
    1915      106609 :   if (_is_serial)
    1916           0 :     return;
    1917      106607 :   MeshCommunication().allgather(*this);
    1918      106607 :   _is_serial = true;
    1919      106607 :   _is_serial_on_proc_0 = true;
    1920             : 
    1921             :   // Make sure our caches are up to date and our
    1922             :   // DofObjects are well packed
    1923             : #ifdef DEBUG
    1924          48 :   libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
    1925          48 :   libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
    1926          48 :   const dof_id_type pmax_node_id = this->parallel_max_node_id();
    1927          48 :   const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
    1928          48 :   libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
    1929          48 :   libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
    1930             : 
    1931             :   // If we've disabled renumbering we can't be sure we're contiguous
    1932             :   // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
    1933             :   // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
    1934             : 
    1935             :   // Make sure our neighbor links are all fine
    1936          48 :   MeshTools::libmesh_assert_valid_neighbors(*this);
    1937             : 
    1938             :   // Make sure our ids and flags are consistent
    1939          48 :   this->libmesh_assert_valid_parallel_ids();
    1940          48 :   this->libmesh_assert_valid_parallel_flags();
    1941             : #endif
    1942             : }
    1943             : 
    1944       18885 : void DistributedMesh::gather_to_zero()
    1945             : {
    1946       18885 :   if (_is_serial_on_proc_0)
    1947           6 :     return;
    1948             : 
    1949       14576 :   _is_serial_on_proc_0 = true;
    1950       14576 :   MeshCommunication().gather(0, *this);
    1951             : }
    1952             : 
    1953             : 
    1954             : } // namespace libMesh

Generated by: LCOV version 1.14