libMesh
Loading...
Searching...
No Matches
mesh_communication.C
Go to the documentation of this file.
1// The libMesh Finite Element Library.
2// Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3
4// This library is free software; you can redistribute it and/or
5// modify it under the terms of the GNU Lesser General Public
6// License as published by the Free Software Foundation; either
7// version 2.1 of the License, or (at your option) any later version.
8
9// This library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12// Lesser General Public License for more details.
13
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
18
19
20// Local Includes
21#include "libmesh/boundary_info.h"
22#include "libmesh/distributed_mesh.h"
23#include "libmesh/elem.h"
24#include "libmesh/ghosting_functor.h"
25#include "libmesh/libmesh_config.h"
26#include "libmesh/libmesh_common.h"
27#include "libmesh/libmesh_logging.h"
28#include "libmesh/mesh_base.h"
29#include "libmesh/mesh_communication.h"
30#include "libmesh/null_output_iterator.h"
31#include "libmesh/mesh_tools.h"
32#include "libmesh/parallel.h"
33#include "libmesh/parallel_elem.h"
34#include "libmesh/parallel_node.h"
35#include "libmesh/parallel_ghost_sync.h"
36#include "libmesh/utility.h"
37#include "libmesh/remote_elem.h"
38#include "libmesh/int_range.h"
39#include "libmesh/elem_side_builder.h"
40
41// C++ Includes
42#include <numeric>
43#include <set>
44#include <unordered_set>
45#include <unordered_map>
46
47
48
49//-----------------------------------------------
50// anonymous namespace for implementation details
51namespace {
52
53using namespace libMesh;
54
55struct SyncNeighbors
56{
57 typedef std::vector<dof_id_type> datum;
58
59 SyncNeighbors(MeshBase & _mesh) :
60 mesh(_mesh) {}
61
62 MeshBase & mesh;
63
64 // Find the neighbor ids for each requested element
65 void gather_data (const std::vector<dof_id_type> & ids,
66 std::vector<datum> & neighbors) const
67 {
68 neighbors.resize(ids.size());
69
70 for (auto i : index_range(ids))
71 {
72 // Look for this element in the mesh
73 // We'd better find every element we're asked for
74 const Elem & elem = mesh.elem_ref(ids[i]);
75
76 // Return the element's neighbors
77 const unsigned int n_neigh = elem.n_neighbors();
78 neighbors[i].resize(n_neigh);
79 for (unsigned int n = 0; n != n_neigh; ++n)
80 {
81 const Elem * neigh = elem.neighbor_ptr(n);
82 if (neigh)
83 {
84 libmesh_assert_not_equal_to(neigh, remote_elem);
85 neighbors[i][n] = neigh->id();
86 }
87 else
88 neighbors[i][n] = DofObject::invalid_id;
89 }
90 }
91 }
92
93 void act_on_data (const std::vector<dof_id_type> & ids,
94 const std::vector<datum> & neighbors) const
95 {
96 for (auto i : index_range(ids))
97 {
98 Elem & elem = mesh.elem_ref(ids[i]);
99
100 const datum & new_neigh = neighbors[i];
101
102 const unsigned int n_neigh = elem.n_neighbors();
103 libmesh_assert_equal_to (n_neigh, new_neigh.size());
104
105 for (unsigned int n = 0; n != n_neigh; ++n)
106 {
107 const dof_id_type new_neigh_id = new_neigh[n];
108 const Elem * old_neigh = elem.neighbor_ptr(n);
109 if (old_neigh && old_neigh != remote_elem)
110 {
111 libmesh_assert_equal_to(old_neigh->id(), new_neigh_id);
112 }
113 else if (new_neigh_id == DofObject::invalid_id)
114 {
115 libmesh_assert (!old_neigh);
116 }
117 else
118 {
119 Elem * neigh = mesh.query_elem_ptr(new_neigh_id);
120 if (neigh)
121 elem.set_neighbor(n, neigh);
122 else
123 elem.set_neighbor(n, const_cast<RemoteElem *>(remote_elem));
124 }
125 }
126 }
127 }
128};
129
130
131void
132connect_element_families(const connected_elem_set_type & connected_elements,
133 const connected_elem_set_type & new_connected_elements,
134 connected_elem_set_type & newer_connected_elements,
135 const MeshBase * mesh)
136{
137 // mesh was an optional parameter for API backwards compatibility
138 if (mesh && !mesh->get_constraint_rows().empty())
139 {
140 // We start with the constraint connections, not ancestors,
141 // because we don't need constraining nodes of elements'
142 // ancestors' constrained nodes.
143 const auto & constraint_rows = mesh->get_constraint_rows();
144
145 std::unordered_set<const Elem *> constraining_nodes_elems;
146 for (const Elem * elem : connected_elements)
147 {
148 for (const Node & node : elem->node_ref_range())
149 {
150 // Retain all elements containing constraining nodes
151 if (const auto it = constraint_rows.find(&node);
152 it != constraint_rows.end())
153 for (auto & p : it->second)
154 {
155 const Elem * constraining_elem = p.first.first;
156 libmesh_assert(constraining_elem ==
157 mesh->elem_ptr(constraining_elem->id()));
158 if (!connected_elements.count(constraining_elem) &&
159 !new_connected_elements.count(constraining_elem))
160 constraining_nodes_elems.insert(constraining_elem);
161 }
162 }
163 }
164
165 newer_connected_elements.insert(constraining_nodes_elems.begin(),
166 constraining_nodes_elems.end());
167 }
168
169#ifdef LIBMESH_ENABLE_AMR
170
171 // Because our set is sorted by ascending level, we can traverse it
172 // in reverse order, adding parents as we go, and end up with all
173 // ancestors added. This is safe for std::set where insert doesn't
174 // invalidate iterators.
175 //
176 // This only works because we do *not* cache
177 // connected_elements.rend(), whose value can change when we insert
178 // elements which are sorted before the original rend.
179 //
180 // We're also going to get subactive descendents here, when any
181 // exist. We're iterating in the wrong direction to do that
182 // non-recursively, so we'll cop out and rely on total_family_tree.
183 // Iterating backwards does mean that we won't be querying the newly
184 // inserted subactive elements redundantly.
185
186 connected_elem_set_type::reverse_iterator
187 elem_rit = new_connected_elements.rbegin();
188
189 for (; elem_rit != new_connected_elements.rend(); ++elem_rit)
190 {
191 const Elem * elem = *elem_rit;
192 libmesh_assert(elem);
193
194 // We let ghosting functors worry about only active elements,
195 // but the remote processor needs all its semilocal elements'
196 // ancestors and active semilocal elements' descendants too.
197 for (const Elem * parent = elem->parent(); parent;
198 parent = parent->parent())
199 if (!connected_elements.count(parent) &&
200 !new_connected_elements.count(parent))
201 newer_connected_elements.insert (parent);
202
203 auto total_family_insert =
204 [&connected_elements, &new_connected_elements,
205 &newer_connected_elements]
206 (const Elem * e)
207 {
208 if (e->active() && e->has_children())
209 {
210 std::vector<const Elem *> subactive_family;
211 e->total_family_tree(subactive_family);
212 for (const auto & f : subactive_family)
213 {
215 if (!connected_elements.count(f) &&
216 !new_connected_elements.count(f))
217 newer_connected_elements.insert(f);
218 }
219 }
220 };
221
222 total_family_insert(elem);
223
224 // We also need any interior parents on this mesh, which will
225 // then need their own ancestors and descendants.
226 const Elem * interior_parent = elem->interior_parent();
227
228 // Don't try to grab interior parents from other meshes, e.g. if
229 // this was a BoundaryMesh associated with a separate Mesh.
230
231 // We can't test this if someone's using the pre-mesh-ptr API
232 libmesh_assert(!interior_parent || mesh);
233
234 if (interior_parent &&
235 interior_parent == mesh->query_elem_ptr(interior_parent->id()) &&
236 !connected_elements.count(interior_parent) &&
237 !new_connected_elements.count(interior_parent))
238 {
239 newer_connected_elements.insert (interior_parent);
240 total_family_insert(interior_parent);
241 }
242 }
243
244# ifdef DEBUG
245 // Let's be paranoid and make sure that all our ancestors
246 // really did get inserted. I screwed this up the first time
247 // by caching rend, and I can easily imagine screwing it up in
248 // the future by changing containers.
249 auto check_elem =
250 [&connected_elements,
251 &new_connected_elements,
252 &newer_connected_elements]
253 (const Elem * elem)
254 {
255 libmesh_assert(elem);
256 const Elem * parent = elem->parent();
257 if (parent)
258 libmesh_assert(connected_elements.count(parent) ||
259 new_connected_elements.count(parent) ||
260 newer_connected_elements.count(parent));
261 };
262
263 for (const auto & elem : connected_elements)
264 check_elem(elem);
265 for (const auto & elem : new_connected_elements)
266 check_elem(elem);
267 for (const auto & elem : newer_connected_elements)
268 check_elem(elem);
269# endif // DEBUG
270
271#endif // LIBMESH_ENABLE_AMR
272}
273
274
275
276void connect_nodes (const connected_elem_set_type & new_connected_elements,
277 const connected_node_set_type & connected_nodes,
278 const connected_node_set_type & new_connected_nodes,
279 connected_node_set_type & newer_connected_nodes)
280{
281 for (const auto & elem : new_connected_elements)
282 for (auto & n : elem->node_ref_range())
283 if (!connected_nodes.count(&n) &&
284 !new_connected_nodes.count(&n))
285 newer_connected_nodes.insert(&n);
286}
287
288
289} // anonymous namespace
290
291
292
293namespace libMesh
294{
295
296
301 connected_elem_set_type & connected_elements)
302{
303 for (auto & gf :
306 {
307 GhostingFunctor::map_type elements_to_ghost;
308 libmesh_assert(gf);
309 (*gf)(elem_it, elem_end, pid, elements_to_ghost);
310
311 // We can ignore the CouplingMatrix in ->second, but we
312 // need to ghost all the elements in ->first.
313 for (auto & pr : elements_to_ghost)
314 {
315 const Elem * elem = pr.first;
317 libmesh_assert(mesh.elem_ptr(elem->id()) == elem);
318 connected_elements.insert(elem);
319 }
320 }
321
322 // The GhostingFunctors won't be telling us about the elements from
323 // pid; we need to add those ourselves.
324 for (; elem_it != elem_end; ++elem_it)
325 connected_elements.insert(*elem_it);
326}
327
328
332 connected_elem_set_type & connected_elements)
333{
334 // None of these parameters are used when !LIBMESH_ENABLE_AMR.
335 libmesh_ignore(mesh, elem_it, elem_end, connected_elements);
336
337#ifdef LIBMESH_ENABLE_AMR
338 // Our XdrIO output needs inactive local elements to not have any
339 // remote_elem children. Let's make sure that doesn't happen.
340 //
341 for (const auto & elem : as_range(elem_it, elem_end))
342 {
343 if (elem->has_children())
344 for (auto & child : elem->child_ref_range())
345 if (&child != remote_elem)
346 connected_elements.insert(&child);
347 }
348#endif // LIBMESH_ENABLE_AMR
349}
350
351
352void reconnect_nodes (connected_elem_set_type & connected_elements,
353 connected_node_set_type & connected_nodes)
354{
355 // We're done using the nodes list for element decisions; now
356 // let's reuse it for nodes of the elements we've decided on.
357 connected_nodes.clear();
358
359 // Use the newer API
360 connect_nodes(connected_elements, connected_nodes, connected_nodes,
361 connected_nodes);
362}
363
364
366 connected_elem_set_type & connected_elements,
367 connected_node_set_type & connected_nodes)
368{
369 // We haven't examined any of these inputs for dependencies yet, so
370 // let's mark them all as to be examined now.
371 connected_elem_set_type new_connected_elements;
372 connected_node_set_type new_connected_nodes;
373 new_connected_elements.swap(connected_elements);
374 new_connected_nodes.swap(connected_nodes);
375
376 while (!new_connected_elements.empty() ||
377 !new_connected_nodes.empty())
378 {
379 auto [newer_connected_elements,
380 newer_connected_nodes] =
382 (mesh, connected_elements, connected_nodes,
383 new_connected_elements, new_connected_nodes);
384
385 // These have now been examined
386 connected_elements.merge(new_connected_elements);
387 connected_nodes.merge(new_connected_nodes);
388
389 // merge() doesn't guarantee empty() unless there are no
390 // duplicates, which there shouldn't be
391 libmesh_assert(new_connected_elements.empty());
392 libmesh_assert(new_connected_nodes.empty());
393
394 // These now need to be examined
395 new_connected_elements.swap(newer_connected_elements);
396 new_connected_nodes.swap(newer_connected_nodes);
397 }
398}
399
400
401std::pair<connected_elem_set_type, connected_node_set_type>
403 const connected_elem_set_type & connected_elements,
404 const connected_node_set_type & connected_nodes,
405 const connected_elem_set_type & new_connected_elements,
406 const connected_node_set_type & new_connected_nodes)
407{
408 std::pair<connected_elem_set_type, connected_node_set_type> returnval;
409 auto & [newer_connected_elements, newer_connected_nodes] = returnval;
410 connect_element_families(connected_elements, new_connected_elements,
411 newer_connected_elements, &mesh);
412
413 connect_nodes(new_connected_elements, connected_nodes,
414 new_connected_nodes, newer_connected_nodes);
415
416 return returnval;
417}
418
419
420
421
422// ------------------------------------------------------------
423// MeshCommunication class members
425{
426 // _neighboring_processors.clear();
427}
428
429
430
431#ifndef LIBMESH_HAVE_MPI // avoid spurious gcc warnings
432// ------------------------------------------------------------
434{
435 // no MPI == one processor, no redistribution
436 return;
437}
438
439#else
440// ------------------------------------------------------------
442 bool newly_coarsened_only) const
443{
444 // This method will be called after a new partitioning has been
445 // assigned to the elements. This partitioning was defined in
446 // terms of the active elements, and "trickled down" to the
447 // parents and nodes as to be consistent.
448 //
449 // The point is that the entire concept of local elements is
450 // kinda shaky in this method. Elements which were previously
451 // local may now be assigned to other processors, so we need to
452 // send those off. Similarly, we need to accept elements from
453 // other processors.
454
455 // This method is also useful in the more limited case of
456 // post-coarsening redistribution: if elements are only ghosting
457 // neighbors of their active elements, but adaptive coarsening
458 // causes an inactive element to become active, then we may need a
459 // copy of that inactive element's neighbors.
460
461 // The approach is as follows:
462 // (1) send all relevant elements we have stored to their proper homes
463 // (2) receive elements from all processors, watching for duplicates
464 // (3) deleting all nonlocal elements elements
465 // (4) obtaining required ghost elements from neighboring processors
466 libmesh_parallel_only(mesh.comm());
468 libmesh_assert (MeshTools::n_elem(mesh.unpartitioned_elements_begin(),
469 mesh.unpartitioned_elements_end()) == 0);
470
471 LOG_SCOPE("redistribute()", "MeshCommunication");
472
473 // Be compatible with both deprecated and corrected MeshBase iterator types
474 typedef std::remove_const<MeshBase::const_element_iterator::value_type>::type nc_v_t;
475
476 // We're going to sort elements-to-send by pid in one pass, to avoid
477 // sending predicated iterators through the whole mesh N_p times
478 std::unordered_map<processor_id_type, std::vector<nc_v_t>> send_to_pid;
479
480 const MeshBase::const_element_iterator send_elems_begin =
481#ifdef LIBMESH_ENABLE_AMR
482 newly_coarsened_only ?
483 mesh.flagged_elements_begin(Elem::JUST_COARSENED) :
484#endif
485 mesh.active_elements_begin();
486
487 const MeshBase::const_element_iterator send_elems_end =
488#ifdef LIBMESH_ENABLE_AMR
489 newly_coarsened_only ?
490 mesh.flagged_elements_end(Elem::JUST_COARSENED) :
491#endif
492 mesh.active_elements_end();
493
494 // See what should get sent where. We don't send to ourselves.
495 for (auto & elem : as_range(send_elems_begin, send_elems_end))
496 if (elem->processor_id() != mesh.processor_id())
497 send_to_pid[elem->processor_id()].push_back(elem);
498
499 std::map<processor_id_type, std::vector<const Node *>> all_nodes_to_send;
500 std::map<processor_id_type, std::vector<const Elem *>> all_elems_to_send;
501
502 // We may need to send constraint rows too.
503 //
504 // We know we're not changing n_constraint_rows(), so we're using
505 // friend class status here rather than using the setter that
506 // invalidates caches.
507 auto & constraint_rows = mesh._constraint_rows;
508 bool have_constraint_rows = !constraint_rows.empty();
509 mesh.comm().broadcast(have_constraint_rows);
510
511#ifdef DEBUG
512 const dof_id_type n_constraint_rows =
513 have_constraint_rows ? mesh.n_constraint_rows() : 0;
514#endif
515
516 typedef std::vector<std::pair<std::pair<dof_id_type, unsigned int>, Real>>
517 serialized_row_type;
518 std::unordered_map<processor_id_type,
519 std::vector<std::pair<dof_id_type, serialized_row_type>>>
520 all_constraint_rows_to_send;
521
522 // If we don't have any just-coarsened elements to send to a
523 // pid, then there won't be any nodes or any elements pulled
524 // in by ghosting either, and we're done with this pid.
525 for (const auto & [pid, p_elements] : send_to_pid)
526 {
527 // Build up a list of nodes and elements to send to processor pid.
528 // We will certainly send all the elements assigned to this processor,
529 // but we will also ship off any elements which are required
530 // to be ghosted and any nodes which are used by any of the
531 // above.
532
533 libmesh_assert(!p_elements.empty());
534
535 // Be compatible with both deprecated and
536 // corrected MeshBase iterator types
538
539 v_t * elempp = p_elements.data();
540 v_t * elemend = elempp + p_elements.size();
541
542#ifndef LIBMESH_ENABLE_AMR
543 // This parameter is not used when !LIBMESH_ENABLE_AMR.
544 libmesh_ignore(newly_coarsened_only);
545 libmesh_assert(!newly_coarsened_only);
546#endif
547
550 (elempp, elemend, Predicates::NotNull<v_t *>());
551
552 const MeshBase::const_element_iterator elem_end =
554 (elemend, elemend, Predicates::NotNull<v_t *>());
555
556 connected_elem_set_type elements_to_send;
557
558 // See which to-be-ghosted elements we need to send
559 query_ghosting_functors (mesh, pid, elem_it, elem_end,
560 elements_to_send);
561
562 // The inactive elements we need to send should have their
563 // immediate children present.
564 connect_children(mesh, mesh.pid_elements_begin(pid),
565 mesh.pid_elements_end(pid),
566 elements_to_send);
567
568 // Now see which other elements and nodes they depend on
569 connected_node_set_type connected_nodes;
570 connect_element_dependencies(mesh, elements_to_send,
571 connected_nodes);
572
573 all_nodes_to_send[pid].assign(connected_nodes.begin(),
574 connected_nodes.end());
575
576 all_elems_to_send[pid].assign(elements_to_send.begin(),
577 elements_to_send.end());
578
579 for (auto & [node, row] : constraint_rows)
580 {
581 if (!connected_nodes.count(node))
582 continue;
583
584 serialized_row_type serialized_row;
585 for (auto [elem_and_node, coef] : row)
586 serialized_row.emplace_back(std::make_pair(elem_and_node.first->id(),
587 elem_and_node.second),
588 coef);
589
590 all_constraint_rows_to_send[pid].emplace_back
591 (node->id(), std::move(serialized_row));
592 }
593 }
594
595 // Elem/Node unpack() automatically adds them to the given mesh
596 auto null_node_action = [](processor_id_type, const std::vector<const Node*>&){};
597 auto null_elem_action = [](processor_id_type, const std::vector<const Elem*>&){};
598
599 // Communicate nodes first since elements will need to attach to them
600 TIMPI::push_parallel_packed_range(mesh.comm(), all_nodes_to_send, &mesh,
601 null_node_action);
602
603 TIMPI::push_parallel_packed_range(mesh.comm(), all_elems_to_send, &mesh,
604 null_elem_action);
605
606 // At this point we have all the nodes and elems we need, so we can
607 // communicate any constraint rows that our targets will need.
608 if (have_constraint_rows)
609 {
610 auto constraint_row_action =
611 [&mesh, &constraint_rows]
612 (processor_id_type /* src_pid */,
613 const std::vector<std::pair<dof_id_type, serialized_row_type>> rows)
614 {
615 for (auto & [node_id, serialized_row] : rows)
616 {
618 for (auto [elem_and_node, coef] : serialized_row)
619 row.emplace_back(std::make_pair(mesh.elem_ptr(elem_and_node.first),
620 elem_and_node.second),
621 coef);
622
623 constraint_rows[mesh.node_ptr(node_id)] = row;
624 }
625 };
626
628 all_constraint_rows_to_send,
629 constraint_row_action);
630
631 }
632
633 // Check on the redistribution consistency
634#ifdef DEBUG
636
638
639 const dof_id_type new_n_constraint_rows =
640 have_constraint_rows ? mesh.n_constraint_rows() : 0;
641
642 libmesh_assert_equal_to(n_constraint_rows, new_n_constraint_rows);
643
645#endif
646
647 // If we had a point locator, it's invalid now that there are new
648 // elements it can't locate.
650
651 // Let the mesh handle any other post-redistribute() tasks, like
652 // notifying GhostingFunctors. Be sure we're just calling the base
653 // class method so we don't recurse back into ourselves here.
654 mesh.MeshBase::redistribute();
655}
656#endif // LIBMESH_HAVE_MPI
657
658
659
660#ifndef LIBMESH_HAVE_MPI // avoid spurious gcc warnings
661// ------------------------------------------------------------
663{
664 // no MPI == one processor, no need for this method...
665 return;
666}
667#else
668// ------------------------------------------------------------
670{
671 // Don't need to do anything if there is
672 // only one processor.
673 if (mesh.n_processors() == 1)
674 return;
675
676 // This function must be run on all processors at once
677 libmesh_parallel_only(mesh.comm());
678
679 LOG_SCOPE("gather_neighboring_elements()", "MeshCommunication");
680
681 //------------------------------------------------------------------
682 // The purpose of this function is to provide neighbor data structure
683 // consistency for a parallel, distributed mesh. In libMesh we require
684 // that each local element have access to a full set of valid face
685 // neighbors. In some cases this requires us to store "ghost elements" -
686 // elements that belong to other processors but we store to provide
687 // data structure consistency. Also, it is assumed that any element
688 // with a nullptr neighbor resides on a physical domain boundary. So,
689 // even our "ghost elements" must have non-nullptr neighbors. To handle
690 // this the concept of "RemoteElem" is used - a special construct which
691 // is used to denote that an element has a face neighbor, but we do
692 // not actually store detailed information about that neighbor. This
693 // is required to prevent data structure explosion.
694 //
695 // So when this method is called we should have only local elements.
696 // These local elements will then find neighbors among the local
697 // element set. After this is completed, any element with a nullptr
698 // neighbor has either (i) a face on the physical boundary of the mesh,
699 // or (ii) a neighboring element which lives on a remote processor.
700 // To handle case (ii), we communicate the global node indices connected
701 // to all such faces to our neighboring processors. They then send us
702 // all their elements with a nullptr neighbor that are connected to any
703 // of the nodes in our list.
704 //------------------------------------------------------------------
705
706 // Let's begin with finding consistent neighbor data information
707 // for all the elements we currently have. We'll use a clean
708 // slate here - clear any existing information, including RemoteElem's.
709 mesh.find_neighbors (/* reset_remote_elements = */ true,
710 /* reset_current_list = */ true);
711
712 // Get a unique message tag to use in communications
714 element_neighbors_tag = mesh.comm().get_unique_tag();
715
716 // Now any element with a nullptr neighbor either
717 // (i) lives on the physical domain boundary, or
718 // (ii) lives on an inter-processor boundary.
719 // We will now gather all the elements from adjacent processors
720 // which are of the same state, which should address all the type (ii)
721 // elements.
722
723 // A list of all the processors which *may* contain neighboring elements.
724 // (for development simplicity, just make this the identity map)
725 std::vector<processor_id_type> adjacent_processors;
726 for (auto pid : make_range(mesh.n_processors()))
727 if (pid != mesh.processor_id())
728 adjacent_processors.push_back (pid);
729
730
731 const processor_id_type n_adjacent_processors =
732 cast_int<processor_id_type>(adjacent_processors.size());
733
734 //-------------------------------------------------------------------------
735 // Let's build a list of all nodes which live on nullptr-neighbor sides.
736 // For simplicity, we will use a set to build the list, then transfer
737 // it to a vector for communication.
738 std::vector<dof_id_type> my_interface_node_list;
739 std::vector<const Elem *> my_interface_elements;
740 {
741 std::set<dof_id_type> my_interface_node_set;
742
743 // For avoiding extraneous element side construction
744 ElemSideBuilder side_builder;
745
746 // since parent nodes are a subset of children nodes, this should be sufficient
747 for (const auto & elem : mesh.active_local_element_ptr_range())
748 {
749 libmesh_assert(elem);
750
751 if (elem->on_boundary()) // denotes *any* side has a nullptr neighbor
752 {
753 my_interface_elements.push_back(elem); // add the element, but only once, even
754 // if there are multiple nullptr neighbors
755 for (auto s : elem->side_index_range())
756 if (elem->neighbor_ptr(s) == nullptr)
757 {
758 const Elem & side = side_builder(*elem, s);
759
760 for (auto n : make_range(side.n_vertices()))
761 my_interface_node_set.insert (side.node_id(n));
762 }
763 }
764 }
765
766 my_interface_node_list.reserve (my_interface_node_set.size());
767 my_interface_node_list.insert (my_interface_node_list.end(),
768 my_interface_node_set.begin(),
769 my_interface_node_set.end());
770 }
771
772 // we will now send my_interface_node_list to all of the adjacent processors.
773 // note that for the time being we will copy the list to a unique buffer for
774 // each processor so that we can use a nonblocking send and not access the
775 // buffer again until the send completes. it is my understanding that the
776 // MPI 2.1 standard seeks to remove this restriction as unnecessary, so in
777 // the future we should change this to send the same buffer to each of the
778 // adjacent processors. - BSK 11/17/2008
779 std::vector<std::vector<dof_id_type>>
780 my_interface_node_xfer_buffers (n_adjacent_processors, my_interface_node_list);
781 std::map<processor_id_type, unsigned char> n_comm_steps;
782
783 std::vector<Parallel::Request> send_requests (3*n_adjacent_processors);
784 unsigned int current_request = 0;
785
786 for (unsigned int comm_step=0; comm_step<n_adjacent_processors; comm_step++)
787 {
788 n_comm_steps[adjacent_processors[comm_step]]=1;
789 mesh.comm().send (adjacent_processors[comm_step],
790 my_interface_node_xfer_buffers[comm_step],
791 send_requests[current_request++],
792 element_neighbors_tag);
793 }
794
795 //-------------------------------------------------------------------------
796 // processor pairings are symmetric - I expect to receive an interface node
797 // list from each processor in adjacent_processors as well!
798 // now we will catch an incoming node list for each of our adjacent processors.
799 //
800 // we are done with the adjacent_processors list - note that it is in general
801 // a superset of the processors we truly share elements with. so let's
802 // clear the superset list, and we will fill it with the true list.
803 adjacent_processors.clear();
804
805 std::vector<dof_id_type> common_interface_node_list;
806
807 // we expect two classes of messages -
808 // (1) incoming interface node lists, to which we will reply with our elements
809 // touching nodes in the list, and
810 // (2) replies from the requests we sent off previously.
811 // (2.a) - nodes
812 // (2.b) - elements
813 // so we expect 3 communications from each adjacent processor.
814 // by structuring the communication in this way we hopefully impose no
815 // order on the handling of the arriving messages. in particular, we
816 // should be able to handle the case where we receive a request and
817 // all replies from processor A before even receiving a request from
818 // processor B.
819
820 for (unsigned int comm_step=0; comm_step<3*n_adjacent_processors; comm_step++)
821 {
822 //------------------------------------------------------------------
823 // catch incoming node list
826 element_neighbors_tag));
828 source_pid_idx = cast_int<processor_id_type>(status.source()),
829 dest_pid_idx = source_pid_idx;
830
831 //------------------------------------------------------------------
832 // first time - incoming request
833 if (n_comm_steps[source_pid_idx] == 1)
834 {
835 n_comm_steps[source_pid_idx]++;
836
837 mesh.comm().receive (source_pid_idx,
838 common_interface_node_list,
839 element_neighbors_tag);
840
841 // const std::size_t
842 // their_interface_node_list_size = common_interface_node_list.size();
843
844 // we now have the interface node list from processor source_pid_idx.
845 // now we can find all of our elements which touch any of these nodes
846 // and send copies back to this processor. however, we can make our
847 // search more efficient by first excluding all the nodes in
848 // their list which are not also contained in
849 // my_interface_node_list. we can do this in place as a set
850 // intersection.
851 common_interface_node_list.erase
852 (std::set_intersection (my_interface_node_list.begin(),
853 my_interface_node_list.end(),
854 common_interface_node_list.begin(),
855 common_interface_node_list.end(),
856 common_interface_node_list.begin()),
857 common_interface_node_list.end());
858
859 // if (false)
860 // libMesh::out << "[" << mesh.processor_id() << "] "
861 // << "my_interface_node_list.size()=" << my_interface_node_list.size()
862 // << ", [" << source_pid_idx << "] "
863 // << "their_interface_node_list.size()=" << their_interface_node_list_size
864 // << ", common_interface_node_list.size()=" << common_interface_node_list.size()
865 // << std::endl;
866
867 // Now we need to see which of our elements touch the nodes in the list.
868 // We will certainly send all the active elements which intersect source_pid_idx,
869 // but we will also ship off the other elements in the same family tree
870 // as the active ones for data structure consistency.
871 //
872 // FIXME - shipping full family trees is unnecessary and inefficient.
873 //
874 // We also ship any nodes connected to these elements. Note
875 // some of these nodes and elements may be replicated from
876 // other processors, but that is OK.
877 connected_elem_set_type elements_to_send;
878
879 // Technically we're not doing reconnect_nodes on this, but
880 // we might as well use the same data structure since the
881 // performance questions ought to be similar
882 connected_node_set_type connected_nodes;
883
884 // Check for quick return?
885 if (common_interface_node_list.empty())
886 {
887 // let's try to be smart here - if we have no nodes in common,
888 // we cannot share elements. so post the messages expected
889 // from us here and go on about our business.
890 // note that even though these are nonblocking sends
891 // they should complete essentially instantly, because
892 // in all cases the send buffers are empty
893 mesh.comm().send_packed_range (dest_pid_idx,
894 &mesh,
895 connected_nodes.begin(),
896 connected_nodes.end(),
897 send_requests[current_request++],
898 element_neighbors_tag);
899
900 mesh.comm().send_packed_range (dest_pid_idx,
901 &mesh,
902 elements_to_send.begin(),
903 elements_to_send.end(),
904 send_requests[current_request++],
905 element_neighbors_tag);
906
907 continue;
908 }
909 // otherwise, this really *is* an adjacent processor.
910 adjacent_processors.push_back(source_pid_idx);
911
912 std::vector<const Elem *> family_tree;
913
914 for (auto & elem : my_interface_elements)
915 {
916 std::size_t n_shared_nodes = 0;
917
918 for (auto n : make_range(elem->n_vertices()))
919 if (std::binary_search (common_interface_node_list.begin(),
920 common_interface_node_list.end(),
921 elem->node_id(n)))
922 {
923 n_shared_nodes++;
924
925 // TBD - how many nodes do we need to share
926 // before we care? certainly 2, but 1? not
927 // sure, so let's play it safe...
928 if (n_shared_nodes > 0) break;
929 }
930
931 if (n_shared_nodes) // share at least one node?
932 {
933 elem = elem->top_parent();
934
935 // avoid a lot of duplicated effort -- if we already have elem
936 // in the set its entire family tree is already in the set.
937 if (!elements_to_send.count(elem))
938 {
939#ifdef LIBMESH_ENABLE_AMR
940 elem->family_tree(family_tree);
941#else
942 family_tree.clear();
943 family_tree.push_back(elem);
944#endif
945 for (const auto & f : family_tree)
946 {
947 elem = f;
948 elements_to_send.insert (elem);
949
950 for (auto & n : elem->node_ref_range())
951 connected_nodes.insert (&n);
952 }
953 }
954 }
955 }
956
957 // The elements_to_send and connected_nodes sets now contain all
958 // the elements and nodes we need to send to this processor.
959 // All that remains is to pack up the objects (along with
960 // any boundary conditions) and send the messages off.
961 {
962 libmesh_assert (connected_nodes.empty() || !elements_to_send.empty());
963 libmesh_assert (!connected_nodes.empty() || elements_to_send.empty());
964
965 // send the nodes off to the destination processor
966 mesh.comm().send_packed_range (dest_pid_idx,
967 &mesh,
968 connected_nodes.begin(),
969 connected_nodes.end(),
970 send_requests[current_request++],
971 element_neighbors_tag);
972
973 // send the elements off to the destination processor
974 mesh.comm().send_packed_range (dest_pid_idx,
975 &mesh,
976 elements_to_send.begin(),
977 elements_to_send.end(),
978 send_requests[current_request++],
979 element_neighbors_tag);
980 }
981 }
982 //------------------------------------------------------------------
983 // second time - reply of nodes
984 else if (n_comm_steps[source_pid_idx] == 2)
985 {
986 n_comm_steps[source_pid_idx]++;
987
988 mesh.comm().receive_packed_range (source_pid_idx,
989 &mesh,
991 (Node**)nullptr,
992 element_neighbors_tag);
993 }
994 //------------------------------------------------------------------
995 // third time - reply of elements
996 else if (n_comm_steps[source_pid_idx] == 3)
997 {
998 n_comm_steps[source_pid_idx]++;
999
1000 mesh.comm().receive_packed_range (source_pid_idx,
1001 &mesh,
1003 (Elem**)nullptr,
1004 element_neighbors_tag);
1005 }
1006 //------------------------------------------------------------------
1007 // fourth time - shouldn't happen
1008 else
1009 {
1010 libMesh::err << "ERROR: unexpected number of replies: "
1011 << n_comm_steps[source_pid_idx]
1012 << std::endl;
1013 }
1014 } // done catching & processing replies associated with tag ~ 100,000pi
1015
1016 // allow any pending requests to complete
1017 Parallel::wait (send_requests);
1018
1019 // If we had a point locator, it's invalid now that there are new
1020 // elements it can't locate.
1022
1023 // We can now find neighbor information for the interfaces between
1024 // local elements and ghost elements.
1025 mesh.find_neighbors (/* reset_remote_elements = */ true,
1026 /* reset_current_list = */ false);
1027
1028 // Ghost elements may not have correct remote_elem neighbor links,
1029 // and we may not be able to locally infer correct neighbor links to
1030 // remote elements. So we synchronize ghost element neighbor links.
1031 SyncNeighbors nsync(mesh);
1032
1034 (mesh.comm(), mesh.elements_begin(), mesh.elements_end(), nsync);
1035}
1036#endif // LIBMESH_HAVE_MPI
1037
1038
1039#ifndef LIBMESH_HAVE_MPI // avoid spurious gcc warnings
1040// ------------------------------------------------------------
1042{
1043 // no MPI == one processor, no need for this method...
1044 return;
1045}
1046#else
1048{
1049
1050 // Don't need to do anything if all processors already ghost all non-local
1051 // elements.
1052 if (mesh.is_serial())
1053 return;
1054
1055 // This algorithm uses the MeshBase::flagged_pid_elements_begin/end iterators
1056 // which are only available when AMR is enabled.
1057#ifndef LIBMESH_ENABLE_AMR
1058 libmesh_error_msg("Calling MeshCommunication::send_coarse_ghosts() requires AMR to be enabled. "
1059 "Please configure libmesh with --enable-amr.");
1060#else
1061 // When we coarsen elements on a DistributedMesh, we make their
1062 // parents active. This may increase the ghosting requirements on
1063 // the processor which owns the newly-activated parent element. To
1064 // ensure ghosting requirements are satisfied, processors which
1065 // coarsen an element will send all the associated ghosted elements
1066 // to all processors which own any of the coarsened-away-element's
1067 // siblings.
1068 typedef std::unordered_map<processor_id_type, std::vector<Elem *>> ghost_map;
1069 ghost_map coarsening_elements_to_ghost;
1070
1071 const processor_id_type proc_id = mesh.processor_id();
1072 // Look for just-coarsened elements
1073 for (auto elem : as_range(mesh.flagged_pid_elements_begin(Elem::COARSEN, proc_id),
1074 mesh.flagged_pid_elements_end(Elem::COARSEN, proc_id)))
1075 {
1076 // If it's flagged for coarsening it had better have a parent
1077 libmesh_assert(elem->parent());
1078
1079 // On a distributed mesh:
1080 // If we don't own this element's parent but we do own it, then
1081 // there is a chance that we are aware of ghost elements which
1082 // the parent's owner needs us to send them.
1083 const processor_id_type their_proc_id = elem->parent()->processor_id();
1084 if (their_proc_id != proc_id)
1085 coarsening_elements_to_ghost[their_proc_id].push_back(elem);
1086 }
1087
1088 std::map<processor_id_type, std::vector<const Node *>> all_nodes_to_send;
1089 std::map<processor_id_type, std::vector<const Elem *>> all_elems_to_send;
1090
1091 const processor_id_type n_proc = mesh.n_processors();
1092
1093 for (processor_id_type p=0; p != n_proc; ++p)
1094 {
1095 if (p == proc_id)
1096 continue;
1097
1098 connected_elem_set_type elements_to_send;
1099 std::set<const Node *> nodes_to_send;
1100
1101 if (const auto it = std::as_const(coarsening_elements_to_ghost).find(p);
1102 it != coarsening_elements_to_ghost.end())
1103 {
1104 const std::vector<Elem *> & elems = it->second;
1105 libmesh_assert(elems.size());
1106
1107 // Make some fake element iterators defining this vector of
1108 // elements
1109 Elem * const * elempp = const_cast<Elem * const *>(elems.data());
1110 Elem * const * elemend = elempp+elems.size();
1111 const MeshBase::const_element_iterator elem_it =
1113 const MeshBase::const_element_iterator elem_end =
1115
1116 for (auto & gf : as_range(mesh.ghosting_functors_begin(),
1117 mesh.ghosting_functors_end()))
1118 {
1119 GhostingFunctor::map_type elements_to_ghost;
1120 libmesh_assert(gf);
1121 (*gf)(elem_it, elem_end, p, elements_to_ghost);
1122
1123 // We can ignore the CouplingMatrix in ->second, but we
1124 // need to ghost all the elements in ->first.
1125 for (auto & pr : elements_to_ghost)
1126 {
1127 const Elem * elem = pr.first;
1128 libmesh_assert(elem);
1129 while (elem)
1130 {
1131 libmesh_assert(elem != remote_elem);
1132 elements_to_send.insert(elem);
1133 for (auto & n : elem->node_ref_range())
1134 nodes_to_send.insert(&n);
1135 elem = elem->parent();
1136 }
1137 }
1138 }
1139
1140 all_nodes_to_send[p].assign(nodes_to_send.begin(), nodes_to_send.end());
1141 all_elems_to_send[p].assign(elements_to_send.begin(), elements_to_send.end());
1142 }
1143 }
1144
1145 // Elem/Node unpack() automatically adds them to the given mesh
1146 auto null_node_action = [](processor_id_type, const std::vector<const Node*>&){};
1147 auto null_elem_action = [](processor_id_type, const std::vector<const Elem*>&){};
1148
1149 // Communicate nodes first since elements will need to attach to them
1150 TIMPI::push_parallel_packed_range(mesh.comm(), all_nodes_to_send, &mesh,
1151 null_node_action);
1152
1153 TIMPI::push_parallel_packed_range(mesh.comm(), all_elems_to_send, &mesh,
1154 null_elem_action);
1155
1156#endif // LIBMESH_ENABLE_AMR
1157}
1158
1159#endif // LIBMESH_HAVE_MPI
1160
1161#ifndef LIBMESH_HAVE_MPI // avoid spurious gcc warnings
1162// ------------------------------------------------------------
1164{
1165 // no MPI == one processor, no need for this method...
1166 return;
1167}
1168#else
1169// ------------------------------------------------------------
1171{
1172 // Don't need to do anything if there is
1173 // only one processor.
1174 if (mesh.n_processors() == 1)
1175 return;
1176
1177 // This function must be run on all processors at once
1178 libmesh_parallel_only(mesh.comm());
1179
1180 LOG_SCOPE("broadcast()", "MeshCommunication");
1181
1182 // Explicitly clear the mesh on all but processor 0.
1183 if (mesh.processor_id() != 0)
1184 mesh.clear();
1185
1186 // We may have set extra data only on processor 0 in a read()
1191
1192 // We may have set mapping data only on processor 0 in a read()
1193 unsigned char map_type = mesh.default_mapping_type();
1194 unsigned char map_data = mesh.default_mapping_data();
1195 mesh.comm().broadcast(map_type);
1196 mesh.comm().broadcast(map_data);
1199
1200 // Broadcast nodes
1202 mesh.nodes_begin(),
1203 mesh.nodes_end(),
1204 &mesh,
1206
1207 // Broadcast elements from coarsest to finest, so that child
1208 // elements will see their parents already in place.
1209 //
1210 // When restarting from a checkpoint, we may have elements which are
1211 // assigned to a processor but which have not yet been sent to that
1212 // processor, so we need to use a paranoid n_levels() count and not
1213 // the usual fast algorithm.
1214 const unsigned int n_levels = MeshTools::paranoid_n_levels(mesh);
1215
1216 for (unsigned int l=0; l != n_levels; ++l)
1218 mesh.level_elements_begin(l),
1219 mesh.level_elements_end(l),
1220 &mesh,
1222
1223 // Make sure mesh_dimension and elem_dimensions are consistent.
1225
1226 // Make sure mesh id counts are consistent.
1228
1229 // We may have constraint rows on IsoGeometric Analysis meshes. We
1230 // don't want to send these along with constrained nodes (like we
1231 // send boundary info for those nodes) because the associated rows'
1232 // elements may not exist at that point.
1233 //
1234 // We know we're not changing n_constraint_rows(), so we're using
1235 // friend class status here rather than using the setter that
1236 // invalidates caches.
1237 auto & constraint_rows = mesh._constraint_rows;
1238
1239 bool have_constraint_rows = !constraint_rows.empty();
1240 mesh.comm().broadcast(have_constraint_rows);
1241 if (have_constraint_rows)
1242 {
1243 std::map<dof_id_type,
1244 std::vector<std::tuple<dof_id_type, unsigned int, Real>>>
1245 serialized_rows;
1246
1247 for (auto & row : constraint_rows)
1248 {
1249 const Node * node = row.first;
1250 const dof_id_type rowid = node->id();
1251 libmesh_assert(node == mesh.node_ptr(rowid));
1252
1253 std::vector<std::tuple<dof_id_type, unsigned int, Real>>
1254 serialized_row;
1255 for (auto & entry : row.second)
1256 serialized_row.push_back
1257 (std::make_tuple(entry.first.first->id(),
1258 entry.first.second, entry.second));
1259
1260 serialized_rows.emplace(rowid, std::move(serialized_row));
1261 }
1262
1263 mesh.comm().broadcast(serialized_rows);
1264 if (mesh.processor_id() != 0)
1265 {
1266 constraint_rows.clear();
1267
1268 for (auto & row : serialized_rows)
1269 {
1270 const dof_id_type rowid = row.first;
1271 const Node * node = mesh.node_ptr(rowid);
1272
1273 std::vector<std::pair<std::pair<const Elem *, unsigned int>, Real>>
1274 deserialized_row;
1275 for (auto & entry : row.second)
1276 deserialized_row.push_back
1277 (std::make_pair(std::make_pair(mesh.elem_ptr(std::get<0>(entry)),
1278 std::get<1>(entry)),
1279 std::get<2>(entry)));
1280
1281 constraint_rows.emplace(node, deserialized_row);
1282 }
1283 }
1284 }
1285
1286 // Broadcast all of the named entity information
1290
1291 // If we had a point locator, it's invalid now that there are new
1292 // elements it can't locate.
1294
1299
1300#ifdef DEBUG
1301 MeshTools::libmesh_assert_valid_procids<Elem>(mesh);
1302 MeshTools::libmesh_assert_valid_procids<Node>(mesh);
1303#endif
1304}
1305#endif // LIBMESH_HAVE_MPI
1306
1307
1308
1309#ifndef LIBMESH_HAVE_MPI // avoid spurious gcc warnings
1310// ------------------------------------------------------------
1312{
1313 // no MPI == one processor, no need for this method...
1314 return;
1315}
1316#else
1317// ------------------------------------------------------------
1318void MeshCommunication::gather (const processor_id_type root_id, MeshBase & mesh) const
1319{
1320 // Check for quick return
1321 if (mesh.n_processors() == 1)
1322 return;
1323
1324 // This function must be run on all processors at once
1325 libmesh_parallel_only(mesh.comm());
1326
1327 LOG_SCOPE("(all)gather()", "MeshCommunication");
1328
1329 // Ensure we don't build too big a buffer at once
1330 static const std::size_t approx_total_buffer_size = 1e8;
1331 const std::size_t approx_each_buffer_size =
1332 approx_total_buffer_size / mesh.comm().size();
1333
1334 (root_id == DofObject::invalid_processor_id) ?
1335
1337 mesh.nodes_begin(),
1338 mesh.nodes_end(),
1340 approx_each_buffer_size) :
1341
1342 mesh.comm().gather_packed_range (root_id,
1343 &mesh,
1344 mesh.nodes_begin(),
1345 mesh.nodes_end(),
1347 approx_each_buffer_size);
1348
1349 // Gather elements from coarsest to finest, so that child
1350 // elements will see their parents already in place.
1351 const unsigned int n_levels = MeshTools::n_levels(mesh);
1352
1353 for (unsigned int l=0; l != n_levels; ++l)
1354 (root_id == DofObject::invalid_processor_id) ?
1355
1357 mesh.level_elements_begin(l),
1358 mesh.level_elements_end(l),
1360 approx_each_buffer_size) :
1361
1362 mesh.comm().gather_packed_range (root_id,
1363 &mesh,
1364 mesh.level_elements_begin(l),
1365 mesh.level_elements_end(l),
1367 approx_each_buffer_size);
1368
1369 // If we had a point locator, it's invalid now that there are new
1370 // elements it can't locate.
1372
1373 // We may have constraint rows on IsoGeometric Analysis meshes. We
1374 // don't want to send these along with constrained nodes (like we
1375 // send boundary info for those nodes) because the associated rows'
1376 // elements may not exist at that point.
1377 //
1378 // We know we're not changing n_constraint_rows(), so we're using
1379 // friend class status here rather than using the setter that
1380 // invalidates caches.
1381 auto & constraint_rows = mesh._constraint_rows;
1382 bool have_constraint_rows = !constraint_rows.empty();
1383 mesh.comm().max(have_constraint_rows);
1384 if (have_constraint_rows)
1385 {
1386 std::map<dof_id_type,
1387 std::vector<std::tuple<dof_id_type, unsigned int, Real>>>
1388 serialized_rows;
1389
1390 for (auto & row : constraint_rows)
1391 {
1392 const Node * node = row.first;
1393 const dof_id_type rowid = node->id();
1394 libmesh_assert(node == mesh.node_ptr(rowid));
1395
1396 std::vector<std::tuple<dof_id_type, unsigned int, Real>>
1397 serialized_row;
1398 for (auto & entry : row.second)
1399 serialized_row.push_back
1400 (std::make_tuple(entry.first.first->id(),
1401 entry.first.second, entry.second));
1402
1403 serialized_rows.emplace(rowid, std::move(serialized_row));
1404 }
1405
1406 if (root_id == DofObject::invalid_processor_id)
1407 mesh.comm().set_union(serialized_rows);
1408 else
1409 mesh.comm().set_union(serialized_rows, root_id);
1410
1411 if (root_id == DofObject::invalid_processor_id ||
1412 root_id == mesh.processor_id())
1413 {
1414 for (auto & row : serialized_rows)
1415 {
1416 const dof_id_type rowid = row.first;
1417 const Node * node = mesh.node_ptr(rowid);
1418
1419 std::vector<std::pair<std::pair<const Elem *, unsigned int>, Real>>
1420 deserialized_row;
1421 for (auto & entry : row.second)
1422 deserialized_row.push_back
1423 (std::make_pair(std::make_pair(mesh.elem_ptr(std::get<0>(entry)),
1424 std::get<1>(entry)),
1425 std::get<2>(entry)));
1426
1427 constraint_rows.emplace(node, deserialized_row);
1428 }
1429 }
1430#ifdef DEBUG
1432#endif
1433 }
1434
1435
1436 // If we are doing an allgather(), perform sanity check on the result.
1437 if (root_id == DofObject::invalid_processor_id)
1438 {
1441 }
1442
1443 // Inform new elements of their neighbors,
1444 // while resetting all remote_elem links on
1445 // the ranks which did the gather.
1448 root_id == mesh.processor_id());
1449
1450 // All done, but let's make sure it's done correctly
1451
1452#ifdef DEBUG
1454#endif
1455}
1456#endif // LIBMESH_HAVE_MPI
1457
1458
1459
1460// Functor for make_elems_parallel_consistent and
1461// make_node_ids_parallel_consistent
1462namespace {
1463
1464struct SyncIds
1465{
1466 typedef dof_id_type datum;
1467 typedef void (MeshBase::*renumber_obj)(dof_id_type, dof_id_type);
1468
1469 SyncIds(MeshBase & _mesh, renumber_obj _renumberer) :
1470 mesh(_mesh),
1471 renumber(_renumberer) {}
1472
1474 renumber_obj renumber;
1475 // renumber_obj & renumber;
1476
1477 // Find the id of each requested DofObject -
1478 // Parallel::sync_* already did the work for us
1479 void gather_data (const std::vector<dof_id_type> & ids,
1480 std::vector<datum> & ids_out) const
1481 {
1482 ids_out = ids;
1483 }
1484
1485 void act_on_data (const std::vector<dof_id_type> & old_ids,
1486 const std::vector<datum> & new_ids) const
1487 {
1488 for (auto i : index_range(old_ids))
1489 if (old_ids[i] != new_ids[i])
1490 (mesh.*renumber)(old_ids[i], new_ids[i]);
1491 }
1492};
1493
1494
1495struct SyncNodeIds
1496{
1497 typedef dof_id_type datum;
1498
1499 SyncNodeIds(MeshBase & _mesh) :
1500 mesh(_mesh) {}
1501
1502 MeshBase & mesh;
1503
1504 // We only know a Node id() is definitive if we own the Node or if
1505 // we're told it's definitive. We keep track of the latter cases by
1506 // putting definitively id'd ghost nodes into this set.
1507 typedef std::unordered_set<const Node *> uset_type;
1509
1510 // We should never be told two different definitive ids for the same
1511 // node, but let's check on that in debug mode.
1512#ifdef DEBUG
1513 typedef std::unordered_map<dof_id_type, dof_id_type> umap_type;
1515#endif
1516
1517 // Find the id of each requested DofObject -
1518 // Parallel::sync_* already tried to do the work for us, but we can
1519 // only say the result is definitive if we own the DofObject or if
1520 // we were given the definitive result from another processor.
1521 void gather_data (const std::vector<dof_id_type> & ids,
1522 std::vector<datum> & ids_out) const
1523 {
1524 ids_out.clear();
1525 ids_out.resize(ids.size(), DofObject::invalid_id);
1526
1527 for (auto i : index_range(ids))
1528 {
1529 const dof_id_type id = ids[i];
1530 const Node * node = mesh.query_node_ptr(id);
1531 if (node && (node->processor_id() == mesh.processor_id() ||
1532 definitive_ids.count(node)))
1533 ids_out[i] = id;
1534 }
1535 }
1536
1537 bool act_on_data (const std::vector<dof_id_type> & old_ids,
1538 const std::vector<datum> & new_ids)
1539 {
1540 bool data_changed = false;
1541 for (auto i : index_range(old_ids))
1542 {
1543 const dof_id_type new_id = new_ids[i];
1544
1545 const dof_id_type old_id = old_ids[i];
1546
1547 Node * node = mesh.query_node_ptr(old_id);
1548
1549 // If we can't find the node we were asking about, another
1550 // processor must have already given us the definitive id
1551 // for it
1552 if (!node)
1553 {
1554 // But let's check anyway in debug mode
1555#ifdef DEBUG
1557 (definitive_renumbering.count(old_id));
1558 libmesh_assert_equal_to
1559 (new_id, definitive_renumbering[old_id]);
1560#endif
1561 continue;
1562 }
1563
1564 // If we asked for an id but there's no definitive id ready
1565 // for us yet, then we can't quit trying to sync yet.
1566 if (new_id == DofObject::invalid_id)
1567 {
1568 // But we might have gotten a definitive id from a
1569 // different request
1570 if (!definitive_ids.count(mesh.node_ptr(old_id)))
1571 data_changed = true;
1572 }
1573 else
1574 {
1575 if (node->processor_id() != mesh.processor_id())
1576 definitive_ids.insert(node);
1577 if (old_id != new_id)
1578 {
1579#ifdef DEBUG
1581 (!definitive_renumbering.count(old_id));
1582 definitive_renumbering[old_id] = new_id;
1583#endif
1584 mesh.renumber_node(old_id, new_id);
1585 data_changed = true;
1586 }
1587 }
1588 }
1589 return data_changed;
1590 }
1591};
1592
1593
1594#ifdef LIBMESH_ENABLE_AMR
1595struct SyncPLevels
1596{
1597 typedef std::pair<unsigned char,unsigned char> datum;
1598
1599 SyncPLevels(MeshBase & _mesh) :
1600 mesh(_mesh) {}
1601
1602 MeshBase & mesh;
1603
1604 // Find the p_level of each requested Elem
1605 void gather_data (const std::vector<dof_id_type> & ids,
1606 std::vector<datum> & ids_out) const
1607 {
1608 ids_out.reserve(ids.size());
1609
1610 for (const auto & id : ids)
1611 {
1612 Elem & elem = mesh.elem_ref(id);
1613 ids_out.push_back
1614 (std::make_pair(cast_int<unsigned char>(elem.p_level()),
1615 static_cast<unsigned char>(elem.p_refinement_flag())));
1616 }
1617 }
1618
1619 void act_on_data (const std::vector<dof_id_type> & old_ids,
1620 const std::vector<datum> & new_p_levels) const
1621 {
1622 for (auto i : index_range(old_ids))
1623 {
1624 Elem & elem = mesh.elem_ref(old_ids[i]);
1625 // Make sure these are consistent
1627 (new_p_levels[i].first,
1628 static_cast<Elem::RefinementState>(new_p_levels[i].second));
1629 // Make sure parents' levels are consistent
1630 elem.set_p_level(new_p_levels[i].first);
1631 }
1632 }
1633};
1634#endif // LIBMESH_ENABLE_AMR
1635
1636
1637#ifdef LIBMESH_ENABLE_UNIQUE_ID
1638template <typename DofObjSubclass>
1639struct SyncUniqueIds
1640{
1641 typedef unique_id_type datum;
1642 typedef DofObjSubclass* (MeshBase::*query_obj)(const dof_id_type);
1643
1644 SyncUniqueIds(MeshBase &_mesh, query_obj _querier) :
1645 mesh(_mesh),
1646 query(_querier) {}
1647
1648 MeshBase & mesh;
1649 query_obj query;
1650
1651 // Find the id of each requested DofObject -
1652 // Parallel::sync_* already did the work for us
1653 void gather_data (const std::vector<dof_id_type> & ids,
1654 std::vector<datum> & ids_out) const
1655 {
1656 ids_out.reserve(ids.size());
1657
1658 for (const auto & id : ids)
1659 {
1660 DofObjSubclass * d = (mesh.*query)(id);
1661 libmesh_assert(d);
1662 ids_out.push_back(d->unique_id());
1663 }
1664 }
1665
1666 void act_on_data (const std::vector<dof_id_type> & ids,
1667 const std::vector<datum> & unique_ids) const
1668 {
1669 for (auto i : index_range(ids))
1670 {
1671 DofObjSubclass * d = (mesh.*query)(ids[i]);
1672 libmesh_assert(d);
1673 d->set_unique_id(unique_ids[i]);
1674 }
1675 }
1676};
1677#endif // LIBMESH_ENABLE_UNIQUE_ID
1678
1679template <typename DofObjSubclass>
1680struct SyncBCIds
1681{
1682 typedef std::vector<boundary_id_type> datum;
1683
1684 SyncBCIds(MeshBase &_mesh) :
1685 mesh(_mesh) {}
1686
1687 MeshBase & mesh;
1688
1689 // Find the id of each requested DofObject -
1690 // Parallel::sync_* already did the work for us
1691 void gather_data (const std::vector<dof_id_type> & ids,
1692 std::vector<datum> & ids_out) const
1693 {
1694 ids_out.reserve(ids.size());
1695
1696 const BoundaryInfo & boundary_info = mesh.get_boundary_info();
1697
1698 for (const auto & id : ids)
1699 {
1700 Node * n = mesh.query_node_ptr(id);
1701 libmesh_assert(n);
1702 std::vector<boundary_id_type> bcids;
1703 boundary_info.boundary_ids(n, bcids);
1704 ids_out.push_back(std::move(bcids));
1705 }
1706 }
1707
1708 void act_on_data (const std::vector<dof_id_type> & ids,
1709 const std::vector<datum> & bcids) const
1710 {
1711 BoundaryInfo & boundary_info = mesh.get_boundary_info();
1712
1713 for (auto i : index_range(ids))
1714 {
1715 Node * n = mesh.query_node_ptr(ids[i]);
1716 libmesh_assert(n);
1717 boundary_info.add_node(n, bcids[i]);
1718 }
1719 }
1720};
1721
1722}
1723
1724
1725
1726// ------------------------------------------------------------
1728{
1729 // This function must be run on all processors at once
1730 libmesh_parallel_only(mesh.comm());
1731
1732 // We need to agree on which processor owns every node, but we can't
1733 // easily assert that here because we don't currently agree on which
1734 // id every node has, and some of our temporary ids on unrelated
1735 // nodes will "overlap".
1736//#ifdef DEBUG
1737// MeshTools::libmesh_assert_parallel_consistent_procids<Node> (mesh);
1738//#endif // DEBUG
1739
1740 LOG_SCOPE ("make_node_ids_parallel_consistent()", "MeshCommunication");
1741
1742 SyncNodeIds syncids(mesh);
1744 (mesh, mesh.elements_begin(), mesh.elements_end(),
1746
1747 // At this point, with both ids and processor ids synced, we can
1748 // finally check for topological consistency of node processor ids.
1749#ifdef DEBUG
1751#endif
1752}
1753
1754
1755
1757{
1758 // Avoid unused variable warnings if unique ids aren't enabled.
1760
1761 // This function must be run on all processors at once
1762 libmesh_parallel_only(mesh.comm());
1763
1764#ifdef LIBMESH_ENABLE_UNIQUE_ID
1765 LOG_SCOPE ("make_node_unique_ids_parallel_consistent()", "MeshCommunication");
1766
1767 SyncUniqueIds<Node> syncuniqueids(mesh, &MeshBase::query_node_ptr);
1769 mesh.nodes_begin(),
1770 mesh.nodes_end(),
1771 syncuniqueids);
1772
1773#endif
1774}
1775
1776
1778{
1779 // Avoid unused variable warnings if unique ids aren't enabled.
1781
1782 // This function must be run on all processors at once
1783 libmesh_parallel_only(mesh.comm());
1784
1785 LOG_SCOPE ("make_node_bcids_parallel_consistent()", "MeshCommunication");
1786
1787 SyncBCIds<Node> syncbcids(mesh);
1789 mesh.nodes_begin(),
1790 mesh.nodes_end(),
1791 syncbcids);
1792}
1793
1794
1795
1796
1797
1798// ------------------------------------------------------------
1800{
1801 // This function must be run on all processors at once
1802 libmesh_parallel_only(mesh.comm());
1803
1804 LOG_SCOPE ("make_elems_parallel_consistent()", "MeshCommunication");
1805
1806 SyncIds syncids(mesh, &MeshBase::renumber_elem);
1808 (mesh, mesh.active_elements_begin(),
1809 mesh.active_elements_end(), syncids);
1810
1811#ifdef LIBMESH_ENABLE_UNIQUE_ID
1812 SyncUniqueIds<Elem> syncuniqueids(mesh, &MeshBase::query_elem_ptr);
1814 (mesh.comm(), mesh.active_elements_begin(),
1815 mesh.active_elements_end(), syncuniqueids);
1816#endif
1817}
1818
1819
1820
1821// ------------------------------------------------------------
1822#ifdef LIBMESH_ENABLE_AMR
1824{
1825 // This function must be run on all processors at once
1826 libmesh_parallel_only(mesh.comm());
1827
1828 LOG_SCOPE ("make_p_levels_parallel_consistent()", "MeshCommunication");
1829
1830 SyncPLevels syncplevels(mesh);
1832 (mesh.comm(), mesh.elements_begin(), mesh.elements_end(),
1833 syncplevels);
1834}
1835#endif // LIBMESH_ENABLE_AMR
1836
1837
1838
1839// Functors for make_node_proc_ids_parallel_consistent
1840namespace {
1841
1842struct SyncProcIds
1843{
1844 typedef processor_id_type datum;
1845
1846 SyncProcIds(MeshBase & _mesh) : mesh(_mesh) {}
1847
1848 MeshBase & mesh;
1849
1850 // ------------------------------------------------------------
1851 void gather_data (const std::vector<dof_id_type> & ids,
1852 std::vector<datum> & data)
1853 {
1854 // Find the processor id of each requested node
1855 data.resize(ids.size());
1856
1857 for (auto i : index_range(ids))
1858 {
1859 // Look for this point in the mesh
1860 if (ids[i] != DofObject::invalid_id)
1861 {
1862 Node & node = mesh.node_ref(ids[i]);
1863
1864 // Return the node's correct processor id,
1865 data[i] = node.processor_id();
1866 }
1867 else
1869 }
1870 }
1871
1872 // ------------------------------------------------------------
1873 bool act_on_data (const std::vector<dof_id_type> & ids,
1874 const std::vector<datum> proc_ids)
1875 {
1876 bool data_changed = false;
1877
1878 // Set the ghost node processor ids we've now been informed of
1879 for (auto i : index_range(ids))
1880 {
1881 Node & node = mesh.node_ref(ids[i]);
1882
1883 // We may not have ids synched when this synchronization is done, so we
1884 // *can't* use id to load-balance processor id properly; we have to use
1885 // the old heuristic of choosing the smallest valid processor id.
1886 //
1887 // If someone tells us our node processor id is too low, then
1888 // they're wrong. If they tell us our node processor id is
1889 // too high, then we're wrong.
1890 if (node.processor_id() > proc_ids[i])
1891 {
1892 data_changed = true;
1893 node.processor_id() = proc_ids[i];
1894 }
1895 }
1896
1897 return data_changed;
1898 }
1899};
1900
1901
1902struct ElemNodesMaybeNew
1903{
1904 ElemNodesMaybeNew() {}
1905
1906 bool operator() (const Elem * elem) const
1907 {
1908 // If this element was just refined then it may have new nodes we
1909 // need to work on
1910#ifdef LIBMESH_ENABLE_AMR
1911 if (elem->refinement_flag() == Elem::JUST_REFINED)
1912 return true;
1913#endif
1914
1915 // If this element has remote_elem neighbors then there may have
1916 // been refinement of those neighbors that affect its nodes'
1917 // processor_id()
1918 for (auto neigh : elem->neighbor_ptr_range())
1919 if (neigh == remote_elem)
1920 return true;
1921 return false;
1922 }
1923};
1924
1925
1926struct NodeWasNew
1927{
1928 NodeWasNew(const MeshBase & mesh)
1929 {
1930 for (const auto & node : mesh.node_ptr_range())
1931 if (node->processor_id() == DofObject::invalid_processor_id)
1932 was_new.insert(node);
1933 }
1934
1935 bool operator() (const Elem * elem, unsigned int local_node_num) const
1936 {
1937 if (was_new.count(elem->node_ptr(local_node_num)))
1938 return true;
1939 return false;
1940 }
1941
1942 std::unordered_set<const Node *> was_new;
1943};
1944
1945}
1946
1947
1948
1949// ------------------------------------------------------------
1951{
1952 LOG_SCOPE ("make_node_proc_ids_parallel_consistent()", "MeshCommunication");
1953
1954 // This function must be run on all processors at once
1955 libmesh_parallel_only(mesh.comm());
1956
1957 // When this function is called, each section of a parallelized mesh
1958 // should be in the following state:
1959 //
1960 // All nodes should have the exact same physical location on every
1961 // processor where they exist.
1962 //
1963 // Local nodes should have unique authoritative ids,
1964 // and processor ids consistent with all processors which own
1965 // an element touching them.
1966 //
1967 // Ghost nodes touching local elements should have processor ids
1968 // consistent with all processors which own an element touching
1969 // them.
1970 SyncProcIds sync(mesh);
1972 (mesh, mesh.elements_begin(), mesh.elements_end(),
1974}
1975
1976
1977
1978// ------------------------------------------------------------
1980{
1981 LOG_SCOPE ("make_new_node_proc_ids_parallel_consistent()", "MeshCommunication");
1982
1983 // This function must be run on all processors at once
1984 libmesh_parallel_only(mesh.comm());
1985
1986 // When this function is called, each section of a parallelized mesh
1987 // should be in the following state:
1988 //
1989 // Local nodes should have unique authoritative ids,
1990 // and new nodes should be unpartitioned.
1991 //
1992 // New ghost nodes touching local elements should be unpartitioned.
1993
1994 // We may not have consistent processor ids for new nodes (because a
1995 // node may be old and partitioned on one processor but new and
1996 // unpartitioned on another) when we start
1997#ifdef DEBUG
1999 // MeshTools::libmesh_assert_parallel_consistent_new_node_procids(mesh);
2000#endif
2001
2002 // We have two kinds of new nodes. *NEW* nodes are unpartitioned on
2003 // all processors: we need to use a id-independent (i.e. dumb)
2004 // heuristic to partition them. But "new" nodes are newly created
2005 // on some processors (when ghost elements are refined) yet
2006 // correspond to existing nodes on other processors: we need to use
2007 // the existing processor id for them.
2008 //
2009 // A node which is "new" on one processor will be associated with at
2010 // least one ghost element, and we can just query that ghost
2011 // element's owner to find out the correct processor id.
2012
2013 auto node_unpartitioned =
2014 [](const Elem * elem, unsigned int local_node_num)
2015 { return elem->node_ref(local_node_num).processor_id() ==
2017
2018 SyncProcIds sync(mesh);
2019
2020 sync_node_data_by_element_id_once
2021 (mesh, mesh.not_local_elements_begin(),
2022 mesh.not_local_elements_end(), Parallel::SyncEverything(),
2023 node_unpartitioned, sync);
2024
2025 // Nodes should now be unpartitioned iff they are truly new; those
2026 // are the *only* nodes we will touch.
2027#ifdef DEBUG
2029#endif
2030
2031 NodeWasNew node_was_new(mesh);
2032
2033 // Set the lowest processor id we can on truly new nodes
2034 for (auto & elem : mesh.element_ptr_range())
2035 for (auto & node : elem->node_ref_range())
2036 if (node_was_new.was_new.count(&node))
2037 {
2038 processor_id_type & pid = node.processor_id();
2039 pid = std::min(pid, elem->processor_id());
2040 }
2041
2042 // Then finally see if other processors have a lower option
2044 (mesh, mesh.elements_begin(), mesh.elements_end(),
2045 ElemNodesMaybeNew(), node_was_new, sync);
2046
2047 // We should have consistent processor ids when we're done.
2048#ifdef DEBUG
2051#endif
2052}
2053
2054
2055
2056// ------------------------------------------------------------
2058{
2059 // This function must be run on all processors at once
2060 libmesh_parallel_only(mesh.comm());
2061
2062 // When this function is called, each section of a parallelized mesh
2063 // should be in the following state:
2064 //
2065 // Element ids and locations should be consistent on every processor
2066 // where they exist.
2067 //
2068 // All nodes should have the exact same physical location on every
2069 // processor where they exist.
2070 //
2071 // Local nodes should have unique authoritative ids,
2072 // and processor ids consistent with all processors which own
2073 // an element touching them.
2074 //
2075 // Ghost nodes touching local elements should have processor ids
2076 // consistent with all processors which own an element touching
2077 // them.
2078 //
2079 // Ghost nodes should have ids which are either already correct
2080 // or which are in the "unpartitioned" id space.
2081
2082 // First, let's sync up processor ids. Some of these processor ids
2083 // may be "wrong" from coarsening, but they're right in the sense
2084 // that they'll tell us who has the authoritative dofobject ids for
2085 // each node.
2086
2088
2089 // Second, sync up dofobject ids.
2091
2092 // Third, sync up dofobject unique_ids if applicable.
2094
2095 // Finally, correct the processor ids to make DofMap happy
2097}
2098
2099
2100
2101// ------------------------------------------------------------
2103{
2104 // This function must be run on all processors at once
2105 libmesh_parallel_only(mesh.comm());
2106
2107 // When this function is called, each section of a parallelized mesh
2108 // should be in the following state:
2109 //
2110 // All nodes should have the exact same physical location on every
2111 // processor where they exist.
2112 //
2113 // Local nodes should have unique authoritative ids,
2114 // and new nodes should be unpartitioned.
2115 //
2116 // New ghost nodes touching local elements should be unpartitioned.
2117 //
2118 // New ghost nodes should have ids which are either already correct
2119 // or which are in the "unpartitioned" id space.
2120 //
2121 // Non-new nodes should have correct ids and processor ids already.
2122
2123 // First, let's sync up new nodes' processor ids.
2124
2126
2127 // Second, sync up dofobject ids.
2129
2130 // Third, sync up dofobject unique_ids if applicable.
2132
2133 // Fourth, sync up any nodal boundary conditions
2135
2136 // Finally, correct the processor ids to make DofMap happy
2138}
2139
2140
2141
2142// ------------------------------------------------------------
2143void
2145 const std::set<Elem *> & extra_ghost_elem_ids) const
2146{
2147 // The mesh should know it's about to be parallelized
2149
2150 LOG_SCOPE("delete_remote_elements()", "MeshCommunication");
2151
2152#ifdef DEBUG
2153 // We expect maximum ids to be in sync so we can use them to size
2154 // vectors
2157 const dof_id_type par_max_node_id = mesh.parallel_max_node_id();
2158 const dof_id_type par_max_elem_id = mesh.parallel_max_elem_id();
2159 libmesh_assert_equal_to (par_max_node_id, mesh.max_node_id());
2160 libmesh_assert_equal_to (par_max_elem_id, mesh.max_elem_id());
2161
2162 // n_constraint_rows might not be cached, but we're in dbg mode,
2163 // let's just (re)count them
2164 const dof_id_type n_constraint_rows = mesh.count_constraint_rows();
2165#endif
2166
2167 connected_elem_set_type elements_to_keep;
2168
2169 // Don't delete elements that we were explicitly told not to
2170 for (const auto & elem : extra_ghost_elem_ids)
2171 {
2172 std::vector<const Elem *> active_family;
2173#ifdef LIBMESH_ENABLE_AMR
2174 if (!elem->subactive())
2175 elem->active_family_tree(active_family);
2176 else
2177#endif
2178 active_family.push_back(elem);
2179
2180 for (const auto & f : active_family)
2181 elements_to_keep.insert(f);
2182 }
2183
2184 // See which elements we still need to keep ghosted, given that
2185 // we're keeping local and unpartitioned elements.
2188 mesh.active_pid_elements_begin(mesh.processor_id()),
2189 mesh.active_pid_elements_end(mesh.processor_id()),
2190 elements_to_keep);
2193 mesh.active_pid_elements_begin(DofObject::invalid_processor_id),
2194 mesh.active_pid_elements_end(DofObject::invalid_processor_id),
2195 elements_to_keep);
2196
2197 // The inactive elements we need to send should have their
2198 // immediate children present.
2199 connect_children(mesh, mesh.pid_elements_begin(mesh.processor_id()),
2200 mesh.pid_elements_end(mesh.processor_id()),
2201 elements_to_keep);
2203 mesh.pid_elements_begin(DofObject::invalid_processor_id),
2204 mesh.pid_elements_end(DofObject::invalid_processor_id),
2205 elements_to_keep);
2206
2207 // And see which elements and nodes they depend on
2208 connected_node_set_type connected_nodes;
2209 connect_element_dependencies(mesh, elements_to_keep, connected_nodes);
2210
2211 // Delete all the elements we have no reason to save,
2212 // starting with the most refined so that the mesh
2213 // is valid at all intermediate steps
2214 unsigned int n_levels = MeshTools::n_levels(mesh);
2215
2216 for (int l = n_levels - 1; l >= 0; --l)
2217 for (auto & elem : as_range(mesh.level_elements_begin(l),
2218 mesh.level_elements_end(l)))
2219 {
2220 libmesh_assert (elem);
2221 // Make sure we don't leave any invalid pointers
2222 const bool keep_me = elements_to_keep.count(elem);
2223
2224 if (!keep_me)
2226
2227 // delete_elem doesn't currently invalidate element
2228 // iterators... that had better not change
2229 if (!keep_me)
2230 mesh.delete_elem(elem);
2231 }
2232
2233 // Delete all the nodes we have no reason to save
2234 for (auto & node : mesh.node_ptr_range())
2235 {
2236 libmesh_assert(node);
2237 if (!connected_nodes.count(node))
2238 {
2239 libmesh_assert_not_equal_to(node->processor_id(),
2240 mesh.processor_id());
2241 mesh.delete_node(node);
2242 }
2243 }
2244
2245 // If we had a point locator, it's invalid now that some of the
2246 // elements it pointed to have been deleted.
2248
2249 // We now have all remote elements and nodes deleted; our ghosting
2250 // functors should be ready to delete any now-redundant cached data
2251 // they use too.
2253 gf->delete_remote_elements();
2254
2255#ifdef DEBUG
2256 const dof_id_type n_new_constraint_rows = mesh.count_constraint_rows();
2257 libmesh_assert_equal_to(n_constraint_rows, n_new_constraint_rows);
2258
2261#endif
2262}
2263
2264} // namespace libMesh
void max(const T &r, T &o, Request &req) const
void allgather_packed_range(Context *context, Iter range_begin, const Iter range_end, OutputIter out, std::size_t approx_buffer_size=1000000) const
processor_id_type size() const
void broadcast_packed_range(const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out, const unsigned int root_id=0, std::size_t approx_buffer_size=1000000) const
MessageTag get_unique_tag(int tagvalue=MessageTag::invalid_tag) const
void receive_packed_range(const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag) const
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
timpi_pure bool verify(const T &r) const
void set_union(T &data, const unsigned int root_id) const
void gather_packed_range(const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out, std::size_t approx_buffer_size=1000000) const
void broadcast(T &data, const unsigned int root_id=0, const bool identical_sizes=false) const
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag, std::size_t approx_buffer_size=1000000) const
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
The BoundaryInfo class contains information relevant to boundary conditions including storing faces,...
void boundary_ids(const Node *node, std::vector< boundary_id_type > &vec_to_fill) const
Fills a user-provided std::vector with the boundary ids associated with Node node.
std::map< boundary_id_type, std::string > & set_sideset_name_map()
void add_node(const Node *node, const boundary_id_type id)
Add Node node with boundary id id to the boundary information data structures.
std::map< boundary_id_type, std::string > & set_nodeset_name_map()
The DistributedMesh class is derived from the MeshBase class, and is intended to provide identical fu...
The DofObject defines an abstract base class for objects that have degrees of freedom associated with...
Definition dof_object.h:55
processor_id_type processor_id() const
Definition dof_object.h:895
static constexpr dof_id_type invalid_id
An invalid id to distinguish an uninitialized DofObject.
Definition dof_object.h:473
dof_id_type id() const
Definition dof_object.h:833
static constexpr processor_id_type invalid_processor_id
An invalid processor_id to distinguish DoFs that have not been assigned to a processor.
Definition dof_object.h:484
Helper for building element sides that minimizes the construction of new elements.
This is the base class from which all geometric element types are derived.
Definition elem.h:96
void family_tree(std::vector< const Elem * > &family, bool reset=true) const
Fills the vector family with the children of this element, recursively.
Definition elem.C:2212
RefinementState refinement_flag() const
Definition elem.h:3348
void hack_p_level_and_refinement_flag(const unsigned int p, RefinementState pflag)
Sets the value of the p-refinement level for the element without altering the p-level of its ancestor...
Definition elem.h:3413
const Node & node_ref(const unsigned int i) const
Definition elem.h:2659
bool has_children() const
Definition elem.h:3117
const Elem * parent() const
Definition elem.h:3168
void set_neighbor(const unsigned int i, Elem *n)
Assigns n as the neighbor.
Definition elem.h:2756
unsigned int n_neighbors() const
Definition elem.h:812
SimpleRange< NodeRefIter > node_ref_range()
Returns a range with all nodes of an element, usable in range-based for loops.
Definition elem.h:2803
RefinementState
Enumeration of possible element refinement states.
Definition elem.h:1567
@ JUST_COARSENED
Definition elem.h:1571
SimpleRange< ChildRefIter > child_ref_range()
Returns a range with all children of a parent element, usable in range-based for loops.
Definition elem.h:2476
void make_links_to_me_remote()
Resets this element's neighbors' appropriate neighbor pointers and its parent's and children's approp...
Definition elem.C:1482
bool on_boundary() const
Definition elem.h:3047
const Node * node_ptr(const unsigned int i) const
Definition elem.h:2637
void active_family_tree(std::vector< const Elem * > &active_family, bool reset=true) const
Same as the family_tree() member, but only adds the active children.
Definition elem.C:2244
const Elem * interior_parent() const
Definition elem.C:1100
bool subactive() const
Definition elem.h:3097
unsigned int p_level() const
Definition elem.h:3246
RefinementState p_refinement_flag() const
Definition elem.h:3364
void set_p_level(const unsigned int p)
Sets the value of the p-refinement level for the element.
const Elem * neighbor_ptr(unsigned int i) const
Definition elem.h:2736
const Elem * top_parent() const
Definition elem.h:3194
std::map< const Elem *, const CouplingMatrix *, CompareDofObjectsByPIDAndThenID > map_type
What elements do we care about and what variables do we care about on each element?
This is the MeshBase class.
Definition mesh_base.h:81
virtual const Node & node_ref(const dof_id_type i) const
Definition mesh_base.h:742
virtual bool is_serial() const
Definition mesh_base.h:357
void set_default_mapping_data(const unsigned char data)
Set the default master space to physical space mapping basis functions to be used on newly added elem...
Definition mesh_base.h:965
const BoundaryInfo & get_boundary_info() const
The information about boundary ids on the mesh.
Definition mesh_base.h:170
virtual const Node * node_ptr(const dof_id_type i) const =0
virtual dof_id_type n_elem() const =0
virtual void renumber_elem(dof_id_type old_id, dof_id_type new_id)=0
Changes the id of element old_id, both by changing elem(old_id)->id() and by moving elem(old_id) in t...
void allow_find_neighbors(bool allow)
If false is passed then this mesh will no longer work to find element neighbors when being prepared f...
Definition mesh_base.h:1366
virtual void delete_node(Node *n)=0
Removes the Node n from the mesh.
virtual dof_id_type n_nodes() const =0
virtual void delete_elem(Elem *e)=0
Removes element e from the mesh.
ElemMappingType default_mapping_type() const
Returns the default master space to physical space mapping basis functions to be used on newly added ...
Definition mesh_base.h:938
std::vector< dof_id_type > _node_integer_default_values
The array of default initialization values for integer data associated with each node in the mesh.
Definition mesh_base.h:2401
virtual dof_id_type max_node_id() const =0
virtual const Node * query_node_ptr(const dof_id_type i) const =0
std::vector< std::pair< std::pair< const Elem *, unsigned int >, Real > > constraint_rows_mapped_type
Definition mesh_base.h:1933
GhostingFunctorIterator ghosting_functors_begin() const
Beginning of range of ghosting functors.
Definition mesh_base.h:1476
virtual const Elem * elem_ptr(const dof_id_type i) const =0
virtual void renumber_node(dof_id_type old_id, dof_id_type new_id)=0
Changes the id of node old_id, both by changing node(old_id)->id() and by moving node(old_id) in the ...
std::vector< std::string > _elem_integer_names
The array of names for integer data associated with each element in the mesh.
Definition mesh_base.h:2383
std::vector< dof_id_type > _elem_integer_default_values
The array of default initialization values for integer data associated with each element in the mesh.
Definition mesh_base.h:2389
virtual dof_id_type max_elem_id() const =0
virtual void clear()
Deletes all the element and node data that is currently stored.
Definition mesh_base.C:1036
void set_default_mapping_type(const ElemMappingType type)
Set the default master space to physical space mapping basis functions to be used on newly added elem...
Definition mesh_base.h:947
constraint_rows_type _constraint_rows
Definition mesh_base.h:2454
constraint_rows_type & get_constraint_rows()
Constraint rows accessors.
Definition mesh_base.h:1940
virtual const Elem * query_elem_ptr(const dof_id_type i) const =0
dof_id_type n_constraint_rows() const
Returns how many total constraint rows we have.
Definition mesh_base.h:2691
virtual void find_neighbors(const bool reset_remote_elements=false, const bool reset_current_list=true, const bool assert_valid=true, const bool check_non_remote=true)=0
Locate element face (edge in 2D) neighbors.
virtual const Elem & elem_ref(const dof_id_type i) const
Definition mesh_base.h:785
unsigned char default_mapping_data() const
Returns any default data value used by the master space to physical space mapping.
Definition mesh_base.h:956
void clear_point_locator()
Releases the current PointLocator object.
Definition mesh_base.C:1881
GhostingFunctorIterator ghosting_functors_end() const
End of range of ghosting functors.
Definition mesh_base.h:1482
std::vector< std::string > _node_integer_names
The array of names for integer data associated with each node in the mesh.
Definition mesh_base.h:2395
std::map< subdomain_id_type, std::string > & set_subdomain_name_map()
Definition mesh_base.h:1928
virtual void update_parallel_id_counts()=0
Updates parallel caches so that methods like n_elem() accurately reflect changes on other processors.
dof_id_type count_constraint_rows()
Helper function for subclasses to use to set _n_constraint_rows.
Definition mesh_base.C:2487
void make_node_unique_ids_parallel_consistent(MeshBase &)
Assuming all unique_ids on local nodes are globally unique, and assuming all processor ids are parall...
void delete_remote_elements(DistributedMesh &, const std::set< Elem * > &) const
This method takes an input DistributedMesh which may be distributed among all the processors.
void clear()
Clears all data structures and resets to a pristine state.
void redistribute(DistributedMesh &mesh, bool newly_coarsened_only=false) const
This method takes a parallel distributed mesh and redistributes the elements.
void make_elems_parallel_consistent(MeshBase &)
Copy ids of ghost elements from their local processors.
void make_node_bcids_parallel_consistent(MeshBase &)
Assuming all processor ids are parallel consistent, this function makes all ghost boundary ids on nod...
void send_coarse_ghosts(MeshBase &) const
Examine a just-coarsened mesh, and for any newly-coarsened elements, send the associated ghosted elem...
void make_p_levels_parallel_consistent(MeshBase &)
Copy p levels of ghost elements from their local processors.
void make_node_ids_parallel_consistent(MeshBase &)
Assuming all ids on local nodes are globally unique, and assuming all processor ids are parallel cons...
void make_node_proc_ids_parallel_consistent(MeshBase &)
Assuming all processor ids on nodes touching local elements are parallel consistent,...
void make_nodes_parallel_consistent(MeshBase &)
Copy processor_ids and ids on ghost nodes from their local processors.
void gather_neighboring_elements(DistributedMesh &) const
void gather(const processor_id_type root_id, MeshBase &) const
This method takes an input DistributedMesh which may be distributed among all the processors.
void make_new_node_proc_ids_parallel_consistent(MeshBase &)
Assuming all processor ids on nodes touching local elements are parallel consistent,...
void make_new_nodes_parallel_consistent(MeshBase &)
Copy processor_ids and ids on new nodes from their local processors.
void broadcast(MeshBase &) const
This method takes a mesh (which is assumed to reside on processor 0) and broadcasts it to all the oth...
A Node is like a Point, but with more information.
Definition node.h:55
processor_id_type processor_id() const
const Parallel::Communicator & comm() const
processor_id_type n_processors() const
In parallel meshes where a ghost element has neighbors which do not exist on the local processor,...
Definition remote_elem.h:61
renumber_obj renumber
std::unordered_set< const Node * > was_new
uset_type definitive_ids
query_obj query
umap_type definitive_renumbering
MeshBase & mesh
Status wait(Request &r)
MPI_Status status
void push_parallel_vector_data(const Communicator &comm, MapToVectors &&data, const ActionFunctor &act_on_data)
const unsigned int any_source
void push_parallel_packed_range(const Communicator &comm, MapToContainers &&data, Context *context, const ActionFunctor &act_on_data)
void family_tree(T elem, std::vector< T > &family, bool reset=true)
LIBMESH_DEVICE_INLINE ResultVector row(const TensorLike &T_in, const unsigned int row_index)
dof_id_type n_elem(const MeshBase::const_element_iterator &begin, const MeshBase::const_element_iterator &end)
Count up the number of elements of a specific type (as defined by an iterator range).
void libmesh_assert_parallel_consistent_new_node_procids(const MeshBase &mesh)
A function for verifying that processor assignment is parallel consistent (every processor agrees on ...
void libmesh_assert_valid_boundary_ids(const MeshBase &mesh)
A function for verifying that boundary condition ids match across processors.
void libmesh_assert_valid_constraint_rows(const MeshBase &mesh)
A function for verifying that all mesh constraint rows express relations between nodes and elements t...
void correct_node_proc_ids(MeshBase &)
Changes the processor ids on each node so be the same as the id of the lowest element touching that n...
void libmesh_assert_equal_n_systems(const MeshBase &mesh)
The following functions, only available in builds with NDEBUG undefined, are for asserting internal c...
void libmesh_assert_valid_refinement_tree(const MeshBase &mesh)
A function for verifying that elements on this processor have valid descendants and consistent active...
void libmesh_assert_topology_consistent_procids< Node >(const MeshBase &mesh)
unsigned int n_levels(const MeshBase &mesh)
Definition mesh_tools.C:826
unsigned int paranoid_n_levels(const MeshBase &mesh)
Definition mesh_tools.C:850
void libmesh_assert_parallel_consistent_procids< Node >(const MeshBase &mesh)
void sync_element_data_by_parent_id(MeshBase &mesh, const Iterator &range_begin, const Iterator &range_end, SyncFunctor &sync)
Request data about a range of ghost elements uniquely identified by their parent id and which child t...
void sync_node_data_by_element_id(MeshBase &mesh, const MeshBase::const_element_iterator &range_begin, const MeshBase::const_element_iterator &range_end, const ElemCheckFunctor &elem_check, const NodeCheckFunctor &node_check, SyncFunctor &sync)
Synchronize data about a range of ghost nodes uniquely identified by an element id and local node id,...
void sync_dofobject_data_by_id(const Communicator &comm, const Iterator &range_begin, const Iterator &range_end, SyncFunctor &sync)
Request data about a range of ghost dofobjects uniquely identified by their id.
The libMesh namespace provides an interface to certain functionality in the library.
OStreamProxy err
uint8_t unique_id_type
Definition id_types.h:86
void reconnect_nodes(connected_elem_set_type &connected_elements, connected_node_set_type &connected_nodes)
SimpleRange< IndexType > as_range(const std::pair< IndexType, IndexType > &p)
Helper function that allows us to treat a homogenous pair as a range.
auto index_range(const T &sizable)
Helper function that returns an IntRange<std::size_t> representing all the indices of the passed-in v...
Definition int_range.h:153
std::set< const Elem *, CompareElemIdsByLevel > connected_elem_set_type
void query_ghosting_functors(const MeshBase &mesh, processor_id_type pid, MeshBase::const_element_iterator elem_it, MeshBase::const_element_iterator elem_end, connected_elem_set_type &connected_elements)
void libmesh_ignore(const Args &...)
ElemMappingType
Enumeration of possible element master->physical mapping types.
libmesh_assert(ctx)
const RemoteElem * remote_elem
Definition remote_elem.C:57
std::set< const Node * > connected_node_set_type
void connect_element_dependencies(const MeshBase &mesh, connected_elem_set_type &connected_elements, connected_node_set_type &connected_nodes)
uint8_t dof_id_type
Definition id_types.h:67
void connect_children(const MeshBase &mesh, MeshBase::const_element_iterator elem_it, MeshBase::const_element_iterator elem_end, connected_elem_set_type &connected_elements)
DIE A HORRIBLE DEATH HERE typedef LIBMESH_DEFAULT_SCALAR_TYPE Real
uint8_t processor_id_type
Definition id_types.h:104
IntRange< T > make_range(T beg, T end)
The 2-parameter make_range() helper function returns an IntRange<T> when both input parameters are of...
Definition int_range.h:176
The definition of the const_element_iterator struct.
Definition mesh_base.h:2575
Used to iterate over non-nullptr entries in a container.
A do-nothing class for templated methods that expect output iterator arguments.