Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2025 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : #ifndef LIBMESH_DISTRIBUTED_MESH_H
21 : #define LIBMESH_DISTRIBUTED_MESH_H
22 :
23 : // Local Includes
24 : #include "libmesh_config.h"
25 :
26 : #if LIBMESH_MAPVECTOR_CHUNK_SIZE == 1
27 : # include "libmesh/mapvector.h"
28 : #else
29 : # include "libmesh/chunked_mapvector.h"
30 : #endif
31 :
32 : #include "libmesh/unstructured_mesh.h"
33 :
34 : // C++ Includes
35 : #include <cstddef>
36 : #include <memory>
37 : #include <set>
38 :
39 : namespace libMesh
40 : {
41 :
42 : // Forward declarations
43 : class Elem;
44 : class Node;
45 :
46 :
47 : /**
48 : * The \p DistributedMesh class is derived from the \p MeshBase class,
49 : * and is intended to provide identical functionality to the user
50 : * but be distributed rather than replicated across distributed-memory
51 : * systems.
52 : *
53 : * \author Roy Stogner
54 : * \date 2007
55 : * \brief Mesh data structure which is distributed across all processors.
56 : */
57 : class DistributedMesh : public UnstructuredMesh
58 : {
59 : public:
60 :
61 : template <typename Obj>
62 : #if LIBMESH_MAPVECTOR_CHUNK_SIZE == 1
63 : using dofobject_container = mapvector<Obj *, dof_id_type>;
64 : #else
65 : using dofobject_container = chunked_mapvector<Obj *, dof_id_type, LIBMESH_MAPVECTOR_CHUNK_SIZE>;
66 : #endif
67 :
68 : /**
69 : * Constructor. Takes \p dim, the dimension of the mesh.
70 : * The mesh dimension can be changed (and may automatically be
71 : * changed by mesh generation/loading) later.
72 : */
73 : explicit
74 : DistributedMesh (const Parallel::Communicator & comm_in,
75 : unsigned char dim=1);
76 :
77 : /**
78 : * Copy-constructor. This should be able to take a
79 : * replicated or distributed mesh.
80 : */
81 : DistributedMesh (const MeshBase & other_mesh);
82 :
83 : /**
84 : * Copy-constructor, possibly specialized for a
85 : * distributed mesh.
86 : */
87 : DistributedMesh (const DistributedMesh & other_mesh);
88 :
89 : /**
90 : * Move-constructor deleted in MeshBase.
91 : */
92 : DistributedMesh(DistributedMesh &&) = delete;
93 :
94 : /**
95 : * Copy assignment is not allowed.
96 : */
97 : DistributedMesh & operator= (const DistributedMesh &) = delete;
98 :
99 : /**
100 : * Overloaded operator= will move contents of other_mesh to calling
101 : * DistributedMesh object.
102 : */
103 : DistributedMesh & operator= (DistributedMesh && other_mesh);
104 :
105 : /**
106 : * Shim to call the move assignment operator for this class
107 : */
108 : virtual MeshBase & assign(MeshBase && other_mesh) override;
109 :
110 : /**
111 : * Shim to allow operator == (&) to behave like a virtual function
112 : * without having to be one.
113 : */
114 : virtual bool subclass_locally_equals (const MeshBase & other_mesh) const override;
115 :
116 : /**
117 : * Virtual copy-constructor, creates a copy of this mesh
118 : */
119 476 : virtual std::unique_ptr<MeshBase> clone () const override
120 : {
121 484 : auto returnval = std::make_unique<DistributedMesh>(*this);
122 : #ifdef DEBUG
123 8 : libmesh_assert(*returnval == *this);
124 : #endif
125 484 : return returnval;
126 460 : }
127 :
128 : /**
129 : * Destructor.
130 : */
131 : virtual ~DistributedMesh();
132 :
133 : /**
134 : * Clear all internal data.
135 : */
136 : virtual void clear() override;
137 :
138 : /**
139 : * Clear internal Elem data.
140 : */
141 : virtual void clear_elems() override;
142 :
143 : /**
144 : * Redistribute elements between processors. This gets called
145 : * automatically by the Partitioner, and is a no-op in the case of a
146 : * serialized mesh.
147 : */
148 : virtual void redistribute () override;
149 :
150 : /**
151 : * Recalculate cached data after elements and nodes have been
152 : * repartitioned.
153 : */
154 : virtual void update_post_partitioning () override;
155 :
156 : /**
157 : * \returns \p true if all elements and nodes of the mesh
158 : * exist on the current processor, \p false otherwise
159 : */
160 13500 : virtual bool is_serial () const override final
161 506108 : { return _is_serial; }
162 :
163 : /**
164 : * \returns \p true if all elements and nodes of the mesh
165 : * exist on the processor 0, \p false otherwise
166 : */
167 57 : virtual bool is_serial_on_zero () const override final
168 323 : { return _is_serial || _is_serial_on_proc_0; }
169 :
170 : /**
171 : * Asserts that not all elements and nodes of the mesh necessarily
172 : * exist on the current processor.
173 : */
174 402 : virtual void set_distributed () override final
175 402 : { _is_serial = false;
176 402 : _is_serial_on_proc_0 = false; }
177 :
178 : /**
179 : * \returns \p true if new elements and nodes can and should be
180 : * created in synchronization on all processors, \p false otherwise
181 : */
182 81536 : virtual bool is_replicated () const override final
183 81536 : { return false; }
184 :
185 : /**
186 : * Verify id, processor_id, and if applicable unique_id consistency
187 : * of a parallel objects container.
188 : * Calls libmesh_assert() on each possible failure in that container.
189 : */
190 : template <typename T>
191 : void libmesh_assert_valid_parallel_object_ids(const dofobject_container<T> &) const;
192 :
193 : /**
194 : * Verify id and processor_id consistency of our elements and
195 : * nodes containers.
196 : * Calls libmesh_assert() on each possible failure.
197 : */
198 : virtual void libmesh_assert_valid_parallel_ids() const override;
199 :
200 : /**
201 : * Verify p_level consistency of our elements containers.
202 : * Calls libmesh_assert() on each possible failure.
203 : */
204 : void libmesh_assert_valid_parallel_p_levels() const;
205 :
206 : /**
207 : * Verify refinement_flag and p_refinement_flag consistency of our
208 : * elements containers.
209 : * Calls libmesh_assert() on each possible failure.
210 : */
211 : void libmesh_assert_valid_parallel_flags() const;
212 :
213 : /**
214 : * Renumber a parallel objects container.
215 : *
216 : * \returns The smallest globally unused id for that container.
217 : */
218 : template <typename T>
219 : dof_id_type renumber_dof_objects (dofobject_container<T> &);
220 :
221 : /**
222 : * Remove nullptr elements from arrays.
223 : */
224 : virtual void renumber_nodes_and_elements () override;
225 :
226 : /**
227 : * Gathers all elements and nodes of the mesh onto
228 : * every processor
229 : */
230 : virtual void allgather() override;
231 :
232 : /**
233 : * Gathers all elements and nodes of the mesh onto
234 : * processor zero
235 : */
236 : virtual void gather_to_zero() override;
237 :
238 : /**
239 : * Deletes all nonlocal elements of the mesh
240 : * except for "ghosts" which touch a local element, and deletes
241 : * all nodes which are not part of a local or ghost element
242 : */
243 : virtual void delete_remote_elements() override;
244 :
245 : /**
246 : * Inserts the element _and_ adds it to a list of elements that
247 : * should not get deleted or have their descendants deleted by
248 : * delete_remote_elements. This is handy for inserting otherwise
249 : * off-processor elements that you want to keep track of on this
250 : * processor.
251 : */
252 : virtual void add_extra_ghost_elem(Elem * e);
253 :
254 : /**
255 : * Clears extra ghost elements.
256 : */
257 0 : virtual void clear_extra_ghost_elems() { _extra_ghost_elems.clear(); }
258 :
259 : /**
260 : * Clears specified extra ghost elements
261 : */
262 : virtual void clear_extra_ghost_elems(const std::set<Elem *> & extra_ghost_elems);
263 :
264 : /**
265 : * Const accessor to the ghosted elements
266 : */
267 : const std::set<Elem *> & extra_ghost_elems() const { return _extra_ghost_elems; }
268 :
269 : // Cached methods that can be called in serial
270 76932 : virtual dof_id_type n_nodes () const override final { return _n_nodes; }
271 17188 : virtual dof_id_type max_node_id () const override final { return _max_node_id; }
272 2270 : virtual void reserve_nodes (const dof_id_type) override final {}
273 61532 : virtual dof_id_type n_elem () const override final { return _n_elem; }
274 : virtual dof_id_type n_active_elem () const override final;
275 19358 : virtual dof_id_type max_elem_id () const override final { return _max_elem_id; }
276 2151 : virtual void reserve_elem (const dof_id_type) override final {}
277 :
278 : // Parallel only method to update the caches
279 : virtual void update_parallel_id_counts () override;
280 :
281 : // And more parallel only methods to test non-cached values
282 : virtual dof_id_type parallel_n_nodes () const override;
283 : dof_id_type parallel_max_node_id () const;
284 : virtual dof_id_type parallel_n_elem () const override;
285 : dof_id_type parallel_max_elem_id () const;
286 :
287 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
288 : virtual unique_id_type parallel_max_unique_id () const override;
289 : virtual void set_next_unique_id(unique_id_type id) override;
290 : #endif
291 :
292 : virtual const Point & point (const dof_id_type i) const override final;
293 :
294 : virtual const Node * node_ptr (const dof_id_type i) const override final;
295 : virtual Node * node_ptr (const dof_id_type i) override final;
296 :
297 : virtual const Node * query_node_ptr (const dof_id_type i) const override final;
298 : virtual Node * query_node_ptr (const dof_id_type i) override final;
299 :
300 : virtual const Elem * elem_ptr (const dof_id_type i) const override final;
301 : virtual Elem * elem_ptr (const dof_id_type i) override final;
302 :
303 : virtual const Elem * query_elem_ptr (const dof_id_type i) const override final;
304 : virtual Elem * query_elem_ptr (const dof_id_type i) override final;
305 :
306 : /**
307 : * functions for adding /deleting nodes elements.
308 : */
309 : virtual Node * add_point (const Point & p,
310 : const dof_id_type id = DofObject::invalid_id,
311 : const processor_id_type proc_id = DofObject::invalid_processor_id) override final;
312 : virtual Node * add_node (Node * n) override final;
313 : virtual Node * add_node (std::unique_ptr<Node> n) override final;
314 :
315 : #ifdef LIBMESH_ENABLE_DEPRECATED
316 : /**
317 : * These methods are deprecated. Please use \p add_node instead
318 : * Calls add_node().
319 : */
320 : virtual Node * insert_node(Node * n) override final;
321 : virtual Node * insert_node(std::unique_ptr<Node> n) override final;
322 : #endif
323 :
324 : /**
325 : * Takes ownership of node \p n on this partition of a distributed
326 : * mesh, by setting n.processor_id() to this->processor_id(), as
327 : * well as changing n.id() and moving it in the mesh's internal
328 : * container to give it a new authoritative id.
329 : */
330 : virtual void own_node (Node & n) override final;
331 :
332 : virtual void delete_node (Node * n) override final;
333 : virtual void renumber_node (dof_id_type old_id, dof_id_type new_id) override final;
334 : virtual Elem * add_elem (Elem * e) override final;
335 : virtual Elem * add_elem (std::unique_ptr<Elem> e) override final;
336 : virtual Elem * insert_elem (Elem * e) override final;
337 : virtual Elem * insert_elem (std::unique_ptr<Elem> e) override final;
338 : virtual void delete_elem (Elem * e) override final;
339 : virtual void renumber_elem (dof_id_type old_id, dof_id_type new_id) override final;
340 :
341 : /**
342 : * There is no reason for a user to ever call this function.
343 : *
344 : * This function restores a previously broken element/node numbering such that
345 : * \p mesh.node_ref(n).id() == n.
346 : */
347 : virtual void fix_broken_node_and_element_numbering () override;
348 :
349 : public:
350 : /**
351 : * Elem and Node iterator accessor functions. See MeshBase for
352 : * documentation.
353 : */
354 16238 : DECLARE_ELEM_ITERATORS(,,);
355 28696 : DECLARE_ELEM_ITERATORS(active_,,);
356 0 : DECLARE_ELEM_ITERATORS(ancestor_,,)
357 0 : DECLARE_ELEM_ITERATORS(subactive_,,)
358 0 : DECLARE_ELEM_ITERATORS(local_,,)
359 0 : DECLARE_ELEM_ITERATORS(unpartitioned_,,)
360 0 : DECLARE_ELEM_ITERATORS(facelocal_,,)
361 0 : DECLARE_ELEM_ITERATORS(level_, unsigned int level, level)
362 0 : DECLARE_ELEM_ITERATORS(pid_, processor_id_type pid, pid)
363 0 : DECLARE_ELEM_ITERATORS(type_, ElemType type, type)
364 :
365 0 : DECLARE_ELEM_ITERATORS(active_subdomain_, subdomain_id_type sid, sid)
366 0 : DECLARE_ELEM_ITERATORS(active_subdomain_set_, std::set<subdomain_id_type> ss, ss)
367 :
368 : // Backwards compatibility
369 0 : virtual SimpleRange<element_iterator> active_subdomain_elements_ptr_range(subdomain_id_type sid) override final { return active_subdomain_element_ptr_range(sid); }
370 0 : virtual SimpleRange<const_element_iterator> active_subdomain_elements_ptr_range(subdomain_id_type sid) const override final { return active_subdomain_element_ptr_range(sid); }
371 0 : virtual SimpleRange<element_iterator> active_local_subdomain_elements_ptr_range(subdomain_id_type sid) override final { return active_local_subdomain_element_ptr_range(sid); }
372 0 : virtual SimpleRange<const_element_iterator> active_local_subdomain_elements_ptr_range(subdomain_id_type sid) const override final { return active_local_subdomain_element_ptr_range(sid); }
373 0 : virtual SimpleRange<element_iterator> active_subdomain_set_elements_ptr_range(std::set<subdomain_id_type> ss) override final { return active_subdomain_set_element_ptr_range(ss); }
374 0 : virtual SimpleRange<const_element_iterator> active_subdomain_set_elements_ptr_range(std::set<subdomain_id_type> ss) const override final { return active_subdomain_set_element_ptr_range(ss); }
375 :
376 0 : DECLARE_ELEM_ITERATORS(not_active_,,);
377 0 : DECLARE_ELEM_ITERATORS(not_ancestor_,,);
378 0 : DECLARE_ELEM_ITERATORS(not_subactive_,,);
379 0 : DECLARE_ELEM_ITERATORS(not_local_,,);
380 0 : DECLARE_ELEM_ITERATORS(not_level_, unsigned int level, level)
381 :
382 9727 : DECLARE_ELEM_ITERATORS(active_local_,,)
383 0 : DECLARE_ELEM_ITERATORS(active_not_local_,,)
384 190 : DECLARE_ELEM_ITERATORS(active_unpartitioned_,,)
385 0 : DECLARE_ELEM_ITERATORS(active_type_, ElemType type, type)
386 0 : DECLARE_ELEM_ITERATORS(active_pid_, processor_id_type pid, pid)
387 0 : DECLARE_ELEM_ITERATORS(local_level_, unsigned int level, level)
388 0 : DECLARE_ELEM_ITERATORS(local_not_level_, unsigned int level, level)
389 0 : DECLARE_ELEM_ITERATORS(active_local_subdomain_, subdomain_id_type sid, sid)
390 0 : DECLARE_ELEM_ITERATORS(active_local_subdomain_set_, std::set<subdomain_id_type> ss, ss)
391 :
392 0 : DECLARE_ELEM_ITERATORS(semilocal_,,)
393 0 : DECLARE_ELEM_ITERATORS(ghost_,,)
394 0 : DECLARE_ELEM_ITERATORS(active_semilocal_,,)
395 :
396 0 : DECLARE_ELEM_ITERATORS(evaluable_, const DofMap & dof_map LIBMESH_COMMA unsigned int var_num = libMesh::invalid_uint, dof_map LIBMESH_COMMA var_num)
397 0 : DECLARE_ELEM_ITERATORS(multi_evaluable_, std::vector<const DofMap *> dof_maps, dof_maps)
398 :
399 : #ifdef LIBMESH_ENABLE_AMR
400 0 : DECLARE_ELEM_ITERATORS(flagged_, unsigned char rflag, rflag)
401 :
402 : // Elem::refinement_flag() == rflag && Elem::processor_id() == pid
403 0 : DECLARE_ELEM_ITERATORS(flagged_pid_, unsigned char rflag LIBMESH_COMMA processor_id_type pid, rflag LIBMESH_COMMA pid)
404 : #endif
405 :
406 11272 : DECLARE_NODE_ITERATORS(,,)
407 0 : DECLARE_NODE_ITERATORS(active_,,)
408 9532 : DECLARE_NODE_ITERATORS(local_,,)
409 0 : DECLARE_NODE_ITERATORS(bnd_,,)
410 0 : DECLARE_NODE_ITERATORS(pid_, processor_id_type pid, pid)
411 0 : DECLARE_NODE_ITERATORS(bid_, boundary_id_type bid, bid)
412 :
413 0 : DECLARE_NODE_ITERATORS(evaluable_, const DofMap & dof_map LIBMESH_COMMA unsigned int var_num = libMesh::invalid_uint, dof_map LIBMESH_COMMA var_num)
414 :
415 0 : DECLARE_NODE_ITERATORS(multi_evaluable_, std::vector<const DofMap *> dof_maps, dof_maps)
416 :
417 :
418 : protected:
419 :
420 : /**
421 : * Move node and elements from a DistributedMesh.
422 : */
423 : virtual void move_nodes_and_elements(MeshBase && other_mesh) override;
424 :
425 : /**
426 : * The vertices (spatial coordinates) of the mesh.
427 : */
428 : dofobject_container<Node> _nodes;
429 :
430 : /**
431 : * The elements in the mesh.
432 : */
433 : dofobject_container<Elem> _elements;
434 :
435 : /**
436 : * A boolean remembering whether we're serialized or not
437 : */
438 : bool _is_serial;
439 :
440 : /**
441 : * A boolean remembering whether we're serialized to proc 0 or not
442 : */
443 : bool _is_serial_on_proc_0;
444 :
445 : /**
446 : * A boolean remembering whether we've recently deleted top-level elements or
447 : * not. If so, we'll need to be extra careful when deleting "unused" nodes
448 : * they used to have, because those nodes might still be used by ghost elements.
449 : */
450 : bool _deleted_coarse_elements;
451 :
452 : /**
453 : * Cached data from the last renumber_nodes_and_elements call
454 : */
455 : dof_id_type _n_nodes, _n_elem, _max_node_id, _max_elem_id;
456 :
457 : /**
458 : * Guaranteed globally unused IDs for use when adding new
459 : * nodes or elements.
460 : */
461 : dof_id_type _next_free_local_node_id,
462 : _next_free_local_elem_id;
463 : dof_id_type _next_free_unpartitioned_node_id,
464 : _next_free_unpartitioned_elem_id;
465 :
466 : #ifdef LIBMESH_ENABLE_UNIQUE_ID
467 : /**
468 : * The next available unique id for assigning ids to unpartitioned DOF objects
469 : */
470 : unique_id_type _next_unpartitioned_unique_id;
471 : #endif
472 :
473 : /**
474 : * These are extra ghost elements that we want to make sure
475 : * not to delete when we call delete_remote_elements()
476 : */
477 : std::set<Elem *> _extra_ghost_elems;
478 :
479 : private:
480 :
481 : /**
482 : * Typedefs for the container implementation.
483 : */
484 : typedef dofobject_container<Elem>::veclike_iterator elem_iterator_imp;
485 : typedef dofobject_container<Elem>::const_veclike_iterator const_elem_iterator_imp;
486 :
487 : /**
488 : * Typedefs for the container implementation.
489 : */
490 : typedef dofobject_container<Node>::veclike_iterator node_iterator_imp;
491 : typedef dofobject_container<Node>::const_veclike_iterator const_node_iterator_imp;
492 : };
493 :
494 :
495 : } // namespace libMesh
496 :
497 : #endif // LIBMESH_DISTRIBUTED_MESH_H
|
