Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2025 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : #ifndef LIBMESH_PARALLEL_GHOST_SYNC_H
21 : #define LIBMESH_PARALLEL_GHOST_SYNC_H
22 :
23 : // libMesh includes
24 : #include "libmesh/elem.h"
25 : #include "libmesh/int_range.h"
26 : #include "libmesh/location_maps.h"
27 : #include "libmesh/mesh_base.h"
28 : #include "libmesh/parallel_algebra.h"
29 :
30 : // TIMPI includes
31 : #include "timpi/communicator.h"
32 : #include "timpi/parallel_sync.h"
33 :
34 : // C++ includes
35 : #include <map> // FIXME - pid > comm.size() breaks with unordered_map
36 : #include <vector>
37 :
38 :
39 : namespace libMesh
40 : {
41 :
42 :
43 :
44 : //--------------------------------------------------------------------------
45 : namespace Parallel {
46 :
47 : //------------------------------------------------------------------------
48 : /**
49 : * Request data about a range of ghost nodes uniquely identified by
50 : * their xyz location or a range of active ghost elements uniquely
51 : * identified by their vertex averages' xyz location. Fulfill requests
52 : * with
53 : * sync.gather_data(const std::vector<unsigned int> & ids,
54 : * std::vector<sync::datum> & data),
55 : * by resizing and setting the values of the data vector.
56 : * Respond to fulfillment with
57 : * sync.act_on_data(const std::vector<unsigned int> & ids,
58 : * std::vector<sync::datum> & data)
59 : * The user must define Parallel::StandardType<sync::datum> if
60 : * sync::datum isn't a built-in type.
61 : * The user-provided location_map should be already filled.
62 : *
63 : * This method may fail in cases of overlapping nodes or vertex
64 : * averages, e.g. with slit meshes and/or overset meshes.
65 : */
66 : template <typename Iterator,
67 : typename DofObjType,
68 : typename SyncFunctor>
69 : void sync_dofobject_data_by_xyz(const Communicator & comm,
70 : const Iterator & range_begin,
71 : const Iterator & range_end,
72 : LocationMap<DofObjType> & location_map,
73 : SyncFunctor & sync);
74 :
75 : //------------------------------------------------------------------------
76 : /**
77 : * Request data about a range of ghost dofobjects uniquely
78 : * identified by their id. Fulfill requests with
79 : * sync.gather_data(const std::vector<dof_id_type> & ids,
80 : * std::vector<sync::datum> & data),
81 : * by resizing and setting the values of the data vector.
82 : * Respond to fulfillment with
83 : * sync.act_on_data(const std::vector<dof_id_type> & ids,
84 : * std::vector<sync::datum> & data)
85 : * The user must define Parallel::StandardType<sync::datum> if
86 : * sync::datum isn't a built-in type.
87 : */
88 : template <typename Iterator,
89 : typename SyncFunctor>
90 : void sync_dofobject_data_by_id(const Communicator & comm,
91 : const Iterator & range_begin,
92 : const Iterator & range_end,
93 : SyncFunctor & sync);
94 :
95 : /**
96 : * Request data about a range of ghost dofobjects uniquely
97 : * identified by their id.
98 : *
99 : * Elements within the range can be excluded from the request by
100 : * returning false from dofobj_check(dof_object)
101 : */
102 : template <typename Iterator,
103 : typename DofObjectCheckFunctor,
104 : typename SyncFunctor>
105 : void sync_dofobject_data_by_id(const Communicator & comm,
106 : const Iterator & range_begin,
107 : const Iterator & range_end,
108 : const DofObjectCheckFunctor & dofobj_check,
109 : SyncFunctor & sync);
110 :
111 : //------------------------------------------------------------------------
112 : /**
113 : * Request data about a range of ghost elements uniquely
114 : * identified by their parent id and which child they are.
115 : * Fulfill requests with
116 : * sync.gather_data(const std::vector<unsigned int> & ids,
117 : * std::vector<sync::datum> & data),
118 : * by resizing and setting the values of the data vector.
119 : * Respond to fulfillment with
120 : * sync.act_on_data(const std::vector<unsigned int> & ids,
121 : * std::vector<sync::datum> & data)
122 : * The user must define Parallel::StandardType<sync::datum> if
123 : * sync::datum isn't a built-in type.
124 : */
125 : template <typename Iterator,
126 : typename SyncFunctor>
127 : void sync_element_data_by_parent_id(MeshBase & mesh,
128 : const Iterator & range_begin,
129 : const Iterator & range_end,
130 : SyncFunctor & sync);
131 :
132 : //------------------------------------------------------------------------
133 : /**
134 : * Synchronize data about a range of ghost nodes uniquely identified
135 : * by an element id and local node id, assuming a single
136 : * synchronization pass is necessary.
137 : *
138 : * Data for all nodes connected to elements in the given range of
139 : * *element* iterators will be requested.
140 : *
141 : * Elements can be further excluded from the request by returning
142 : * false from element_check(elem)
143 : *
144 : * Nodes can be further excluded from the request by returning false
145 : * from node_check(elem, local_node_num)
146 : *
147 : * Fulfill requests with
148 : * sync.gather_data(const std::vector<unsigned int> & ids,
149 : * std::vector<sync::datum> & data),
150 : * by resizing and setting the values of the data vector.
151 : * Respond to fulfillment with
152 : * bool sync.act_on_data(const std::vector<unsigned int> & ids,
153 : * std::vector<sync::datum> & data)
154 : * and return true iff the response changed any data.
155 : *
156 : * The user must define Parallel::StandardType<sync::datum> if
157 : * sync::datum isn't a built-in type.
158 : *
159 : * This method returns true iff the sync pass changed any data on any
160 : * processor.
161 : */
162 : template <typename ElemCheckFunctor,
163 : typename NodeCheckFunctor,
164 : typename SyncFunctor>
165 : bool sync_node_data_by_element_id_once(MeshBase & mesh,
166 : const MeshBase::const_element_iterator & range_begin,
167 : const MeshBase::const_element_iterator & range_end,
168 : const ElemCheckFunctor & elem_check,
169 : const NodeCheckFunctor & node_check,
170 : SyncFunctor & sync);
171 :
172 :
173 :
174 : //------------------------------------------------------------------------
175 : /**
176 : * Synchronize data about a range of ghost nodes uniquely identified
177 : * by an element id and local node id, iterating until data is
178 : * completely in sync and further synchronization passes cause no
179 : * changes.
180 : *
181 : * Imagine a vertex surrounded by triangles, each on a different
182 : * processor, with a ghosting policy that include only face neighbors
183 : * and not point neighbors. Then the only way for authoritative
184 : * information to trickle out from that vertex is by being passed
185 : * along, one neighbor at a time, to processors who mostly don't even
186 : * see the node's true owner!
187 : *
188 : * Data for all nodes connected to elements in the given range of
189 : * *element* iterators will be requested.
190 : *
191 : * Elements can be further excluded from the request by returning
192 : * false from element_check(elem)
193 : *
194 : * Nodes can be further excluded from the request by returning false
195 : * from node_check(elem, local_node_num)
196 : *
197 : * Fulfill requests with
198 : * sync.gather_data(const std::vector<unsigned int> & ids,
199 : * std::vector<sync::datum> & data),
200 : * by resizing and setting the values of the data vector.
201 : * Respond to fulfillment with
202 : * bool sync.act_on_data(const std::vector<unsigned int> & ids,
203 : * std::vector<sync::datum> & data)
204 : * and return true iff the response changed any data.
205 : *
206 : * The user must define Parallel::StandardType<sync::datum> if
207 : * sync::datum isn't a built-in type.
208 : */
209 : template <typename ElemCheckFunctor,
210 : typename NodeCheckFunctor,
211 : typename SyncFunctor>
212 : void sync_node_data_by_element_id(MeshBase & mesh,
213 : const MeshBase::const_element_iterator & range_begin,
214 : const MeshBase::const_element_iterator & range_end,
215 : const ElemCheckFunctor & elem_check,
216 : const NodeCheckFunctor & node_check,
217 : SyncFunctor & sync);
218 :
219 :
220 : //------------------------------------------------------------------------
221 : // Parallel members
222 :
223 :
224 : // "Check" Functor to perform sync operations with no exclusions
225 : struct SyncEverything
226 : {
227 6598 : SyncEverything() {}
228 :
229 6078442 : bool operator() (const DofObject *) const { return true; }
230 :
231 312780 : bool operator() (const Elem *, unsigned int) const
232 312780 : { return true; }
233 : };
234 :
235 :
236 :
237 : template <typename Iterator,
238 : typename DofObjType,
239 : typename SyncFunctor>
240 : void sync_dofobject_data_by_xyz(const Communicator & comm,
241 : const Iterator & range_begin,
242 : const Iterator & range_end,
243 : LocationMap<DofObjType> & location_map,
244 : SyncFunctor & sync)
245 : {
246 : // This function must be run on all processors at once
247 : libmesh_parallel_only(comm);
248 :
249 : // We need a valid location_map
250 : #ifdef DEBUG
251 : bool need_map_update = (range_begin != range_end && location_map.empty());
252 : comm.max(need_map_update);
253 : libmesh_assert(!need_map_update);
254 : #endif
255 :
256 : // Count the objects to ask each processor about
257 : std::map<processor_id_type, dof_id_type>
258 : ghost_objects_from_proc;
259 :
260 : for (Iterator it = range_begin; it != range_end; ++it)
261 : {
262 : DofObjType * obj = *it;
263 : libmesh_assert (obj);
264 : processor_id_type obj_procid = obj->processor_id();
265 : if (obj_procid != DofObject::invalid_processor_id)
266 : ghost_objects_from_proc[obj_procid]++;
267 : }
268 :
269 : // Request sets to send to each processor
270 : std::map<processor_id_type, std::vector<Point>>
271 : requested_objs_pt;
272 : // Corresponding ids to keep track of
273 : std::map<processor_id_type, std::vector<dof_id_type>>
274 : requested_objs_id;
275 :
276 : // We know how many objects live on each processor, so reserve()
277 : // space for each.
278 : for (auto pair : ghost_objects_from_proc)
279 : {
280 : const processor_id_type p = pair.first;
281 : if (p != comm.rank())
282 : {
283 : requested_objs_pt[p].reserve(pair.second);
284 : requested_objs_id[p].reserve(pair.second);
285 : }
286 : }
287 :
288 : for (Iterator it = range_begin; it != range_end; ++it)
289 : {
290 : DofObjType * obj = *it;
291 : processor_id_type obj_procid = obj->processor_id();
292 : if (obj_procid == comm.rank() ||
293 : obj_procid == DofObject::invalid_processor_id)
294 : continue;
295 :
296 : Point p = location_map.point_of(*obj);
297 : requested_objs_pt[obj_procid].push_back(p);
298 : requested_objs_id[obj_procid].push_back(obj->id());
299 : }
300 :
301 : std::map<const std::vector<Point> *, processor_id_type>
302 : requested_objs_pt_inv;
303 : for (auto & pair : requested_objs_pt)
304 : requested_objs_pt_inv[&pair.second] = pair.first;
305 :
306 : auto gather_functor =
307 : [&location_map, &sync]
308 : (processor_id_type /*pid*/, const std::vector<Point> & pts,
309 : std::vector<typename SyncFunctor::datum> & data)
310 : {
311 : // Find the local id of each requested object
312 : std::size_t query_size = pts.size();
313 : std::vector<dof_id_type> query_id(query_size);
314 : for (std::size_t i=0; i != query_size; ++i)
315 : {
316 : Point pt = pts[i];
317 :
318 : // Look for this object in the multimap
319 : DofObjType * obj = location_map.find(pt);
320 :
321 : // We'd better find every object we're asked for
322 : libmesh_assert (obj);
323 :
324 : // Return the object's correct processor id,
325 : // and our (correct if it's local) id for it.
326 : query_id[i] = obj->id();
327 : }
328 :
329 : // Gather whatever data the user wants
330 : sync.gather_data(query_id, data);
331 : };
332 :
333 : auto action_functor =
334 : [&sync, &requested_objs_id,
335 : &requested_objs_pt_inv]
336 : (processor_id_type /* pid */, const std::vector<Point> & point_request,
337 : const std::vector<typename SyncFunctor::datum> & data)
338 : {
339 : // With splits working on more pids than ranks, query_pid may not equal pid
340 : const processor_id_type query_pid =
341 : requested_objs_pt_inv[&point_request];
342 :
343 : // Let the user process the results
344 : sync.act_on_data(requested_objs_id[query_pid], data);
345 : };
346 :
347 : // Trade requests with other processors
348 : typename SyncFunctor::datum * ex = nullptr;
349 : pull_parallel_vector_data
350 : (comm, requested_objs_pt, gather_functor, action_functor, ex);
351 : }
352 :
353 :
354 :
355 : template <typename Iterator,
356 : typename SyncFunctor>
357 5924 : void sync_dofobject_data_by_id(const Communicator & comm,
358 : const Iterator & range_begin,
359 : const Iterator & range_end,
360 : SyncFunctor & sync)
361 : {
362 806179 : sync_dofobject_data_by_id(comm, range_begin, range_end, SyncEverything(), sync);
363 5924 : }
364 :
365 : template <typename Iterator,
366 : typename DofObjectCheckFunctor,
367 : typename SyncFunctor>
368 814158 : void sync_dofobject_data_by_id(const Communicator & comm,
369 : const Iterator & range_begin,
370 : const Iterator & range_end,
371 : const DofObjectCheckFunctor & dofobj_check,
372 : SyncFunctor & sync)
373 : {
374 : // This function must be run on all processors at once
375 6082 : libmesh_parallel_only(comm);
376 :
377 : // Count the objects to ask each processor about
378 : std::map<processor_id_type, dof_id_type>
379 12164 : ghost_objects_from_proc;
380 :
381 224018769 : for (Iterator it = range_begin; it != range_end; ++it)
382 : {
383 114390872 : const DofObject * obj = *it;
384 3238509 : libmesh_assert (obj);
385 :
386 : // We may want to pass Elem* or Node* to the check function, not
387 : // just DofObject*
388 7196292 : if (!dofobj_check(*it))
389 2610840 : continue;
390 :
391 111780032 : processor_id_type obj_procid = obj->processor_id();
392 111780032 : if (obj_procid != DofObject::invalid_processor_id)
393 111776208 : ghost_objects_from_proc[obj_procid]++;
394 : }
395 :
396 : // Request sets to send to each processor
397 : std::map<processor_id_type, std::vector<dof_id_type>>
398 12164 : requested_objs_id;
399 :
400 : // We know how many objects live on each processor, so reserve()
401 : // space for each.
402 5179650 : for (auto pair : ghost_objects_from_proc)
403 : {
404 4365492 : const processor_id_type p = pair.first;
405 4365492 : if (p != comm.rank())
406 3824145 : requested_objs_id[p].reserve(pair.second);
407 : }
408 :
409 224018769 : for (Iterator it = range_begin; it != range_end; ++it)
410 : {
411 114390872 : const DofObject * obj = *it;
412 :
413 7196292 : if (!dofobj_check(*it))
414 33707943 : continue;
415 :
416 111780032 : processor_id_type obj_procid = obj->processor_id();
417 113298091 : if (obj_procid == comm.rank() ||
418 1518059 : obj_procid == DofObject::invalid_processor_id)
419 2386132 : continue;
420 :
421 80682929 : requested_objs_id[obj_procid].push_back(obj->id());
422 : }
423 :
424 825704 : auto gather_functor =
425 3813994 : [&sync]
426 : (processor_id_type, const std::vector<dof_id_type> & ids,
427 5773 : std::vector<typename SyncFunctor::datum> & data)
428 : {
429 3824464 : sync.gather_data(ids, data);
430 : };
431 :
432 826154 : auto action_functor =
433 3973023 : [&sync]
434 : (processor_id_type, const std::vector<dof_id_type> & ids,
435 5998 : const std::vector<typename SyncFunctor::datum> & data)
436 : {
437 : // Let the user process the results
438 3983674 : sync.act_on_data(ids, data);
439 : };
440 :
441 : // Trade requests with other processors
442 6082 : typename SyncFunctor::datum * ex = nullptr;
443 : pull_parallel_vector_data
444 814158 : (comm, requested_objs_id, gather_functor, action_functor, ex);
445 814158 : }
446 :
447 :
448 :
449 : // If there's no refined elements, there's nothing to sync
450 : #ifdef LIBMESH_ENABLE_AMR
451 : template <typename Iterator,
452 : typename SyncFunctor>
453 21505 : void sync_element_data_by_parent_id(MeshBase & mesh,
454 : const Iterator & range_begin,
455 : const Iterator & range_end,
456 : SyncFunctor & sync)
457 : {
458 76 : const Communicator & comm (mesh.comm());
459 :
460 : // This function must be run on all processors at once
461 38 : libmesh_parallel_only(comm);
462 :
463 : // Count the objects to ask each processor about
464 : std::map<processor_id_type, dof_id_type>
465 76 : ghost_objects_from_proc;
466 :
467 7741847 : for (Iterator it = range_begin; it != range_end; ++it)
468 : {
469 3864119 : Elem * elem = *it;
470 3864119 : processor_id_type obj_procid = elem->processor_id();
471 3865351 : if (obj_procid == comm.rank() ||
472 1232 : obj_procid == DofObject::invalid_processor_id)
473 1221561 : continue;
474 2464 : const Elem * parent = elem->parent();
475 2660708 : if (!parent || !elem->active())
476 18150 : continue;
477 :
478 2642558 : ghost_objects_from_proc[obj_procid]++;
479 : }
480 :
481 : // Request sets to send to each processor
482 : std::map<processor_id_type, std::vector<dof_id_type>>
483 76 : requested_objs_id;
484 : std::map<processor_id_type, std::vector<std::pair<dof_id_type,unsigned char>>>
485 76 : requested_objs_parent_id_child_num;
486 :
487 : // We know how many objects live on each processor, so reserve()
488 : // space for each.
489 86015 : for (auto pair : ghost_objects_from_proc)
490 : {
491 64510 : const processor_id_type p = pair.first;
492 64510 : if (p != comm.rank())
493 : {
494 64510 : requested_objs_id[p].reserve(pair.second);
495 64510 : requested_objs_parent_id_child_num[p].reserve(pair.second);
496 : }
497 : }
498 :
499 7741847 : for (Iterator it = range_begin; it != range_end; ++it)
500 : {
501 3864119 : Elem * elem = *it;
502 3864119 : processor_id_type obj_procid = elem->processor_id();
503 3865351 : if (obj_procid == comm.rank() ||
504 1232 : obj_procid == DofObject::invalid_processor_id)
505 1221561 : continue;
506 2464 : const Elem * parent = elem->parent();
507 2660708 : if (!parent || !elem->active())
508 18150 : continue;
509 :
510 2642558 : requested_objs_id[obj_procid].push_back(elem->id());
511 : requested_objs_parent_id_child_num[obj_procid].emplace_back
512 2642558 : (parent->id(), cast_int<unsigned char>(parent->which_child_am_i(elem)));
513 : }
514 :
515 : std::map<const std::vector<std::pair<dof_id_type,unsigned char>> *, processor_id_type>
516 76 : requested_objs_parent_id_child_num_inv;
517 86015 : for (auto & pair : requested_objs_parent_id_child_num)
518 64510 : requested_objs_parent_id_child_num_inv[&pair.second] = pair.first;
519 :
520 21497 : auto gather_functor =
521 128884 : [&mesh, &sync]
522 : (processor_id_type,
523 : const std::vector<std::pair<dof_id_type, unsigned char>> & parent_id_child_num,
524 68 : std::vector<typename SyncFunctor::datum> & data)
525 : {
526 : // Find the id of each requested element
527 68 : std::size_t query_size = parent_id_child_num.size();
528 64544 : std::vector<dof_id_type> query_id(query_size);
529 2707068 : for (std::size_t i=0; i != query_size; ++i)
530 : {
531 2643790 : Elem & parent = mesh.elem_ref(parent_id_child_num[i].first);
532 1232 : libmesh_assert(parent.has_children());
533 2643790 : Elem * child = parent.child_ptr(parent_id_child_num[i].second);
534 1232 : libmesh_assert(child);
535 1232 : libmesh_assert(child->active());
536 2643790 : query_id[i] = child->id();
537 : }
538 :
539 : // Gather whatever data the user wants
540 34 : sync.gather_data(query_id, data);
541 : };
542 :
543 21463 : auto action_functor =
544 193326 : [&sync, &requested_objs_id,
545 : &requested_objs_parent_id_child_num_inv]
546 : (processor_id_type /* pid */,
547 : const std::vector<std::pair<dof_id_type, unsigned char>> & parent_id_child_num_request,
548 68 : const std::vector<typename SyncFunctor::datum> & data)
549 : {
550 : // With splits working on more pids than ranks, query_pid may not equal pid
551 128952 : const processor_id_type query_pid =
552 64442 : requested_objs_parent_id_child_num_inv[&parent_id_child_num_request];
553 :
554 : // Let the user process the results
555 64510 : sync.act_on_data(requested_objs_id[query_pid], data);
556 : };
557 :
558 : // Trade requests with other processors
559 38 : typename SyncFunctor::datum * ex = nullptr;
560 : pull_parallel_vector_data
561 21505 : (comm, requested_objs_parent_id_child_num, gather_functor,
562 : action_functor, ex);
563 21505 : }
564 : #else
565 : template <typename Iterator,
566 : typename SyncFunctor>
567 : void sync_element_data_by_parent_id(MeshBase &,
568 : const Iterator &,
569 : const Iterator &,
570 : SyncFunctor &)
571 : {
572 : }
573 : #endif // LIBMESH_ENABLE_AMR
574 :
575 :
576 :
577 : template <typename ElemCheckFunctor,
578 : typename NodeCheckFunctor,
579 : typename SyncFunctor>
580 306426 : bool sync_node_data_by_element_id_once(MeshBase & mesh,
581 : const MeshBase::const_element_iterator & range_begin,
582 : const MeshBase::const_element_iterator & range_end,
583 : const ElemCheckFunctor & elem_check,
584 : const NodeCheckFunctor & node_check,
585 : SyncFunctor & sync)
586 : {
587 430 : const Communicator & comm (mesh.comm());
588 :
589 : // Count the objects to ask each processor about
590 : std::map<processor_id_type, dof_id_type>
591 860 : ghost_objects_from_proc;
592 :
593 77183878 : for (const auto & elem : as_range(range_begin, range_end))
594 : {
595 42840 : libmesh_assert (elem);
596 :
597 38154150 : if (!elem_check(elem))
598 1255564 : continue;
599 :
600 36898586 : const processor_id_type proc_id = elem->processor_id();
601 :
602 41536 : bool i_have_elem =
603 36898586 : (proc_id == comm.rank() ||
604 : proc_id == DofObject::invalid_processor_id);
605 :
606 31956416 : if (elem->active() && i_have_elem)
607 8409619 : continue;
608 :
609 288447904 : for (auto n : elem->node_index_range())
610 : {
611 259958937 : if (!node_check(elem, n))
612 35520899 : continue;
613 :
614 232647819 : const processor_id_type node_pid =
615 170262 : elem->node_ref(n).processor_id();
616 :
617 232647819 : if (i_have_elem && (node_pid == comm.rank()))
618 42266 : continue;
619 :
620 224438038 : if (i_have_elem)
621 : {
622 2116 : libmesh_assert_not_equal_to
623 : (node_pid, DofObject::invalid_processor_id);
624 1729803 : ghost_objects_from_proc[node_pid]++;
625 : }
626 : else
627 : {
628 222708235 : const processor_id_type request_pid =
629 206409528 : (node_pid == DofObject::invalid_processor_id) ?
630 : proc_id : node_pid;
631 222708235 : ghost_objects_from_proc[request_pid]++;
632 : }
633 : }
634 : }
635 :
636 : // Now repeat that iteration, filling request sets this time.
637 :
638 : // Request sets to send to each processor
639 : std::map<processor_id_type, std::vector<std::pair<dof_id_type, unsigned char>>>
640 430 : requested_objs_elem_id_node_num;
641 :
642 : // We know how many objects live on each processor, so reserve()
643 : // space for each.
644 2015531 : for (auto pair : ghost_objects_from_proc)
645 : {
646 1709105 : const processor_id_type p = pair.first;
647 1709105 : if (p != comm.rank())
648 1514493 : requested_objs_elem_id_node_num[p].reserve(ghost_objects_from_proc[p]);
649 : }
650 :
651 77183878 : for (const auto & elem : as_range(range_begin, range_end))
652 : {
653 42840 : libmesh_assert (elem);
654 :
655 38154150 : if (!elem_check(elem))
656 9665183 : continue;
657 :
658 36898586 : const processor_id_type proc_id = elem->processor_id();
659 :
660 41536 : bool i_have_elem =
661 36898586 : (proc_id == comm.rank() ||
662 : proc_id == DofObject::invalid_processor_id);
663 :
664 31956416 : if (elem->active() && i_have_elem)
665 1865020 : continue;
666 :
667 28488967 : const dof_id_type elem_id = elem->id();
668 :
669 288447904 : for (auto n : elem->node_index_range())
670 : {
671 259958937 : if (!node_check(elem, n))
672 35520899 : continue;
673 :
674 170262 : const Node & node = elem->node_ref(n);
675 232647819 : const processor_id_type node_pid = node.processor_id();
676 :
677 232647819 : if (i_have_elem && (node_pid == comm.rank()))
678 42266 : continue;
679 :
680 224438038 : if (i_have_elem)
681 : {
682 2116 : libmesh_assert_not_equal_to
683 : (node_pid, DofObject::invalid_processor_id);
684 : requested_objs_elem_id_node_num[node_pid].emplace_back
685 1729803 : (elem_id, cast_int<unsigned char>(n));
686 : }
687 : else
688 : {
689 222708235 : const processor_id_type request_pid =
690 206409528 : (node_pid == DofObject::invalid_processor_id) ?
691 : proc_id : node_pid;
692 : requested_objs_elem_id_node_num[request_pid].emplace_back
693 222708235 : (elem_id,cast_int<unsigned char>(n));
694 : }
695 : }
696 : }
697 :
698 307370 : auto gather_functor =
699 3417006 : [&mesh, &sync]
700 : (processor_id_type,
701 : const std::vector<std::pair<dof_id_type, unsigned char>> & elem_id_node_num,
702 1204 : std::vector<typename SyncFunctor::datum> & data)
703 : {
704 : // Find the id of each requested element
705 1204 : std::size_t request_size = elem_id_node_num.size();
706 1710309 : std::vector<dof_id_type> query_id(request_size);
707 226275741 : for (std::size_t i=0; i != request_size; ++i)
708 : {
709 : // We might now get queries about remote elements, in which
710 : // case we'll have to ignore them and wait for the query
711 : // answer to filter to the querier via another source.
712 224694030 : const Elem * elem = mesh.query_elem_ptr(elem_id_node_num[i].first);
713 :
714 224566034 : if (elem)
715 : {
716 224678471 : const unsigned int n = elem_id_node_num[i].second;
717 127996 : libmesh_assert_less (n, elem->n_nodes());
718 :
719 127996 : const Node & node = elem->node_ref(n);
720 :
721 : // This isn't a safe assertion in the case where we're
722 : // syncing processor ids
723 : // libmesh_assert_equal_to (node->processor_id(), comm.rank());
724 :
725 224678471 : query_id[i] = node.id();
726 : }
727 : else
728 15559 : query_id[i] = DofObject::invalid_id;
729 : }
730 :
731 : // Gather whatever data the user wants
732 1709707 : sync.gather_data(query_id, data);
733 : };
734 :
735 306426 : bool data_changed = false;
736 :
737 307370 : auto action_functor =
738 3417006 : [&sync, &mesh, &data_changed]
739 : (processor_id_type /* pid */,
740 : const std::vector<std::pair<dof_id_type, unsigned char>> & elem_id_node_num,
741 1204 : const std::vector<typename SyncFunctor::datum> & data)
742 : {
743 1204 : const std::size_t data_size = data.size();
744 :
745 602 : libmesh_assert_equal_to(elem_id_node_num.size(), data_size);
746 :
747 1710309 : std::vector<dof_id_type> requested_objs_id(data.size());
748 :
749 226275741 : for (auto i : IntRange<std::size_t>(0,data_size))
750 : {
751 224694030 : const Elem & elem = mesh.elem_ref(elem_id_node_num[i].first);
752 224694030 : const Node & node = elem.node_ref(elem_id_node_num[i].second);
753 224694030 : requested_objs_id[i] = node.id();
754 : }
755 :
756 : // Let the user process the results. If any of the results
757 : // were different than what the user expected, then we may
758 : // need to sync again just in case this processor has to
759 : // pass on the changes to yet another processor.
760 3143578 : if (sync.act_on_data(requested_objs_id, data))
761 203018 : data_changed = true;
762 : };
763 :
764 : // Trade requests with other processors
765 430 : typename SyncFunctor::datum * ex = nullptr;
766 : pull_parallel_vector_data
767 306426 : (comm, requested_objs_elem_id_node_num, gather_functor,
768 : action_functor, ex);
769 :
770 306426 : comm.max(data_changed);
771 :
772 612852 : return data_changed;
773 : }
774 :
775 :
776 :
777 : template <typename ElemCheckFunctor,
778 : typename NodeCheckFunctor,
779 : typename SyncFunctor>
780 356 : void sync_node_data_by_element_id(MeshBase & mesh,
781 : const MeshBase::const_element_iterator & range_begin,
782 : const MeshBase::const_element_iterator & range_end,
783 : const ElemCheckFunctor & elem_check,
784 : const NodeCheckFunctor & node_check,
785 : SyncFunctor & sync)
786 : {
787 : // This function must be run on all processors at once
788 356 : libmesh_parallel_only(mesh.comm());
789 :
790 356 : bool need_sync = false;
791 :
792 284995 : do
793 : {
794 : need_sync =
795 : sync_node_data_by_element_id_once
796 285351 : (mesh, range_begin, range_end, elem_check, node_check,
797 : sync);
798 : } while (need_sync);
799 356 : }
800 :
801 :
802 : }
803 :
804 :
805 :
806 : // This struct can be created and passed to the
807 : // Parallel::sync_dofobject_data_by_id() function.
808 : struct SyncNodalPositions
809 : {
810 : // The constructor. You need a reference to the mesh where you will
811 : // be setting/getting nodal positions.
812 : explicit
813 : SyncNodalPositions(MeshBase & m);
814 :
815 : // The datum typedef is required of this functor, so that the
816 : // Parallel::sync_dofobject_data_by_id() function can create e.g.
817 : // std::vector<datum>.
818 : typedef Point datum;
819 :
820 : // First required interface. This function must fill up the data vector for the
821 : // ids specified in the ids vector.
822 : void gather_data (const std::vector<dof_id_type> & ids, std::vector<datum> & data) const;
823 :
824 : // Second required interface. This function must do something with the data in
825 : // the data vector for the ids in the ids vector.
826 : void act_on_data (const std::vector<dof_id_type> & ids, const std::vector<datum> & data) const;
827 :
828 : MeshBase & mesh;
829 : };
830 :
831 : // This struct can be created and passed to the
832 : // Parallel::sync_dofobject_data_by_id() function.
833 : struct SyncSubdomainIds
834 : {
835 : // The constructor. You need a reference to the mesh where you will
836 : // be setting/getting element subdomain IDs.
837 : explicit
838 : SyncSubdomainIds(MeshBase & m);
839 :
840 : // The datum typedef is required of this functor, so that the
841 : // Parallel::sync_dofobject_data_by_id() function can create e.g.
842 : // std::vector<datum>.
843 : typedef subdomain_id_type datum;
844 :
845 : // First required interface. This function must fill up the data vector for the
846 : // ids specified in the ids vector.
847 : void gather_data (const std::vector<dof_id_type> & ids, std::vector<datum> & data) const;
848 :
849 : // Second required interface. This function must do something with the data in
850 : // the data vector for the ids in the ids vector.
851 : void act_on_data (const std::vector<dof_id_type> & ids, const std::vector<datum> & data) const;
852 :
853 : MeshBase & mesh;
854 : };
855 :
856 : // This struct can be created and passed to the
857 : // Parallel::sync_dofobject_data_by_id() function
858 : // for sync element integers.
859 : struct SyncElementIntegers
860 : {
861 : // The constructor. You need a reference to the mesh where you will
862 : // be setting/getting element integers and an existing element integer name.
863 : explicit
864 : SyncElementIntegers(MeshBase & m, const std::string & integer_name);
865 :
866 : // The datum typedef is required of this functor, so that the
867 : // Parallel::sync_dofobject_data_by_id() function can create e.g.
868 : // std::vector<datum>.
869 : typedef dof_id_type datum;
870 :
871 : // First required interface. This function must fill up the data vector for the
872 : // ids specified in the ids vector.
873 : void gather_data(const std::vector<dof_id_type> & ids, std::vector<datum> & data) const;
874 :
875 : // Second required interface. This function must do something with the data in
876 : // the data vector for the ids in the ids vector.
877 : void act_on_data(const std::vector<dof_id_type> & ids, const std::vector<datum> & data) const;
878 :
879 : MeshBase & mesh;
880 : unsigned int ind;
881 : };
882 :
883 : } // namespace libMesh
884 :
885 : #endif // LIBMESH_PARALLEL_GHOST_SYNC_H
|
