libMesh
communicator.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2019 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_COMMUNICATOR_H
20 #define LIBMESH_COMMUNICATOR_H
21 
22 // Parallel includes
23 #include "libmesh/data_type.h"
24 #include "libmesh/message_tag.h"
25 #include "libmesh/request.h"
26 #include "libmesh/status.h"
27 #include "libmesh/post_wait_dereference_shared_ptr.h"
28 
29 // libMesh Includes
30 #include "libmesh/libmesh_common.h"
31 
32 // C++ includes
33 #include <map>
34 #include <string>
35 #include <vector>
36 
37 // C++ includes needed for parallel_communicator_specializations
38 //
39 // These could be forward declarations if only that wasn't illegal
40 #include <numeric> // complex
41 #include <set>
42 
43 namespace libMesh
44 {
45 
53 namespace Parallel
54 {
55 
56 template <typename T>
57 class Packing;
58 
59 #ifdef LIBMESH_HAVE_MPI
60 
61 //-------------------------------------------------------------------
65 typedef MPI_Comm communicator;
66 
70 const unsigned int any_source =
71  static_cast<unsigned int>(MPI_ANY_SOURCE);
72 
73 #else
74 
75 // These shouldn't actually be needed, but must be
76 // unique types for function overloading to work
77 // properly.
78 typedef int communicator; // Must match petsc-nompi definition
79 
80 const unsigned int any_source=0;
81 
82 #endif // LIBMESH_HAVE_MPI
83 
84 //-------------------------------------------------------------------
93 {
94  // Basic operations:
95 public:
96 
100  Communicator ();
101 
102  /*
103  * Constructor from MPI_Comm
104  */
105  explicit Communicator (const communicator & comm);
106 
107  /*
108  * Don't use copy construction or assignment, just copy by reference
109  * or pointer - it's too hard to keep a common used_tag_values if
110  * each communicator is shared by more than one Communicator
111  */
112  Communicator (const Communicator &) = delete;
113  Communicator & operator= (const Communicator &) = delete;
114 
115  /*
116  * Move constructor and assignment operator
117  */
118  Communicator (Communicator &&) = default;
119  Communicator & operator= (Communicator &&) = default;
120 
121  /*
122  * NON-VIRTUAL destructor
123  */
124  ~Communicator ();
125 
126  /*
127  * Create a new communicator between some subset of \p this
128  */
129  void split(int color, int key, Communicator & target) const;
130 
131  /*
132  * Create a new duplicate of \p this communicator
133  */
134  void duplicate(const Communicator & comm);
135 
136  /*
137  * Create a new duplicate of an MPI communicator
138  */
139  void duplicate(const communicator & comm);
140 
141  communicator & get() { return _communicator; }
142 
143  const communicator & get() const { return _communicator; }
144 
161  MessageTag get_unique_tag(int tagvalue = MessageTag::invalid_tag) const;
162 
167  void reference_unique_tag(int tagvalue) const;
168 
173  void dereference_unique_tag(int tagvalue) const;
174 
178  void clear();
179 
180  Communicator & operator= (const communicator & comm);
181 
182  processor_id_type rank() const { return _rank; }
183 
184  processor_id_type size() const { return _size; }
185 
190 
191 private:
192 
197  void assign(const communicator & comm);
198 
202 
203  // mutable used_tag_values and tag_queue - not thread-safe, but then
204  // Parallel:: isn't thread-safe in general.
205  mutable std::map<int, unsigned int> used_tag_values;
206  mutable int _next_tag;
207 
208  int _max_tag;
209 
210  // Keep track of duplicate/split operations so we know when to free
212 
213  // Communication operations:
214 public:
215 
219  void send_mode (const SendMode sm) { _send_mode = sm; }
220 
224  SendMode send_mode() const { return _send_mode; }
225 
229  void barrier () const;
230 
234  void nonblocking_barrier (Request & req) const;
235 
240  template <typename T>
241  bool verify(const T & r) const;
242 
248  template <typename T>
249  bool semiverify(const T * r) const;
250 
255  template <typename T>
256  void min(T & r) const;
257 
263  template <typename T>
264  void minloc(T & r,
265  unsigned int & min_id) const;
266 
272  template <typename T, typename A1, typename A2>
273  void minloc(std::vector<T,A1> & r,
274  std::vector<unsigned int,A2> & min_id) const;
275 
280  template <typename T>
281  void max(T & r) const;
282 
288  template <typename T>
289  void maxloc(T & r,
290  unsigned int & max_id) const;
291 
297  template <typename T, typename A1, typename A2>
298  void maxloc(std::vector<T,A1> & r,
299  std::vector<unsigned int,A2> & max_id) const;
300 
305  template <typename T>
306  void sum(T & r) const;
307 
313  template <typename T>
314  void set_union(T & data, const unsigned int root_id) const;
315 
320  template <typename T>
321  void set_union(T & data) const;
322 
327  status probe (const unsigned int src_processor_id,
328  const MessageTag & tag=any_tag) const;
329 
342  template <typename T>
343  Status packed_range_probe (const unsigned int src_processor_id,
344  const MessageTag & tag,
345  bool & flag) const;
346 
350  template <typename T>
351  void send (const unsigned int dest_processor_id,
352  const T & buf,
353  const MessageTag & tag=no_tag) const;
354 
358  template <typename T>
359  void send (const unsigned int dest_processor_id,
360  const T & buf,
361  Request & req,
362  const MessageTag & tag=no_tag) const;
363 
371  template <typename T>
372  void send (const unsigned int dest_processor_id,
373  const T & buf,
374  const DataType & type,
375  const MessageTag & tag=no_tag) const;
376 
384  template <typename T>
385  void send (const unsigned int dest_processor_id,
386  const T & buf,
387  const DataType & type,
388  Request & req,
389  const MessageTag & tag=no_tag) const;
390 
394  template <typename T>
395  Status receive (const unsigned int dest_processor_id,
396  T & buf,
397  const MessageTag & tag=any_tag) const;
398 
402  template <typename T>
403  void receive (const unsigned int dest_processor_id,
404  T & buf,
405  Request & req,
406  const MessageTag & tag=any_tag) const;
407 
415  template <typename T>
416  Status receive (const unsigned int dest_processor_id,
417  T & buf,
418  const DataType & type,
419  const MessageTag & tag=any_tag) const;
420 
428  template <typename T>
429  void receive (const unsigned int dest_processor_id,
430  T & buf,
431  const DataType & type,
432  Request & req,
433  const MessageTag & tag=any_tag) const;
434 
458  template <typename T, typename A>
459  bool possibly_receive (unsigned int & src_processor_id,
460  std::vector<T,A> & buf,
461  const DataType & type,
462  Request & req,
463  const MessageTag & tag) const;
464 
478  template <typename Context, typename Iter>
479  void send_packed_range (const unsigned int dest_processor_id,
480  const Context * context,
481  Iter range_begin,
482  const Iter range_end,
483  const MessageTag & tag=no_tag) const;
484 
498  template <typename Context, typename Iter>
499  void send_packed_range (const unsigned int dest_processor_id,
500  const Context * context,
501  Iter range_begin,
502  const Iter range_end,
503  Request & req,
504  const MessageTag & tag=no_tag) const;
505 
514  template <typename Context, typename Iter>
515  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
516  const Context * context,
517  Iter range_begin,
518  const Iter range_end,
519  Request & req,
520  const MessageTag & tag=no_tag) const;
521 
522 
531  template <typename Context, typename Iter>
532  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
533  const Context * context,
534  Iter range_begin,
535  const Iter range_end,
536  Request & req,
537  std::shared_ptr<std::vector<typename Parallel::Packing<typename std::iterator_traits<Iter>::value_type>::buffer_type>> & buffer,
538  const MessageTag & tag=no_tag) const;
539 
566  template <typename Context, typename OutputIter, typename T>
567  void receive_packed_range (const unsigned int dest_processor_id,
568  Context * context,
569  OutputIter out,
570  const T * output_type, // used only to infer T
571  const MessageTag & tag=any_tag) const;
572 
585  template <typename Context, typename OutputIter, typename T>
586  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
587  Context * context,
588  OutputIter out,
589  const T * output_type,
590  Request & req,
591  Status & stat,
592  const MessageTag & tag=any_tag) const;
593 
606  template <typename Context, typename OutputIter, typename T>
607  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
608  Context * context,
609  OutputIter out,
610  const T * output_type,
611  Request & req,
612  Status & stat,
613  std::shared_ptr<std::vector<typename Parallel::Packing<T>::buffer_type>> & buffer,
614  const MessageTag & tag=any_tag
615  ) const;
616 
621  template <typename T1, typename T2>
622  void send_receive(const unsigned int dest_processor_id,
623  const T1 & send,
624  const unsigned int source_processor_id,
625  T2 & recv,
626  const MessageTag & send_tag = no_tag,
627  const MessageTag & recv_tag = any_tag) const;
628 
665  template <typename Context1, typename RangeIter, typename Context2,
666  typename OutputIter, typename T>
667  void send_receive_packed_range(const unsigned int dest_processor_id,
668  const Context1 * context1,
669  RangeIter send_begin,
670  const RangeIter send_end,
671  const unsigned int source_processor_id,
672  Context2 * context2,
673  OutputIter out,
674  const T * output_type, // used only to infer T
675  const MessageTag & send_tag = no_tag,
676  const MessageTag & recv_tag = any_tag) const;
677 
683  template <typename T1, typename T2>
684  void send_receive(const unsigned int dest_processor_id,
685  const T1 & send,
686  const DataType & type1,
687  const unsigned int source_processor_id,
688  T2 & recv,
689  const DataType & type2,
690  const MessageTag & send_tag = no_tag,
691  const MessageTag & recv_tag = any_tag) const;
692 
697  template <typename T, typename A>
698  inline void gather(const unsigned int root_id,
699  const T & send,
700  std::vector<T,A> & recv) const;
701 
707  template <typename T, typename A>
708  inline void gather(const unsigned int root_id,
709  const std::basic_string<T> & send,
710  std::vector<std::basic_string<T>,A> & recv,
711  const bool identical_buffer_sizes=false) const;
712 
735  template <typename T, typename A>
736  inline void gather(const unsigned int root_id,
737  std::vector<T,A> & r) const;
738 
743  template <typename T, typename A>
744  inline void allgather(const T & send,
745  std::vector<T,A> & recv) const;
746 
752  template <typename T, typename A>
753  inline void allgather(const std::basic_string<T> & send,
754  std::vector<std::basic_string<T>,A> & recv,
755  const bool identical_buffer_sizes=false) const;
756 
781  template <typename T, typename A>
782  inline void allgather(std::vector<T,A> & r,
783  const bool identical_buffer_sizes = false) const;
784 
788  template <typename T, typename A>
789  inline void allgather(std::vector<std::basic_string<T>,A> & r,
790  const bool identical_buffer_sizes = false) const;
791 
792  //-------------------------------------------------------------------
797  template <typename T, typename A>
798  inline void scatter(const std::vector<T,A> & data,
799  T & recv,
800  const unsigned int root_id=0) const;
801 
808  template <typename T, typename A>
809  inline void scatter(const std::vector<T,A> & data,
810  std::vector<T,A> & recv,
811  const unsigned int root_id=0) const;
812 
819  template <typename T, typename A1, typename A2>
820  inline void scatter(const std::vector<T,A1> & data,
821  const std::vector<int,A2> counts,
822  std::vector<T,A1> & recv,
823  const unsigned int root_id=0) const;
824 
830  template <typename T, typename A1, typename A2>
831  inline void scatter(const std::vector<std::vector<T,A1>,A2> & data,
832  std::vector<T,A1> & recv,
833  const unsigned int root_id=0,
834  const bool identical_buffer_sizes=false) const;
835 
836  //-------------------------------------------------------------------
841  template <typename Context, typename Iter, typename OutputIter>
842  inline void gather_packed_range (const unsigned int root_id,
843  Context * context,
844  Iter range_begin,
845  const Iter range_end,
846  OutputIter out) const;
847 
852  template <typename Context, typename Iter, typename OutputIter>
853  inline void allgather_packed_range (Context * context,
854  Iter range_begin,
855  const Iter range_end,
856  OutputIter out) const;
857 
863  template <typename T, typename A>
864  inline void alltoall(std::vector<T,A> & r) const;
865 
874  template <typename T>
875  inline void broadcast(T & data, const unsigned int root_id=0) const;
876 
894  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
895  inline void broadcast_packed_range (const Context * context1,
896  Iter range_begin,
897  const Iter range_end,
898  OutputContext * context2,
899  OutputIter out,
900  const unsigned int root_id = 0) const;
901 
909 #include "libmesh/parallel_communicator_specializations"
910 
911 }; // class Communicator
912 
913 
914 } // namespace Parallel
915 
916 } // namespace libMesh
917 
918 #endif // LIBMESH_COMMUNICATOR_H
void barrier() const
Pause execution until all processors reach a certain point.
void set_union(T &data, const unsigned int root_id) const
Take a container of local variables on each processor, and collect their union over all processors...
SendMode
Whether to use default or synchronous sends?
Definition: communicator.h:189
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Blocking-send to one processor with data-defined type.
void send_receive(const unsigned int dest_processor_id, const T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
Send data send to one processor while simultaneously receiving other data recv from a (potentially di...
void maxloc(T &r, unsigned int &max_id) const
Take a local variable and replace it with the maximum of it&#39;s values on all processors, returning the minimum rank of a processor which originally held the maximum value.
Encapsulates the MPI_Comm object.
Definition: communicator.h:92
const unsigned int any_source
Processor id meaning "Accept from any source".
Definition: communicator.h:70
Encapsulates the MPI_Status struct.
Definition: status.h:59
void max(T &r) const
Take a local variable and replace it with the maximum of it&#39;s values on all processors.
Definition: assembly.h:98
processor_id_type size() const
Definition: communicator.h:184
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag) const
Blocking-send range-of-pointers to one processor.
void min(T &r) const
Take a local variable and replace it with the minimum of it&#39;s values on all processors.
MPI_Comm communicator
Communicator object for talking with subsets of processors.
Definition: communicator.h:57
void allgather(const T &send, std::vector< T, A > &recv) const
Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that proc...
void receive_packed_range(const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const
Blocking-receive range-of-pointers from one processor.
void gather(const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
Take a vector of length comm.size(), and on processor root_id fill in recv[processor_id] = the value ...
void alltoall(std::vector< T, A > &r) const
Effectively transposes the input vector across all processors.
void nonblocking_send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, const MessageTag &tag=no_tag) const
Similar to the above Nonblocking send_packed_range with a few important differences: ...
void send_mode(const SendMode sm)
Explicitly sets the SendMode type used for send operations.
Definition: communicator.h:219
The libMesh namespace provides an interface to certain functionality in the library.
Define data types and (un)serialization functions for use when encoding a potentially-variable-size o...
Definition: communicator.h:57
uint8_t processor_id_type
Definition: id_types.h:104
bool semiverify(const T *r) const
Verify that a local pointer points to the same value on all processors where it is not nullptr...
void minloc(T &r, unsigned int &min_id) const
Take a local variable and replace it with the minimum of it&#39;s values on all processors, returning the minimum rank of a processor which originally held the minimum value.
processor_id_type rank() const
Definition: communicator.h:182
void send_receive_packed_range(const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
Send a range-of-pointers to one processor while simultaneously receiving another range from a (potent...
Encapsulates the MPI tag integers.
Definition: message_tag.h:47
void assign(const communicator &comm)
Utility function for setting our member variables from an MPI communicator.
void sum(T &r) const
Take a local variable and replace it with the sum of it&#39;s values on all processors.
void scatter(const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
Take a vector of local variables and scatter the ith item to the ith processor in the communicator...
void duplicate(const Communicator &comm)
OStreamProxy out
Communicator()
Default Constructor.
void nonblocking_receive_packed_range(const unsigned int src_processor_id, Context *context, OutputIter out, const T *output_type, Request &req, Status &stat, const MessageTag &tag=any_tag) const
Non-Blocking-receive range-of-pointers from one processor.
bool possibly_receive(unsigned int &src_processor_id, std::vector< T, A > &buf, const DataType &type, Request &req, const MessageTag &tag) const
Nonblocking-receive from one processor with user-defined type.
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag) const
Blocking message probe.
MessageTag get_unique_tag(int tagvalue=MessageTag::invalid_tag) const
Get a tag that is unique to this Communicator.
void allgather_packed_range(Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
Take a range of local variables, combine it with ranges from all processors, and write the output to ...
Encapsulates the MPI_Datatype.
Definition: data_type.h:63
const MessageTag no_tag
Definition: message_tag.h:120
Communicator & operator=(const Communicator &)=delete
void clear()
Free and reset this communicator.
static const int invalid_tag
Invalid tag, to allow for default construction.
Definition: message_tag.h:54
Status packed_range_probe(const unsigned int src_processor_id, const MessageTag &tag, bool &flag) const
Non-Blocking message probe for a packed range message.
void dereference_unique_tag(int tagvalue) const
Dereference an already-acquired tag, and see if we can re-release it.
Encapsulates the MPI_Request.
Definition: request.h:63
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
Blocking-receive from one processor with data-defined type.
const MessageTag any_tag
Default message tag ids.
Definition: message_tag.h:115
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:205
IterBase * data
Ideally this private member data should have protected access.
void nonblocking_barrier(Request &req) const
Start a barrier that doesn&#39;t block.
void reference_unique_tag(int tagvalue) const
Reference an already-acquired tag, so that we know it will be dereferenced multiple times before we c...
void gather_packed_range(const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
Take a range of local variables, combine it with ranges from all processors, and write the output to ...
void broadcast(T &data, const unsigned int root_id=0) const
Take a local value and broadcast it to all processors.
void split(int color, int key, Communicator &target) const
void broadcast_packed_range(const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out, const unsigned int root_id=0) const
Blocking-broadcast range-of-pointers to one processor.
SendMode send_mode() const
Gets the user-requested SendMode.
Definition: communicator.h:224
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.