libMesh
Public Types | Public Member Functions | Private Member Functions | Private Attributes | List of all members
libMesh::Parallel::Communicator Class Reference

Encapsulates the MPI_Comm object. More...

#include <communicator.h>

Public Types

enum  SendMode { DEFAULT =0, SYNCHRONOUS }
 Whether to use default or synchronous sends? More...
 

Public Member Functions

 Communicator ()
 Default Constructor. More...
 
 Communicator (const communicator &comm)
 
 Communicator (const Communicator &)=delete
 
Communicatoroperator= (const Communicator &)=delete
 
 Communicator (Communicator &&)=default
 
Communicatoroperator= (Communicator &&)=default
 
 ~Communicator ()
 
void split (int color, int key, Communicator &target) const
 
void duplicate (const Communicator &comm)
 
void duplicate (const communicator &comm)
 
communicatorget ()
 
const communicatorget () const
 
MessageTag get_unique_tag (int tagvalue) const
 Get a tag that is unique to this Communicator. More...
 
void reference_unique_tag (int tagvalue) const
 Reference an already-acquired tag, so that we know it will be dereferenced multiple times before we can re-release it. More...
 
void dereference_unique_tag (int tagvalue) const
 Dereference an already-acquired tag, and see if we can re-release it. More...
 
void clear ()
 Free and reset this communicator. More...
 
Communicatoroperator= (const communicator &comm)
 
processor_id_type rank () const
 
processor_id_type size () const
 
void send_mode (const SendMode sm)
 Explicitly sets the SendMode type used for send operations. More...
 
SendMode send_mode () const
 Gets the user-requested SendMode. More...
 
void barrier () const
 Pause execution until all processors reach a certain point. More...
 
template<typename T >
bool verify (const T &r) const
 Verify that a local variable has the same value on all processors. More...
 
template<typename T >
bool semiverify (const T *r) const
 Verify that a local pointer points to the same value on all processors where it is not nullptr. More...
 
template<typename T >
void min (T &r) const
 Take a local variable and replace it with the minimum of it's values on all processors. More...
 
template<typename T >
void minloc (T &r, unsigned int &min_id) const
 Take a local variable and replace it with the minimum of it's values on all processors, returning the minimum rank of a processor which originally held the minimum value. More...
 
template<typename T , typename A1 , typename A2 >
void minloc (std::vector< T, A1 > &r, std::vector< unsigned int, A2 > &min_id) const
 Take a vector of local variables and replace each entry with the minimum of it's values on all processors. More...
 
template<typename T >
void max (T &r) const
 Take a local variable and replace it with the maximum of it's values on all processors. More...
 
template<typename T >
void maxloc (T &r, unsigned int &max_id) const
 Take a local variable and replace it with the maximum of it's values on all processors, returning the minimum rank of a processor which originally held the maximum value. More...
 
template<typename T , typename A1 , typename A2 >
void maxloc (std::vector< T, A1 > &r, std::vector< unsigned int, A2 > &max_id) const
 Take a vector of local variables and replace each entry with the maximum of it's values on all processors. More...
 
template<typename T >
void sum (T &r) const
 Take a local variable and replace it with the sum of it's values on all processors. More...
 
template<typename T >
void set_union (T &data, const unsigned int root_id) const
 Take a container of local variables on each processor, and collect their union over all processors, replacing the set on processor 0. More...
 
template<typename T >
void set_union (T &data) const
 Take a container of local variables on each processor, and replace it with their union over all processors. More...
 
status probe (const unsigned int src_processor_id, const MessageTag &tag=any_tag) const
 Blocking message probe. More...
 
template<typename T >
Status packed_range_probe (const unsigned int src_processor_id, const MessageTag &tag, bool &flag) const
 Non-Blocking message probe for a packed range message. More...
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
 Blocking-send to one processor with data-defined type. More...
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, Request &req, const MessageTag &tag=no_tag) const
 Nonblocking-send to one processor with data-defined type. More...
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, const DataType &type, const MessageTag &tag=no_tag) const
 Blocking-send to one processor with user-defined type. More...
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, const DataType &type, Request &req, const MessageTag &tag=no_tag) const
 Nonblocking-send to one processor with user-defined type. More...
 
template<typename T >
Status receive (const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
 Blocking-receive from one processor with data-defined type. More...
 
template<typename T >
void receive (const unsigned int dest_processor_id, T &buf, Request &req, const MessageTag &tag=any_tag) const
 Nonblocking-receive from one processor with data-defined type. More...
 
template<typename T >
Status receive (const unsigned int dest_processor_id, T &buf, const DataType &type, const MessageTag &tag=any_tag) const
 Blocking-receive from one processor with user-defined type. More...
 
template<typename T >
void receive (const unsigned int dest_processor_id, T &buf, const DataType &type, Request &req, const MessageTag &tag=any_tag) const
 Nonblocking-receive from one processor with user-defined type. More...
 
template<typename Context , typename Iter >
void send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag) const
 Blocking-send range-of-pointers to one processor. More...
 
template<typename Context , typename Iter >
void send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, const MessageTag &tag=no_tag) const
 Nonblocking-send range-of-pointers to one processor. More...
 
template<typename Context , typename Iter >
void nonblocking_send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, const MessageTag &tag=no_tag) const
 Similar to the above Nonblocking send_packed_range with a few important differences: More...
 
template<typename Context , typename Iter >
void nonblocking_send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, std::shared_ptr< std::vector< typename Parallel::Packing< typename std::iterator_traits< Iter >::value_type >::buffer_type >> &buffer, const MessageTag &tag=no_tag) const
 Similar to the above Nonblocking send_packed_range with a few important differences: More...
 
template<typename Context , typename OutputIter , typename T >
void receive_packed_range (const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const
 Blocking-receive range-of-pointers from one processor. More...
 
template<typename Context , typename OutputIter , typename T >
void nonblocking_receive_packed_range (const unsigned int src_processor_id, Context *context, OutputIter out, const T *output_type, Request &req, Status &stat, const MessageTag &tag=any_tag) const
 Non-Blocking-receive range-of-pointers from one processor. More...
 
template<typename Context , typename OutputIter , typename T >
void nonblocking_receive_packed_range (const unsigned int src_processor_id, Context *context, OutputIter out, const T *output_type, Request &req, Status &stat, std::shared_ptr< std::vector< typename Parallel::Packing< T >::buffer_type >> &buffer, const MessageTag &tag=any_tag) const
 Non-Blocking-receive range-of-pointers from one processor. More...
 
template<typename T1 , typename T2 >
void send_receive (const unsigned int dest_processor_id, const T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
 Send data send to one processor while simultaneously receiving other data recv from a (potentially different) processor. More...
 
template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void send_receive_packed_range (const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
 Send a range-of-pointers to one processor while simultaneously receiving another range from a (potentially different) processor. More...
 
template<typename T1 , typename T2 >
void send_receive (const unsigned int dest_processor_id, const T1 &send, const DataType &type1, const unsigned int source_processor_id, T2 &recv, const DataType &type2, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
 Send data send to one processor while simultaneously receiving other data recv from a (potentially different) processor, using a user-specified MPI Dataype. More...
 
template<typename T , typename A >
void gather (const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
 Take a vector of length comm.size(), and on processor root_id fill in recv[processor_id] = the value of send on processor processor_id. More...
 
template<typename T , typename A >
void gather (const unsigned int root_id, const std::basic_string< T > &send, std::vector< std::basic_string< T >, A > &recv, const bool identical_buffer_sizes=false) const
 The gather overload for string types has an optional identical_buffer_sizes optimization for when all strings are the same length. More...
 
template<typename T , typename A >
void gather (const unsigned int root_id, std::vector< T, A > &r) const
 
Take a vector of local variables and expand it on processor root_id to include values from all processors More...
 
template<typename T , typename A >
void allgather (const T &send, std::vector< T, A > &recv) const
 Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that processor. More...
 
template<typename T , typename A >
void allgather (const std::basic_string< T > &send, std::vector< std::basic_string< T >, A > &recv, const bool identical_buffer_sizes=false) const
 The allgather overload for string types has an optional identical_buffer_sizes optimization for when all strings are the same length. More...
 
template<typename T , typename A >
void allgather (std::vector< T, A > &r, const bool identical_buffer_sizes=false) const
 
Take a vector of local variables and expand it to include values from all processors. More...
 
template<typename T , typename A >
void allgather (std::vector< std::basic_string< T >, A > &r, const bool identical_buffer_sizes=false) const
 AllGather overload for vectors of string types. More...
 
template<typename T , typename A >
void scatter (const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
 Take a vector of local variables and scatter the ith item to the ith processor in the communicator. More...
 
template<typename T , typename A >
void scatter (const std::vector< T, A > &data, std::vector< T, A > &recv, const unsigned int root_id=0) const
 Take a vector of local variables and scatter the ith equal-sized chunk to the ith processor in the communicator. More...
 
template<typename T , typename A1 , typename A2 >
void scatter (const std::vector< T, A1 > &data, const std::vector< int, A2 > counts, std::vector< T, A1 > &recv, const unsigned int root_id=0) const
 Take a vector of local variables and scatter the ith variable-sized chunk to the ith processor in the communicator. More...
 
template<typename T , typename A1 , typename A2 >
void scatter (const std::vector< std::vector< T, A1 >, A2 > &data, std::vector< T, A1 > &recv, const unsigned int root_id=0, const bool identical_buffer_sizes=false) const
 Take a vector of vectors and scatter the ith inner vector to the ith processor in the communicator. More...
 
template<typename Context , typename Iter , typename OutputIter >
void gather_packed_range (const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
 Take a range of local variables, combine it with ranges from all processors, and write the output to the output iterator on rank root. More...
 
template<typename Context , typename Iter , typename OutputIter >
void allgather_packed_range (Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
 Take a range of local variables, combine it with ranges from all processors, and write the output to the output iterator. More...
 
template<typename T , typename A >
void alltoall (std::vector< T, A > &r) const
 Effectively transposes the input vector across all processors. More...
 
template<typename T >
void broadcast (T &data, const unsigned int root_id=0) const
 Take a local value and broadcast it to all processors. More...
 
template<typename Context , typename OutputContext , typename Iter , typename OutputIter >
void broadcast_packed_range (const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out, const unsigned int root_id=0) const
 Blocking-broadcast range-of-pointers to one processor. More...
 
template<typename T >
void send (const unsigned int dest_processor_id, const std::basic_string< T > &buf, const MessageTag &tag) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const std::basic_string< T > &buf, Request &req, const MessageTag &tag) const
 
template<typename T >
Status receive (const unsigned int src_processor_id, std::basic_string< T > &buf, const MessageTag &tag) const
 
template<typename T >
void receive (const unsigned int src_processor_id, std::basic_string< T > &buf, Request &req, const MessageTag &tag) const
 
template<typename T , typename A >
void send_receive (const unsigned int dest_processor_id, const std::vector< T, A > &sendvec, const unsigned int source_processor_id, std::vector< T, A > &recv, const MessageTag &send_tag, const MessageTag &recv_tag) const
 
template<>
void broadcast (bool &data, const unsigned int root_id) const
 
template<typename T >
void broadcast (std::basic_string< T > &data, const unsigned int root_id) const
 
template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void send_receive_packed_range (const unsigned int libmesh_dbg_var(dest_processor_id), const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int libmesh_dbg_var(source_processor_id), Context2 *context2, OutputIter out_iter, const T *output_type, const MessageTag &, const MessageTag &) const
 Send-receive range-of-pointers from one processor. More...
 
template<typename T >
void min (T &libmesh_mpi_var(r)) const
 
template<typename A >
void min (std::vector< bool, A > &r) const
 
template<typename A1 , typename A2 >
void minloc (std::vector< bool, A1 > &r, std::vector< unsigned int, A2 > &min_id) const
 
template<typename T >
void max (T &libmesh_mpi_var(r)) const
 
template<typename A >
void max (std::vector< bool, A > &r) const
 
template<typename A1 , typename A2 >
void maxloc (std::vector< bool, A1 > &r, std::vector< unsigned int, A2 > &max_id) const
 
template<typename T >
void sum (T &libmesh_mpi_var(r)) const
 
template<typename T >
void sum (std::complex< T > &libmesh_mpi_var(r)) const
 
template<typename T >
void broadcast (T &libmesh_mpi_var(data), const unsigned int root_id) const
 

Private Member Functions

void assign (const communicator &comm)
 Utility function for setting our member variables from an MPI communicator. More...
 

Private Attributes

communicator _communicator
 
processor_id_type _rank
 
processor_id_type _size
 
SendMode _send_mode
 
std::map< int, unsigned int > used_tag_values
 
bool _I_duped_it
 

Detailed Description

Encapsulates the MPI_Comm object.

Allows the size of the group and this process's position in the group to be determined.

Methods of this object are the preferred way to perform distributed-memory parallel operations.

Definition at line 92 of file communicator.h.

Member Enumeration Documentation

◆ SendMode

Whether to use default or synchronous sends?

Enumerator
DEFAULT 
SYNCHRONOUS 

Definition at line 180 of file communicator.h.

Constructor & Destructor Documentation

◆ Communicator() [1/4]

libMesh::Parallel::Communicator::Communicator ( )

Default Constructor.

◆ Communicator() [2/4]

libMesh::Parallel::Communicator::Communicator ( const communicator comm)
explicit

◆ Communicator() [3/4]

libMesh::Parallel::Communicator::Communicator ( const Communicator )
delete

◆ Communicator() [4/4]

libMesh::Parallel::Communicator::Communicator ( Communicator &&  )
default

◆ ~Communicator()

libMesh::Parallel::Communicator::~Communicator ( )

Member Function Documentation

◆ allgather() [1/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( const T &  send,
std::vector< T, A > &  recv 
) const

Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that processor.

Definition at line 2746 of file parallel_implementation.h.

References size().

Referenced by allgather(), allgather_packed_range(), and gather().

2748 {
2749  LOG_SCOPE ("allgather()","Parallel");
2750 
2751  libmesh_assert(this->size());
2752  recv.resize(this->size());
2753 
2754  unsigned int comm_size = this->size();
2755  if (comm_size > 1)
2756  {
2757  StandardType<T> send_type(&sendval);
2758 
2759  libmesh_call_mpi
2760  (MPI_Allgather (const_cast<T*>(&sendval), 1, send_type, &recv[0], 1,
2761  send_type, this->get()));
2762  }
2763  else if (comm_size > 0)
2764  recv[0] = sendval;
2765 }
processor_id_type size() const
Definition: communicator.h:175

◆ allgather() [2/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( const std::basic_string< T > &  send,
std::vector< std::basic_string< T >, A > &  recv,
const bool  identical_buffer_sizes = false 
) const

The allgather overload for string types has an optional identical_buffer_sizes optimization for when all strings are the same length.

Definition at line 1374 of file parallel_implementation.h.

References allgather(), and size().

1377 {
1378  LOG_SCOPE ("allgather()","Parallel");
1379 
1380  libmesh_assert(this->size());
1381  recv.assign(this->size(), "");
1382 
1383  // serial case
1384  if (this->size() < 2)
1385  {
1386  recv.resize(1);
1387  recv[0] = sendval;
1388  return;
1389  }
1390 
1391  std::vector<int>
1392  sendlengths (this->size(), 0),
1393  displacements(this->size(), 0);
1394 
1395  const int mysize = static_cast<int>(sendval.size());
1396 
1397  if (identical_buffer_sizes)
1398  sendlengths.assign(this->size(), mysize);
1399  else
1400  // first comm step to determine buffer sizes from all processors
1401  this->allgather(mysize, sendlengths);
1402 
1403  // Find the total size of the final array and
1404  // set up the displacement offsets for each processor
1405  unsigned int globalsize = 0;
1406  for (unsigned int i=0; i != this->size(); ++i)
1407  {
1408  displacements[i] = globalsize;
1409  globalsize += sendlengths[i];
1410  }
1411 
1412  // Check for quick return
1413  if (globalsize == 0)
1414  return;
1415 
1416  // monolithic receive buffer
1417  std::string r(globalsize, 0);
1418 
1419  // and get the data from the remote processors.
1420  libmesh_call_mpi
1421  (MPI_Allgatherv (const_cast<T*>(mysize ? &sendval[0] : nullptr),
1422  mysize, StandardType<T>(),
1423  &r[0], &sendlengths[0], &displacements[0],
1424  StandardType<T>(), this->get()));
1425 
1426  // slice receive buffer up
1427  for (unsigned int i=0; i != this->size(); ++i)
1428  recv[i] = r.substr(displacements[i], sendlengths[i]);
1429 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const
Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that proc...

◆ allgather() [3/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( std::vector< T, A > &  r,
const bool  identical_buffer_sizes = false 
) const


Take a vector of local variables and expand it to include values from all processors.

By default, each processor is allowed to have its own unique input buffer length. If it is known that all processors have the same input sizes additional communication can be avoided.

Specifically, this function transforms this:

* Processor 0: [ ... N_0 ]
* Processor 1: [ ....... N_1 ]
* ...
* Processor M: [ .. N_M]
* 

into this:

* [ [ ... N_0 ] [ ....... N_1 ] ... [ .. N_M] ]
* 

on each processor. This function is collective and therefore must be called by all processors in the Communicator.

Definition at line 2770 of file parallel_implementation.h.

References allgather(), size(), and verify().

2772 {
2773  if (this->size() < 2)
2774  return;
2775 
2776  LOG_SCOPE("allgather()", "Parallel");
2777 
2778  if (identical_buffer_sizes)
2779  {
2780  if (r.empty())
2781  return;
2782 
2783  libmesh_assert(this->verify(r.size()));
2784 
2785  std::vector<T,A> r_src(r.size()*this->size());
2786  r_src.swap(r);
2787  StandardType<T> send_type(&r_src[0]);
2788 
2789  libmesh_call_mpi
2790  (MPI_Allgather (&r_src[0], cast_int<int>(r_src.size()),
2791  send_type, &r[0], cast_int<int>(r_src.size()),
2792  send_type, this->get()));
2793  // libmesh_assert(this->verify(r));
2794  return;
2795  }
2796 
2797  std::vector<int>
2798  sendlengths (this->size(), 0),
2799  displacements(this->size(), 0);
2800 
2801  const int mysize = static_cast<int>(r.size());
2802  this->allgather(mysize, sendlengths);
2803 
2804  // Find the total size of the final array and
2805  // set up the displacement offsets for each processor.
2806  unsigned int globalsize = 0;
2807  for (unsigned int i=0; i != this->size(); ++i)
2808  {
2809  displacements[i] = globalsize;
2810  globalsize += sendlengths[i];
2811  }
2812 
2813  // Check for quick return
2814  if (globalsize == 0)
2815  return;
2816 
2817  // copy the input buffer
2818  std::vector<T,A> r_src(globalsize);
2819  r_src.swap(r);
2820 
2821  StandardType<T> send_type(&r[0]);
2822 
2823  // and get the data from the remote processors.
2824  // Pass nullptr if our vector is empty.
2825  libmesh_call_mpi
2826  (MPI_Allgatherv (r_src.empty() ? nullptr : &r_src[0], mysize,
2827  send_type, &r[0], &sendlengths[0],
2828  &displacements[0], send_type, this->get()));
2829 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const
Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that proc...
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ allgather() [4/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( std::vector< std::basic_string< T >, A > &  r,
const bool  identical_buffer_sizes = false 
) const

AllGather overload for vectors of string types.

Definition at line 2834 of file parallel_implementation.h.

References allgather(), end, size(), and verify().

2836 {
2837  if (this->size() < 2)
2838  return;
2839 
2840  LOG_SCOPE("allgather()", "Parallel");
2841 
2842  if (identical_buffer_sizes)
2843  {
2844  libmesh_assert(this->verify(r.size()));
2845 
2846  // identical_buffer_sizes doesn't buy us much since we have to
2847  // communicate the lengths of strings within each buffer anyway
2848  if (r.empty())
2849  return;
2850  }
2851 
2852  // Concatenate the input buffer into a send buffer, and keep track
2853  // of input string lengths
2854  std::vector<int> mystrlengths (r.size());
2855  std::vector<T> concat_src;
2856 
2857  int myconcatsize = 0;
2858  for (unsigned int i=0; i != r.size(); ++i)
2859  {
2860  int stringlen = cast_int<int>(r[i].size());
2861  mystrlengths[i] = stringlen;
2862  myconcatsize += stringlen;
2863  }
2864  concat_src.reserve(myconcatsize);
2865  for (unsigned int i=0; i != r.size(); ++i)
2866  concat_src.insert
2867  (concat_src.end(), r[i].begin(), r[i].end());
2868 
2869  // Get the string lengths from all other processors
2870  std::vector<int> strlengths = mystrlengths;
2871  this->allgather(strlengths, identical_buffer_sizes);
2872 
2873  // We now know how many strings we'll be receiving
2874  r.resize(strlengths.size());
2875 
2876  // Get the concatenated data sizes from all other processors
2877  std::vector<int> concat_sizes;
2878  this->allgather(myconcatsize, concat_sizes);
2879 
2880  // Find the total size of the final concatenated array and
2881  // set up the displacement offsets for each processor.
2882  std::vector<int> displacements(this->size(), 0);
2883  unsigned int globalsize = 0;
2884  for (unsigned int i=0; i != this->size(); ++i)
2885  {
2886  displacements[i] = globalsize;
2887  globalsize += concat_sizes[i];
2888  }
2889 
2890  // Check for quick return
2891  if (globalsize == 0)
2892  return;
2893 
2894  // Get the concatenated data from the remote processors.
2895  // Pass nullptr if our vector is empty.
2896  std::vector<T> concat(globalsize);
2897 
2898  // We may have concat_src.empty(), but we know concat has at least
2899  // one element we can use as an example for StandardType
2900  StandardType<T> send_type(&concat[0]);
2901 
2902  libmesh_call_mpi
2903  (MPI_Allgatherv (concat_src.empty() ?
2904  nullptr : &concat_src[0], myconcatsize,
2905  send_type, &concat[0], &concat_sizes[0],
2906  &displacements[0], send_type, this->get()));
2907 
2908  // Finally, split concatenated data into strings
2909  const T * begin = &concat[0];
2910  for (unsigned int i=0; i != r.size(); ++i)
2911  {
2912  const T * end = begin + strlengths[i];
2913  r[i].assign(begin, end);
2914  begin = end;
2915  }
2916 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const
Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that proc...
IterBase * end
Also have a polymorphic pointer to the end object, this prevents iterating past the end...
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ allgather_packed_range()

template<typename Context , typename Iter , typename OutputIter >
void libMesh::Parallel::Communicator::allgather_packed_range ( Context *  context,
Iter  range_begin,
const Iter  range_end,
OutputIter  out 
) const

Take a range of local variables, combine it with ranges from all processors, and write the output to the output iterator.

Definition at line 3176 of file parallel_implementation.h.

References allgather(), max(), libMesh::Parallel::pack_range(), and libMesh::Parallel::unpack_range().

3180 {
3181  typedef typename std::iterator_traits<Iter>::value_type T;
3182  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
3183 
3184  bool nonempty_range = (range_begin != range_end);
3185  this->max(nonempty_range);
3186 
3187  while (nonempty_range)
3188  {
3189  // We will serialize variable size objects from *range_begin to
3190  // *range_end as a sequence of ints in this buffer
3191  std::vector<buffer_t> buffer;
3192 
3193  range_begin = Parallel::pack_range
3194  (context, range_begin, range_end, buffer);
3195 
3196  this->allgather(buffer, false);
3197 
3198  libmesh_assert(buffer.size());
3199 
3201  (buffer, context, out_iter, (T*)nullptr);
3202 
3203  nonempty_range = (range_begin != range_end);
3204  this->max(nonempty_range);
3205  }
3206 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
void max(T &r) const
Take a local variable and replace it with the maximum of it&#39;s values on all processors.
void allgather(const T &send, std::vector< T, A > &recv) const
Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that proc...
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
Decode a range of potentially-variable-size objects from a data array.

◆ alltoall()

template<typename T , typename A >
void libMesh::Parallel::Communicator::alltoall ( std::vector< T, A > &  r) const

Effectively transposes the input vector across all processors.

The jth entry on processor i is replaced with the ith entry from processor j.

Definition at line 3092 of file parallel_implementation.h.

References libMesh::libmesh_ignore(), size(), and verify().

Referenced by libMesh::Parallel::push_parallel_vector_data().

3093 {
3094  if (this->size() < 2 || buf.empty())
3095  return;
3096 
3097  LOG_SCOPE("alltoall()", "Parallel");
3098 
3099  // the per-processor size. this is the same for all
3100  // processors using MPI_Alltoall, could be variable
3101  // using MPI_Alltoallv
3102  const int size_per_proc =
3103  cast_int<int>(buf.size()/this->size());
3104  libmesh_ignore(size_per_proc);
3105 
3106  libmesh_assert_equal_to (buf.size()%this->size(), 0);
3107 
3108  libmesh_assert(this->verify(size_per_proc));
3109 
3110  StandardType<T> send_type(&buf[0]);
3111 
3112  libmesh_call_mpi
3113  (MPI_Alltoall (MPI_IN_PLACE, size_per_proc, send_type, &buf[0],
3114  size_per_proc, send_type, this->get()));
3115 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ assign()

void libMesh::Parallel::Communicator::assign ( const communicator comm)
private

Utility function for setting our member variables from an MPI communicator.

◆ barrier()

void libMesh::Parallel::Communicator::barrier ( ) const

Pause execution until all processors reach a certain point.

◆ broadcast() [1/4]

template<typename T >
void libMesh::Parallel::Communicator::broadcast ( T &  data,
const unsigned int  root_id = 0 
) const

Take a local value and broadcast it to all processors.

Optionally takes the root_id processor, which specifies the processor initiating the broadcast. If data is a vector, the user is responsible for resizing it on all processors, except in the case when data is a vector of strings.

Referenced by broadcast(), broadcast_packed_range(), and scatter().

◆ broadcast() [2/4]

template<>
void libMesh::Parallel::Communicator::broadcast ( bool &  data,
const unsigned int  root_id 
) const

Definition at line 1434 of file parallel_implementation.h.

References data, rank(), and size().

1435 {
1436  if (this->size() == 1)
1437  {
1438  libmesh_assert (!this->rank());
1439  libmesh_assert (!root_id);
1440  return;
1441  }
1442 
1443  libmesh_assert_less (root_id, this->size());
1444 
1445  LOG_SCOPE("broadcast()", "Parallel");
1446 
1447  // We don't want to depend on MPI-2 or C++ MPI, so we don't have
1448  // MPI::BOOL available
1449  char char_data = data;
1450 
1451  // Spread data to remote processors.
1452  libmesh_call_mpi
1453  (MPI_Bcast (&char_data, 1, StandardType<char>(&char_data),
1454  root_id, this->get()));
1455 
1456  data = char_data;
1457 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
Ideally this private member data should have protected access.

◆ broadcast() [3/4]

template<typename T >
void libMesh::Parallel::Communicator::broadcast ( std::basic_string< T > &  data,
const unsigned int  root_id 
) const

Definition at line 1461 of file parallel_implementation.h.

References broadcast(), data, rank(), and size().

1463 {
1464  if (this->size() == 1)
1465  {
1466  libmesh_assert (!this->rank());
1467  libmesh_assert (!root_id);
1468  return;
1469  }
1470 
1471  libmesh_assert_less (root_id, this->size());
1472 
1473  LOG_SCOPE("broadcast()", "Parallel");
1474 
1475  std::size_t data_size = data.size();
1476  this->broadcast(data_size, root_id);
1477 
1478  std::vector<T> data_c(data_size);
1479 #ifndef NDEBUG
1480  std::string orig(data);
1481 #endif
1482 
1483  if (this->rank() == root_id)
1484  for (std::size_t i=0; i<data.size(); i++)
1485  data_c[i] = data[i];
1486 
1487  this->broadcast (data_c, root_id);
1488 
1489  data.assign(data_c.begin(), data_c.end());
1490 
1491 #ifndef NDEBUG
1492  if (this->rank() == root_id)
1493  libmesh_assert_equal_to (data, orig);
1494 #endif
1495 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
Ideally this private member data should have protected access.
void broadcast(T &data, const unsigned int root_id=0) const
Take a local value and broadcast it to all processors.

◆ broadcast() [4/4]

template<typename T >
void libMesh::Parallel::Communicator::broadcast ( T &  libmesh_mpi_vardata,
const unsigned int  root_id 
) const

Definition at line 3120 of file parallel_implementation.h.

References data, libMesh::libmesh_ignore(), rank(), and size().

3121 {
3122  libmesh_ignore(root_id); // Only needed for MPI and/or dbg/devel
3123  if (this->size() == 1)
3124  {
3125  libmesh_assert (!this->rank());
3126  libmesh_assert (!root_id);
3127  return;
3128  }
3129 
3130  libmesh_assert_less (root_id, this->size());
3131 
3132  LOG_SCOPE("broadcast()", "Parallel");
3133 
3134  // Spread data to remote processors.
3135  libmesh_call_mpi
3136  (MPI_Bcast (&data, 1, StandardType<T>(&data), root_id,
3137  this->get()));
3138 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
Ideally this private member data should have protected access.

◆ broadcast_packed_range()

template<typename Context , typename OutputContext , typename Iter , typename OutputIter >
void libMesh::Parallel::Communicator::broadcast_packed_range ( const Context *  context1,
Iter  range_begin,
const Iter  range_end,
OutputContext *  context2,
OutputIter  out,
const unsigned int  root_id = 0 
) const

Blocking-broadcast range-of-pointers to one processor.

This function does not send the raw pointers, but rather constructs new objects at the other end whose contents match the objects pointed to by the sender.

void Parallel::pack(const T *, vector<int> & data, const Context *) is used to serialize type T onto the end of a data vector.

unsigned int Parallel::packable_size(const T *, const Context *) is used to allow data vectors to reserve memory, and for additional error checking

unsigned int Parallel::packed_size(const T *, vector<int>::const_iterator) is used to advance to the beginning of the next object's data.

Definition at line 1672 of file parallel_implementation.h.

References broadcast(), libMesh::Parallel::pack_range(), rank(), and libMesh::Parallel::unpack_range().

1678 {
1679  typedef typename std::iterator_traits<Iter>::value_type T;
1680  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1681 
1682  do
1683  {
1684  // We will serialize variable size objects from *range_begin to
1685  // *range_end as a sequence of ints in this buffer
1686  std::vector<buffer_t> buffer;
1687 
1688  if (this->rank() == root_id)
1689  range_begin = Parallel::pack_range
1690  (context1, range_begin, range_end, buffer);
1691 
1692  // this->broadcast(vector) requires the receiving vectors to
1693  // already be the appropriate size
1694  std::size_t buffer_size = buffer.size();
1695  this->broadcast (buffer_size, root_id);
1696 
1697  // We continue until there's nothing left to broadcast
1698  if (!buffer_size)
1699  break;
1700 
1701  buffer.resize(buffer_size);
1702 
1703  // Broadcast the packed data
1704  this->broadcast (buffer, root_id);
1705 
1706  if (this->rank() != root_id)
1708  (buffer, context2, out_iter, (T*)nullptr);
1709  } while (true); // break above when we reach buffer_size==0
1710 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
processor_id_type rank() const
Definition: communicator.h:173
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
Decode a range of potentially-variable-size objects from a data array.
void broadcast(T &data, const unsigned int root_id=0) const
Take a local value and broadcast it to all processors.

◆ clear()

void libMesh::Parallel::Communicator::clear ( )

Free and reset this communicator.

◆ dereference_unique_tag()

void libMesh::Parallel::Communicator::dereference_unique_tag ( int  tagvalue) const

Dereference an already-acquired tag, and see if we can re-release it.

◆ duplicate() [1/2]

void libMesh::Parallel::Communicator::duplicate ( const Communicator comm)

◆ duplicate() [2/2]

void libMesh::Parallel::Communicator::duplicate ( const communicator comm)

◆ gather() [1/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::gather ( const unsigned int  root_id,
const T &  send,
std::vector< T, A > &  recv 
) const

Take a vector of length comm.size(), and on processor root_id fill in recv[processor_id] = the value of send on processor processor_id.

Definition at line 2606 of file parallel_implementation.h.

References rank(), and size().

Referenced by gather(), and gather_packed_range().

2609 {
2610  libmesh_assert_less (root_id, this->size());
2611 
2612  if (this->rank() == root_id)
2613  recv.resize(this->size());
2614 
2615  if (this->size() > 1)
2616  {
2617  LOG_SCOPE("gather()", "Parallel");
2618 
2619  StandardType<T> send_type(&sendval);
2620 
2621  libmesh_call_mpi
2622  (MPI_Gather(const_cast<T*>(&sendval), 1, send_type,
2623  recv.empty() ? nullptr : &recv[0], 1, send_type,
2624  root_id, this->get()));
2625  }
2626  else
2627  recv[0] = sendval;
2628 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173

◆ gather() [2/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::gather ( const unsigned int  root_id,
const std::basic_string< T > &  send,
std::vector< std::basic_string< T >, A > &  recv,
const bool  identical_buffer_sizes = false 
) const

The gather overload for string types has an optional identical_buffer_sizes optimization for when all strings are the same length.

Definition at line 2686 of file parallel_implementation.h.

References gather(), rank(), and size().

2690 {
2691  libmesh_assert_less (root_id, this->size());
2692 
2693  if (this->rank() == root_id)
2694  recv.resize(this->size());
2695 
2696  if (this->size() > 1)
2697  {
2698  LOG_SCOPE ("gather()","Parallel");
2699 
2700  std::vector<int>
2701  sendlengths (this->size(), 0),
2702  displacements(this->size(), 0);
2703 
2704  const int mysize = static_cast<int>(sendval.size());
2705 
2706  if (identical_buffer_sizes)
2707  sendlengths.assign(this->size(), mysize);
2708  else
2709  // first comm step to determine buffer sizes from all processors
2710  this->gather(root_id, mysize, sendlengths);
2711 
2712  // Find the total size of the final array and
2713  // set up the displacement offsets for each processor
2714  unsigned int globalsize = 0;
2715  for (unsigned int i=0; i < this->size(); ++i)
2716  {
2717  displacements[i] = globalsize;
2718  globalsize += sendlengths[i];
2719  }
2720 
2721  // monolithic receive buffer
2722  std::string r;
2723  if (this->rank() == root_id)
2724  r.resize(globalsize, 0);
2725 
2726  // and get the data from the remote processors.
2727  libmesh_call_mpi
2728  (MPI_Gatherv (const_cast<T*>(&sendval[0]),
2729  mysize, StandardType<T>(),
2730  this->rank() == root_id ? &r[0] : nullptr,
2731  &sendlengths[0], &displacements[0],
2732  StandardType<T>(), root_id, this->get()));
2733 
2734  // slice receive buffer up
2735  if (this->rank() == root_id)
2736  for (unsigned int i=0; i != this->size(); ++i)
2737  recv[i] = r.substr(displacements[i], sendlengths[i]);
2738  }
2739  else
2740  recv[0] = sendval;
2741 }
processor_id_type size() const
Definition: communicator.h:175
void gather(const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
Take a vector of length comm.size(), and on processor root_id fill in recv[processor_id] = the value ...
processor_id_type rank() const
Definition: communicator.h:173

◆ gather() [3/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::gather ( const unsigned int  root_id,
std::vector< T, A > &  r 
) const


Take a vector of local variables and expand it on processor root_id to include values from all processors

This handles the case where the lengths of the vectors may vary. Specifically, this function transforms this:

* Processor 0: [ ... N_0 ]
* Processor 1: [ ....... N_1 ]
* ...
* Processor M: [ .. N_M]
* 

into this:

* [ [ ... N_0 ] [ ....... N_1 ] ... [ .. N_M] ]
* 

on processor root_id. This function is collective and therefore must be called by all processors in the Communicator.

Definition at line 2633 of file parallel_implementation.h.

References allgather(), rank(), and size().

2635 {
2636  if (this->size() == 1)
2637  {
2638  libmesh_assert (!this->rank());
2639  libmesh_assert (!root_id);
2640  return;
2641  }
2642 
2643  libmesh_assert_less (root_id, this->size());
2644 
2645  std::vector<int>
2646  sendlengths (this->size(), 0),
2647  displacements(this->size(), 0);
2648 
2649  const int mysize = static_cast<int>(r.size());
2650  this->allgather(mysize, sendlengths);
2651 
2652  LOG_SCOPE("gather()", "Parallel");
2653 
2654  // Find the total size of the final array and
2655  // set up the displacement offsets for each processor.
2656  unsigned int globalsize = 0;
2657  for (unsigned int i=0; i != this->size(); ++i)
2658  {
2659  displacements[i] = globalsize;
2660  globalsize += sendlengths[i];
2661  }
2662 
2663  // Check for quick return
2664  if (globalsize == 0)
2665  return;
2666 
2667  // copy the input buffer
2668  std::vector<T,A> r_src(r);
2669 
2670  // now resize it to hold the global data
2671  // on the receiving processor
2672  if (root_id == this->rank())
2673  r.resize(globalsize);
2674 
2675  // and get the data from the remote processors
2676  libmesh_call_mpi
2677  (MPI_Gatherv (r_src.empty() ? nullptr : &r_src[0], mysize,
2678  StandardType<T>(), r.empty() ? nullptr : &r[0],
2679  &sendlengths[0], &displacements[0],
2680  StandardType<T>(), root_id, this->get()));
2681 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const
Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that proc...
processor_id_type rank() const
Definition: communicator.h:173

◆ gather_packed_range()

template<typename Context , typename Iter , typename OutputIter >
void libMesh::Parallel::Communicator::gather_packed_range ( const unsigned int  root_id,
Context *  context,
Iter  range_begin,
const Iter  range_end,
OutputIter  out 
) const

Take a range of local variables, combine it with ranges from all processors, and write the output to the output iterator on rank root.

Definition at line 3143 of file parallel_implementation.h.

References gather(), max(), libMesh::Parallel::pack_range(), and libMesh::Parallel::unpack_range().

3148 {
3149  typedef typename std::iterator_traits<Iter>::value_type T;
3150  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
3151 
3152  bool nonempty_range = (range_begin != range_end);
3153  this->max(nonempty_range);
3154 
3155  while (nonempty_range)
3156  {
3157  // We will serialize variable size objects from *range_begin to
3158  // *range_end as a sequence of ints in this buffer
3159  std::vector<buffer_t> buffer;
3160 
3161  range_begin = Parallel::pack_range
3162  (context, range_begin, range_end, buffer);
3163 
3164  this->gather(root_id, buffer);
3165 
3167  (buffer, context, out_iter, (T*)(nullptr));
3168 
3169  nonempty_range = (range_begin != range_end);
3170  this->max(nonempty_range);
3171  }
3172 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
void max(T &r) const
Take a local variable and replace it with the maximum of it&#39;s values on all processors.
void gather(const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
Take a vector of length comm.size(), and on processor root_id fill in recv[processor_id] = the value ...
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
Decode a range of potentially-variable-size objects from a data array.

◆ get() [1/2]

communicator& libMesh::Parallel::Communicator::get ( )

Definition at line 141 of file communicator.h.

References _communicator.

141 { return _communicator; }

◆ get() [2/2]

const communicator& libMesh::Parallel::Communicator::get ( ) const

Definition at line 143 of file communicator.h.

References _communicator.

143 { return _communicator; }

◆ get_unique_tag()

MessageTag libMesh::Parallel::Communicator::get_unique_tag ( int  tagvalue) const

Get a tag that is unique to this Communicator.

Note
If people are also using magic numbers or copying communicators around then we can't guarantee the tag is unique to this MPI_Comm.

Referenced by libMesh::Parallel::pull_parallel_vector_data(), and libMesh::Parallel::push_parallel_vector_data().

◆ max() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::max ( T &  r) const

Take a local variable and replace it with the maximum of it's values on all processors.

Containers are replaced element-wise.

Referenced by allgather_packed_range(), gather_packed_range(), semiverify(), libMesh::Parallel::sync_dofobject_data_by_xyz(), and verify().

◆ max() [2/3]

template<typename T >
void libMesh::Parallel::Communicator::max ( T &  libmesh_mpi_varr) const

Definition at line 2280 of file parallel_implementation.h.

References size().

2281 {
2282  if (this->size() > 1)
2283  {
2284  LOG_SCOPE("max(scalar)", "Parallel");
2285 
2286  libmesh_call_mpi
2287  (MPI_Allreduce (MPI_IN_PLACE, &r, 1,
2288  StandardType<T>(&r),
2290  this->get()));
2291  }
2292 }
processor_id_type size() const
Definition: communicator.h:175
long double max(long double a, double b)

◆ max() [3/3]

template<typename A >
void libMesh::Parallel::Communicator::max ( std::vector< bool, A > &  r) const

Definition at line 2332 of file parallel_implementation.h.

References size(), and verify().

2333 {
2334  if (this->size() > 1 && !r.empty())
2335  {
2336  LOG_SCOPE("max(vector<bool>)", "Parallel");
2337 
2338  libmesh_assert(this->verify(r.size()));
2339 
2340  std::vector<unsigned int> ruint;
2341  pack_vector_bool(r, ruint);
2342  std::vector<unsigned int> temp(ruint.size());
2343  libmesh_call_mpi
2344  (MPI_Allreduce (&ruint[0], &temp[0],
2345  cast_int<int>(ruint.size()),
2346  StandardType<unsigned int>(), MPI_BOR,
2347  this->get()));
2348  unpack_vector_bool(temp, r);
2349  }
2350 }
processor_id_type size() const
Definition: communicator.h:175
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ maxloc() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::maxloc ( T &  r,
unsigned int &  max_id 
) const

Take a local variable and replace it with the maximum of it's values on all processors, returning the minimum rank of a processor which originally held the maximum value.

Definition at line 2354 of file parallel_implementation.h.

References libMesh::libmesh_ignore(), libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), and libMesh::Parallel::DataPlusInt< T >::val.

2356 {
2357  if (this->size() > 1)
2358  {
2359  LOG_SCOPE("maxloc(scalar)", "Parallel");
2360 
2361  DataPlusInt<T> data_in;
2362  libmesh_ignore(data_in); // unused ifndef LIBMESH_HAVE_MPI
2363  data_in.val = r;
2364  data_in.rank = this->rank();
2365  libmesh_call_mpi
2366  (MPI_Allreduce (MPI_IN_PLACE, &data_in, 1,
2367  dataplusint_type<T>(),
2368  OpFunction<T>::max_location(),
2369  this->get()));
2370  r = data_in.val;
2371  max_id = data_in.rank;
2372  }
2373  else
2374  max_id = this->rank();
2375 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173

◆ maxloc() [2/3]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::maxloc ( std::vector< T, A1 > &  r,
std::vector< unsigned int, A2 > &  max_id 
) const

Take a vector of local variables and replace each entry with the maximum of it's values on all processors.

Set each min_id entry to the minimum rank where a corresponding maximum was found.

Definition at line 2404 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2406 {
2407  if (this->size() > 1 && !r.empty())
2408  {
2409  LOG_SCOPE("maxloc(vector)", "Parallel");
2410 
2411  libmesh_assert(this->verify(r.size()));
2412 
2413  std::vector<DataPlusInt<T>> data_in(r.size());
2414  for (std::size_t i=0; i != r.size(); ++i)
2415  {
2416  data_in[i].val = r[i];
2417  data_in[i].rank = this->rank();
2418  }
2419  std::vector<DataPlusInt<T>> data_out(r.size());
2420  libmesh_call_mpi
2421  (MPI_Allreduce (&data_in[0], &data_out[0],
2422  cast_int<int>(r.size()),
2423  dataplusint_type<T>(),
2424  OpFunction<T>::max_location(),
2425  this->get()));
2426  for (std::size_t i=0; i != r.size(); ++i)
2427  {
2428  r[i] = data_out[i].val;
2429  max_id[i] = data_out[i].rank;
2430  }
2431  }
2432  else if (!r.empty())
2433  {
2434  for (std::size_t i=0; i != r.size(); ++i)
2435  max_id[i] = this->rank();
2436  }
2437 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ maxloc() [3/3]

template<typename A1 , typename A2 >
void libMesh::Parallel::Communicator::maxloc ( std::vector< bool, A1 > &  r,
std::vector< unsigned int, A2 > &  max_id 
) const

Definition at line 2441 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2443 {
2444  if (this->size() > 1 && !r.empty())
2445  {
2446  LOG_SCOPE("maxloc(vector<bool>)", "Parallel");
2447 
2448  libmesh_assert(this->verify(r.size()));
2449 
2450  std::vector<DataPlusInt<int>> data_in(r.size());
2451  for (std::size_t i=0; i != r.size(); ++i)
2452  {
2453  data_in[i].val = r[i];
2454  data_in[i].rank = this->rank();
2455  }
2456  std::vector<DataPlusInt<int>> data_out(r.size());
2457  libmesh_call_mpi
2458  (MPI_Allreduce (&data_in[0], &data_out[0],
2459  cast_int<int>(r.size()),
2460  StandardType<int>(),
2461  OpFunction<int>::max_location(),
2462  this->get()));
2463  for (std::size_t i=0; i != r.size(); ++i)
2464  {
2465  r[i] = data_out[i].val;
2466  max_id[i] = data_out[i].rank;
2467  }
2468  }
2469  else if (!r.empty())
2470  {
2471  for (std::size_t i=0; i != r.size(); ++i)
2472  max_id[i] = this->rank();
2473  }
2474 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ min() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::min ( T &  r) const

Take a local variable and replace it with the minimum of it's values on all processors.

Containers are replaced element-wise.

Referenced by semiverify(), and verify().

◆ min() [2/3]

template<typename T >
void libMesh::Parallel::Communicator::min ( T &  libmesh_mpi_varr) const

Definition at line 2087 of file parallel_implementation.h.

References size().

2088 {
2089  if (this->size() > 1)
2090  {
2091  LOG_SCOPE("min(scalar)", "Parallel");
2092 
2093  libmesh_call_mpi
2094  (MPI_Allreduce (MPI_IN_PLACE, &r, 1,
2095  StandardType<T>(&r), OpFunction<T>::min(),
2096  this->get()));
2097  }
2098 }
processor_id_type size() const
Definition: communicator.h:175
long double min(long double a, double b)

◆ min() [3/3]

template<typename A >
void libMesh::Parallel::Communicator::min ( std::vector< bool, A > &  r) const

Definition at line 2139 of file parallel_implementation.h.

References size(), and verify().

2140 {
2141  if (this->size() > 1 && !r.empty())
2142  {
2143  LOG_SCOPE("min(vector<bool>)", "Parallel");
2144 
2145  libmesh_assert(this->verify(r.size()));
2146 
2147  std::vector<unsigned int> ruint;
2148  pack_vector_bool(r, ruint);
2149  std::vector<unsigned int> temp(ruint.size());
2150  libmesh_call_mpi
2151  (MPI_Allreduce (&ruint[0], &temp[0],
2152  cast_int<int>(ruint.size()),
2153  StandardType<unsigned int>(), MPI_BAND,
2154  this->get()));
2155  unpack_vector_bool(temp, r);
2156  }
2157 }
processor_id_type size() const
Definition: communicator.h:175
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ minloc() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::minloc ( T &  r,
unsigned int &  min_id 
) const

Take a local variable and replace it with the minimum of it's values on all processors, returning the minimum rank of a processor which originally held the minimum value.

Definition at line 2161 of file parallel_implementation.h.

References libMesh::libmesh_ignore(), libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), and libMesh::Parallel::DataPlusInt< T >::val.

2163 {
2164  if (this->size() > 1)
2165  {
2166  LOG_SCOPE("minloc(scalar)", "Parallel");
2167 
2168  DataPlusInt<T> data_in;
2169  libmesh_ignore(data_in); // unused ifndef LIBMESH_HAVE_MPI
2170  data_in.val = r;
2171  data_in.rank = this->rank();
2172  libmesh_call_mpi
2173  (MPI_Allreduce (MPI_IN_PLACE, &data_in, 1, dataplusint_type<T>(),
2174  OpFunction<T>::min_location(), this->get()));
2175  r = data_in.val;
2176  min_id = data_in.rank;
2177  }
2178  else
2179  min_id = this->rank();
2180 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173

◆ minloc() [2/3]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::minloc ( std::vector< T, A1 > &  r,
std::vector< unsigned int, A2 > &  min_id 
) const

Take a vector of local variables and replace each entry with the minimum of it's values on all processors.

Set each min_id entry to the minimum rank where a corresponding minimum was found.

Definition at line 2208 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2210 {
2211  if (this->size() > 1 && !r.empty())
2212  {
2213  LOG_SCOPE("minloc(vector)", "Parallel");
2214 
2215  libmesh_assert(this->verify(r.size()));
2216 
2217  std::vector<DataPlusInt<T>> data_in(r.size());
2218  for (std::size_t i=0; i != r.size(); ++i)
2219  {
2220  data_in[i].val = r[i];
2221  data_in[i].rank = this->rank();
2222  }
2223  std::vector<DataPlusInt<T>> data_out(r.size());
2224  libmesh_call_mpi
2225  (MPI_Allreduce (&data_in[0], &data_out[0],
2226  cast_int<int>(r.size()),
2227  dataplusint_type<T>(),
2228  OpFunction<T>::min_location(), this->get()));
2229  for (std::size_t i=0; i != r.size(); ++i)
2230  {
2231  r[i] = data_out[i].val;
2232  min_id[i] = data_out[i].rank;
2233  }
2234  }
2235  else if (!r.empty())
2236  {
2237  for (std::size_t i=0; i != r.size(); ++i)
2238  min_id[i] = this->rank();
2239  }
2240 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ minloc() [3/3]

template<typename A1 , typename A2 >
void libMesh::Parallel::Communicator::minloc ( std::vector< bool, A1 > &  r,
std::vector< unsigned int, A2 > &  min_id 
) const

Definition at line 2244 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2246 {
2247  if (this->size() > 1 && !r.empty())
2248  {
2249  LOG_SCOPE("minloc(vector<bool>)", "Parallel");
2250 
2251  libmesh_assert(this->verify(r.size()));
2252 
2253  std::vector<DataPlusInt<int>> data_in(r.size());
2254  for (std::size_t i=0; i != r.size(); ++i)
2255  {
2256  data_in[i].val = r[i];
2257  data_in[i].rank = this->rank();
2258  }
2259  std::vector<DataPlusInt<int>> data_out(r.size());
2260  libmesh_call_mpi
2261  (MPI_Allreduce (&data_in[0], &data_out[0],
2262  cast_int<int>(r.size()),
2263  StandardType<int>(),
2264  OpFunction<int>::min_location(), this->get()));
2265  for (std::size_t i=0; i != r.size(); ++i)
2266  {
2267  r[i] = data_out[i].val;
2268  min_id[i] = data_out[i].rank;
2269  }
2270  }
2271  else if (!r.empty())
2272  {
2273  for (std::size_t i=0; i != r.size(); ++i)
2274  min_id[i] = this->rank();
2275  }
2276 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
bool verify(const T &r) const
Verify that a local variable has the same value on all processors.

◆ nonblocking_receive_packed_range() [1/2]

template<typename Context , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::nonblocking_receive_packed_range ( const unsigned int  src_processor_id,
Context *  context,
OutputIter  out,
const T *  output_type,
Request req,
Status stat,
const MessageTag tag = any_tag 
) const

Non-Blocking-receive range-of-pointers from one processor.

This is meant to receive messages from nonblocking_send_packed_range

Similar in design to the above receive_packed_range. However, this version requires a Request and a Status.

The Status must be a positively tested Status for a message of this type (i.e. a message does exist). It should most likely be generated by Communicator::packed_range_probe.

Definition at line 1117 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::out, receive(), and libMesh::Parallel::Status::size().

1124 {
1125  libmesh_experimental();
1126 
1127  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1128 
1129  // Receive serialized variable size objects as a sequence of
1130  // buffer_t.
1131  // Allocate a buffer on the heap so we don't have to free it until
1132  // after the Request::wait()
1133  std::vector<buffer_t> * buffer = new std::vector<buffer_t>(stat.size());
1134  this->receive(src_processor_id, *buffer, req, tag);
1135 
1136  // Make the Request::wait() handle unpacking the buffer
1137  req.add_post_wait_work
1138  (new Parallel::PostWaitUnpackBuffer<std::vector<buffer_t>, Context, OutputIter, T>(*buffer, context, out));
1139 
1140  // Make the Request::wait() then handle deleting the buffer
1141  req.add_post_wait_work
1142  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t>>(buffer));
1143 }
OStreamProxy out
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
Blocking-receive from one processor with data-defined type.

◆ nonblocking_receive_packed_range() [2/2]

template<typename Context , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::nonblocking_receive_packed_range ( const unsigned int  src_processor_id,
Context *  context,
OutputIter  out,
const T *  output_type,
Request req,
Status stat,
std::shared_ptr< std::vector< typename Parallel::Packing< T >::buffer_type >> &  buffer,
const MessageTag tag = any_tag 
) const

Non-Blocking-receive range-of-pointers from one processor.

This is meant to receive messages from nonblocking_send_packed_range

Similar in design to the above receive_packed_range. However, this version requires a Request and a Status.

The Status must be a positively tested Status for a message of this type (i.e. a message does exist). It should most likely be generated by Communicator::packed_range_probe.

Definition at line 1715 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::out, receive(), and libMesh::Parallel::Status::size().

1723 {
1724  libmesh_experimental();
1725 
1726  // If they didn't pass in a buffer - let's make one
1727  if (buffer == nullptr)
1728  buffer = std::make_shared<std::vector<typename Parallel::Packing<T>::buffer_type>>();
1729  else
1730  buffer->clear();
1731 
1732  // Receive serialized variable size objects as a sequence of
1733  // buffer_t.
1734  // Allocate a buffer on the heap so we don't have to free it until
1735  // after the Request::wait()
1736  buffer->resize(stat.size());
1737  this->receive(src_processor_id, *buffer, req, tag);
1738 
1739  // Make the Request::wait() handle unpacking the buffer
1740  req.add_post_wait_work
1741  (new Parallel::PostWaitUnpackBuffer<std::vector<typename Parallel::Packing<T>::buffer_type>, Context, OutputIter, T>(*buffer, context, out));
1742 
1743  // Make it dereference the shared pointer (possibly freeing the buffer)
1744  req.add_post_wait_work
1745  (new Parallel::PostWaitDereferenceSharedPtr<std::vector<typename Parallel::Packing<T>::buffer_type>>(buffer));
1746 }
OStreamProxy out
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
Blocking-receive from one processor with data-defined type.

◆ nonblocking_send_packed_range() [1/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::nonblocking_send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
Request req,
const MessageTag tag = no_tag 
) const

Similar to the above Nonblocking send_packed_range with a few important differences:

  1. The total size of the packed buffer MUST be less than std::numeric_limits<int>::max()
  2. Only one message is generated
  3. On the receiving end the message should be tested for using Communicator::packed_range_probe()
  4. The message must be received by Communicator::nonblocking_receive_packed_range()

Definition at line 686 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), std::max(), libMesh::Parallel::pack_range(), and send().

692 {
693  libmesh_experimental();
694 
695  // Allocate a buffer on the heap so we don't have to free it until
696  // after the Request::wait()
697  typedef typename std::iterator_traits<Iter>::value_type T;
698  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
699 
700  if (range_begin != range_end)
701  {
702  std::vector<buffer_t> * buffer = new std::vector<buffer_t>();
703 
704  range_begin =
705  Parallel::pack_range(context,
706  range_begin,
707  range_end,
708  *buffer,
709  // MPI-2 can only use integers for size
711 
712  if (range_begin != range_end)
713  libmesh_error_msg("Non-blocking packed range sends cannot exceed " << std::numeric_limits<int>::max() << "in size");
714 
715  // Make the Request::wait() handle deleting the buffer
716  req.add_post_wait_work
717  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t>>
718  (buffer));
719 
720  // Non-blocking send of the buffer
721  this->send(dest_processor_id, *buffer, req, tag);
722  }
723 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Blocking-send to one processor with data-defined type.
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
long double max(long double a, double b)

◆ nonblocking_send_packed_range() [2/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::nonblocking_send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
Request req,
std::shared_ptr< std::vector< typename Parallel::Packing< typename std::iterator_traits< Iter >::value_type >::buffer_type >> &  buffer,
const MessageTag tag = no_tag 
) const

Similar to the above Nonblocking send_packed_range with a few important differences:

  1. The total size of the packed buffer MUST be less than std::numeric_limits<int>::max()
  2. Only one message is generated
  3. On the receiving end the message should be tested for using Communicator::packed_range_probe()
  4. The message must be received by Communicator::nonblocking_receive_packed_range()

Definition at line 1329 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), std::max(), libMesh::Parallel::pack_range(), and send().

1336 {
1337  libmesh_experimental();
1338 
1339  // Allocate a buffer on the heap so we don't have to free it until
1340  // after the Request::wait()
1341  typedef typename std::iterator_traits<Iter>::value_type T;
1342  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1343 
1344  if (range_begin != range_end)
1345  {
1346  if (buffer == nullptr)
1347  buffer = std::make_shared<std::vector<buffer_t>>();
1348  else
1349  buffer->clear();
1350 
1351  range_begin =
1352  Parallel::pack_range(context,
1353  range_begin,
1354  range_end,
1355  *buffer,
1356  // MPI-2 can only use integers for size
1358 
1359  if (range_begin != range_end)
1360  libmesh_error_msg("Non-blocking packed range sends cannot exceed " << std::numeric_limits<int>::max() << "in size");
1361 
1362  // Make it dereference the shared pointer (possibly freeing the buffer)
1363  req.add_post_wait_work
1364  (new Parallel::PostWaitDereferenceSharedPtr<std::vector<buffer_t>>(buffer));
1365 
1366  // Non-blocking send of the buffer
1367  this->send(dest_processor_id, *buffer, req, tag);
1368  }
1369 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Blocking-send to one processor with data-defined type.
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
long double max(long double a, double b)

◆ operator=() [1/3]

Communicator& libMesh::Parallel::Communicator::operator= ( const Communicator )
delete

◆ operator=() [2/3]

Communicator& libMesh::Parallel::Communicator::operator= ( Communicator &&  )
default

◆ operator=() [3/3]

Communicator& libMesh::Parallel::Communicator::operator= ( const communicator comm)

◆ packed_range_probe()

template<typename T >
Status libMesh::Parallel::Communicator::packed_range_probe ( const unsigned int  src_processor_id,
const MessageTag tag,
bool &  flag 
) const

Non-Blocking message probe for a packed range message.

Allows information about a message to be examined before the message is actually received.

Template type must match the object type that will be in the packed range

Parameters
src_processor_idThe processor the message is expected from or Parallel::any_source
tagThe message tag or Parallel::any_tag
flagOutput. True if a message exists. False otherwise.

Definition at line 188 of file parallel_implementation.h.

References libMesh::Parallel::Status::get(), and libMesh::Parallel::MessageTag::value().

191 {
192  LOG_SCOPE("packed_range_probe()", "Parallel");
193 
194  libmesh_experimental();
195 
196  Status stat((StandardType<typename Packing<T>::buffer_type>()));
197 
198  int int_flag;
199 
200  libmesh_call_mpi(MPI_Iprobe(src_processor_id,
201  tag.value(),
202  this->get(),
203  &int_flag,
204  stat.get()));
205 
206  flag = int_flag;
207 
208  return stat;
209 }

◆ probe()

status libMesh::Parallel::Communicator::probe ( const unsigned int  src_processor_id,
const MessageTag tag = any_tag 
) const

Blocking message probe.

We do not currently support probes on one processor without MPI.

Allows information about a message to be examined before the message is actually received.

Definition at line 174 of file parallel_implementation.h.

References libMesh::Parallel::MessageTag::value().

Referenced by libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), and receive().

176 {
177  LOG_SCOPE("probe()", "Parallel");
178 
179  status stat;
180 
181  libmesh_call_mpi
182  (MPI_Probe (src_processor_id, tag.value(), this->get(), &stat));
183 
184  return stat;
185 }
MPI_Status status
Status object for querying messages.
Definition: status.h:41

◆ rank()

processor_id_type libMesh::Parallel::Communicator::rank ( ) const

◆ receive() [1/6]

template<typename T >
Status libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
const MessageTag tag = any_tag 
) const

Blocking-receive from one processor with data-defined type.

We do not currently support receives on one processor without MPI.

Definition at line 771 of file parallel_implementation.h.

References libMesh::Parallel::Status::get(), probe(), and libMesh::Parallel::MessageTag::value().

Referenced by nonblocking_receive_packed_range(), libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), receive(), and receive_packed_range().

774 {
775  LOG_SCOPE("receive()", "Parallel");
776 
777  // Get the status of the message, explicitly provide the
778  // datatype so we can later query the size
779  Status stat(this->probe(src_processor_id, tag), StandardType<T>(&buf));
780 
781  libmesh_call_mpi
782  (MPI_Recv (&buf, 1, StandardType<T>(&buf), src_processor_id,
783  tag.value(), this->get(), stat.get()));
784 
785  return stat;
786 }
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag) const
Blocking message probe.

◆ receive() [2/6]

template<typename T >
void libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
Request req,
const MessageTag tag = any_tag 
) const

Nonblocking-receive from one processor with data-defined type.

Definition at line 791 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::get(), and libMesh::Parallel::MessageTag::value().

795 {
796  LOG_SCOPE("receive()", "Parallel");
797 
798  libmesh_call_mpi
799  (MPI_Irecv (&buf, 1, StandardType<T>(&buf), src_processor_id,
800  tag.value(), this->get(), req.get()));
801 
802  // The MessageTag should stay registered for the Request lifetime
803  req.add_post_wait_work
804  (new Parallel::PostWaitDereferenceTag(tag));
805 }

◆ receive() [3/6]

template<typename T >
Status libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
const DataType type,
const MessageTag tag = any_tag 
) const

Blocking-receive from one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1823 of file parallel_implementation.h.

1827 { libmesh_not_implemented(); return Status(); }

◆ receive() [4/6]

template<typename T >
void libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
const DataType type,
Request req,
const MessageTag tag = any_tag 
) const

Nonblocking-receive from one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1830 of file parallel_implementation.h.

1835 { libmesh_not_implemented(); }

◆ receive() [5/6]

template<typename T >
Status libMesh::Parallel::Communicator::receive ( const unsigned int  src_processor_id,
std::basic_string< T > &  buf,
const MessageTag tag 
) const

Definition at line 727 of file parallel_implementation.h.

References receive().

730 {
731  std::vector<T> tempbuf; // Officially C++ won't let us get a
732  // modifiable array from a string
733 
734  Status stat = this->receive(src_processor_id, tempbuf, tag);
735  buf.assign(tempbuf.begin(), tempbuf.end());
736  return stat;
737 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
Blocking-receive from one processor with data-defined type.

◆ receive() [6/6]

template<typename T >
void libMesh::Parallel::Communicator::receive ( const unsigned int  src_processor_id,
std::basic_string< T > &  buf,
Request req,
const MessageTag tag 
) const

Definition at line 742 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), and receive().

746 {
747  // Officially C++ won't let us get a modifiable array from a
748  // string, and we can't even put one on the stack for the
749  // non-blocking case.
750  std::vector<T> * tempbuf = new std::vector<T>();
751 
752  // We can clear the string, but the Request::wait() will need to
753  // handle copying our temporary buffer to it
754  buf.clear();
755 
756  req.add_post_wait_work
757  (new Parallel::PostWaitCopyBuffer<std::vector<T>,
758  std::back_insert_iterator<std::basic_string<T>>>
759  (tempbuf, std::back_inserter(buf)));
760 
761  // Make the Request::wait() then handle deleting the buffer
762  req.add_post_wait_work
763  (new Parallel::PostWaitDeleteBuffer<std::vector<T>>(tempbuf));
764 
765  this->receive(src_processor_id, tempbuf, req, tag);
766 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
Blocking-receive from one processor with data-defined type.

◆ receive_packed_range()

template<typename Context , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::receive_packed_range ( const unsigned int  dest_processor_id,
Context *  context,
OutputIter  out,
const T *  output_type,
const MessageTag tag = any_tag 
) const

Blocking-receive range-of-pointers from one processor.

This function does not receive raw pointers, but rather constructs new objects whose contents match the objects pointed to by the sender.

The objects will be of type T = iterator_traits<OutputIter>::value_type.

Using std::back_inserter as the output iterator allows receive to fill any container type. Using libMesh::null_output_iterator allows the receive to be dealt with solely by Parallel::unpack(), for objects whose unpack() is written so as to not leak memory when used in this fashion.

A future version of this method should be created to preallocate memory when receiving vectors...

void Parallel::unpack(vector<int>::iterator in, T ** out, Context *) is used to unserialize type T, typically into a new heap-allocated object whose pointer is returned as *out.

unsigned int Parallel::packed_size(const T *, vector<int>::const_iterator) is used to advance to the beginning of the next object's data.

Definition at line 1060 of file parallel_implementation.h.

References receive(), libMesh::Parallel::Status::source(), libMesh::Parallel::Status::tag(), and libMesh::Parallel::unpack_range().

Referenced by send_receive_packed_range().

1065 {
1066  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1067 
1068  // Receive serialized variable size objects as sequences of buffer_t
1069  std::size_t total_buffer_size = 0;
1070  Status stat = this->receive(src_processor_id, total_buffer_size, tag);
1071 
1072  // Use stat.source() and stat.tag() in subsequent receives - if
1073  // src_processor_id is or tag is "any" then we want to be sure we
1074  // try to receive messages all corresponding to the same send.
1075 
1076  std::size_t received_buffer_size = 0;
1077  while (received_buffer_size < total_buffer_size)
1078  {
1079  std::vector<buffer_t> buffer;
1080  this->receive(stat.source(), buffer, MessageTag(stat.tag()));
1081  received_buffer_size += buffer.size();
1083  (buffer, context, out_iter, output_type);
1084  }
1085 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
Blocking-receive from one processor with data-defined type.
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
Decode a range of potentially-variable-size objects from a data array.

◆ reference_unique_tag()

void libMesh::Parallel::Communicator::reference_unique_tag ( int  tagvalue) const

Reference an already-acquired tag, so that we know it will be dereferenced multiple times before we can re-release it.

◆ scatter() [1/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::scatter ( const std::vector< T, A > &  data,
T &  recv,
const unsigned int  root_id = 0 
) const

Take a vector of local variables and scatter the ith item to the ith processor in the communicator.

The result is saved into recv.

Definition at line 2921 of file parallel_implementation.h.

References data, libMesh::libmesh_ignore(), rank(), and size().

Referenced by scatter().

2924 {
2925  libmesh_ignore(root_id); // Only needed for MPI and/or dbg/devel
2926  libmesh_assert_less (root_id, this->size());
2927 
2928  // Do not allow the root_id to scatter a nullptr vector.
2929  // That would leave recv in an indeterminate state.
2930  libmesh_assert (this->rank() != root_id || this->size() == data.size());
2931 
2932  if (this->size() == 1)
2933  {
2934  libmesh_assert (!this->rank());
2935  libmesh_assert (!root_id);
2936  recv = data[0];
2937  return;
2938  }
2939 
2940  LOG_SCOPE("scatter()", "Parallel");
2941 
2942  T * data_ptr = const_cast<T*>(data.empty() ? nullptr : &data[0]);
2943  libmesh_ignore(data_ptr); // unused ifndef LIBMESH_HAVE_MPI
2944 
2945  libmesh_call_mpi
2946  (MPI_Scatter (data_ptr, 1, StandardType<T>(data_ptr),
2947  &recv, 1, StandardType<T>(&recv), root_id, this->get()));
2948 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
Ideally this private member data should have protected access.

◆ scatter() [2/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::scatter ( const std::vector< T, A > &  data,
std::vector< T, A > &  recv,
const unsigned int  root_id = 0 
) const

Take a vector of local variables and scatter the ith equal-sized chunk to the ith processor in the communicator.

The data size must be a multiple of the communicator size. The result is saved into recv buffer. The recv buffer does not have to be sized prior to this operation.

Definition at line 2953 of file parallel_implementation.h.

References broadcast(), data, libMesh::libmesh_ignore(), rank(), and size().

2956 {
2957  libmesh_assert_less (root_id, this->size());
2958 
2959  if (this->size() == 1)
2960  {
2961  libmesh_assert (!this->rank());
2962  libmesh_assert (!root_id);
2963  recv.assign(data.begin(), data.end());
2964  return;
2965  }
2966 
2967  LOG_SCOPE("scatter()", "Parallel");
2968 
2969  int recv_buffer_size = 0;
2970  if (this->rank() == root_id)
2971  {
2972  libmesh_assert(data.size() % this->size() == 0);
2973  recv_buffer_size = cast_int<int>(data.size() / this->size());
2974  }
2975 
2976  this->broadcast(recv_buffer_size);
2977  recv.resize(recv_buffer_size);
2978 
2979  T * data_ptr = const_cast<T*>(data.empty() ? nullptr : &data[0]);
2980  T * recv_ptr = recv.empty() ? nullptr : &recv[0];
2981  libmesh_ignore(data_ptr, recv_ptr); // unused ifndef LIBMESH_HAVE_MPI
2982 
2983  libmesh_call_mpi
2984  (MPI_Scatter (data_ptr, recv_buffer_size, StandardType<T>(data_ptr),
2985  recv_ptr, recv_buffer_size, StandardType<T>(recv_ptr), root_id, this->get()));
2986 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
Ideally this private member data should have protected access.
void broadcast(T &data, const unsigned int root_id=0) const
Take a local value and broadcast it to all processors.

◆ scatter() [3/4]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::scatter ( const std::vector< T, A1 > &  data,
const std::vector< int, A2 counts,
std::vector< T, A1 > &  recv,
const unsigned int  root_id = 0 
) const

Take a vector of local variables and scatter the ith variable-sized chunk to the ith processor in the communicator.

The counts vector should contain the number of items for each processor. The result is saved into recv buffer. The recv buffer does not have to be sized prior to this operation.

Definition at line 2991 of file parallel_implementation.h.

References data, libMesh::libmesh_ignore(), rank(), scatter(), and size().

2995 {
2996  libmesh_assert_less (root_id, this->size());
2997 
2998  if (this->size() == 1)
2999  {
3000  libmesh_assert (!this->rank());
3001  libmesh_assert (!root_id);
3002  libmesh_assert (counts.size() == this->size());
3003  recv.assign(data.begin(), data.begin() + counts[0]);
3004  return;
3005  }
3006 
3007  std::vector<int,A2> displacements(this->size(), 0);
3008  if (root_id == this->rank())
3009  {
3010  libmesh_assert(counts.size() == this->size());
3011 
3012  // Create a displacements vector from the incoming counts vector
3013  unsigned int globalsize = 0;
3014  for (unsigned int i=0; i < this->size(); ++i)
3015  {
3016  displacements[i] = globalsize;
3017  globalsize += counts[i];
3018  }
3019 
3020  libmesh_assert(data.size() == globalsize);
3021  }
3022 
3023  LOG_SCOPE("scatter()", "Parallel");
3024 
3025  // Scatter the buffer sizes to size remote buffers
3026  int recv_buffer_size = 0;
3027  this->scatter(counts, recv_buffer_size, root_id);
3028  recv.resize(recv_buffer_size);
3029 
3030  T * data_ptr = const_cast<T*>(data.empty() ? nullptr : &data[0]);
3031  int * count_ptr = const_cast<int*>(counts.empty() ? nullptr : &counts[0]);
3032  T * recv_ptr = recv.empty() ? nullptr : &recv[0];
3033  libmesh_ignore(data_ptr, count_ptr, recv_ptr); // unused ifndef LIBMESH_HAVE_MPI
3034 
3035  // Scatter the non-uniform chunks
3036  libmesh_call_mpi
3037  (MPI_Scatterv (data_ptr, count_ptr, &displacements[0], StandardType<T>(data_ptr),
3038  recv_ptr, recv_buffer_size, StandardType<T>(recv_ptr), root_id, this->get()));
3039 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
void scatter(const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
Take a vector of local variables and scatter the ith item to the ith processor in the communicator...
IterBase * data
Ideally this private member data should have protected access.

◆ scatter() [4/4]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::scatter ( const std::vector< std::vector< T, A1 >, A2 > &  data,
std::vector< T, A1 > &  recv,
const unsigned int  root_id = 0,
const bool  identical_buffer_sizes = false 
) const

Take a vector of vectors and scatter the ith inner vector to the ith processor in the communicator.

The result is saved into recv buffer. The recv buffer does not have to be sized prior to this operation.

Definition at line 3044 of file parallel_implementation.h.

References data, end, rank(), scatter(), and size().

3048 {
3049  libmesh_assert_less (root_id, this->size());
3050 
3051  if (this->size() == 1)
3052  {
3053  libmesh_assert (!this->rank());
3054  libmesh_assert (!root_id);
3055  libmesh_assert (data.size() == this->size());
3056  recv.assign(data[0].begin(), data[0].end());
3057  return;
3058  }
3059 
3060  std::vector<T,A1> stacked_data;
3061  std::vector<int> counts;
3062 
3063  if (root_id == this->rank())
3064  {
3065  libmesh_assert (data.size() == this->size());
3066 
3067  if (!identical_buffer_sizes)
3068  counts.resize(this->size());
3069 
3070  for (std::size_t i=0; i < data.size(); ++i)
3071  {
3072  if (!identical_buffer_sizes)
3073  counts[i] = cast_int<int>(data[i].size());
3074 #ifndef NDEBUG
3075  else
3076  // Check that buffer sizes are indeed equal
3077  libmesh_assert(!i || data[i-1].size() == data[i].size());
3078 #endif
3079  std::copy(data[i].begin(), data[i].end(), std::back_inserter(stacked_data));
3080  }
3081  }
3082 
3083  if (identical_buffer_sizes)
3084  this->scatter(stacked_data, recv, root_id);
3085  else
3086  this->scatter(stacked_data, counts, recv, root_id);
3087 }
processor_id_type size() const
Definition: communicator.h:175
IterBase * end
Also have a polymorphic pointer to the end object, this prevents iterating past the end...
processor_id_type rank() const
Definition: communicator.h:173
void scatter(const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
Take a vector of local variables and scatter the ith item to the ith processor in the communicator...
IterBase * data
Ideally this private member data should have protected access.

◆ semiverify()

template<typename T >
bool libMesh::Parallel::Communicator::semiverify ( const T *  r) const

Verify that a local pointer points to the same value on all processors where it is not nullptr.

Containers must have the same value in every entry.

Definition at line 1948 of file parallel_implementation.h.

References max(), min(), libMesh::Parallel::Attributes< T >::set_highest(), libMesh::Parallel::Attributes< T >::set_lowest(), and size().

1949 {
1950  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1951  {
1952  T tempmin, tempmax;
1953  if (r)
1954  tempmin = tempmax = *r;
1955  else
1956  {
1957  Attributes<T>::set_highest(tempmin);
1958  Attributes<T>::set_lowest(tempmax);
1959  }
1960  this->min(tempmin);
1961  this->max(tempmax);
1962  bool invalid = r && ((*r != tempmin) ||
1963  (*r != tempmax));
1964  this->max(invalid);
1965  return !invalid;
1966  }
1967 
1968  static_assert(Attributes<T>::has_min_max,
1969  "Tried to semiverify an unverifiable type");
1970 
1971  return true;
1972 }
void max(T &r) const
Take a local variable and replace it with the maximum of it&#39;s values on all processors.
processor_id_type size() const
Definition: communicator.h:175
static void set_lowest(T &)
Definition: attributes.h:45
void min(T &r) const
Take a local variable and replace it with the minimum of it&#39;s values on all processors.
static void set_highest(T &)
Definition: attributes.h:46
static const bool has_min_max
Definition: attributes.h:44

◆ send() [1/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const std::basic_string< T > &  buf,
const MessageTag tag 
) const

Definition at line 213 of file parallel_implementation.h.

References send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

216 {
217  LOG_SCOPE("send()", "Parallel");
218 
219  T * dataptr = buf.empty() ? nullptr : const_cast<T *>(buf.data());
220 
221  libmesh_call_mpi
222  (((this->send_mode() == SYNCHRONOUS) ?
223  MPI_Ssend : MPI_Send) (dataptr,
224  cast_int<int>(buf.size()),
225  StandardType<T>(dataptr),
226  dest_processor_id,
227  tag.value(),
228  this->get()));
229 }
SendMode send_mode() const
Gets the user-requested SendMode.
Definition: communicator.h:210

◆ send() [2/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const std::basic_string< T > &  buf,
Request req,
const MessageTag tag 
) const

Definition at line 234 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::get(), send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

238 {
239  LOG_SCOPE("send()", "Parallel");
240 
241  T * dataptr = buf.empty() ? nullptr : const_cast<T *>(buf.data());
242 
243  libmesh_call_mpi
244  (((this->send_mode() == SYNCHRONOUS) ?
245  MPI_Issend : MPI_Isend) (dataptr,
246  cast_int<int>(buf.size()),
247  StandardType<T>(dataptr),
248  dest_processor_id,
249  tag.value(),
250  this->get(),
251  req.get()));
252 
253  // The MessageTag should stay registered for the Request lifetime
254  req.add_post_wait_work
255  (new Parallel::PostWaitDereferenceTag(tag));
256 }
SendMode send_mode() const
Gets the user-requested SendMode.
Definition: communicator.h:210

◆ send() [3/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
const MessageTag tag = no_tag 
) const

Blocking-send to one processor with data-defined type.

We do not currently support sends on one processor without MPI.

Definition at line 261 of file parallel_implementation.h.

References send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

Referenced by nonblocking_send_packed_range(), libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), and send_packed_range().

264 {
265  LOG_SCOPE("send()", "Parallel");
266 
267  T * dataptr = const_cast<T*> (&buf);
268 
269  libmesh_call_mpi
270  (((this->send_mode() == SYNCHRONOUS) ?
271  MPI_Ssend : MPI_Send) (dataptr,
272  1,
273  StandardType<T>(dataptr),
274  dest_processor_id,
275  tag.value(),
276  this->get()));
277 }
SendMode send_mode() const
Gets the user-requested SendMode.
Definition: communicator.h:210

◆ send() [4/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
Request req,
const MessageTag tag = no_tag 
) const

Nonblocking-send to one processor with data-defined type.

Definition at line 282 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::get(), send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

286 {
287  LOG_SCOPE("send()", "Parallel");
288 
289  T * dataptr = const_cast<T*>(&buf);
290 
291  libmesh_call_mpi
292  (((this->send_mode() == SYNCHRONOUS) ?
293  MPI_Issend : MPI_Isend) (dataptr,
294  1,
295  StandardType<T>(dataptr),
296  dest_processor_id,
297  tag.value(),
298  this->get(),
299  req.get()));
300 
301  // The MessageTag should stay registered for the Request lifetime
302  req.add_post_wait_work
303  (new Parallel::PostWaitDereferenceTag(tag));
304 }
SendMode send_mode() const
Gets the user-requested SendMode.
Definition: communicator.h:210

◆ send() [5/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
const DataType type,
const MessageTag tag = no_tag 
) const

Blocking-send to one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1775 of file parallel_implementation.h.

1779 { libmesh_not_implemented(); }

◆ send() [6/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
const DataType type,
Request req,
const MessageTag tag = no_tag 
) const

Nonblocking-send to one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1782 of file parallel_implementation.h.

1787 { libmesh_not_implemented(); }

◆ send_mode() [1/2]

void libMesh::Parallel::Communicator::send_mode ( const SendMode  sm)

Explicitly sets the SendMode type used for send operations.

Definition at line 205 of file communicator.h.

References _send_mode.

205 { _send_mode = sm; }

◆ send_mode() [2/2]

SendMode libMesh::Parallel::Communicator::send_mode ( ) const

Gets the user-requested SendMode.

Definition at line 210 of file communicator.h.

References _send_mode.

Referenced by send().

210 { return _send_mode; }

◆ send_packed_range() [1/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
const MessageTag tag = no_tag 
) const

Blocking-send range-of-pointers to one processor.

This function does not send the raw pointers, but rather constructs new objects at the other end whose contents match the objects pointed to by the sender.

void Parallel::pack(const T *, vector<int> & data, const Context *) is used to serialize type T onto the end of a data vector.

unsigned int Parallel::packable_size(const T *, const Context *) is used to allow data vectors to reserve memory, and for additional error checking

Definition at line 562 of file parallel_implementation.h.

References libMesh::Parallel::pack_range(), libMesh::Parallel::packed_range_size(), and send().

Referenced by send_receive_packed_range().

567 {
568  // We will serialize variable size objects from *range_begin to
569  // *range_end as a sequence of plain data (e.g. ints) in this buffer
570  typedef typename std::iterator_traits<Iter>::value_type T;
571 
572  std::size_t total_buffer_size =
573  Parallel::packed_range_size (context, range_begin, range_end);
574 
575  this->send(dest_processor_id, total_buffer_size, tag);
576 
577 #ifdef DEBUG
578  std::size_t used_buffer_size = 0;
579 #endif
580 
581  while (range_begin != range_end)
582  {
583  libmesh_assert_greater (std::distance(range_begin, range_end), 0);
584 
585  std::vector<typename Parallel::Packing<T>::buffer_type> buffer;
586 
587  const Iter next_range_begin = Parallel::pack_range
588  (context, range_begin, range_end, buffer);
589 
590  libmesh_assert_greater (std::distance(range_begin, next_range_begin), 0);
591 
592  range_begin = next_range_begin;
593 
594 #ifdef DEBUG
595  used_buffer_size += buffer.size();
596 #endif
597 
598  // Blocking send of the buffer
599  this->send(dest_processor_id, buffer, tag);
600  }
601 
602 #ifdef DEBUG
603  libmesh_assert_equal_to(used_buffer_size, total_buffer_size);
604 #endif
605 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Blocking-send to one processor with data-defined type.
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
Return the total buffer size needed to encode a range of potentially-variable-size objects to a data ...
Definition: packing.h:118

◆ send_packed_range() [2/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
Request req,
const MessageTag tag = no_tag 
) const

Nonblocking-send range-of-pointers to one processor.

This function does not send the raw pointers, but rather constructs new objects at the other end whose contents match the objects pointed to by the sender.

void Parallel::pack(const T *, vector<int> & data, const Context *) is used to serialize type T onto the end of a data vector.

unsigned int Parallel::packable_size(const T *, const Context *) is used to allow data vectors to reserve memory, and for additional error checking

Definition at line 609 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::add_prior_request(), libMesh::Parallel::pack_range(), libMesh::Parallel::packed_range_size(), and send().

615 {
616  // Allocate a buffer on the heap so we don't have to free it until
617  // after the Request::wait()
618  typedef typename std::iterator_traits<Iter>::value_type T;
619  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
620 
621  std::size_t total_buffer_size =
622  Parallel::packed_range_size (context, range_begin, range_end);
623 
624  // That local variable will be gone soon; we need a send buffer that
625  // will stick around. I heard you like buffering so I put a buffer
626  // for your buffer size so you can buffer the size of your buffer.
627  std::size_t * total_buffer_size_buffer = new std::size_t;
628  *total_buffer_size_buffer = total_buffer_size;
629 
630  // Delete the buffer size's buffer when we're done
631  Request intermediate_req = request();
632  intermediate_req.add_post_wait_work
633  (new Parallel::PostWaitDeleteBuffer<std::size_t>(total_buffer_size_buffer));
634  this->send(dest_processor_id, *total_buffer_size_buffer, intermediate_req, tag);
635 
636  // And don't finish up the full request until we're done with its
637  // dependencies
638  req.add_prior_request(intermediate_req);
639 
640 #ifdef DEBUG
641  std::size_t used_buffer_size = 0;
642 #endif
643 
644  while (range_begin != range_end)
645  {
646  libmesh_assert_greater (std::distance(range_begin, range_end), 0);
647 
648  std::vector<buffer_t> * buffer = new std::vector<buffer_t>();
649 
650  const Iter next_range_begin =
651  Parallel::pack_range(context, range_begin, range_end,
652  *buffer);
653 
654  libmesh_assert_greater (std::distance(range_begin, next_range_begin), 0);
655 
656  range_begin = next_range_begin;
657 
658 #ifdef DEBUG
659  used_buffer_size += buffer->size();
660 #endif
661 
662  Request next_intermediate_req;
663 
664  Request * my_req = (range_begin == range_end) ? &req : &next_intermediate_req;
665 
666  // Make the Request::wait() handle deleting the buffer
667  my_req->add_post_wait_work
668  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t>>
669  (buffer));
670 
671  // Non-blocking send of the buffer
672  this->send(dest_processor_id, *buffer, *my_req, tag);
673 
674  if (range_begin != range_end)
675  req.add_prior_request(*my_req);
676  }
677 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Blocking-send to one processor with data-defined type.
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
MPI_Request request
Request object for non-blocking I/O.
Definition: request.h:40
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
Return the total buffer size needed to encode a range of potentially-variable-size objects to a data ...
Definition: packing.h:118

◆ send_receive() [1/3]

template<typename T1 , typename T2 >
void libMesh::Parallel::Communicator::send_receive ( const unsigned int  dest_processor_id,
const T1 &  send,
const unsigned int  source_processor_id,
T2 &  recv,
const MessageTag send_tag = no_tag,
const MessageTag recv_tag = any_tag 
) const

Send data send to one processor while simultaneously receiving other data recv from a (potentially different) processor.

Send-receive data from one processor.

Definition at line 1178 of file parallel_implementation.h.

References rank(), and libMesh::Parallel::MessageTag::value().

Referenced by send_receive().

1184 {
1185  LOG_SCOPE("send_receive()", "Parallel");
1186 
1187  if (dest_processor_id == this->rank() &&
1188  source_processor_id == this->rank())
1189  {
1190  recv = sendvec;
1191  return;
1192  }
1193 
1194  // MPI_STATUS_IGNORE is from MPI-2; using it with some versions of
1195  // MPICH may cause a crash:
1196  // https://bugzilla.mcs.anl.gov/globus/show_bug.cgi?id=1798
1197  libmesh_call_mpi
1198  (MPI_Sendrecv(const_cast<T1*>(&sendvec), 1, StandardType<T1>(&sendvec),
1199  dest_processor_id, send_tag.value(), &recv, 1,
1200  StandardType<T2>(&recv), source_processor_id,
1201  recv_tag.value(), this->get(), MPI_STATUS_IGNORE));
1202 }
processor_id_type rank() const
Definition: communicator.h:173

◆ send_receive() [2/3]

template<typename T1 , typename T2 >
void libMesh::Parallel::Communicator::send_receive ( const unsigned int  dest_processor_id,
const T1 &  send,
const DataType type1,
const unsigned int  source_processor_id,
T2 &  recv,
const DataType type2,
const MessageTag send_tag = no_tag,
const MessageTag recv_tag = any_tag 
) const

Send data send to one processor while simultaneously receiving other data recv from a (potentially different) processor, using a user-specified MPI Dataype.

◆ send_receive() [3/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::send_receive ( const unsigned int  dest_processor_id,
const std::vector< T, A > &  sendvec,
const unsigned int  source_processor_id,
std::vector< T, A > &  recv,
const MessageTag send_tag,
const MessageTag recv_tag 
) const

Definition at line 1214 of file parallel_implementation.h.

References rank(), and send_receive().

1220 {
1221  if (dest_processor_id == this->rank() &&
1222  source_processor_id == this->rank())
1223  {
1224  LOG_SCOPE("send_receive()", "Parallel");
1225  recv = sendvec;
1226  return;
1227  }
1228 
1229  const T* example = sendvec.empty() ?
1230  (recv.empty() ? nullptr : &recv[0]) : &sendvec[0];
1231 
1232  // Call the user-defined type version with automatic
1233  // type conversion based on template argument:
1234  this->send_receive (dest_processor_id, sendvec,
1235  StandardType<T>(example),
1236  source_processor_id, recv,
1237  StandardType<T>(example),
1238  send_tag, recv_tag);
1239 }
void send_receive(const unsigned int dest_processor_id, const T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
Send data send to one processor while simultaneously receiving other data recv from a (potentially di...
processor_id_type rank() const
Definition: communicator.h:173

◆ send_receive_packed_range() [1/2]

template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::send_receive_packed_range ( const unsigned int  dest_processor_id,
const Context1 *  context1,
RangeIter  send_begin,
const RangeIter  send_end,
const unsigned int  source_processor_id,
Context2 *  context2,
OutputIter  out,
const T *  output_type,
const MessageTag send_tag = no_tag,
const MessageTag recv_tag = any_tag 
) const

Send a range-of-pointers to one processor while simultaneously receiving another range from a (potentially different) processor.

This function does not send or receive raw pointers, but rather constructs new objects at each receiver whose contents match the objects pointed to by the sender.

The objects being sent will be of type T1 = iterator_traits<RangeIter>::value_type, and the objects being received will be of type T2 = iterator_traits<OutputIter>::value_type

void Parallel::pack(const T1*, vector<int> & data, const Context1*) is used to serialize type T1 onto the end of a data vector.

Using std::back_inserter as the output iterator allows send_receive to fill any container type. Using libMesh::null_output_iterator allows the receive to be dealt with solely by Parallel::unpack(), for objects whose unpack() is written so as to not leak memory when used in this fashion.

A future version of this method should be created to preallocate memory when receiving vectors...

void Parallel::unpack(vector<int>::iterator in, T2** out, Context *) is used to unserialize type T2, typically into a new heap-allocated object whose pointer is returned as *out.

unsigned int Parallel::packable_size(const T1*, const Context1*) is used to allow data vectors to reserve memory, and for additional error checking.

unsigned int Parallel::packed_size(const T2*, vector<int>::const_iterator) is used to advance to the beginning of the next object's data.

Definition at line 1302 of file parallel_implementation.h.

References receive_packed_range(), send_packed_range(), and libMesh::Parallel::Request::wait().

1312 {
1313  LOG_SCOPE("send_receive()", "Parallel");
1314 
1315  Parallel::Request req;
1316 
1317  this->send_packed_range (dest_processor_id, context1, send_begin, send_end,
1318  req, send_tag);
1319 
1320  this->receive_packed_range (source_processor_id, context2, out_iter,
1321  output_type, recv_tag);
1322 
1323  req.wait();
1324 }
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag) const
Blocking-send range-of-pointers to one processor.
void receive_packed_range(const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const
Blocking-receive range-of-pointers from one processor.

◆ send_receive_packed_range() [2/2]

template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::send_receive_packed_range ( const unsigned int   libmesh_dbg_vardest_processor_id,
const Context1 *  context1,
RangeIter  send_begin,
const RangeIter  send_end,
const unsigned int   libmesh_dbg_varsource_processor_id,
Context2 *  context2,
OutputIter  out_iter,
const T *  output_type,
const MessageTag ,
const MessageTag  
) const

Send-receive range-of-pointers from one processor.

If you call this without MPI you might be making a mistake, but we'll support it.

Definition at line 1876 of file parallel_implementation.h.

References libMesh::Parallel::pack_range(), and libMesh::Parallel::unpack_range().

1886 {
1887  // This makes no sense on one processor unless we're deliberately
1888  // sending to ourself.
1889  libmesh_assert_equal_to(dest_processor_id, 0);
1890  libmesh_assert_equal_to(source_processor_id, 0);
1891 
1892  // On one processor, we just need to pack the range and then unpack
1893  // it again.
1894  typedef typename std::iterator_traits<RangeIter>::value_type T1;
1895  typedef typename Parallel::Packing<T1>::buffer_type buffer_t;
1896 
1897  while (send_begin != send_end)
1898  {
1899  libmesh_assert_greater (std::distance(send_begin, send_end), 0);
1900 
1901  // We will serialize variable size objects from *range_begin to
1902  // *range_end as a sequence of ints in this buffer
1903  std::vector<buffer_t> buffer;
1904 
1905  const RangeIter next_send_begin = Parallel::pack_range
1906  (context1, send_begin, send_end, buffer);
1907 
1908  libmesh_assert_greater (std::distance(send_begin, next_send_begin), 0);
1909 
1910  send_begin = next_send_begin;
1911 
1913  (buffer, context2, out_iter, output_type);
1914  }
1915 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Encode a range of potentially-variable-size objects to a data array.
Definition: packing.h:139
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
Decode a range of potentially-variable-size objects from a data array.

◆ set_union() [1/2]

template<typename T >
void libMesh::Parallel::Communicator::set_union ( T &  data,
const unsigned int  root_id 
) const

Take a container of local variables on each processor, and collect their union over all processors, replacing the set on processor 0.

◆ set_union() [2/2]

template<typename T >
void libMesh::Parallel::Communicator::set_union ( T &  data) const

Take a container of local variables on each processor, and replace it with their union over all processors.

◆ size()

processor_id_type libMesh::Parallel::Communicator::size ( ) const

◆ split()

void libMesh::Parallel::Communicator::split ( int  color,
int  key,
Communicator target 
) const

◆ sum() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::sum ( T &  r) const

Take a local variable and replace it with the sum of it's values on all processors.

Containers are replaced element-wise.

◆ sum() [2/3]

template<typename T >
void libMesh::Parallel::Communicator::sum ( T &  libmesh_mpi_varr) const

Definition at line 2478 of file parallel_implementation.h.

References size().

2479 {
2480  if (this->size() > 1)
2481  {
2482  LOG_SCOPE("sum()", "Parallel");
2483 
2484  libmesh_call_mpi
2485  (MPI_Allreduce (MPI_IN_PLACE, &r, 1,
2486  StandardType<T>(&r),
2487  OpFunction<T>::sum(),
2488  this->get()));
2489  }
2490 }
processor_id_type size() const
Definition: communicator.h:175

◆ sum() [3/3]

template<typename T >
void libMesh::Parallel::Communicator::sum ( std::complex< T > &  libmesh_mpi_varr) const

Definition at line 2515 of file parallel_implementation.h.

References size().

2516 {
2517  if (this->size() > 1)
2518  {
2519  LOG_SCOPE("sum()", "Parallel");
2520 
2521  libmesh_call_mpi
2522  (MPI_Allreduce (MPI_IN_PLACE, &r, 2,
2523  StandardType<T>(),
2524  OpFunction<T>::sum(),
2525  this->get()));
2526  }
2527 }
processor_id_type size() const
Definition: communicator.h:175

◆ verify()

template<typename T >
bool libMesh::Parallel::Communicator::verify ( const T &  r) const

Verify that a local variable has the same value on all processors.

Containers must have the same value in every entry.

Definition at line 1928 of file parallel_implementation.h.

References max(), min(), and size().

Referenced by allgather(), alltoall(), max(), maxloc(), min(), and minloc().

1929 {
1930  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1931  {
1932  T tempmin = r, tempmax = r;
1933  this->min(tempmin);
1934  this->max(tempmax);
1935  bool verified = (r == tempmin) &&
1936  (r == tempmax);
1937  this->min(verified);
1938  return verified;
1939  }
1940 
1941  static_assert(Attributes<T>::has_min_max,
1942  "Tried to verify an unverifiable type");
1943 
1944  return true;
1945 }
void max(T &r) const
Take a local variable and replace it with the maximum of it&#39;s values on all processors.
processor_id_type size() const
Definition: communicator.h:175
void min(T &r) const
Take a local variable and replace it with the minimum of it&#39;s values on all processors.
static const bool has_min_max
Definition: attributes.h:44

Member Data Documentation

◆ _communicator

communicator libMesh::Parallel::Communicator::_communicator
private

Definition at line 190 of file communicator.h.

Referenced by get(), and libMesh::ParallelObject::operator=().

◆ _I_duped_it

bool libMesh::Parallel::Communicator::_I_duped_it
private

Definition at line 197 of file communicator.h.

◆ _rank

processor_id_type libMesh::Parallel::Communicator::_rank
private

Definition at line 191 of file communicator.h.

Referenced by rank().

◆ _send_mode

SendMode libMesh::Parallel::Communicator::_send_mode
private

Definition at line 192 of file communicator.h.

Referenced by send_mode().

◆ _size

processor_id_type libMesh::Parallel::Communicator::_size
private

Definition at line 191 of file communicator.h.

Referenced by size().

◆ used_tag_values

std::map<int, unsigned int> libMesh::Parallel::Communicator::used_tag_values
mutableprivate

Definition at line 196 of file communicator.h.


The documentation for this class was generated from the following files: