communicator.h

Go to the documentation of this file.

// The libMesh Finite Element Library.
// Copyright (C) 2002-2019 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


#ifndef LIBMESH_COMMUNICATOR_H
#define LIBMESH_COMMUNICATOR_H

// Parallel includes
#include "libmesh/data_type.h"
#include "libmesh/message_tag.h"
#include "libmesh/request.h"
#include "libmesh/status.h"
#include "libmesh/post_wait_dereference_shared_ptr.h"

// libMesh Includes
#include "libmesh/libmesh_common.h"

// C++ includes
#include <map>
#include <string>
#include <vector>

// C++ includes needed for parallel_communicator_specializations
//
// These could be forward declarations if only that wasn't illegal
#include <numeric> // complex
#include <set>

namespace libMesh
{

namespace Parallel
{

template <typename T>
class Packing;

#ifdef LIBMESH_HAVE_MPI

//-------------------------------------------------------------------
typedef MPI_Comm communicator;

const unsigned int any_source =
  static_cast<unsigned int>(MPI_ANY_SOURCE);

#else

// These shouldn't actually be needed, but must be
// unique types for function overloading to work
// properly.
typedef int communicator; // Must match petsc-nompi definition

const unsigned int any_source=0;

#endif // LIBMESH_HAVE_MPI

//-------------------------------------------------------------------
class Communicator
{
  // Basic operations:
public:

  Communicator ();

  /*
   * Constructor from MPI_Comm
   */
  explicit Communicator (const communicator & comm);

  /*
   * Don't use copy construction or assignment, just copy by reference
   * or pointer - it's too hard to keep a common used_tag_values if
   * each communicator is shared by more than one Communicator
   */
  Communicator (const Communicator &) = delete;
  Communicator & operator= (const Communicator &) = delete;

  /*
   * Move constructor and assignment operator
   */
  Communicator (Communicator &&) = default;
  Communicator & operator= (Communicator &&) = default;

  /*
   * NON-VIRTUAL destructor
   */
  ~Communicator ();

  /*
   * Create a new communicator between some subset of \p this
   */
  void split(int color, int key, Communicator & target) const;

  /*
   * Create a new duplicate of \p this communicator
   */
  void duplicate(const Communicator & comm);

  /*
   * Create a new duplicate of an MPI communicator
   */
  void duplicate(const communicator & comm);

  communicator & get() { return _communicator; }

  const communicator & get() const { return _communicator; }

  MessageTag get_unique_tag(int tagvalue = MessageTag::invalid_tag) const;

  void reference_unique_tag(int tagvalue) const;

  void dereference_unique_tag(int tagvalue) const;

  void clear();

  Communicator & operator= (const communicator & comm);

  processor_id_type rank() const { return _rank; }

  processor_id_type size() const { return _size; }

  enum SendMode { DEFAULT=0, SYNCHRONOUS };

private:

  void assign(const communicator & comm);

  communicator  _communicator;
  processor_id_type _rank, _size;
  SendMode _send_mode;

  // mutable used_tag_values and tag_queue - not thread-safe, but then
  // Parallel:: isn't thread-safe in general.
  mutable std::map<int, unsigned int> used_tag_values;
  mutable int _next_tag;

  int _max_tag;

  // Keep track of duplicate/split operations so we know when to free
  bool _I_duped_it;

  // Communication operations:
public:

  void send_mode (const SendMode sm) { _send_mode = sm; }

  SendMode send_mode() const { return _send_mode; }

  void barrier () const;

  void nonblocking_barrier (Request & req) const;

  template <typename T>
  bool verify(const T & r) const;

  template <typename T>
  bool semiverify(const T * r) const;

  template <typename T>
  void min(T & r) const;

  template <typename T>
  void minloc(T & r,
              unsigned int & min_id) const;

  template <typename T, typename A1, typename A2>
  void minloc(std::vector<T,A1> & r,
              std::vector<unsigned int,A2> & min_id) const;

  template <typename T>
  void max(T & r) const;

  template <typename T>
  void maxloc(T & r,
              unsigned int & max_id) const;

  template <typename T, typename A1, typename A2>
  void maxloc(std::vector<T,A1> & r,
              std::vector<unsigned int,A2> & max_id) const;

  template <typename T>
  void sum(T & r) const;

  template <typename T>
  void set_union(T & data, const unsigned int root_id) const;

  template <typename T>
  void set_union(T & data) const;

  status probe (const unsigned int src_processor_id,
                const MessageTag & tag=any_tag) const;

  template <typename T>
  Status packed_range_probe (const unsigned int src_processor_id,
                             const MessageTag & tag,
                             bool & flag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             const T & buf,
             const MessageTag & tag=no_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             const T & buf,
             Request & req,
             const MessageTag & tag=no_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             const T & buf,
             const DataType & type,
             const MessageTag & tag=no_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             const T & buf,
             const DataType & type,
             Request & req,
             const MessageTag & tag=no_tag) const;

  template <typename T>
  Status receive (const unsigned int dest_processor_id,
                  T & buf,
                  const MessageTag & tag=any_tag) const;

  template <typename T>
  void receive (const unsigned int dest_processor_id,
                T & buf,
                Request & req,
                const MessageTag & tag=any_tag) const;

  template <typename T>
  Status receive (const unsigned int dest_processor_id,
                  T & buf,
                  const DataType & type,
                  const MessageTag & tag=any_tag) const;

  template <typename T>
  void receive (const unsigned int dest_processor_id,
                T & buf,
                const DataType & type,
                Request & req,
                const MessageTag & tag=any_tag) const;

  template <typename T, typename A>
  bool possibly_receive (unsigned int & src_processor_id,
                         std::vector<T,A> & buf,
                         const DataType & type,
                         Request & req,
                         const MessageTag & tag) const;

  template <typename Context, typename Iter>
  void send_packed_range (const unsigned int dest_processor_id,
                          const Context * context,
                          Iter range_begin,
                          const Iter range_end,
                          const MessageTag & tag=no_tag) const;

  template <typename Context, typename Iter>
  void send_packed_range (const unsigned int dest_processor_id,
                          const Context * context,
                          Iter range_begin,
                          const Iter range_end,
                          Request & req,
                          const MessageTag & tag=no_tag) const;

  template <typename Context, typename Iter>
  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
                                      const Context * context,
                                      Iter range_begin,
                                      const Iter range_end,
                                      Request & req,
                                      const MessageTag & tag=no_tag) const;


  template <typename Context, typename Iter>
  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
                                      const Context * context,
                                      Iter range_begin,
                                      const Iter range_end,
                                      Request & req,
                                      std::shared_ptr<std::vector<typename Parallel::Packing<typename std::iterator_traits<Iter>::value_type>::buffer_type>> & buffer,
                                      const MessageTag & tag=no_tag) const;

  template <typename Context, typename OutputIter, typename T>
  void receive_packed_range (const unsigned int dest_processor_id,
                             Context * context,
                             OutputIter out,
                             const T * output_type, // used only to infer T
                             const MessageTag & tag=any_tag) const;

  template <typename Context, typename OutputIter, typename T>
  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
                                         Context * context,
                                         OutputIter out,
                                         const T * output_type,
                                         Request & req,
                                         Status & stat,
                                         const MessageTag & tag=any_tag) const;

  template <typename Context, typename OutputIter, typename T>
  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
                                         Context * context,
                                         OutputIter out,
                                         const T * output_type,
                                         Request & req,
                                         Status & stat,
                                         std::shared_ptr<std::vector<typename Parallel::Packing<T>::buffer_type>> & buffer,
                                         const MessageTag & tag=any_tag
                                         ) const;

  template <typename T1, typename T2>
  void send_receive(const unsigned int dest_processor_id,
                    const T1 & send,
                    const unsigned int source_processor_id,
                    T2 & recv,
                    const MessageTag & send_tag = no_tag,
                    const MessageTag & recv_tag = any_tag) const;

  template <typename Context1, typename RangeIter, typename Context2,
            typename OutputIter, typename T>
  void send_receive_packed_range(const unsigned int dest_processor_id,
                                 const Context1 * context1,
                                 RangeIter send_begin,
                                 const RangeIter send_end,
                                 const unsigned int source_processor_id,
                                 Context2 * context2,
                                 OutputIter out,
                                 const T * output_type, // used only to infer T
                                 const MessageTag & send_tag = no_tag,
                                 const MessageTag & recv_tag = any_tag) const;

  template <typename T1, typename T2>
  void send_receive(const unsigned int dest_processor_id,
                    const T1 & send,
                    const DataType & type1,
                    const unsigned int source_processor_id,
                    T2 & recv,
                    const DataType & type2,
                    const MessageTag & send_tag = no_tag,
                    const MessageTag & recv_tag = any_tag) const;

  template <typename T, typename A>
  inline void gather(const unsigned int root_id,
                     const T & send,
                     std::vector<T,A> & recv) const;

  template <typename T, typename A>
  inline void gather(const unsigned int root_id,
                     const std::basic_string<T> & send,
                     std::vector<std::basic_string<T>,A> & recv,
                     const bool identical_buffer_sizes=false) const;

  template <typename T, typename A>
  inline void gather(const unsigned int root_id,
                     std::vector<T,A> & r) const;

  template <typename T, typename A>
  inline void allgather(const T & send,
                        std::vector<T,A> & recv) const;

  template <typename T, typename A>
  inline void allgather(const std::basic_string<T> & send,
                        std::vector<std::basic_string<T>,A> & recv,
                        const bool identical_buffer_sizes=false) const;

  template <typename T, typename A>
  inline void allgather(std::vector<T,A> & r,
                        const bool identical_buffer_sizes = false) const;

  template <typename T, typename A>
  inline void allgather(std::vector<std::basic_string<T>,A> & r,
                        const bool identical_buffer_sizes = false) const;

  //-------------------------------------------------------------------
  template <typename T, typename A>
  inline void scatter(const std::vector<T,A> & data,
                      T & recv,
                      const unsigned int root_id=0) const;

  template <typename T, typename A>
  inline void scatter(const std::vector<T,A> & data,
                      std::vector<T,A> & recv,
                      const unsigned int root_id=0) const;

  template <typename T, typename A1, typename A2>
  inline void scatter(const std::vector<T,A1> & data,
                      const std::vector<int,A2> counts,
                      std::vector<T,A1> & recv,
                      const unsigned int root_id=0) const;

  template <typename T, typename A1, typename A2>
  inline void scatter(const std::vector<std::vector<T,A1>,A2> & data,
                      std::vector<T,A1> & recv,
                      const unsigned int root_id=0,
                      const bool identical_buffer_sizes=false) const;

  //-------------------------------------------------------------------
  template <typename Context, typename Iter, typename OutputIter>
  inline void gather_packed_range (const unsigned int root_id,
                                   Context * context,
                                   Iter range_begin,
                                   const Iter range_end,
                                   OutputIter out) const;

  template <typename Context, typename Iter, typename OutputIter>
  inline void allgather_packed_range (Context * context,
                                      Iter range_begin,
                                      const Iter range_end,
                                      OutputIter out) const;

  template <typename T, typename A>
  inline void alltoall(std::vector<T,A> & r) const;

  template <typename T>
  inline void broadcast(T & data, const unsigned int root_id=0) const;

  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
  inline void broadcast_packed_range (const Context * context1,
                                      Iter range_begin,
                                      const Iter range_end,
                                      OutputContext * context2,
                                      OutputIter out,
                                      const unsigned int root_id = 0) const;

#include "libmesh/parallel_communicator_specializations"

}; // class Communicator


} // namespace Parallel

} // namespace libMesh

#endif // LIBMESH_COMMUNICATOR_H