LCOV - code coverage report
Current view: top level - src/numerics - petsc_matrix.C (source / functions) Hit Total Coverage
Test: libMesh/libmesh: #4573 (924896) with base 3f0912 Lines: 238 405 58.8 %
Date: 2026-11-18 21:52:26 Functions: 45 66 68.2 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : // The libMesh Finite Element Library.
       2             : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
       3             : 
       4             : // This library is free software; you can redistribute it and/or
       5             : // modify it under the terms of the GNU Lesser General Public
       6             : // License as published by the Free Software Foundation; either
       7             : // version 2.1 of the License, or (at your option) any later version.
       8             : 
       9             : // This library is distributed in the hope that it will be useful,
      10             : // but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      12             : // Lesser General Public License for more details.
      13             : 
      14             : // You should have received a copy of the GNU Lesser General Public
      15             : // License along with this library; if not, write to the Free Software
      16             : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
      17             : 
      18             : 
      19             : 
      20             : #include "libmesh/libmesh_config.h"
      21             : 
      22             : #ifdef LIBMESH_HAVE_PETSC
      23             : 
      24             : // Local includes
      25             : #include "libmesh/petsc_matrix.h"
      26             : 
      27             : // libMesh includes
      28             : #include "libmesh/dof_map.h"
      29             : #include "libmesh/dense_matrix.h"
      30             : #include "libmesh/libmesh_logging.h"
      31             : #include "libmesh/petsc_vector.h"
      32             : #include "libmesh/parallel.h"
      33             : #include "libmesh/utility.h"
      34             : #include "libmesh/wrapped_petsc.h"
      35             : 
      36             : // C++ includes
      37             : #ifdef LIBMESH_HAVE_UNISTD_H
      38             : #include <unistd.h> // mkstemp
      39             : #endif
      40             : #include <fstream>
      41             : 
      42             : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
      43             : 
      44             : namespace
      45             : {
      46             : using namespace libMesh;
      47             : 
      48             : // historic libMesh n_nz & n_oz arrays are set up for PETSc's AIJ format.
      49             : // however, when the blocksize is >1, we need to transform these into
      50             : // their BAIJ counterparts.
      51             : inline
      52             : void transform_preallocation_arrays (const PetscInt blocksize,
      53             :                                      const std::vector<numeric_index_type> & n_nz,
      54             :                                      const std::vector<numeric_index_type> & n_oz,
      55             :                                      std::vector<numeric_index_type>       & b_n_nz,
      56             :                                      std::vector<numeric_index_type>       & b_n_oz)
      57             : {
      58             :   libmesh_assert_equal_to (n_nz.size(), n_oz.size());
      59             :   libmesh_assert_equal_to (n_nz.size()%blocksize, 0);
      60             : 
      61             :   b_n_nz.clear(); /**/ b_n_nz.reserve(n_nz.size()/blocksize);
      62             :   b_n_oz.clear(); /**/ b_n_oz.reserve(n_oz.size()/blocksize);
      63             : 
      64             :   for (std::size_t nn=0, nnzs=n_nz.size(); nn<nnzs; nn += blocksize)
      65             :     {
      66             :       b_n_nz.push_back (n_nz[nn]/blocksize);
      67             :       b_n_oz.push_back (n_oz[nn]/blocksize);
      68             :     }
      69             : }
      70             : }
      71             : 
      72             : #endif
      73             : 
      74             : 
      75             : 
      76             : namespace libMesh
      77             : {
      78             : 
      79             : 
      80             : //-----------------------------------------------------------------------
      81             : // PetscMatrix members
      82             : 
      83             : 
      84             : // Constructor
      85             : template <typename T>
      86        2828 : PetscMatrix<T>::PetscMatrix(const Parallel::Communicator & comm_in) :
      87        2828 :   PetscMatrixBase<T>(comm_in), _mat_type(AIJ)
      88             : {
      89        2828 : }
      90             : 
      91             : 
      92             : 
      93             : // Constructor taking an existing Mat but not the responsibility
      94             : // for destroying it
      95             : template <typename T>
      96          92 : PetscMatrix<T>::PetscMatrix(Mat mat_in,
      97             :                             const Parallel::Communicator & comm_in,
      98             :                             const bool destroy_on_exit) :
      99          92 :   PetscMatrixBase<T>(mat_in, comm_in, destroy_on_exit)
     100             : {
     101             :   MatType mat_type;
     102          92 :   LibmeshPetscCall(MatGetType(mat_in, &mat_type));
     103             :   PetscBool is_hypre;
     104          92 :   LibmeshPetscCall(PetscStrcmp(mat_type, MATHYPRE, &is_hypre));
     105          92 :   if (is_hypre == PETSC_TRUE)
     106           0 :     _mat_type = HYPRE;
     107             :   else
     108          92 :     _mat_type = AIJ;
     109          92 : }
     110             : 
     111             : 
     112             : // Constructor taking global and local dimensions and, optionally,
     113             : // number of on- and off-diagonal non-zeros and a block size
     114             : template <typename T>
     115          40 : PetscMatrix<T>::PetscMatrix(const Parallel::Communicator & comm_in,
     116             :                             const numeric_index_type m_in,
     117             :                             const numeric_index_type n_in,
     118             :                             const numeric_index_type m_l,
     119             :                             const numeric_index_type n_l,
     120             :                             const numeric_index_type n_nz,
     121             :                             const numeric_index_type n_oz,
     122             :                             const numeric_index_type blocksize_in) :
     123          40 :   PetscMatrixBase<T>(comm_in), _mat_type(AIJ)
     124             : {
     125          40 :   this->init(m_in, n_in, m_l, n_l, n_nz, n_oz, blocksize_in);
     126          40 : }
     127             : 
     128             : 
     129             : // Destructor
     130             : template <typename T>
     131        1048 : PetscMatrix<T>::~PetscMatrix() = default;
     132             : 
     133             : template <typename T>
     134             : void
     135        4682 : PetscMatrix<T>::init_without_preallocation (const numeric_index_type m_in,
     136             :                                             const numeric_index_type n_in,
     137             :                                             const numeric_index_type m_l,
     138             :                                             const numeric_index_type n_l,
     139             :                                             const numeric_index_type blocksize_in)
     140             : {
     141             :   // So compilers don't warn when !LIBMESH_ENABLE_BLOCKED_STORAGE
     142        1578 :   libmesh_ignore(blocksize_in);
     143             : 
     144             :   // Clear initialized matrices
     145        4682 :   if (this->initialized())
     146          24 :     this->clear();
     147             : 
     148        4682 :   PetscInt m_global   = static_cast<PetscInt>(m_in);
     149        4682 :   PetscInt n_global   = static_cast<PetscInt>(n_in);
     150        4682 :   PetscInt m_local    = static_cast<PetscInt>(m_l);
     151        4682 :   PetscInt n_local    = static_cast<PetscInt>(n_l);
     152             : 
     153        4682 :   LibmeshPetscCall(MatCreate(this->comm().get(), &this->_mat));
     154        4682 :   LibmeshPetscCall(MatSetSizes(this->_mat, m_local, n_local, m_global, n_global));
     155        4682 :   PetscInt blocksize  = static_cast<PetscInt>(blocksize_in);
     156        4682 :   LibmeshPetscCall(MatSetBlockSize(this->_mat,blocksize));
     157        4682 :   LibmeshPetscCall(MatSetOptionsPrefix(this->_mat, this->options_prefix().c_str()));
     158             : 
     159             : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
     160             :   if (blocksize > 1)
     161             :     {
     162             :       // specified blocksize, bs>1.
     163             :       // double check sizes.
     164             :       libmesh_assert_equal_to (m_local  % blocksize, 0);
     165             :       libmesh_assert_equal_to (n_local  % blocksize, 0);
     166             :       libmesh_assert_equal_to (m_global % blocksize, 0);
     167             :       libmesh_assert_equal_to (n_global % blocksize, 0);
     168             :       libmesh_assert_equal_to (n_nz     % blocksize, 0);
     169             :       libmesh_assert_equal_to (n_oz     % blocksize, 0);
     170             : 
     171             :       LibmeshPetscCall(MatSetType(this->_mat, MATBAIJ)); // Automatically chooses seqbaij or mpibaij
     172             : 
     173             :       // MatSetFromOptions needs to happen before Preallocation routines
     174             :       // since MatSetFromOptions can change matrix type and remove incompatible
     175             :       // preallocation
     176             :       LibmeshPetscCall(MatSetFromOptions(this->_mat));
     177             :     }
     178             :   else
     179             : #endif
     180             :     {
     181        4682 :       switch (this->_mat_type) {
     182        4682 :         case AIJ:
     183        4682 :           LibmeshPetscCall(MatSetType(this->_mat, MATAIJ)); // Automatically chooses seqaij or mpiaij
     184             : 
     185             :           // MatSetFromOptions needs to happen before Preallocation routines
     186             :           // since MatSetFromOptions can change matrix type and remove incompatible
     187             :           // preallocation
     188        4682 :           LibmeshPetscCall(MatSetFromOptions(this->_mat));
     189        1578 :           break;
     190             : 
     191           0 :         case HYPRE:
     192             : #if !PETSC_VERSION_LESS_THAN(3,9,4) && LIBMESH_HAVE_PETSC_HYPRE
     193           0 :           LibmeshPetscCall(MatSetType(this->_mat, MATHYPRE));
     194             : 
     195             :           // MatSetFromOptions needs to happen before Preallocation routines
     196             :           // since MatSetFromOptions can change matrix type and remove incompatible
     197             :           // preallocation
     198           0 :           LibmeshPetscCall(MatSetFromOptions(this->_mat));
     199             : #else
     200             :           libmesh_error_msg("PETSc 3.9.4 or higher with hypre is required for MatHypre");
     201             : #endif
     202           0 :           break;
     203             : 
     204           0 :         default: libmesh_error_msg("Unsupported petsc matrix type");
     205             :       }
     206             :     }
     207             : 
     208        4682 :   this->set_context ();
     209        4682 : }
     210             : 
     211             : 
     212             : 
     213             : template <typename T>
     214         322 : void PetscMatrix<T>::init (const numeric_index_type m_in,
     215             :                            const numeric_index_type n_in,
     216             :                            const numeric_index_type m_l,
     217             :                            const numeric_index_type n_l,
     218             :                            const numeric_index_type nnz,
     219             :                            const numeric_index_type noz,
     220             :                            const numeric_index_type blocksize_in)
     221             : {
     222         322 :   this->init_without_preallocation(m_in, n_in, m_l, n_l, blocksize_in);
     223             : 
     224         322 :   PetscInt n_nz       = static_cast<PetscInt>(nnz);
     225         322 :   PetscInt n_oz       = static_cast<PetscInt>(noz);
     226             : 
     227             : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
     228             :   if (blocksize > 1)
     229             :     {
     230             :       LibmeshPetscCall(MatSeqBAIJSetPreallocation(this->_mat, blocksize, n_nz/blocksize, NULL));
     231             :       LibmeshPetscCall(MatMPIBAIJSetPreallocation(this->_mat, blocksize,
     232             :                                                   n_nz/blocksize, NULL,
     233             :                                                   n_oz/blocksize, NULL));
     234             :     }
     235             :   else
     236             : #endif
     237             :     {
     238         322 :       switch (this->_mat_type) {
     239         322 :         case AIJ:
     240         322 :           LibmeshPetscCall(MatSeqAIJSetPreallocation(this->_mat, n_nz, NULL));
     241         322 :           LibmeshPetscCall(MatMPIAIJSetPreallocation(this->_mat, n_nz, NULL, n_oz, NULL));
     242          90 :           break;
     243             : 
     244           0 :         case HYPRE:
     245             : #if !PETSC_VERSION_LESS_THAN(3,9,4) && LIBMESH_HAVE_PETSC_HYPRE
     246           0 :           LibmeshPetscCall(MatHYPRESetPreallocation(this->_mat, n_nz, NULL, n_oz, NULL));
     247             : #else
     248             :           libmesh_error_msg("PETSc 3.9.4 or higher with hypre is required for MatHypre");
     249             : #endif
     250           0 :           break;
     251             : 
     252           0 :         default: libmesh_error_msg("Unsupported petsc matrix type");
     253             :       }
     254             :     }
     255             : 
     256         322 :   this->finish_initialization();
     257         322 : }
     258             : 
     259             : template <typename T>
     260        4356 : void PetscMatrix<T>::preallocate(const numeric_index_type libmesh_dbg_var(m_l),
     261             :                                  const std::vector<numeric_index_type> & n_nz,
     262             :                                  const std::vector<numeric_index_type> & n_oz,
     263             :                                  const numeric_index_type blocksize_in)
     264             : {
     265             :   // Make sure the sparsity pattern isn't empty unless the matrix is 0x0
     266        1488 :   libmesh_assert_equal_to (n_nz.size(), m_l);
     267        1488 :   libmesh_assert_equal_to (n_oz.size(), m_l);
     268             :   // Avoid unused warnings when not configured with block storage
     269        1488 :   libmesh_ignore(blocksize_in);
     270             : 
     271             : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
     272             :   PetscInt blocksize  = static_cast<PetscInt>(blocksize_in);
     273             : 
     274             :   if (blocksize > 1)
     275             :     {
     276             :       // transform the per-entry n_nz and n_oz arrays into their block counterparts.
     277             :       std::vector<numeric_index_type> b_n_nz, b_n_oz;
     278             : 
     279             :       transform_preallocation_arrays (blocksize,
     280             :                                       n_nz, n_oz,
     281             :                                       b_n_nz, b_n_oz);
     282             : 
     283             :       LibmeshPetscCall(MatSeqBAIJSetPreallocation (this->_mat,
     284             :                                                    blocksize,
     285             :                                                    0,
     286             :                                                    numeric_petsc_cast(b_n_nz.empty() ? nullptr : b_n_nz.data())));
     287             : 
     288             :       LibmeshPetscCall(MatMPIBAIJSetPreallocation (this->_mat,
     289             :                                                    blocksize,
     290             :                                                    0,
     291             :                                                    numeric_petsc_cast(b_n_nz.empty() ? nullptr : b_n_nz.data()),
     292             :                                                    0,
     293             :                                                    numeric_petsc_cast(b_n_oz.empty() ? nullptr : b_n_oz.data())));
     294             :     }
     295             :   else
     296             : #endif
     297             :     {
     298        4356 :       switch (this->_mat_type) {
     299        1488 :         case AIJ:
     300        7148 :           LibmeshPetscCall(MatSeqAIJSetPreallocation (this->_mat,
     301             :                                                       0,
     302             :                                                       numeric_petsc_cast(n_nz.empty() ? nullptr : n_nz.data())));
     303        9940 :           LibmeshPetscCall(MatMPIAIJSetPreallocation (this->_mat,
     304             :                                                       0,
     305             :                                                       numeric_petsc_cast(n_nz.empty() ? nullptr : n_nz.data()),
     306             :                                                       0,
     307             :                                                       numeric_petsc_cast(n_oz.empty() ? nullptr : n_oz.data())));
     308        1488 :           break;
     309             : 
     310           0 :         case HYPRE:
     311             : #if !PETSC_VERSION_LESS_THAN(3,9,4) && LIBMESH_HAVE_PETSC_HYPRE
     312           0 :           LibmeshPetscCall(MatHYPRESetPreallocation (this->_mat,
     313             :                                                      0,
     314             :                                                      numeric_petsc_cast(n_nz.empty() ? nullptr : n_nz.data()),
     315             :                                                      0,
     316             :                                                      numeric_petsc_cast(n_oz.empty() ? nullptr : n_oz.data())));
     317             : #else
     318             :           libmesh_error_msg("PETSc 3.9.4 or higher with hypre is required for MatHypre");
     319             : #endif
     320           0 :           break;
     321             : 
     322           0 :         default: libmesh_error_msg("Unsupported petsc matrix type");
     323             :       }
     324             : 
     325             :     }
     326        4356 : }
     327             : 
     328             : template <typename T>
     329        4682 : void PetscMatrix<T>::finish_initialization()
     330             : {
     331             :   // Make it an error for PETSc to allocate new nonzero entries during assembly. For old PETSc
     332             :   // versions this option must be set after preallocation for MPIAIJ matrices
     333        4682 :   LibmeshPetscCall(MatSetOption(this->_mat, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
     334        4682 :   this->_is_initialized = true;
     335        4682 : }
     336             : 
     337             : template <typename T>
     338         354 : void PetscMatrix<T>::init (const numeric_index_type m_in,
     339             :                            const numeric_index_type n_in,
     340             :                            const numeric_index_type m_l,
     341             :                            const numeric_index_type n_l,
     342             :                            const std::vector<numeric_index_type> & n_nz,
     343             :                            const std::vector<numeric_index_type> & n_oz,
     344             :                            const numeric_index_type blocksize_in)
     345             : {
     346         354 :   this->init_without_preallocation(m_in, n_in, m_l, n_l, blocksize_in);
     347         354 :   this->preallocate(m_l, n_nz, n_oz, blocksize_in);
     348             : 
     349         354 :   this->finish_initialization();
     350         354 : }
     351             : 
     352             : 
     353             : template <typename T>
     354        4006 : void PetscMatrix<T>::init (const ParallelType libmesh_dbg_var(type))
     355             : {
     356        1370 :   libmesh_assert(this->_dof_map);
     357             : 
     358        4006 :   const numeric_index_type m_in = this->_dof_map->n_dofs();
     359        1370 :   const numeric_index_type m_l = this->_dof_map->n_local_dofs();
     360        1370 :   if (m_in != m_l)
     361        1332 :     libmesh_assert(type != SERIAL);
     362             : 
     363        4006 :   const auto blocksize = this->_dof_map->block_size();
     364             : 
     365        4006 :   this->init_without_preallocation(m_in, m_in, m_l, m_l, blocksize);
     366        4006 :   if (!this->_use_hash_table)
     367             :     {
     368        4002 :       const std::vector<numeric_index_type> & n_nz = this->_sp->get_n_nz();
     369        1370 :       const std::vector<numeric_index_type> & n_oz = this->_sp->get_n_oz();
     370             : 
     371        4002 :       this->preallocate(m_l, n_nz, n_oz, blocksize);
     372             :     }
     373             : 
     374        4006 :   this->finish_initialization();
     375        4006 : }
     376             : 
     377             : 
     378             : template <typename T>
     379           0 : void PetscMatrix<T>::update_preallocation_and_zero ()
     380             : {
     381           0 :   libmesh_not_implemented();
     382             : }
     383             : 
     384             : template <typename T>
     385           0 : void PetscMatrix<T>::reset_preallocation()
     386             : {
     387           0 :   semiparallel_only();
     388             : 
     389             : #if !PETSC_VERSION_LESS_THAN(3,9,0)
     390           0 :   libmesh_assert (this->initialized());
     391             : 
     392           0 :   LibmeshPetscCall(MatResetPreallocation(this->_mat));
     393             : #else
     394             :   libmesh_warning("Your version of PETSc doesn't support resetting of "
     395             :                   "preallocation, so we will use your most recent sparsity "
     396             :                   "pattern. This may result in a degradation of performance\n");
     397             : #endif
     398           0 : }
     399             : 
     400             : template <typename T>
     401       40536 : void PetscMatrix<T>::zero ()
     402             : {
     403       12516 :   libmesh_assert (this->initialized());
     404             : 
     405       12516 :   semiparallel_only();
     406             : 
     407             :   // MatZeroEntries() must be called on every rank, including ranks with no
     408             :   // local entries: on a matrix that is not yet preallocated it calls
     409             :   // MatSetUp(), which communicates for some types (e.g. MATHYPRE).
     410       40536 :   LibmeshPetscCall(MatZeroEntries(this->_mat));
     411       40536 : }
     412             : 
     413             : template <typename T>
     414           0 : void PetscMatrix<T>::zero_rows (std::vector<numeric_index_type> & rows, T diag_value)
     415             : {
     416           0 :   libmesh_assert (this->initialized());
     417             : 
     418           0 :   semiparallel_only();
     419             : 
     420             :   // As of petsc-dev at the time of 3.1.0, MatZeroRows now takes two additional
     421             :   // optional arguments.  The optional arguments (x,b) can be used to specify the
     422             :   // solutions for the zeroed rows (x) and right hand side (b) to update.
     423             :   // Could be useful for setting boundary conditions...
     424           0 :   if (!rows.empty())
     425           0 :     LibmeshPetscCall(MatZeroRows(this->_mat, cast_int<PetscInt>(rows.size()),
     426             :                                  numeric_petsc_cast(rows.data()), PS(diag_value),
     427             :                                  NULL, NULL));
     428             :   else
     429           0 :     LibmeshPetscCall(MatZeroRows(this->_mat, 0, NULL, PS(diag_value), NULL, NULL));
     430           0 : }
     431             : 
     432             : template <typename T>
     433           6 : std::unique_ptr<SparseMatrix<T>> PetscMatrix<T>::zero_clone () const
     434             : {
     435           6 :   libmesh_error_msg_if(!this->closed(), "Matrix must be closed before it can be cloned!");
     436             : 
     437             :   // Copy the nonzero pattern only
     438             :   Mat copy;
     439           6 :   LibmeshPetscCall(MatDuplicate(this->_mat, MAT_DO_NOT_COPY_VALUES, &copy));
     440             : 
     441             :   // Call wrapping PetscMatrix constructor, have it take over
     442             :   // ownership.
     443           8 :   auto ret = std::make_unique<PetscMatrix<T>>(copy, this->comm());
     444           6 :   ret->set_destroy_mat_on_exit(true);
     445             : 
     446           8 :   return ret;
     447             : }
     448             : 
     449             : 
     450             : 
     451             : template <typename T>
     452          18 : std::unique_ptr<SparseMatrix<T>> PetscMatrix<T>::clone () const
     453             : {
     454          18 :   libmesh_error_msg_if(!this->closed(), "Matrix must be closed before it can be cloned!");
     455             : 
     456             :   // Copy the nonzero pattern and numerical values
     457             :   Mat copy;
     458          18 :   LibmeshPetscCall(MatDuplicate(this->_mat, MAT_COPY_VALUES, &copy));
     459             : 
     460             :   // Call wrapping PetscMatrix constructor, have it take over
     461             :   // ownership.
     462          24 :   auto ret = std::make_unique<PetscMatrix<T>>(copy, this->comm());
     463          18 :   ret->set_destroy_mat_on_exit(true);
     464             : 
     465          24 :   return ret;
     466             : }
     467             : 
     468             : 
     469             : 
     470             : template <typename T>
     471             : template <NormType N>
     472         156 : Real PetscMatrix<T>::norm () const
     473             : {
     474          74 :   libmesh_assert (this->initialized());
     475             : 
     476          74 :   semiparallel_only();
     477             : 
     478             :   PetscReal petsc_value;
     479             :   Real value;
     480             : 
     481          74 :   libmesh_assert (this->closed());
     482             : 
     483         156 :   LibmeshPetscCall(MatNorm(this->_mat, N, &petsc_value));
     484             : 
     485         156 :   value = static_cast<Real>(petsc_value);
     486             : 
     487         156 :   return value;
     488             : }
     489             : template <typename T>
     490         118 : Real PetscMatrix<T>::l1_norm () const
     491             : {
     492         118 :   return PetscMatrix<T>::norm<NORM_1>();
     493             : }
     494             : template <typename T>
     495           0 : Real PetscMatrix<T>::frobenius_norm () const
     496             : {
     497           0 :   return PetscMatrix<T>::norm<NORM_FROBENIUS>();
     498             : }
     499             : template <typename T>
     500         112 : Real PetscMatrix<T>::linfty_norm () const
     501             : {
     502         112 :   return PetscMatrix<T>::norm<NORM_INFINITY>();
     503             : }
     504             : 
     505             : 
     506             : 
     507             : template <typename T>
     508          12 : void PetscMatrix<T>::print_matlab (const std::string & name) const
     509             : {
     510           4 :   libmesh_assert (this->initialized());
     511             : 
     512           4 :   semiparallel_only();
     513             : 
     514          12 :   if (!this->closed())
     515             :     {
     516             :       libmesh_deprecated();
     517             :       libmesh_warning("The matrix must be assembled before calling PetscMatrix::print_matlab().\n"
     518             :                       "Please update your code, as this warning will become an error in a future release.");
     519           0 :       const_cast<PetscMatrix<T> *>(this)->close();
     520             :     }
     521             : 
     522             :   // Create an ASCII file containing the matrix
     523             :   // if a filename was provided.
     524          12 :   if (name != "")
     525             :     {
     526           8 :       WrappedPetsc<PetscViewer> petsc_viewer;
     527             : 
     528          12 :       LibmeshPetscCall(PetscViewerASCIIOpen( this->comm().get(),
     529             :                                              name.c_str(),
     530             :                                              petsc_viewer.get()));
     531             : 
     532             : #if PETSC_VERSION_LESS_THAN(3,7,0)
     533             :       LibmeshPetscCall(PetscViewerSetFormat (petsc_viewer,
     534             :                                              PETSC_VIEWER_ASCII_MATLAB));
     535             : #else
     536          12 :       LibmeshPetscCall(PetscViewerPushFormat (petsc_viewer,
     537             :                                               PETSC_VIEWER_ASCII_MATLAB));
     538             : #endif
     539             : 
     540          12 :       LibmeshPetscCall(MatView (this->_mat, petsc_viewer));
     541             :     }
     542             : 
     543             :   // Otherwise the matrix will be dumped to the screen.
     544             :   else
     545             :     {
     546             : #if PETSC_VERSION_LESS_THAN(3,7,0)
     547             :       LibmeshPetscCall(PetscViewerSetFormat (PETSC_VIEWER_STDOUT_WORLD,
     548             :                                              PETSC_VIEWER_ASCII_MATLAB));
     549             : #else
     550           0 :       LibmeshPetscCall(PetscViewerPushFormat (PETSC_VIEWER_STDOUT_WORLD,
     551             :                                               PETSC_VIEWER_ASCII_MATLAB));
     552             : #endif
     553             : 
     554           0 :       LibmeshPetscCall(MatView (this->_mat, PETSC_VIEWER_STDOUT_WORLD));
     555             :     }
     556          12 : }
     557             : 
     558             : 
     559             : 
     560             : 
     561             : 
     562             : template <typename T>
     563           0 : void PetscMatrix<T>::print_personal(std::ostream & os) const
     564             : {
     565           0 :   libmesh_assert (this->initialized());
     566             : 
     567             :   // Routine must be called in parallel on parallel matrices
     568             :   // and serial on serial matrices.
     569           0 :   semiparallel_only();
     570             : 
     571             :   // #ifndef NDEBUG
     572             :   //   if (os != std::cout)
     573             :   //     libMesh::err << "Warning! PETSc can only print to std::cout!" << std::endl;
     574             :   // #endif
     575             : 
     576             :   // Matrix must be in an assembled state to be printed
     577           0 :   if (!this->closed())
     578             :     {
     579             :       libmesh_deprecated();
     580             :       libmesh_warning("The matrix must be assembled before calling PetscMatrix::print_personal().\n"
     581             :                       "Please update your code, as this warning will become an error in a future release.");
     582           0 :       const_cast<PetscMatrix<T> *>(this)->close();
     583             :     }
     584             : 
     585             :   // Print to screen if ostream is stdout
     586           0 :   if (os.rdbuf() == std::cout.rdbuf())
     587           0 :     LibmeshPetscCall(MatView(this->_mat, NULL));
     588             : 
     589             :   // Otherwise, print to the requested file, in a roundabout way...
     590             :   else
     591             :     {
     592             :       // We will create a temporary filename, and file, for PETSc to
     593             :       // write to.
     594           0 :       std::string temp_filename;
     595             : 
     596             :       {
     597             :         // Template for temporary filename
     598           0 :         char c[] = "temp_petsc_matrix.XXXXXX";
     599             : 
     600             :         // Generate temporary, unique filename only on processor 0.  We will
     601             :         // use this filename for PetscViewerASCIIOpen, before copying it into
     602             :         // the user's stream
     603           0 :         if (this->processor_id() == 0)
     604             :           {
     605           0 :             int fd = mkstemp(c);
     606             : 
     607             :             // Check to see that mkstemp did not fail.
     608           0 :             libmesh_error_msg_if(fd == -1, "mkstemp failed in PetscMatrix::print_personal()");
     609             : 
     610             :             // mkstemp returns a file descriptor for an open file,
     611             :             // so let's close it before we hand it to PETSc!
     612           0 :             ::close (fd);
     613             :           }
     614             : 
     615             :         // Store temporary filename as string, makes it easier to broadcast
     616           0 :         temp_filename = c;
     617             :       }
     618             : 
     619             :       // Now broadcast the filename from processor 0 to all processors.
     620           0 :       this->comm().broadcast(temp_filename);
     621             : 
     622             :       // PetscViewer object for passing to MatView
     623             :       PetscViewer petsc_viewer;
     624             : 
     625             :       // This PETSc function only takes a string and handles the opening/closing
     626             :       // of the file internally.  Since print_personal() takes a reference to
     627             :       // an ostream, we have to do an extra step...  print_personal() should probably
     628             :       // have a version that takes a string to get rid of this problem.
     629           0 :       LibmeshPetscCall(PetscViewerASCIIOpen( this->comm().get(),
     630             :                                              temp_filename.c_str(),
     631             :                                              &petsc_viewer));
     632             : 
     633             :       // Probably don't need to set the format if it's default...
     634             :       //      ierr = PetscViewerSetFormat (petsc_viewer,
     635             :       //   PETSC_VIEWER_DEFAULT);
     636             :       //      LIBMESH_CHKERR(ierr);
     637             : 
     638             :       // Finally print the matrix using the viewer
     639           0 :       LibmeshPetscCall(MatView (this->_mat, petsc_viewer));
     640             : 
     641           0 :       if (this->processor_id() == 0)
     642             :         {
     643             :           // Now the inefficient bit: open temp_filename as an ostream and copy the contents
     644             :           // into the user's desired ostream.  We can't just do a direct file copy, we don't even have the filename!
     645           0 :           std::ifstream input_stream(temp_filename.c_str());
     646           0 :           os << input_stream.rdbuf();  // The "most elegant" way to copy one stream into another.
     647             :           // os.close(); // close not defined in ostream
     648             : 
     649             :           // Now remove the temporary file
     650           0 :           input_stream.close();
     651           0 :           std::remove(temp_filename.c_str());
     652           0 :         }
     653             :     }
     654           0 : }
     655             : 
     656             : 
     657             : 
     658             : template <typename T>
     659          18 : void PetscMatrix<T>::_petsc_viewer(const std::string & filename,
     660             :                                    PetscViewerType viewertype,
     661             :                                    PetscFileMode filemode)
     662             : {
     663           6 :   parallel_object_only();
     664             : 
     665             :   // We'll get matrix sizes from the file, but we need to at least
     666             :   // have a Mat object
     667          18 :   if (!this->initialized())
     668             :     {
     669           6 :       LibmeshPetscCall(MatCreate(this->comm().get(), &this->_mat));
     670           6 :       this->_is_initialized = true;
     671             :     }
     672             : 
     673             :   PetscViewer viewer;
     674          18 :   LibmeshPetscCall(PetscViewerCreate(this->comm().get(), &viewer));
     675          18 :   LibmeshPetscCall(PetscViewerSetType(viewer, viewertype));
     676          18 :   LibmeshPetscCall(PetscViewerSetFromOptions(viewer));
     677          18 :   LibmeshPetscCall(PetscViewerFileSetMode(viewer, filemode));
     678          18 :   LibmeshPetscCall(PetscViewerFileSetName(viewer, filename.c_str()));
     679          18 :   if (filemode == FILE_MODE_READ)
     680          12 :     LibmeshPetscCall(MatLoad(this->_mat, viewer));
     681             :   else
     682           6 :     LibmeshPetscCall(MatView(this->_mat, viewer));
     683          18 :   LibmeshPetscCall(PetscViewerDestroy(&viewer));
     684          18 : }
     685             : 
     686             : 
     687             : 
     688             : template <typename T>
     689           6 : void PetscMatrix<T>::print_petsc_binary(const std::string & filename) const
     690             : {
     691           2 :   libmesh_assert (this->initialized());
     692             : 
     693             :   // Our helper here isn't const-correct for writes
     694           2 :   PetscMatrix<T> * nonconst_this = const_cast<PetscMatrix<T>*>(this);
     695           6 :   nonconst_this->_petsc_viewer(filename, PETSCVIEWERBINARY, FILE_MODE_WRITE);
     696           6 : }
     697             : 
     698             : 
     699             : 
     700             : template <typename T>
     701           0 : void PetscMatrix<T>::print_petsc_hdf5(const std::string & filename) const
     702             : {
     703           0 :   libmesh_assert (this->initialized());
     704             : 
     705             :   // Our helper here isn't const-correct for writes
     706           0 :   PetscMatrix<T> * nonconst_this = const_cast<PetscMatrix<T>*>(this);
     707           0 :   nonconst_this->_petsc_viewer(filename, PETSCVIEWERHDF5, FILE_MODE_WRITE);
     708           0 : }
     709             : 
     710             : 
     711             : 
     712             : template <typename T>
     713          12 : void PetscMatrix<T>::read_petsc_binary(const std::string & filename)
     714             : {
     715          12 :   LOG_SCOPE("read_petsc_binary()", "PetscMatrix");
     716             : 
     717          12 :   this->_petsc_viewer(filename, PETSCVIEWERBINARY, FILE_MODE_READ);
     718          12 : }
     719             : 
     720             : 
     721             : 
     722             : template <typename T>
     723           0 : void PetscMatrix<T>::read_petsc_hdf5(const std::string & filename)
     724             : {
     725           0 :   LOG_SCOPE("read_petsc_hdf5()", "PetscMatrix");
     726             : 
     727           0 :   this->_petsc_viewer(filename, PETSCVIEWERHDF5, FILE_MODE_READ);
     728           0 : }
     729             : 
     730             : 
     731             : 
     732             : template <typename T>
     733    11538018 : void PetscMatrix<T>::add_matrix(const DenseMatrix<T> & dm,
     734             :                                 const std::vector<numeric_index_type> & rows,
     735             :                                 const std::vector<numeric_index_type> & cols)
     736             : {
     737     3293697 :   libmesh_assert (this->initialized());
     738             : 
     739     6587394 :   const numeric_index_type n_rows = dm.m();
     740     6587394 :   const numeric_index_type n_cols = dm.n();
     741             : 
     742     3293697 :   libmesh_assert_equal_to (rows.size(), n_rows);
     743     3293697 :   libmesh_assert_equal_to (cols.size(), n_cols);
     744             : 
     745    14831715 :   std::scoped_lock lock(this->_petsc_matrix_mutex);
     746    11538018 :   LibmeshPetscCall(MatSetValues(this->_mat,
     747             :                                 n_rows, numeric_petsc_cast(rows.data()),
     748             :                                 n_cols, numeric_petsc_cast(cols.data()),
     749             :                                 pPS(const_cast<T*>(dm.get_values().data())),
     750             :                                 ADD_VALUES));
     751    11538018 : }
     752             : 
     753             : 
     754             : 
     755             : 
     756             : 
     757             : 
     758             : template <typename T>
     759           0 : void PetscMatrix<T>::add_block_matrix(const DenseMatrix<T> & dm,
     760             :                                       const std::vector<numeric_index_type> & brows,
     761             :                                       const std::vector<numeric_index_type> & bcols)
     762             : {
     763           0 :   libmesh_assert (this->initialized());
     764             : 
     765             :   const numeric_index_type n_brows =
     766           0 :     cast_int<numeric_index_type>(brows.size());
     767             :   const numeric_index_type n_bcols =
     768           0 :     cast_int<numeric_index_type>(bcols.size());
     769             : 
     770             : #ifndef NDEBUG
     771           0 :   const numeric_index_type n_rows  =
     772           0 :     cast_int<numeric_index_type>(dm.m());
     773           0 :   const numeric_index_type n_cols  =
     774           0 :     cast_int<numeric_index_type>(dm.n());
     775           0 :   const numeric_index_type blocksize = n_rows / n_brows;
     776             : 
     777           0 :   libmesh_assert_equal_to (n_cols / n_bcols, blocksize);
     778           0 :   libmesh_assert_equal_to (blocksize*n_brows, n_rows);
     779           0 :   libmesh_assert_equal_to (blocksize*n_bcols, n_cols);
     780             : 
     781             :   PetscInt petsc_blocksize;
     782           0 :   LibmeshPetscCall(MatGetBlockSize(this->_mat, &petsc_blocksize));
     783           0 :   libmesh_assert_equal_to (blocksize, static_cast<numeric_index_type>(petsc_blocksize));
     784             : #endif
     785             : 
     786           0 :   std::scoped_lock lock(this->_petsc_matrix_mutex);
     787             :   // These casts are required for PETSc <= 2.1.5
     788           0 :   LibmeshPetscCall(MatSetValuesBlocked(this->_mat,
     789             :                                        n_brows, numeric_petsc_cast(brows.data()),
     790             :                                        n_bcols, numeric_petsc_cast(bcols.data()),
     791             :                                        pPS(const_cast<T*>(dm.get_values().data())),
     792             :                                        ADD_VALUES));
     793           0 : }
     794             : 
     795             : 
     796             : 
     797             : 
     798             : 
     799             : template <typename T>
     800          56 : void PetscMatrix<T>::_get_submatrix(SparseMatrix<T> & submatrix,
     801             :                                     const std::vector<numeric_index_type> & rows,
     802             :                                     const std::vector<numeric_index_type> & cols,
     803             :                                     const bool reuse_submatrix) const
     804             : {
     805          56 :   if (!this->closed())
     806             :     {
     807             :       libmesh_deprecated();
     808             :       libmesh_warning("The matrix must be assembled before calling PetscMatrix::create_submatrix().\n"
     809             :                       "Please update your code, as this warning will become an error in a future release.");
     810           0 :       const_cast<PetscMatrix<T> *>(this)->close();
     811             :     }
     812             : 
     813          16 :   semiparallel_only();
     814             : 
     815             :   // Make sure the SparseMatrix passed in is really a PetscMatrix
     816          16 :   PetscMatrix<T> * petsc_submatrix = cast_ptr<PetscMatrix<T> *>(&submatrix);
     817             : 
     818             :   // If we're not reusing submatrix and submatrix is already initialized
     819             :   // then we need to clear it, otherwise we get a memory leak.
     820          56 :   if (!reuse_submatrix && submatrix.initialized())
     821           0 :     submatrix.clear();
     822             : 
     823             :   // Construct row and column index sets.
     824          32 :   WrappedPetsc<IS> isrow;
     825          72 :   LibmeshPetscCall(ISCreateGeneral(this->comm().get(),
     826             :                                    cast_int<PetscInt>(rows.size()),
     827             :                                    numeric_petsc_cast(rows.data()),
     828             :                                    PETSC_USE_POINTER,
     829             :                                    isrow.get()));
     830             : 
     831          32 :   WrappedPetsc<IS> iscol;
     832          72 :   LibmeshPetscCall(ISCreateGeneral(this->comm().get(),
     833             :                                    cast_int<PetscInt>(cols.size()),
     834             :                                    numeric_petsc_cast(cols.data()),
     835             :                                    PETSC_USE_POINTER,
     836             :                                    iscol.get()));
     837             : 
     838             :   // Extract submatrix
     839          96 :   LibmeshPetscCall(LibMeshCreateSubMatrix(this->_mat,
     840             :                                           isrow,
     841             :                                           iscol,
     842             :                                           (reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
     843             :                                           (&petsc_submatrix->_mat)));
     844             : 
     845             :   // Specify that the new submatrix is initialized and close it.
     846          56 :   petsc_submatrix->_is_initialized = true;
     847          56 :   petsc_submatrix->close();
     848          56 : }
     849             : 
     850             : template <typename T>
     851           0 : void PetscMatrix<T>::create_submatrix_nosort(SparseMatrix<T> & submatrix,
     852             :                                              const std::vector<numeric_index_type> & rows,
     853             :                                              const std::vector<numeric_index_type> & cols) const
     854             : {
     855           0 :   if (!this->closed())
     856             :     {
     857             :       libmesh_deprecated();
     858             :       libmesh_warning("The matrix must be assembled before calling PetscMatrix::create_submatrix_nosort().\n"
     859             :                       "Please update your code, as this warning will become an error in a future release.");
     860           0 :       const_cast<PetscMatrix<T> *>(this)->close();
     861             :     }
     862             : 
     863             :   // Make sure the SparseMatrix passed in is really a PetscMatrix
     864           0 :   PetscMatrix<T> * petsc_submatrix = cast_ptr<PetscMatrix<T> *>(&submatrix);
     865             : 
     866           0 :   LibmeshPetscCall(MatZeroEntries(petsc_submatrix->_mat));
     867             : 
     868           0 :   PetscInt pc_ncols = 0;
     869             :   const PetscInt * pc_cols;
     870             :   const PetscScalar * pc_vals;
     871             : 
     872             :   // // data for creating the submatrix
     873           0 :   std::vector<PetscInt> sub_cols;
     874           0 :   std::vector<PetscScalar> sub_vals;
     875             : 
     876           0 :   for (auto i : index_range(rows))
     877             :   {
     878           0 :     PetscInt sub_rid[] = {static_cast<PetscInt>(i)};
     879           0 :     PetscInt rid = static_cast<PetscInt>(rows[i]);
     880             :     // only get value from local rows, and set values to the corresponding columns in the submatrix
     881           0 :     if (rows[i]>= this->row_start() && rows[i]< this->row_stop())
     882             :     {
     883             :       // get one row of data from the original matrix
     884           0 :       LibmeshPetscCall(MatGetRow(this->_mat, rid, &pc_ncols, &pc_cols, &pc_vals));
     885             :       // extract data from certain cols, save the indices and entries sub_cols and sub_vals
     886           0 :       for (auto j : index_range(cols))
     887             :       {
     888           0 :         for (unsigned int idx = 0; idx< static_cast<unsigned int>(pc_ncols); idx++)
     889             :         {
     890           0 :           if (pc_cols[idx] == static_cast<PetscInt>(cols[j]))
     891             :             {
     892           0 :               sub_cols.push_back(static_cast<PetscInt>(j));
     893           0 :               sub_vals.push_back(pc_vals[idx]);
     894             :             }
     895             :         }
     896             :       }
     897             :       // set values
     898           0 :       LibmeshPetscCall(MatSetValues(petsc_submatrix->_mat,
     899             :                                     1,
     900             :                                     sub_rid,
     901             :                                     static_cast<PetscInt>(sub_vals.size()),
     902             :                                     sub_cols.data(),
     903             :                                     sub_vals.data(),
     904             :                                     INSERT_VALUES));
     905           0 :       LibmeshPetscCall(MatRestoreRow(this->_mat, rid, &pc_ncols, &pc_cols, &pc_vals));
     906             :       // clear data for this row
     907           0 :       sub_cols.clear();
     908           0 :       sub_vals.clear();
     909             :     }
     910             :   }
     911           0 :   MatAssemblyBeginEnd(petsc_submatrix->comm(), petsc_submatrix->_mat, MAT_FINAL_ASSEMBLY);
     912             :   // Specify that the new submatrix is initialized and close it.
     913           0 :   petsc_submatrix->_is_initialized = true;
     914           0 :   petsc_submatrix->close();
     915           0 : }
     916             : 
     917             : 
     918             : template <typename T>
     919           0 : void PetscMatrix<T>::get_diagonal (NumericVector<T> & dest) const
     920             : {
     921             :   // Make sure the NumericVector passed in is really a PetscVector
     922           0 :   PetscVector<T> & petsc_dest = cast_ref<PetscVector<T> &>(dest);
     923             : 
     924             :   // Needs a const_cast since PETSc does not work with const.
     925           0 :   LibmeshPetscCall(MatGetDiagonal(const_cast<PetscMatrix<T> *>(this)->mat(),petsc_dest.vec()));
     926           0 : }
     927             : 
     928             : 
     929             : 
     930             : template <typename T>
     931          24 : void PetscMatrix<T>::get_transpose (SparseMatrix<T> & dest) const
     932             : {
     933             :   // Make sure the SparseMatrix passed in is really a PetscMatrix
     934           8 :   PetscMatrix<T> & petsc_dest = cast_ref<PetscMatrix<T> &>(dest);
     935             : 
     936             :   // If we aren't reusing the matrix then need to clear dest,
     937             :   // otherwise we get a memory leak
     938          24 :   if (&petsc_dest != this)
     939           6 :     dest.clear();
     940             : 
     941          24 :   if (&petsc_dest == this)
     942             :     // The MAT_REUSE_MATRIX flag was replaced by MAT_INPLACE_MATRIX
     943             :     // in PETSc 3.7.0
     944             : #if PETSC_VERSION_LESS_THAN(3,7,0)
     945             :     LibmeshPetscCall(MatTranspose(this->_mat,MAT_REUSE_MATRIX, &petsc_dest._mat));
     946             : #else
     947          18 :     LibmeshPetscCall(MatTranspose(this->_mat, MAT_INPLACE_MATRIX, &petsc_dest._mat));
     948             : #endif
     949             :   else
     950           6 :     LibmeshPetscCall(MatTranspose(this->_mat,MAT_INITIAL_MATRIX, &petsc_dest._mat));
     951             : 
     952             :   // Specify that the transposed matrix is initialized and close it.
     953          24 :   petsc_dest._is_initialized = true;
     954          24 :   petsc_dest.close();
     955          24 : }
     956             : 
     957             : 
     958             : 
     959             : template <typename T>
     960           0 : void PetscMatrix<T>::flush ()
     961             : {
     962           0 :   semiparallel_only();
     963             : 
     964           0 :   MatAssemblyBeginEnd(this->comm(), this->_mat, MAT_FLUSH_ASSEMBLY);
     965           0 : }
     966             : 
     967             : 
     968             : 
     969             : template <typename T>
     970      819345 : void PetscMatrix<T>::set (const numeric_index_type i,
     971             :                           const numeric_index_type j,
     972             :                           const T value)
     973             : {
     974      104140 :   libmesh_assert (this->initialized());
     975             : 
     976      819345 :   PetscInt i_val=i, j_val=j;
     977             : 
     978      819345 :   PetscScalar petsc_value = static_cast<PetscScalar>(value);
     979      923485 :   std::scoped_lock lock(this->_petsc_matrix_mutex);
     980      819345 :   LibmeshPetscCall(MatSetValues(this->_mat, 1, &i_val, 1, &j_val,
     981             :                                 &petsc_value, INSERT_VALUES));
     982      819345 : }
     983             : 
     984             : 
     985             : 
     986             : template <typename T>
     987      125139 : void PetscMatrix<T>::add (const numeric_index_type i,
     988             :                           const numeric_index_type j,
     989             :                           const T value)
     990             : {
     991       41713 :   libmesh_assert (this->initialized());
     992             : 
     993      125139 :   PetscInt i_val=i, j_val=j;
     994             : 
     995      125139 :   PetscScalar petsc_value = static_cast<PetscScalar>(value);
     996      166852 :   std::scoped_lock lock(this->_petsc_matrix_mutex);
     997      125139 :   LibmeshPetscCall(MatSetValues(this->_mat, 1, &i_val, 1, &j_val,
     998             :                                 &petsc_value, ADD_VALUES));
     999      125139 : }
    1000             : 
    1001             : 
    1002             : 
    1003             : template <typename T>
    1004    11372406 : void PetscMatrix<T>::add_matrix(const DenseMatrix<T> & dm,
    1005             :                                 const std::vector<numeric_index_type> & dof_indices)
    1006             : {
    1007    11372406 :   this->add_matrix (dm, dof_indices, dof_indices);
    1008    11372406 : }
    1009             : 
    1010             : 
    1011             : 
    1012             : 
    1013             : 
    1014             : 
    1015             : 
    1016             : template <typename T>
    1017       49008 : void PetscMatrix<T>::add (const T a_in, const SparseMatrix<T> & X_in)
    1018             : {
    1019       16336 :   libmesh_assert (this->initialized());
    1020             : 
    1021             :   // sanity check. but this cannot avoid
    1022             :   // crash due to incompatible sparsity structure...
    1023       16336 :   libmesh_assert_equal_to (this->m(), X_in.m());
    1024       16336 :   libmesh_assert_equal_to (this->n(), X_in.n());
    1025             : 
    1026       16336 :   PetscScalar a = static_cast<PetscScalar>      (a_in);
    1027       16336 :   const PetscMatrix<T> * X = cast_ptr<const PetscMatrix<T> *> (&X_in);
    1028             : 
    1029       16336 :   libmesh_assert (X);
    1030             : 
    1031             :   // the matrix from which we copy the values has to be assembled/closed
    1032       16336 :   libmesh_assert(X->closed());
    1033             : 
    1034       16336 :   semiparallel_only();
    1035             : 
    1036       49008 :   LibmeshPetscCall(MatAXPY(this->_mat, a, X->_mat, DIFFERENT_NONZERO_PATTERN));
    1037       49008 : }
    1038             : 
    1039             : 
    1040             : template <typename T>
    1041           0 : void PetscMatrix<T>::matrix_matrix_mult (SparseMatrix<T> & X_in, SparseMatrix<T> & Y_out, bool reuse)
    1042             : {
    1043           0 :   libmesh_assert (this->initialized());
    1044             : 
    1045             :   // sanity check
    1046             :   // we do not check the Y_out size here as we will initialize & close it at the end
    1047           0 :   libmesh_assert_equal_to (this->n(), X_in.m());
    1048             : 
    1049           0 :   const PetscMatrix<T> * X = cast_ptr<const PetscMatrix<T> *> (&X_in);
    1050           0 :   PetscMatrix<T> * Y = cast_ptr<PetscMatrix<T> *> (&Y_out);
    1051             : 
    1052             :   // the matrix from which we copy the values has to be assembled/closed
    1053           0 :   libmesh_assert(X->closed());
    1054             : 
    1055           0 :   semiparallel_only();
    1056             : 
    1057           0 :   if (reuse)
    1058           0 :     LibmeshPetscCall(MatMatMult(this->_mat, X->_mat, MAT_REUSE_MATRIX, PETSC_DEFAULT, &Y->_mat));
    1059             :   else
    1060             :   {
    1061           0 :     Y->clear();
    1062           0 :     LibmeshPetscCall(MatMatMult(this->_mat, X->_mat, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Y->_mat));
    1063             :   }
    1064             :   // Specify that the new matrix is initialized
    1065             :   // We do not close it here as `MatMatMult` ensures Y being closed
    1066           0 :   Y->_is_initialized = true;
    1067           0 : }
    1068             : 
    1069             : template <typename T>
    1070             : void
    1071           0 : PetscMatrix<T>::add_sparse_matrix (const SparseMatrix<T> & spm,
    1072             :                                    const std::map<numeric_index_type, numeric_index_type> & row_ltog,
    1073             :                                    const std::map<numeric_index_type, numeric_index_type> & col_ltog,
    1074             :                                    const T scalar)
    1075             : {
    1076             :   // size of spm is usually greater than row_ltog and col_ltog in parallel as the indices are owned by the processor
    1077             :   // also, we should allow adding certain parts of spm to _mat
    1078           0 :   libmesh_assert_greater_equal(spm.m(), row_ltog.size());
    1079           0 :   libmesh_assert_greater_equal(spm.n(), col_ltog.size());
    1080             : 
    1081             :   // make sure matrix has larger size than spm
    1082           0 :   libmesh_assert_greater_equal(this->m(), spm.m());
    1083           0 :   libmesh_assert_greater_equal(this->n(), spm.n());
    1084             : 
    1085           0 :   if (!this->closed())
    1086           0 :     this->close();
    1087             : 
    1088           0 :   auto pscm = cast_ptr<const PetscMatrix<T> *>(&spm);
    1089             : 
    1090           0 :   PetscInt ncols = 0;
    1091             : 
    1092             :   const PetscInt * lcols;
    1093             :   const PetscScalar * vals;
    1094             : 
    1095           0 :   std::vector<PetscInt> gcols;
    1096           0 :   std::vector<PetscScalar> values;
    1097             : 
    1098           0 :   for (auto ltog : row_ltog)
    1099             :   {
    1100           0 :     PetscInt grow[] = {static_cast<PetscInt>(ltog.second)}; // global row index
    1101             : 
    1102           0 :     LibmeshPetscCall(MatGetRow(pscm->_mat, static_cast<PetscInt>(ltog.first), &ncols, &lcols, &vals));
    1103             : 
    1104             :     // get global indices (gcols) from lcols, increment values = vals*scalar
    1105           0 :     gcols.resize(ncols);
    1106           0 :     values.resize(ncols);
    1107           0 :     for (auto i : index_range(gcols))
    1108             :     {
    1109           0 :       gcols[i] = libmesh_map_find(col_ltog, lcols[i]);
    1110           0 :       values[i] = PS(scalar) * vals[i];
    1111             :     }
    1112             : 
    1113           0 :     LibmeshPetscCall(MatSetValues(this->_mat, 1, grow, ncols, gcols.data(), values.data(), ADD_VALUES));
    1114           0 :     LibmeshPetscCall(MatRestoreRow(pscm->_mat, static_cast<PetscInt>(ltog.first), &ncols, &lcols, &vals));
    1115             :   }
    1116             :   // Note: We are not closing the matrix because it is expensive to do so when adding multiple sparse matrices.
    1117             :   //       Remember to manually close the matrix once all changes to the matrix have been made.
    1118           0 : }
    1119             : 
    1120             : template <typename T>
    1121         130 : T PetscMatrix<T>::operator () (const numeric_index_type i_in,
    1122             :                                const numeric_index_type j_in) const
    1123             : {
    1124           0 :   libmesh_assert (this->initialized());
    1125             : 
    1126             :   // If the entry is not in the sparse matrix, it is 0.
    1127         130 :   T value=0.;
    1128             : 
    1129             :   PetscInt
    1130         130 :     i_val=static_cast<PetscInt>(i_in),
    1131         130 :     j_val=static_cast<PetscInt>(j_in);
    1132             : 
    1133             : 
    1134             :   // the matrix needs to be closed for this to work
    1135             :   // this->close();
    1136             :   // but closing it is a semiparallel operation; we want operator()
    1137             :   // to run on one processor.
    1138           0 :   libmesh_assert(this->closed());
    1139             : 
    1140         130 :   LibmeshPetscCall(MatGetValue(this->_mat, i_val, j_val, &value));
    1141             : 
    1142         130 :   return value;
    1143             : }
    1144             : 
    1145             : template <typename T>
    1146        1907 : void PetscMatrix<T>::get_row (numeric_index_type i_in,
    1147             :                               std::vector<numeric_index_type> & indices,
    1148             :                               std::vector<T> & values) const
    1149             : {
    1150         629 :   libmesh_assert (this->initialized());
    1151             : 
    1152             :   const PetscScalar * petsc_row;
    1153             :   const PetscInt    * petsc_cols;
    1154             : 
    1155             :   PetscInt
    1156        1907 :     ncols=0,
    1157        1907 :     i_val = static_cast<PetscInt>(i_in);
    1158             : 
    1159             :   // the matrix needs to be closed for this to work
    1160             :   // this->close();
    1161             :   // but closing it is a semiparallel operation; we want operator()
    1162             :   // to run on one processor.
    1163         629 :   libmesh_assert(this->closed());
    1164             : 
    1165             :   // PETSc makes no effort at being thread safe. Helgrind complains about
    1166             :   // possible data races even just in PetscFunctionBegin (due to things
    1167             :   // like stack counter incrementing). Perhaps we could ignore
    1168             :   // this, but there are legitimate data races for Mat data members like
    1169             :   // mat->getrowactive between MatGetRow and MatRestoreRow. Moreover,
    1170             :   // there could be a write into mat->rowvalues during MatGetRow from
    1171             :   // one thread while we are attempting to read from mat->rowvalues
    1172             :   // (through petsc_cols) during data copy in another thread. So
    1173             :   // the safe thing to do is to lock the whole method
    1174             : 
    1175        2536 :   std::lock_guard<std::mutex> lock(_petsc_matrix_mutex);
    1176             : 
    1177        1907 :   LibmeshPetscCall(MatGetRow(this->_mat, i_val, &ncols, &petsc_cols, &petsc_row));
    1178             : 
    1179             :   // Copy the data
    1180        1907 :   indices.resize(static_cast<std::size_t>(ncols));
    1181        1907 :   values.resize(static_cast<std::size_t>(ncols));
    1182             : 
    1183       14064 :   for (auto i : index_range(indices))
    1184             :   {
    1185       12157 :     indices[i] = static_cast<numeric_index_type>(petsc_cols[i]);
    1186       16166 :     values[i] = static_cast<T>(petsc_row[i]);
    1187             :   }
    1188             : 
    1189        1907 :   LibmeshPetscCall(MatRestoreRow(this->_mat, i_val,
    1190             :                                  &ncols, &petsc_cols, &petsc_row));
    1191        1907 : }
    1192             : 
    1193             : 
    1194             : 
    1195             : template <typename T>
    1196           8 : PetscMatrix<T> & PetscMatrix<T>::operator= (const PetscMatrix<T> & v)
    1197             : {
    1198           0 :   semiparallel_only();
    1199             : 
    1200           8 :   if (this->_mat)
    1201             :     {
    1202             :       PetscBool assembled;
    1203           8 :       LibmeshPetscCall(MatAssembled(this->_mat, &assembled));
    1204             : #ifndef NDEBUG
    1205           0 :       const bool cxx_assembled = (assembled == PETSC_TRUE) ? true : false;
    1206           0 :       libmesh_assert(this->_communicator.verify(cxx_assembled));
    1207             : #endif
    1208             : 
    1209           8 :       if (!assembled)
    1210             :         // MatCopy does not work with an unassembled matrix. We could use MatDuplicate but then we
    1211             :         // would have to destroy the matrix we manage and others might be relying on that data. So
    1212             :         // we just assemble here regardless of the preceding level of matrix fill
    1213           0 :         this->close();
    1214           8 :       LibmeshPetscCall(MatCopy(v._mat, this->_mat, DIFFERENT_NONZERO_PATTERN));
    1215             :     }
    1216             :   else
    1217           0 :       LibmeshPetscCall(MatDuplicate(v._mat, MAT_COPY_VALUES, &this->_mat));
    1218             : 
    1219           8 :   this->_is_initialized = true;
    1220             : 
    1221           8 :   return *this;
    1222             : }
    1223             : 
    1224             : template <typename T>
    1225           8 : SparseMatrix<T> & PetscMatrix<T>::operator= (const SparseMatrix<T> & v)
    1226             : {
    1227           8 :   *this = cast_ref<const PetscMatrix<T> &>(v);
    1228           8 :   return *this;
    1229             : }
    1230             : 
    1231             : template <typename T>
    1232           6 : void PetscMatrix<T>::scale(const T scale)
    1233             : {
    1234           2 :   libmesh_assert(this->closed());
    1235             : 
    1236           6 :   LibmeshPetscCall(MatScale(this->_mat, PS(scale)));
    1237           6 : }
    1238             : 
    1239             : template <typename T>
    1240          12 : bool PetscMatrix<T>::supports_hash_table() const
    1241             : {
    1242             : #if PETSC_RELEASE_LESS_THAN(3,19,0)
    1243           4 :   return false;
    1244             : #else
    1245           8 :   return true;
    1246             : #endif
    1247             : }
    1248             : 
    1249             : #if PETSC_RELEASE_GREATER_EQUALS(3,23,0)
    1250             : template <typename T>
    1251             : std::unique_ptr<PetscMatrix<T>>
    1252             : PetscMatrix<T>::copy_from_hash ()
    1253             : {
    1254             :   Mat xaij;
    1255             :   libmesh_assert(this->initialized());
    1256             :   libmesh_assert(!this->closed());
    1257             :   LibmeshPetscCall(MatDuplicate(this->_mat, MAT_DO_NOT_COPY_VALUES, &xaij));
    1258             :   LibmeshPetscCall(MatCopyHashToXAIJ(this->_mat, xaij));
    1259             :   return std::make_unique<PetscMatrix<T>>(xaij, this->comm(), /*destroy_on_exit=*/true);
    1260             : }
    1261             : #endif
    1262             : 
    1263             : template <typename T>
    1264             : void
    1265           0 : PetscMatrix<T>::restore_original_nonzero_pattern()
    1266             : {
    1267           0 :   semiparallel_only();
    1268             : 
    1269           0 :   if (this->_use_hash_table)
    1270             : #if PETSC_RELEASE_GREATER_EQUALS(3, 23, 0)
    1271             :     // This performs MatReset plus re-establishes the hash table
    1272             :     LibmeshPetscCall(MatResetHash(this->_mat));
    1273             : #else
    1274           0 :     libmesh_error_msg("Resetting hash tables not supported until PETSc version 3.23");
    1275             : #endif
    1276             :   else
    1277           0 :     this->reset_preallocation();
    1278           0 : }
    1279             : 
    1280             : //------------------------------------------------------------------
    1281             : // Explicit instantiations
    1282             : template class LIBMESH_EXPORT PetscMatrix<Number>;
    1283             : 
    1284             : } // namespace libMesh
    1285             : 
    1286             : 
    1287             : #endif // #ifdef LIBMESH_HAVE_PETSC

Generated by: LCOV version 1.14