Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : #include "libmesh/libmesh_config.h"
21 :
22 : #ifdef LIBMESH_HAVE_PETSC
23 :
24 : // Local includes
25 : #include "libmesh/petsc_matrix.h"
26 :
27 : // libMesh includes
28 : #include "libmesh/dof_map.h"
29 : #include "libmesh/dense_matrix.h"
30 : #include "libmesh/libmesh_logging.h"
31 : #include "libmesh/petsc_vector.h"
32 : #include "libmesh/parallel.h"
33 : #include "libmesh/utility.h"
34 : #include "libmesh/wrapped_petsc.h"
35 :
36 : // C++ includes
37 : #ifdef LIBMESH_HAVE_UNISTD_H
38 : #include <unistd.h> // mkstemp
39 : #endif
40 : #include <fstream>
41 :
42 : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
43 :
44 : namespace
45 : {
46 : using namespace libMesh;
47 :
48 : // historic libMesh n_nz & n_oz arrays are set up for PETSc's AIJ format.
49 : // however, when the blocksize is >1, we need to transform these into
50 : // their BAIJ counterparts.
51 : inline
52 : void transform_preallocation_arrays (const PetscInt blocksize,
53 : const std::vector<numeric_index_type> & n_nz,
54 : const std::vector<numeric_index_type> & n_oz,
55 : std::vector<numeric_index_type> & b_n_nz,
56 : std::vector<numeric_index_type> & b_n_oz)
57 : {
58 : libmesh_assert_equal_to (n_nz.size(), n_oz.size());
59 : libmesh_assert_equal_to (n_nz.size()%blocksize, 0);
60 :
61 : b_n_nz.clear(); /**/ b_n_nz.reserve(n_nz.size()/blocksize);
62 : b_n_oz.clear(); /**/ b_n_oz.reserve(n_oz.size()/blocksize);
63 :
64 : for (std::size_t nn=0, nnzs=n_nz.size(); nn<nnzs; nn += blocksize)
65 : {
66 : b_n_nz.push_back (n_nz[nn]/blocksize);
67 : b_n_oz.push_back (n_oz[nn]/blocksize);
68 : }
69 : }
70 : }
71 :
72 : #endif
73 :
74 :
75 :
76 : namespace libMesh
77 : {
78 :
79 :
80 : //-----------------------------------------------------------------------
81 : // PetscMatrix members
82 :
83 :
84 : // Constructor
85 : template <typename T>
86 33183 : PetscMatrix<T>::PetscMatrix(const Parallel::Communicator & comm_in) :
87 33183 : PetscMatrixBase<T>(comm_in), _mat_type(AIJ)
88 : {
89 33183 : }
90 :
91 :
92 :
93 : // Constructor taking an existing Mat but not the responsibility
94 : // for destroying it
95 : template <typename T>
96 2108 : PetscMatrix<T>::PetscMatrix(Mat mat_in,
97 : const Parallel::Communicator & comm_in,
98 : const bool destroy_on_exit) :
99 2108 : PetscMatrixBase<T>(mat_in, comm_in, destroy_on_exit)
100 : {
101 : MatType mat_type;
102 2108 : LibmeshPetscCall(MatGetType(mat_in, &mat_type));
103 : PetscBool is_hypre;
104 2108 : LibmeshPetscCall(PetscStrcmp(mat_type, MATHYPRE, &is_hypre));
105 2108 : if (is_hypre == PETSC_TRUE)
106 0 : _mat_type = HYPRE;
107 : else
108 2108 : _mat_type = AIJ;
109 2108 : }
110 :
111 :
112 : // Constructor taking global and local dimensions and, optionally,
113 : // number of on- and off-diagonal non-zeros and a block size
114 : template <typename T>
115 808 : PetscMatrix<T>::PetscMatrix(const Parallel::Communicator & comm_in,
116 : const numeric_index_type m_in,
117 : const numeric_index_type n_in,
118 : const numeric_index_type m_l,
119 : const numeric_index_type n_l,
120 : const numeric_index_type n_nz,
121 : const numeric_index_type n_oz,
122 : const numeric_index_type blocksize_in) :
123 808 : PetscMatrixBase<T>(comm_in), _mat_type(AIJ)
124 : {
125 808 : this->init(m_in, n_in, m_l, n_l, n_nz, n_oz, blocksize_in);
126 808 : }
127 :
128 :
129 : // Destructor
130 : template <typename T>
131 34187 : PetscMatrix<T>::~PetscMatrix() = default;
132 :
133 : template <typename T>
134 : void
135 51992 : PetscMatrix<T>::init_without_preallocation (const numeric_index_type m_in,
136 : const numeric_index_type n_in,
137 : const numeric_index_type m_l,
138 : const numeric_index_type n_l,
139 : const numeric_index_type blocksize_in)
140 : {
141 : // So compilers don't warn when !LIBMESH_ENABLE_BLOCKED_STORAGE
142 1578 : libmesh_ignore(blocksize_in);
143 :
144 : // Clear initialized matrices
145 51992 : if (this->initialized())
146 280 : this->clear();
147 :
148 51992 : PetscInt m_global = static_cast<PetscInt>(m_in);
149 51992 : PetscInt n_global = static_cast<PetscInt>(n_in);
150 51992 : PetscInt m_local = static_cast<PetscInt>(m_l);
151 51992 : PetscInt n_local = static_cast<PetscInt>(n_l);
152 :
153 51992 : LibmeshPetscCall(MatCreate(this->comm().get(), &this->_mat));
154 51992 : LibmeshPetscCall(MatSetSizes(this->_mat, m_local, n_local, m_global, n_global));
155 51992 : PetscInt blocksize = static_cast<PetscInt>(blocksize_in);
156 51992 : LibmeshPetscCall(MatSetBlockSize(this->_mat,blocksize));
157 51992 : LibmeshPetscCall(MatSetOptionsPrefix(this->_mat, this->options_prefix().c_str()));
158 :
159 : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
160 : if (blocksize > 1)
161 : {
162 : // specified blocksize, bs>1.
163 : // double check sizes.
164 : libmesh_assert_equal_to (m_local % blocksize, 0);
165 : libmesh_assert_equal_to (n_local % blocksize, 0);
166 : libmesh_assert_equal_to (m_global % blocksize, 0);
167 : libmesh_assert_equal_to (n_global % blocksize, 0);
168 : libmesh_assert_equal_to (n_nz % blocksize, 0);
169 : libmesh_assert_equal_to (n_oz % blocksize, 0);
170 :
171 : LibmeshPetscCall(MatSetType(this->_mat, MATBAIJ)); // Automatically chooses seqbaij or mpibaij
172 :
173 : // MatSetFromOptions needs to happen before Preallocation routines
174 : // since MatSetFromOptions can change matrix type and remove incompatible
175 : // preallocation
176 : LibmeshPetscCall(MatSetFromOptions(this->_mat));
177 : }
178 : else
179 : #endif
180 : {
181 51992 : switch (this->_mat_type) {
182 51992 : case AIJ:
183 51992 : LibmeshPetscCall(MatSetType(this->_mat, MATAIJ)); // Automatically chooses seqaij or mpiaij
184 :
185 : // MatSetFromOptions needs to happen before Preallocation routines
186 : // since MatSetFromOptions can change matrix type and remove incompatible
187 : // preallocation
188 51992 : LibmeshPetscCall(MatSetFromOptions(this->_mat));
189 1578 : break;
190 :
191 0 : case HYPRE:
192 : #if !PETSC_VERSION_LESS_THAN(3,9,4) && LIBMESH_HAVE_PETSC_HYPRE
193 0 : LibmeshPetscCall(MatSetType(this->_mat, MATHYPRE));
194 :
195 : // MatSetFromOptions needs to happen before Preallocation routines
196 : // since MatSetFromOptions can change matrix type and remove incompatible
197 : // preallocation
198 0 : LibmeshPetscCall(MatSetFromOptions(this->_mat));
199 : #else
200 : libmesh_error_msg("PETSc 3.9.4 or higher with hypre is required for MatHypre");
201 : #endif
202 0 : break;
203 :
204 0 : default: libmesh_error_msg("Unsupported petsc matrix type");
205 : }
206 : }
207 :
208 51992 : this->set_context ();
209 51992 : }
210 :
211 :
212 :
213 : template <typename T>
214 4036 : void PetscMatrix<T>::init (const numeric_index_type m_in,
215 : const numeric_index_type n_in,
216 : const numeric_index_type m_l,
217 : const numeric_index_type n_l,
218 : const numeric_index_type nnz,
219 : const numeric_index_type noz,
220 : const numeric_index_type blocksize_in)
221 : {
222 4036 : this->init_without_preallocation(m_in, n_in, m_l, n_l, blocksize_in);
223 :
224 4036 : PetscInt n_nz = static_cast<PetscInt>(nnz);
225 4036 : PetscInt n_oz = static_cast<PetscInt>(noz);
226 :
227 : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
228 : if (blocksize > 1)
229 : {
230 : LibmeshPetscCall(MatSeqBAIJSetPreallocation(this->_mat, blocksize, n_nz/blocksize, NULL));
231 : LibmeshPetscCall(MatMPIBAIJSetPreallocation(this->_mat, blocksize,
232 : n_nz/blocksize, NULL,
233 : n_oz/blocksize, NULL));
234 : }
235 : else
236 : #endif
237 : {
238 4036 : switch (this->_mat_type) {
239 4036 : case AIJ:
240 4036 : LibmeshPetscCall(MatSeqAIJSetPreallocation(this->_mat, n_nz, NULL));
241 4036 : LibmeshPetscCall(MatMPIAIJSetPreallocation(this->_mat, n_nz, NULL, n_oz, NULL));
242 90 : break;
243 :
244 0 : case HYPRE:
245 : #if !PETSC_VERSION_LESS_THAN(3,9,4) && LIBMESH_HAVE_PETSC_HYPRE
246 0 : LibmeshPetscCall(MatHYPRESetPreallocation(this->_mat, n_nz, NULL, n_oz, NULL));
247 : #else
248 : libmesh_error_msg("PETSc 3.9.4 or higher with hypre is required for MatHypre");
249 : #endif
250 0 : break;
251 :
252 0 : default: libmesh_error_msg("Unsupported petsc matrix type");
253 : }
254 : }
255 :
256 4036 : this->finish_initialization();
257 4036 : }
258 :
259 : template <typename T>
260 47824 : void PetscMatrix<T>::preallocate(const numeric_index_type libmesh_dbg_var(m_l),
261 : const std::vector<numeric_index_type> & n_nz,
262 : const std::vector<numeric_index_type> & n_oz,
263 : const numeric_index_type blocksize_in)
264 : {
265 : // Make sure the sparsity pattern isn't empty unless the matrix is 0x0
266 1488 : libmesh_assert_equal_to (n_nz.size(), m_l);
267 1488 : libmesh_assert_equal_to (n_oz.size(), m_l);
268 : // Avoid unused warnings when not configured with block storage
269 1488 : libmesh_ignore(blocksize_in);
270 :
271 : #ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
272 : PetscInt blocksize = static_cast<PetscInt>(blocksize_in);
273 :
274 : if (blocksize > 1)
275 : {
276 : // transform the per-entry n_nz and n_oz arrays into their block counterparts.
277 : std::vector<numeric_index_type> b_n_nz, b_n_oz;
278 :
279 : transform_preallocation_arrays (blocksize,
280 : n_nz, n_oz,
281 : b_n_nz, b_n_oz);
282 :
283 : LibmeshPetscCall(MatSeqBAIJSetPreallocation (this->_mat,
284 : blocksize,
285 : 0,
286 : numeric_petsc_cast(b_n_nz.empty() ? nullptr : b_n_nz.data())));
287 :
288 : LibmeshPetscCall(MatMPIBAIJSetPreallocation (this->_mat,
289 : blocksize,
290 : 0,
291 : numeric_petsc_cast(b_n_nz.empty() ? nullptr : b_n_nz.data()),
292 : 0,
293 : numeric_petsc_cast(b_n_oz.empty() ? nullptr : b_n_oz.data())));
294 : }
295 : else
296 : #endif
297 : {
298 47824 : switch (this->_mat_type) {
299 1488 : case AIJ:
300 86914 : LibmeshPetscCall(MatSeqAIJSetPreallocation (this->_mat,
301 : 0,
302 : numeric_petsc_cast(n_nz.empty() ? nullptr : n_nz.data())));
303 126004 : LibmeshPetscCall(MatMPIAIJSetPreallocation (this->_mat,
304 : 0,
305 : numeric_petsc_cast(n_nz.empty() ? nullptr : n_nz.data()),
306 : 0,
307 : numeric_petsc_cast(n_oz.empty() ? nullptr : n_oz.data())));
308 1488 : break;
309 :
310 0 : case HYPRE:
311 : #if !PETSC_VERSION_LESS_THAN(3,9,4) && LIBMESH_HAVE_PETSC_HYPRE
312 0 : LibmeshPetscCall(MatHYPRESetPreallocation (this->_mat,
313 : 0,
314 : numeric_petsc_cast(n_nz.empty() ? nullptr : n_nz.data()),
315 : 0,
316 : numeric_petsc_cast(n_oz.empty() ? nullptr : n_oz.data())));
317 : #else
318 : libmesh_error_msg("PETSc 3.9.4 or higher with hypre is required for MatHypre");
319 : #endif
320 0 : break;
321 :
322 0 : default: libmesh_error_msg("Unsupported petsc matrix type");
323 : }
324 :
325 : }
326 47824 : }
327 :
328 : template <typename T>
329 51992 : void PetscMatrix<T>::finish_initialization()
330 : {
331 : // Make it an error for PETSc to allocate new nonzero entries during assembly. For old PETSc
332 : // versions this option must be set after preallocation for MPIAIJ matrices
333 51992 : LibmeshPetscCall(MatSetOption(this->_mat, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
334 51992 : this->_is_initialized = true;
335 51992 : }
336 :
337 : template <typename T>
338 4130 : void PetscMatrix<T>::init (const numeric_index_type m_in,
339 : const numeric_index_type n_in,
340 : const numeric_index_type m_l,
341 : const numeric_index_type n_l,
342 : const std::vector<numeric_index_type> & n_nz,
343 : const std::vector<numeric_index_type> & n_oz,
344 : const numeric_index_type blocksize_in)
345 : {
346 4130 : this->init_without_preallocation(m_in, n_in, m_l, n_l, blocksize_in);
347 4130 : this->preallocate(m_l, n_nz, n_oz, blocksize_in);
348 :
349 4130 : this->finish_initialization();
350 4130 : }
351 :
352 :
353 : template <typename T>
354 43826 : void PetscMatrix<T>::init (const ParallelType libmesh_dbg_var(type))
355 : {
356 1370 : libmesh_assert(this->_dof_map);
357 :
358 43826 : const numeric_index_type m_in = this->_dof_map->n_dofs();
359 1370 : const numeric_index_type m_l = this->_dof_map->n_local_dofs();
360 1370 : if (m_in != m_l)
361 1332 : libmesh_assert(type != SERIAL);
362 :
363 43826 : const auto blocksize = this->_dof_map->block_size();
364 :
365 43826 : this->init_without_preallocation(m_in, m_in, m_l, m_l, blocksize);
366 43826 : if (!this->_use_hash_table)
367 : {
368 43694 : const std::vector<numeric_index_type> & n_nz = this->_sp->get_n_nz();
369 1370 : const std::vector<numeric_index_type> & n_oz = this->_sp->get_n_oz();
370 :
371 43694 : this->preallocate(m_l, n_nz, n_oz, blocksize);
372 : }
373 :
374 43826 : this->finish_initialization();
375 43826 : }
376 :
377 :
378 : template <typename T>
379 0 : void PetscMatrix<T>::update_preallocation_and_zero ()
380 : {
381 0 : libmesh_not_implemented();
382 : }
383 :
384 : template <typename T>
385 0 : void PetscMatrix<T>::reset_preallocation()
386 : {
387 0 : semiparallel_only();
388 :
389 : #if !PETSC_VERSION_LESS_THAN(3,9,0)
390 0 : libmesh_assert (this->initialized());
391 :
392 0 : LibmeshPetscCall(MatResetPreallocation(this->_mat));
393 : #else
394 : libmesh_warning("Your version of PETSc doesn't support resetting of "
395 : "preallocation, so we will use your most recent sparsity "
396 : "pattern. This may result in a degradation of performance\n");
397 : #endif
398 0 : }
399 :
400 : template <typename T>
401 468794 : void PetscMatrix<T>::zero ()
402 : {
403 12516 : libmesh_assert (this->initialized());
404 :
405 12516 : semiparallel_only();
406 :
407 : // MatZeroEntries() must be called on every rank, including ranks with no
408 : // local entries: on a matrix that is not yet preallocated it calls
409 : // MatSetUp(), which communicates for some types (e.g. MATHYPRE).
410 468794 : LibmeshPetscCall(MatZeroEntries(this->_mat));
411 468794 : }
412 :
413 : template <typename T>
414 0 : void PetscMatrix<T>::zero_rows (std::vector<numeric_index_type> & rows, T diag_value)
415 : {
416 0 : libmesh_assert (this->initialized());
417 :
418 0 : semiparallel_only();
419 :
420 : // As of petsc-dev at the time of 3.1.0, MatZeroRows now takes two additional
421 : // optional arguments. The optional arguments (x,b) can be used to specify the
422 : // solutions for the zeroed rows (x) and right hand side (b) to update.
423 : // Could be useful for setting boundary conditions...
424 0 : if (!rows.empty())
425 0 : LibmeshPetscCall(MatZeroRows(this->_mat, cast_int<PetscInt>(rows.size()),
426 : numeric_petsc_cast(rows.data()), PS(diag_value),
427 : NULL, NULL));
428 : else
429 0 : LibmeshPetscCall(MatZeroRows(this->_mat, 0, NULL, PS(diag_value), NULL, NULL));
430 0 : }
431 :
432 : template <typename T>
433 70 : std::unique_ptr<SparseMatrix<T>> PetscMatrix<T>::zero_clone () const
434 : {
435 70 : libmesh_error_msg_if(!this->closed(), "Matrix must be closed before it can be cloned!");
436 :
437 : // Copy the nonzero pattern only
438 : Mat copy;
439 70 : LibmeshPetscCall(MatDuplicate(this->_mat, MAT_DO_NOT_COPY_VALUES, ©));
440 :
441 : // Call wrapping PetscMatrix constructor, have it take over
442 : // ownership.
443 72 : auto ret = std::make_unique<PetscMatrix<T>>(copy, this->comm());
444 70 : ret->set_destroy_mat_on_exit(true);
445 :
446 72 : return ret;
447 : }
448 :
449 :
450 :
451 : template <typename T>
452 210 : std::unique_ptr<SparseMatrix<T>> PetscMatrix<T>::clone () const
453 : {
454 210 : libmesh_error_msg_if(!this->closed(), "Matrix must be closed before it can be cloned!");
455 :
456 : // Copy the nonzero pattern and numerical values
457 : Mat copy;
458 210 : LibmeshPetscCall(MatDuplicate(this->_mat, MAT_COPY_VALUES, ©));
459 :
460 : // Call wrapping PetscMatrix constructor, have it take over
461 : // ownership.
462 216 : auto ret = std::make_unique<PetscMatrix<T>>(copy, this->comm());
463 210 : ret->set_destroy_mat_on_exit(true);
464 :
465 216 : return ret;
466 : }
467 :
468 :
469 :
470 : template <typename T>
471 : template <NormType N>
472 2524 : Real PetscMatrix<T>::norm () const
473 : {
474 74 : libmesh_assert (this->initialized());
475 :
476 74 : semiparallel_only();
477 :
478 : PetscReal petsc_value;
479 : Real value;
480 :
481 74 : libmesh_assert (this->closed());
482 :
483 2524 : LibmeshPetscCall(MatNorm(this->_mat, N, &petsc_value));
484 :
485 2524 : value = static_cast<Real>(petsc_value);
486 :
487 2524 : return value;
488 : }
489 : template <typename T>
490 1334 : Real PetscMatrix<T>::l1_norm () const
491 : {
492 1334 : return PetscMatrix<T>::norm<NORM_1>();
493 : }
494 : template <typename T>
495 0 : Real PetscMatrix<T>::frobenius_norm () const
496 : {
497 0 : return PetscMatrix<T>::norm<NORM_FROBENIUS>();
498 : }
499 : template <typename T>
500 1264 : Real PetscMatrix<T>::linfty_norm () const
501 : {
502 1264 : return PetscMatrix<T>::norm<NORM_INFINITY>();
503 : }
504 :
505 :
506 :
507 : template <typename T>
508 140 : void PetscMatrix<T>::print_matlab (const std::string & name) const
509 : {
510 4 : libmesh_assert (this->initialized());
511 :
512 4 : semiparallel_only();
513 :
514 140 : if (!this->closed())
515 : {
516 : libmesh_deprecated();
517 : libmesh_warning("The matrix must be assembled before calling PetscMatrix::print_matlab().\n"
518 : "Please update your code, as this warning will become an error in a future release.");
519 0 : const_cast<PetscMatrix<T> *>(this)->close();
520 : }
521 :
522 : // Create an ASCII file containing the matrix
523 : // if a filename was provided.
524 140 : if (name != "")
525 : {
526 8 : WrappedPetsc<PetscViewer> petsc_viewer;
527 :
528 140 : LibmeshPetscCall(PetscViewerASCIIOpen( this->comm().get(),
529 : name.c_str(),
530 : petsc_viewer.get()));
531 :
532 : #if PETSC_VERSION_LESS_THAN(3,7,0)
533 : LibmeshPetscCall(PetscViewerSetFormat (petsc_viewer,
534 : PETSC_VIEWER_ASCII_MATLAB));
535 : #else
536 140 : LibmeshPetscCall(PetscViewerPushFormat (petsc_viewer,
537 : PETSC_VIEWER_ASCII_MATLAB));
538 : #endif
539 :
540 140 : LibmeshPetscCall(MatView (this->_mat, petsc_viewer));
541 : }
542 :
543 : // Otherwise the matrix will be dumped to the screen.
544 : else
545 : {
546 : #if PETSC_VERSION_LESS_THAN(3,7,0)
547 : LibmeshPetscCall(PetscViewerSetFormat (PETSC_VIEWER_STDOUT_WORLD,
548 : PETSC_VIEWER_ASCII_MATLAB));
549 : #else
550 0 : LibmeshPetscCall(PetscViewerPushFormat (PETSC_VIEWER_STDOUT_WORLD,
551 : PETSC_VIEWER_ASCII_MATLAB));
552 : #endif
553 :
554 0 : LibmeshPetscCall(MatView (this->_mat, PETSC_VIEWER_STDOUT_WORLD));
555 : }
556 140 : }
557 :
558 :
559 :
560 :
561 :
562 : template <typename T>
563 0 : void PetscMatrix<T>::print_personal(std::ostream & os) const
564 : {
565 0 : libmesh_assert (this->initialized());
566 :
567 : // Routine must be called in parallel on parallel matrices
568 : // and serial on serial matrices.
569 0 : semiparallel_only();
570 :
571 : // #ifndef NDEBUG
572 : // if (os != std::cout)
573 : // libMesh::err << "Warning! PETSc can only print to std::cout!" << std::endl;
574 : // #endif
575 :
576 : // Matrix must be in an assembled state to be printed
577 0 : if (!this->closed())
578 : {
579 : libmesh_deprecated();
580 : libmesh_warning("The matrix must be assembled before calling PetscMatrix::print_personal().\n"
581 : "Please update your code, as this warning will become an error in a future release.");
582 0 : const_cast<PetscMatrix<T> *>(this)->close();
583 : }
584 :
585 : // Print to screen if ostream is stdout
586 0 : if (os.rdbuf() == std::cout.rdbuf())
587 0 : LibmeshPetscCall(MatView(this->_mat, NULL));
588 :
589 : // Otherwise, print to the requested file, in a roundabout way...
590 : else
591 : {
592 : // We will create a temporary filename, and file, for PETSc to
593 : // write to.
594 0 : std::string temp_filename;
595 :
596 : {
597 : // Template for temporary filename
598 0 : char c[] = "temp_petsc_matrix.XXXXXX";
599 :
600 : // Generate temporary, unique filename only on processor 0. We will
601 : // use this filename for PetscViewerASCIIOpen, before copying it into
602 : // the user's stream
603 0 : if (this->processor_id() == 0)
604 : {
605 0 : int fd = mkstemp(c);
606 :
607 : // Check to see that mkstemp did not fail.
608 0 : libmesh_error_msg_if(fd == -1, "mkstemp failed in PetscMatrix::print_personal()");
609 :
610 : // mkstemp returns a file descriptor for an open file,
611 : // so let's close it before we hand it to PETSc!
612 0 : ::close (fd);
613 : }
614 :
615 : // Store temporary filename as string, makes it easier to broadcast
616 0 : temp_filename = c;
617 : }
618 :
619 : // Now broadcast the filename from processor 0 to all processors.
620 0 : this->comm().broadcast(temp_filename);
621 :
622 : // PetscViewer object for passing to MatView
623 : PetscViewer petsc_viewer;
624 :
625 : // This PETSc function only takes a string and handles the opening/closing
626 : // of the file internally. Since print_personal() takes a reference to
627 : // an ostream, we have to do an extra step... print_personal() should probably
628 : // have a version that takes a string to get rid of this problem.
629 0 : LibmeshPetscCall(PetscViewerASCIIOpen( this->comm().get(),
630 : temp_filename.c_str(),
631 : &petsc_viewer));
632 :
633 : // Probably don't need to set the format if it's default...
634 : // ierr = PetscViewerSetFormat (petsc_viewer,
635 : // PETSC_VIEWER_DEFAULT);
636 : // LIBMESH_CHKERR(ierr);
637 :
638 : // Finally print the matrix using the viewer
639 0 : LibmeshPetscCall(MatView (this->_mat, petsc_viewer));
640 :
641 0 : if (this->processor_id() == 0)
642 : {
643 : // Now the inefficient bit: open temp_filename as an ostream and copy the contents
644 : // into the user's desired ostream. We can't just do a direct file copy, we don't even have the filename!
645 0 : std::ifstream input_stream(temp_filename.c_str());
646 0 : os << input_stream.rdbuf(); // The "most elegant" way to copy one stream into another.
647 : // os.close(); // close not defined in ostream
648 :
649 : // Now remove the temporary file
650 0 : input_stream.close();
651 0 : std::remove(temp_filename.c_str());
652 0 : }
653 : }
654 0 : }
655 :
656 :
657 :
658 : template <typename T>
659 210 : void PetscMatrix<T>::_petsc_viewer(const std::string & filename,
660 : PetscViewerType viewertype,
661 : PetscFileMode filemode)
662 : {
663 6 : parallel_object_only();
664 :
665 : // We'll get matrix sizes from the file, but we need to at least
666 : // have a Mat object
667 210 : if (!this->initialized())
668 : {
669 70 : LibmeshPetscCall(MatCreate(this->comm().get(), &this->_mat));
670 70 : this->_is_initialized = true;
671 : }
672 :
673 : PetscViewer viewer;
674 210 : LibmeshPetscCall(PetscViewerCreate(this->comm().get(), &viewer));
675 210 : LibmeshPetscCall(PetscViewerSetType(viewer, viewertype));
676 210 : LibmeshPetscCall(PetscViewerSetFromOptions(viewer));
677 210 : LibmeshPetscCall(PetscViewerFileSetMode(viewer, filemode));
678 210 : LibmeshPetscCall(PetscViewerFileSetName(viewer, filename.c_str()));
679 210 : if (filemode == FILE_MODE_READ)
680 140 : LibmeshPetscCall(MatLoad(this->_mat, viewer));
681 : else
682 70 : LibmeshPetscCall(MatView(this->_mat, viewer));
683 210 : LibmeshPetscCall(PetscViewerDestroy(&viewer));
684 210 : }
685 :
686 :
687 :
688 : template <typename T>
689 70 : void PetscMatrix<T>::print_petsc_binary(const std::string & filename) const
690 : {
691 2 : libmesh_assert (this->initialized());
692 :
693 : // Our helper here isn't const-correct for writes
694 2 : PetscMatrix<T> * nonconst_this = const_cast<PetscMatrix<T>*>(this);
695 70 : nonconst_this->_petsc_viewer(filename, PETSCVIEWERBINARY, FILE_MODE_WRITE);
696 70 : }
697 :
698 :
699 :
700 : template <typename T>
701 0 : void PetscMatrix<T>::print_petsc_hdf5(const std::string & filename) const
702 : {
703 0 : libmesh_assert (this->initialized());
704 :
705 : // Our helper here isn't const-correct for writes
706 0 : PetscMatrix<T> * nonconst_this = const_cast<PetscMatrix<T>*>(this);
707 0 : nonconst_this->_petsc_viewer(filename, PETSCVIEWERHDF5, FILE_MODE_WRITE);
708 0 : }
709 :
710 :
711 :
712 : template <typename T>
713 140 : void PetscMatrix<T>::read_petsc_binary(const std::string & filename)
714 : {
715 12 : LOG_SCOPE("read_petsc_binary()", "PetscMatrix");
716 :
717 140 : this->_petsc_viewer(filename, PETSCVIEWERBINARY, FILE_MODE_READ);
718 140 : }
719 :
720 :
721 :
722 : template <typename T>
723 0 : void PetscMatrix<T>::read_petsc_hdf5(const std::string & filename)
724 : {
725 0 : LOG_SCOPE("read_petsc_hdf5()", "PetscMatrix");
726 :
727 0 : this->_petsc_viewer(filename, PETSCVIEWERHDF5, FILE_MODE_READ);
728 0 : }
729 :
730 :
731 :
732 : template <typename T>
733 42375409 : void PetscMatrix<T>::add_matrix(const DenseMatrix<T> & dm,
734 : const std::vector<numeric_index_type> & rows,
735 : const std::vector<numeric_index_type> & cols)
736 : {
737 3293697 : libmesh_assert (this->initialized());
738 :
739 6587394 : const numeric_index_type n_rows = dm.m();
740 6587394 : const numeric_index_type n_cols = dm.n();
741 :
742 3293697 : libmesh_assert_equal_to (rows.size(), n_rows);
743 3293697 : libmesh_assert_equal_to (cols.size(), n_cols);
744 :
745 45669106 : std::scoped_lock lock(this->_petsc_matrix_mutex);
746 42375409 : LibmeshPetscCall(MatSetValues(this->_mat,
747 : n_rows, numeric_petsc_cast(rows.data()),
748 : n_cols, numeric_petsc_cast(cols.data()),
749 : pPS(const_cast<T*>(dm.get_values().data())),
750 : ADD_VALUES));
751 42375409 : }
752 :
753 :
754 :
755 :
756 :
757 :
758 : template <typename T>
759 0 : void PetscMatrix<T>::add_block_matrix(const DenseMatrix<T> & dm,
760 : const std::vector<numeric_index_type> & brows,
761 : const std::vector<numeric_index_type> & bcols)
762 : {
763 0 : libmesh_assert (this->initialized());
764 :
765 : const numeric_index_type n_brows =
766 0 : cast_int<numeric_index_type>(brows.size());
767 : const numeric_index_type n_bcols =
768 0 : cast_int<numeric_index_type>(bcols.size());
769 :
770 : #ifndef NDEBUG
771 0 : const numeric_index_type n_rows =
772 0 : cast_int<numeric_index_type>(dm.m());
773 0 : const numeric_index_type n_cols =
774 0 : cast_int<numeric_index_type>(dm.n());
775 0 : const numeric_index_type blocksize = n_rows / n_brows;
776 :
777 0 : libmesh_assert_equal_to (n_cols / n_bcols, blocksize);
778 0 : libmesh_assert_equal_to (blocksize*n_brows, n_rows);
779 0 : libmesh_assert_equal_to (blocksize*n_bcols, n_cols);
780 :
781 : PetscInt petsc_blocksize;
782 0 : LibmeshPetscCall(MatGetBlockSize(this->_mat, &petsc_blocksize));
783 0 : libmesh_assert_equal_to (blocksize, static_cast<numeric_index_type>(petsc_blocksize));
784 : #endif
785 :
786 0 : std::scoped_lock lock(this->_petsc_matrix_mutex);
787 : // These casts are required for PETSc <= 2.1.5
788 0 : LibmeshPetscCall(MatSetValuesBlocked(this->_mat,
789 : n_brows, numeric_petsc_cast(brows.data()),
790 : n_bcols, numeric_petsc_cast(bcols.data()),
791 : pPS(const_cast<T*>(dm.get_values().data())),
792 : ADD_VALUES));
793 0 : }
794 :
795 :
796 :
797 :
798 :
799 : template <typename T>
800 594 : void PetscMatrix<T>::_get_submatrix(SparseMatrix<T> & submatrix,
801 : const std::vector<numeric_index_type> & rows,
802 : const std::vector<numeric_index_type> & cols,
803 : const bool reuse_submatrix) const
804 : {
805 594 : if (!this->closed())
806 : {
807 : libmesh_deprecated();
808 : libmesh_warning("The matrix must be assembled before calling PetscMatrix::create_submatrix().\n"
809 : "Please update your code, as this warning will become an error in a future release.");
810 0 : const_cast<PetscMatrix<T> *>(this)->close();
811 : }
812 :
813 16 : semiparallel_only();
814 :
815 : // Make sure the SparseMatrix passed in is really a PetscMatrix
816 16 : PetscMatrix<T> * petsc_submatrix = cast_ptr<PetscMatrix<T> *>(&submatrix);
817 :
818 : // If we're not reusing submatrix and submatrix is already initialized
819 : // then we need to clear it, otherwise we get a memory leak.
820 594 : if (!reuse_submatrix && submatrix.initialized())
821 24 : submatrix.clear();
822 :
823 : // Construct row and column index sets.
824 32 : WrappedPetsc<IS> isrow;
825 610 : LibmeshPetscCall(ISCreateGeneral(this->comm().get(),
826 : cast_int<PetscInt>(rows.size()),
827 : numeric_petsc_cast(rows.data()),
828 : PETSC_USE_POINTER,
829 : isrow.get()));
830 :
831 32 : WrappedPetsc<IS> iscol;
832 610 : LibmeshPetscCall(ISCreateGeneral(this->comm().get(),
833 : cast_int<PetscInt>(cols.size()),
834 : numeric_petsc_cast(cols.data()),
835 : PETSC_USE_POINTER,
836 : iscol.get()));
837 :
838 : // Extract submatrix
839 1172 : LibmeshPetscCall(LibMeshCreateSubMatrix(this->_mat,
840 : isrow,
841 : iscol,
842 : (reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
843 : (&petsc_submatrix->_mat)));
844 :
845 : // Specify that the new submatrix is initialized and close it.
846 594 : petsc_submatrix->_is_initialized = true;
847 594 : petsc_submatrix->close();
848 594 : }
849 :
850 : template <typename T>
851 0 : void PetscMatrix<T>::create_submatrix_nosort(SparseMatrix<T> & submatrix,
852 : const std::vector<numeric_index_type> & rows,
853 : const std::vector<numeric_index_type> & cols) const
854 : {
855 0 : if (!this->closed())
856 : {
857 : libmesh_deprecated();
858 : libmesh_warning("The matrix must be assembled before calling PetscMatrix::create_submatrix_nosort().\n"
859 : "Please update your code, as this warning will become an error in a future release.");
860 0 : const_cast<PetscMatrix<T> *>(this)->close();
861 : }
862 :
863 : // Make sure the SparseMatrix passed in is really a PetscMatrix
864 0 : PetscMatrix<T> * petsc_submatrix = cast_ptr<PetscMatrix<T> *>(&submatrix);
865 :
866 0 : LibmeshPetscCall(MatZeroEntries(petsc_submatrix->_mat));
867 :
868 0 : PetscInt pc_ncols = 0;
869 : const PetscInt * pc_cols;
870 : const PetscScalar * pc_vals;
871 :
872 : // // data for creating the submatrix
873 0 : std::vector<PetscInt> sub_cols;
874 0 : std::vector<PetscScalar> sub_vals;
875 :
876 0 : for (auto i : index_range(rows))
877 : {
878 0 : PetscInt sub_rid[] = {static_cast<PetscInt>(i)};
879 0 : PetscInt rid = static_cast<PetscInt>(rows[i]);
880 : // only get value from local rows, and set values to the corresponding columns in the submatrix
881 0 : if (rows[i]>= this->row_start() && rows[i]< this->row_stop())
882 : {
883 : // get one row of data from the original matrix
884 0 : LibmeshPetscCall(MatGetRow(this->_mat, rid, &pc_ncols, &pc_cols, &pc_vals));
885 : // extract data from certain cols, save the indices and entries sub_cols and sub_vals
886 0 : for (auto j : index_range(cols))
887 : {
888 0 : for (unsigned int idx = 0; idx< static_cast<unsigned int>(pc_ncols); idx++)
889 : {
890 0 : if (pc_cols[idx] == static_cast<PetscInt>(cols[j]))
891 : {
892 0 : sub_cols.push_back(static_cast<PetscInt>(j));
893 0 : sub_vals.push_back(pc_vals[idx]);
894 : }
895 : }
896 : }
897 : // set values
898 0 : LibmeshPetscCall(MatSetValues(petsc_submatrix->_mat,
899 : 1,
900 : sub_rid,
901 : static_cast<PetscInt>(sub_vals.size()),
902 : sub_cols.data(),
903 : sub_vals.data(),
904 : INSERT_VALUES));
905 0 : LibmeshPetscCall(MatRestoreRow(this->_mat, rid, &pc_ncols, &pc_cols, &pc_vals));
906 : // clear data for this row
907 0 : sub_cols.clear();
908 0 : sub_vals.clear();
909 : }
910 : }
911 0 : MatAssemblyBeginEnd(petsc_submatrix->comm(), petsc_submatrix->_mat, MAT_FINAL_ASSEMBLY);
912 : // Specify that the new submatrix is initialized and close it.
913 0 : petsc_submatrix->_is_initialized = true;
914 0 : petsc_submatrix->close();
915 0 : }
916 :
917 :
918 : template <typename T>
919 0 : void PetscMatrix<T>::get_diagonal (NumericVector<T> & dest) const
920 : {
921 : // Make sure the NumericVector passed in is really a PetscVector
922 0 : PetscVector<T> & petsc_dest = cast_ref<PetscVector<T> &>(dest);
923 :
924 : // Needs a const_cast since PETSc does not work with const.
925 0 : LibmeshPetscCall(MatGetDiagonal(const_cast<PetscMatrix<T> *>(this)->mat(),petsc_dest.vec()));
926 0 : }
927 :
928 :
929 :
930 : template <typename T>
931 280 : void PetscMatrix<T>::get_transpose (SparseMatrix<T> & dest) const
932 : {
933 : // Make sure the SparseMatrix passed in is really a PetscMatrix
934 8 : PetscMatrix<T> & petsc_dest = cast_ref<PetscMatrix<T> &>(dest);
935 :
936 : // If we aren't reusing the matrix then need to clear dest,
937 : // otherwise we get a memory leak
938 280 : if (&petsc_dest != this)
939 70 : dest.clear();
940 :
941 280 : if (&petsc_dest == this)
942 : // The MAT_REUSE_MATRIX flag was replaced by MAT_INPLACE_MATRIX
943 : // in PETSc 3.7.0
944 : #if PETSC_VERSION_LESS_THAN(3,7,0)
945 : LibmeshPetscCall(MatTranspose(this->_mat,MAT_REUSE_MATRIX, &petsc_dest._mat));
946 : #else
947 210 : LibmeshPetscCall(MatTranspose(this->_mat, MAT_INPLACE_MATRIX, &petsc_dest._mat));
948 : #endif
949 : else
950 70 : LibmeshPetscCall(MatTranspose(this->_mat,MAT_INITIAL_MATRIX, &petsc_dest._mat));
951 :
952 : // Specify that the transposed matrix is initialized and close it.
953 280 : petsc_dest._is_initialized = true;
954 280 : petsc_dest.close();
955 280 : }
956 :
957 :
958 :
959 : template <typename T>
960 0 : void PetscMatrix<T>::flush ()
961 : {
962 0 : semiparallel_only();
963 :
964 0 : MatAssemblyBeginEnd(this->comm(), this->_mat, MAT_FLUSH_ASSEMBLY);
965 0 : }
966 :
967 :
968 :
969 : template <typename T>
970 2250609 : void PetscMatrix<T>::set (const numeric_index_type i,
971 : const numeric_index_type j,
972 : const T value)
973 : {
974 104140 : libmesh_assert (this->initialized());
975 :
976 2250609 : PetscInt i_val=i, j_val=j;
977 :
978 2250609 : PetscScalar petsc_value = static_cast<PetscScalar>(value);
979 2354749 : std::scoped_lock lock(this->_petsc_matrix_mutex);
980 2250609 : LibmeshPetscCall(MatSetValues(this->_mat, 1, &i_val, 1, &j_val,
981 : &petsc_value, INSERT_VALUES));
982 2250609 : }
983 :
984 :
985 :
986 : template <typename T>
987 464756 : void PetscMatrix<T>::add (const numeric_index_type i,
988 : const numeric_index_type j,
989 : const T value)
990 : {
991 41713 : libmesh_assert (this->initialized());
992 :
993 464756 : PetscInt i_val=i, j_val=j;
994 :
995 464756 : PetscScalar petsc_value = static_cast<PetscScalar>(value);
996 506469 : std::scoped_lock lock(this->_petsc_matrix_mutex);
997 464756 : LibmeshPetscCall(MatSetValues(this->_mat, 1, &i_val, 1, &j_val,
998 : &petsc_value, ADD_VALUES));
999 464756 : }
1000 :
1001 :
1002 :
1003 : template <typename T>
1004 41745317 : void PetscMatrix<T>::add_matrix(const DenseMatrix<T> & dm,
1005 : const std::vector<numeric_index_type> & dof_indices)
1006 : {
1007 41745317 : this->add_matrix (dm, dof_indices, dof_indices);
1008 41745317 : }
1009 :
1010 :
1011 :
1012 :
1013 :
1014 :
1015 :
1016 : template <typename T>
1017 571953 : void PetscMatrix<T>::add (const T a_in, const SparseMatrix<T> & X_in)
1018 : {
1019 16336 : libmesh_assert (this->initialized());
1020 :
1021 : // sanity check. but this cannot avoid
1022 : // crash due to incompatible sparsity structure...
1023 16336 : libmesh_assert_equal_to (this->m(), X_in.m());
1024 16336 : libmesh_assert_equal_to (this->n(), X_in.n());
1025 :
1026 16336 : PetscScalar a = static_cast<PetscScalar> (a_in);
1027 16336 : const PetscMatrix<T> * X = cast_ptr<const PetscMatrix<T> *> (&X_in);
1028 :
1029 16336 : libmesh_assert (X);
1030 :
1031 : // the matrix from which we copy the values has to be assembled/closed
1032 16336 : libmesh_assert(X->closed());
1033 :
1034 16336 : semiparallel_only();
1035 :
1036 571953 : LibmeshPetscCall(MatAXPY(this->_mat, a, X->_mat, DIFFERENT_NONZERO_PATTERN));
1037 571953 : }
1038 :
1039 :
1040 : template <typename T>
1041 0 : void PetscMatrix<T>::matrix_matrix_mult (SparseMatrix<T> & X_in, SparseMatrix<T> & Y_out, bool reuse)
1042 : {
1043 0 : libmesh_assert (this->initialized());
1044 :
1045 : // sanity check
1046 : // we do not check the Y_out size here as we will initialize & close it at the end
1047 0 : libmesh_assert_equal_to (this->n(), X_in.m());
1048 :
1049 0 : const PetscMatrix<T> * X = cast_ptr<const PetscMatrix<T> *> (&X_in);
1050 0 : PetscMatrix<T> * Y = cast_ptr<PetscMatrix<T> *> (&Y_out);
1051 :
1052 : // the matrix from which we copy the values has to be assembled/closed
1053 0 : libmesh_assert(X->closed());
1054 :
1055 0 : semiparallel_only();
1056 :
1057 0 : if (reuse)
1058 0 : LibmeshPetscCall(MatMatMult(this->_mat, X->_mat, MAT_REUSE_MATRIX, PETSC_DEFAULT, &Y->_mat));
1059 : else
1060 : {
1061 0 : Y->clear();
1062 0 : LibmeshPetscCall(MatMatMult(this->_mat, X->_mat, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Y->_mat));
1063 : }
1064 : // Specify that the new matrix is initialized
1065 : // We do not close it here as `MatMatMult` ensures Y being closed
1066 0 : Y->_is_initialized = true;
1067 0 : }
1068 :
1069 : template <typename T>
1070 : void
1071 0 : PetscMatrix<T>::add_sparse_matrix (const SparseMatrix<T> & spm,
1072 : const std::map<numeric_index_type, numeric_index_type> & row_ltog,
1073 : const std::map<numeric_index_type, numeric_index_type> & col_ltog,
1074 : const T scalar)
1075 : {
1076 : // size of spm is usually greater than row_ltog and col_ltog in parallel as the indices are owned by the processor
1077 : // also, we should allow adding certain parts of spm to _mat
1078 0 : libmesh_assert_greater_equal(spm.m(), row_ltog.size());
1079 0 : libmesh_assert_greater_equal(spm.n(), col_ltog.size());
1080 :
1081 : // make sure matrix has larger size than spm
1082 0 : libmesh_assert_greater_equal(this->m(), spm.m());
1083 0 : libmesh_assert_greater_equal(this->n(), spm.n());
1084 :
1085 0 : if (!this->closed())
1086 0 : this->close();
1087 :
1088 0 : auto pscm = cast_ptr<const PetscMatrix<T> *>(&spm);
1089 :
1090 0 : PetscInt ncols = 0;
1091 :
1092 : const PetscInt * lcols;
1093 : const PetscScalar * vals;
1094 :
1095 0 : std::vector<PetscInt> gcols;
1096 0 : std::vector<PetscScalar> values;
1097 :
1098 0 : for (auto ltog : row_ltog)
1099 : {
1100 0 : PetscInt grow[] = {static_cast<PetscInt>(ltog.second)}; // global row index
1101 :
1102 0 : LibmeshPetscCall(MatGetRow(pscm->_mat, static_cast<PetscInt>(ltog.first), &ncols, &lcols, &vals));
1103 :
1104 : // get global indices (gcols) from lcols, increment values = vals*scalar
1105 0 : gcols.resize(ncols);
1106 0 : values.resize(ncols);
1107 0 : for (auto i : index_range(gcols))
1108 : {
1109 0 : gcols[i] = libmesh_map_find(col_ltog, lcols[i]);
1110 0 : values[i] = PS(scalar) * vals[i];
1111 : }
1112 :
1113 0 : LibmeshPetscCall(MatSetValues(this->_mat, 1, grow, ncols, gcols.data(), values.data(), ADD_VALUES));
1114 0 : LibmeshPetscCall(MatRestoreRow(pscm->_mat, static_cast<PetscInt>(ltog.first), &ncols, &lcols, &vals));
1115 : }
1116 : // Note: We are not closing the matrix because it is expensive to do so when adding multiple sparse matrices.
1117 : // Remember to manually close the matrix once all changes to the matrix have been made.
1118 0 : }
1119 :
1120 : template <typename T>
1121 130 : T PetscMatrix<T>::operator () (const numeric_index_type i_in,
1122 : const numeric_index_type j_in) const
1123 : {
1124 0 : libmesh_assert (this->initialized());
1125 :
1126 : // If the entry is not in the sparse matrix, it is 0.
1127 130 : T value=0.;
1128 :
1129 : PetscInt
1130 130 : i_val=static_cast<PetscInt>(i_in),
1131 130 : j_val=static_cast<PetscInt>(j_in);
1132 :
1133 :
1134 : // the matrix needs to be closed for this to work
1135 : // this->close();
1136 : // but closing it is a semiparallel operation; we want operator()
1137 : // to run on one processor.
1138 0 : libmesh_assert(this->closed());
1139 :
1140 130 : LibmeshPetscCall(MatGetValue(this->_mat, i_val, j_val, &value));
1141 :
1142 130 : return value;
1143 : }
1144 :
1145 : template <typename T>
1146 8595 : void PetscMatrix<T>::get_row (numeric_index_type i_in,
1147 : std::vector<numeric_index_type> & indices,
1148 : std::vector<T> & values) const
1149 : {
1150 629 : libmesh_assert (this->initialized());
1151 :
1152 : const PetscScalar * petsc_row;
1153 : const PetscInt * petsc_cols;
1154 :
1155 : PetscInt
1156 8595 : ncols=0,
1157 8595 : i_val = static_cast<PetscInt>(i_in);
1158 :
1159 : // the matrix needs to be closed for this to work
1160 : // this->close();
1161 : // but closing it is a semiparallel operation; we want operator()
1162 : // to run on one processor.
1163 629 : libmesh_assert(this->closed());
1164 :
1165 : // PETSc makes no effort at being thread safe. Helgrind complains about
1166 : // possible data races even just in PetscFunctionBegin (due to things
1167 : // like stack counter incrementing). Perhaps we could ignore
1168 : // this, but there are legitimate data races for Mat data members like
1169 : // mat->getrowactive between MatGetRow and MatRestoreRow. Moreover,
1170 : // there could be a write into mat->rowvalues during MatGetRow from
1171 : // one thread while we are attempting to read from mat->rowvalues
1172 : // (through petsc_cols) during data copy in another thread. So
1173 : // the safe thing to do is to lock the whole method
1174 :
1175 9224 : std::lock_guard<std::mutex> lock(_petsc_matrix_mutex);
1176 :
1177 8595 : LibmeshPetscCall(MatGetRow(this->_mat, i_val, &ncols, &petsc_cols, &petsc_row));
1178 :
1179 : // Copy the data
1180 8595 : indices.resize(static_cast<std::size_t>(ncols));
1181 8595 : values.resize(static_cast<std::size_t>(ncols));
1182 :
1183 65076 : for (auto i : index_range(indices))
1184 : {
1185 56481 : indices[i] = static_cast<numeric_index_type>(petsc_cols[i]);
1186 60490 : values[i] = static_cast<T>(petsc_row[i]);
1187 : }
1188 :
1189 8595 : LibmeshPetscCall(MatRestoreRow(this->_mat, i_val,
1190 : &ncols, &petsc_cols, &petsc_row));
1191 8595 : }
1192 :
1193 :
1194 :
1195 : template <typename T>
1196 264 : PetscMatrix<T> & PetscMatrix<T>::operator= (const PetscMatrix<T> & v)
1197 : {
1198 0 : semiparallel_only();
1199 :
1200 264 : if (this->_mat)
1201 : {
1202 : PetscBool assembled;
1203 264 : LibmeshPetscCall(MatAssembled(this->_mat, &assembled));
1204 : #ifndef NDEBUG
1205 0 : const bool cxx_assembled = (assembled == PETSC_TRUE) ? true : false;
1206 0 : libmesh_assert(this->_communicator.verify(cxx_assembled));
1207 : #endif
1208 :
1209 264 : if (!assembled)
1210 : // MatCopy does not work with an unassembled matrix. We could use MatDuplicate but then we
1211 : // would have to destroy the matrix we manage and others might be relying on that data. So
1212 : // we just assemble here regardless of the preceding level of matrix fill
1213 0 : this->close();
1214 264 : LibmeshPetscCall(MatCopy(v._mat, this->_mat, DIFFERENT_NONZERO_PATTERN));
1215 : }
1216 : else
1217 0 : LibmeshPetscCall(MatDuplicate(v._mat, MAT_COPY_VALUES, &this->_mat));
1218 :
1219 264 : this->_is_initialized = true;
1220 :
1221 264 : return *this;
1222 : }
1223 :
1224 : template <typename T>
1225 264 : SparseMatrix<T> & PetscMatrix<T>::operator= (const SparseMatrix<T> & v)
1226 : {
1227 264 : *this = cast_ref<const PetscMatrix<T> &>(v);
1228 264 : return *this;
1229 : }
1230 :
1231 : template <typename T>
1232 70 : void PetscMatrix<T>::scale(const T scale)
1233 : {
1234 2 : libmesh_assert(this->closed());
1235 :
1236 70 : LibmeshPetscCall(MatScale(this->_mat, PS(scale)));
1237 70 : }
1238 :
1239 : template <typename T>
1240 268 : bool PetscMatrix<T>::supports_hash_table() const
1241 : {
1242 : #if PETSC_RELEASE_LESS_THAN(3,19,0)
1243 4 : return false;
1244 : #else
1245 264 : return true;
1246 : #endif
1247 : }
1248 :
1249 : #if PETSC_RELEASE_GREATER_EQUALS(3,23,0)
1250 : template <typename T>
1251 : std::unique_ptr<PetscMatrix<T>>
1252 : PetscMatrix<T>::copy_from_hash ()
1253 : {
1254 : Mat xaij;
1255 : libmesh_assert(this->initialized());
1256 : libmesh_assert(!this->closed());
1257 : LibmeshPetscCall(MatDuplicate(this->_mat, MAT_DO_NOT_COPY_VALUES, &xaij));
1258 : LibmeshPetscCall(MatCopyHashToXAIJ(this->_mat, xaij));
1259 : return std::make_unique<PetscMatrix<T>>(xaij, this->comm(), /*destroy_on_exit=*/true);
1260 : }
1261 : #endif
1262 :
1263 : template <typename T>
1264 : void
1265 0 : PetscMatrix<T>::restore_original_nonzero_pattern()
1266 : {
1267 0 : semiparallel_only();
1268 :
1269 0 : if (this->_use_hash_table)
1270 : #if PETSC_RELEASE_GREATER_EQUALS(3, 23, 0)
1271 : // This performs MatReset plus re-establishes the hash table
1272 : LibmeshPetscCall(MatResetHash(this->_mat));
1273 : #else
1274 0 : libmesh_error_msg("Resetting hash tables not supported until PETSc version 3.23");
1275 : #endif
1276 : else
1277 0 : this->reset_preallocation();
1278 0 : }
1279 :
1280 : //------------------------------------------------------------------
1281 : // Explicit instantiations
1282 : template class LIBMESH_EXPORT PetscMatrix<Number>;
1283 :
1284 : } // namespace libMesh
1285 :
1286 :
1287 : #endif // #ifdef LIBMESH_HAVE_PETSC
|