Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2025 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 :
19 :
20 : // Local includes
21 : #include "libmesh/sparsity_pattern.h"
22 :
23 : // libMesh includes
24 : #include "libmesh/coupling_matrix.h"
25 : #include "libmesh/dof_map.h"
26 : #include "libmesh/elem.h"
27 : #include "libmesh/ghosting_functor.h"
28 : #include "libmesh/hashword.h"
29 : #include "libmesh/parallel_algebra.h"
30 : #include "libmesh/parallel.h"
31 : #include "libmesh/parallel_sync.h"
32 : #include "libmesh/utility.h"
33 : #include "libmesh/static_condensation_dof_map.h"
34 :
35 : // TIMPI includes
36 : #include "timpi/communicator.h"
37 :
38 :
39 : namespace libMesh
40 : {
41 : namespace SparsityPattern
42 : {
43 :
44 : //-------------------------------------------------------
45 : // we need to implement these constructors here so that
46 : // a full DofMap definition is available.
47 38935 : Build::Build (const DofMap & dof_map_in,
48 : const CouplingMatrix * dof_coupling_in,
49 : const std::vector<GhostingFunctor *> & coupling_functors_in,
50 : const bool implicit_neighbor_dofs_in,
51 : const bool need_full_sparsity_pattern_in,
52 : const bool calculate_constrained_in,
53 38935 : const StaticCondensationDofMap * const sc_in) :
54 : ParallelObject(dof_map_in),
55 36455 : dof_map(dof_map_in),
56 36455 : dof_coupling(dof_coupling_in),
57 36455 : coupling_functors(coupling_functors_in),
58 36455 : implicit_neighbor_dofs(implicit_neighbor_dofs_in),
59 36455 : need_full_sparsity_pattern(need_full_sparsity_pattern_in),
60 36455 : calculate_constrained(calculate_constrained_in),
61 36455 : sc(sc_in),
62 : sparsity_pattern(),
63 : nonlocal_pattern(),
64 : n_nz(),
65 38935 : n_oz()
66 38935 : {}
67 :
68 :
69 :
70 2744 : Build::Build (Build & other, Threads::split) :
71 : ParallelObject(other),
72 2744 : dof_map(other.dof_map),
73 2744 : dof_coupling(other.dof_coupling),
74 2744 : coupling_functors(other.coupling_functors),
75 2744 : implicit_neighbor_dofs(other.implicit_neighbor_dofs),
76 2744 : need_full_sparsity_pattern(other.need_full_sparsity_pattern),
77 2744 : calculate_constrained(other.calculate_constrained),
78 2744 : sc(other.sc),
79 958 : hashed_dof_sets(other.hashed_dof_sets),
80 : sparsity_pattern(),
81 : nonlocal_pattern(),
82 : n_nz(),
83 2744 : n_oz()
84 2744 : {}
85 :
86 :
87 :
88 : #if defined(__GNUC__) && (__GNUC__ < 4) && !defined(__INTEL_COMPILER)
89 :
90 : void _dummy_function(void) {}
91 :
92 : #endif
93 :
94 :
95 :
96 7065909 : void Build::sorted_connected_dofs(const Elem * elem,
97 : std::vector<dof_id_type> & dofs_vi,
98 : unsigned int vi)
99 : {
100 7065909 : if (this->sc)
101 : {
102 : // We build a sparsity pattern that will match the size of the condensed system. This is so that
103 : // we have the data necessary to init the reduced system matrix
104 650 : dofs_vi.clear();
105 :
106 : auto total_and_uncondensed_from_scalar_dofs_functor =
107 144 : [&dofs_vi](const Elem & /*elem*/,
108 : std::vector<dof_id_type> & dof_indices,
109 224 : const std::vector<dof_id_type> & scalar_dof_indices)
110 : {
111 192 : dof_indices.insert(dof_indices.end(), scalar_dof_indices.begin(), scalar_dof_indices.end());
112 192 : dofs_vi.insert(dofs_vi.end(), scalar_dof_indices.begin(), scalar_dof_indices.end());
113 6676 : };
114 :
115 : auto total_and_uncondensed_from_field_dofs_functor =
116 34074 : [&dofs_vi, this](const Elem & functor_elem,
117 : const unsigned int node_num,
118 : const unsigned int var_num,
119 : std::vector<dof_id_type> & dof_indices,
120 76146 : const dof_id_type field_dof)
121 : {
122 41646 : dof_indices.push_back(field_dof);
123 44088 : if (this->sc->uncondensed_vars().count(var_num) ||
124 26862 : (node_num != invalid_uint && !functor_elem.is_internal(node_num)))
125 26928 : dofs_vi.push_back(field_dof);
126 48796 : };
127 :
128 9750 : dof_map.dof_indices(elem,
129 7150 : dummy_vec,
130 : vi,
131 : total_and_uncondensed_from_scalar_dofs_functor,
132 : total_and_uncondensed_from_field_dofs_functor,
133 1300 : elem->p_level());
134 : }
135 : else
136 7058759 : dof_map.dof_indices (elem, dofs_vi, vi);
137 :
138 : #ifdef LIBMESH_ENABLE_CONSTRAINTS
139 7065909 : dof_map.find_connected_dofs (dofs_vi);
140 : #endif
141 : // We can be more efficient if we sort the element DOFs into
142 : // increasing order
143 7065909 : std::sort(dofs_vi.begin(), dofs_vi.end());
144 :
145 : // Handle cases where duplicate nodes are intentionally assigned to
146 : // a single element.
147 7065909 : dofs_vi.erase(std::unique(dofs_vi.begin(), dofs_vi.end()), dofs_vi.end());
148 7065909 : }
149 :
150 :
151 :
152 11318723 : void Build::handle_vi_vj(const std::vector<dof_id_type> & element_dofs_i,
153 : const std::vector<dof_id_type> & element_dofs_j)
154 : {
155 : const unsigned int n_dofs_on_element_i =
156 2114871 : cast_int<unsigned int>(element_dofs_i.size());
157 :
158 11318723 : const processor_id_type proc_id = dof_map.processor_id();
159 1057435 : const dof_id_type first_dof_on_proc = dof_map.first_dof(proc_id);
160 1057435 : const dof_id_type end_dof_on_proc = dof_map.end_dof(proc_id);
161 :
162 : std::vector<dof_id_type>
163 2114870 : dofs_to_add;
164 :
165 : const unsigned int n_dofs_on_element_j =
166 2114871 : cast_int<unsigned int>(element_dofs_j.size());
167 :
168 : // It only makes sense to compute hashes and see if we can skip
169 : // doing work when there are a "large" amount of DOFs for a given
170 : // element. The cutoff for "large" is somewhat arbitrarily chosen
171 : // based on a test case with a spider node that resulted in O(10^3)
172 : // entries in element_dofs_i for O(10^3) elements. Making this
173 : // number larger will disable the hashing optimization in more
174 : // cases.
175 1057435 : bool dofs_seen = false;
176 11318723 : if (n_dofs_on_element_j > 0 && n_dofs_on_element_i > 256)
177 : {
178 0 : auto hash_i = Utility::hashword(element_dofs_i);
179 0 : auto hash_j = Utility::hashword(element_dofs_j);
180 0 : auto final_hash = Utility::hashword2(hash_i, hash_j);
181 0 : auto result = hashed_dof_sets.insert(final_hash);
182 : // if insert failed, we have already seen these dofs
183 0 : dofs_seen = !result.second;
184 : }
185 :
186 : // there might be 0 dofs for the other variable on the same element
187 : // (when subdomain variables do not overlap) and that's when we do
188 : // not do anything
189 11318723 : if (n_dofs_on_element_j > 0 && !dofs_seen)
190 : {
191 72519742 : for (unsigned int i=0; i<n_dofs_on_element_i; i++)
192 : {
193 61258369 : const dof_id_type ig = element_dofs_i[i];
194 :
195 : SparsityPattern::Row * row;
196 :
197 : // We save non-local row components for now so we can
198 : // communicate them to other processors later.
199 :
200 61258369 : if ((ig >= first_dof_on_proc) &&
201 5523829 : (ig < end_dof_on_proc))
202 : {
203 : // This is what I mean
204 : // libmesh_assert_greater_equal ((ig - first_dof_on_proc), 0);
205 : // but do the test like this because ig and
206 : // first_dof_on_proc are unsigned ints
207 5394877 : libmesh_assert_greater_equal (ig, first_dof_on_proc);
208 5394877 : libmesh_assert_less (ig, (sparsity_pattern.size() +
209 : first_dof_on_proc));
210 :
211 61055276 : row = &sparsity_pattern[ig - first_dof_on_proc];
212 : }
213 : else
214 : {
215 5597965 : row = &nonlocal_pattern[ig];
216 : }
217 :
218 : // If the row is empty we will add *all*
219 : // the element j DOFs, so just do that.
220 61258369 : if (row->empty())
221 : {
222 8726357 : row->insert(row->end(),
223 : element_dofs_j.begin(),
224 3357430 : element_dofs_j.end());
225 : }
226 : else
227 : {
228 : // Build a list of the DOF indices not found in the
229 : // sparsity pattern
230 4794104 : dofs_to_add.clear();
231 :
232 : // Cache iterators. Low will move forward, subsequent
233 : // searches will be on smaller ranges
234 : SparsityPattern::Row::iterator
235 : low = std::lower_bound
236 46898549 : (row->begin(), row->end(), element_dofs_j.front()),
237 : high = std::upper_bound
238 46898549 : (low, row->end(), element_dofs_j.back());
239 :
240 503416633 : for (unsigned int j=0; j<n_dofs_on_element_j; j++)
241 : {
242 492367640 : const dof_id_type jg = element_dofs_j[j];
243 :
244 : // See if jg is in the sorted range
245 : std::pair<SparsityPattern::Row::iterator,
246 : SparsityPattern::Row::iterator>
247 451723980 : pos = std::equal_range (low, high, jg);
248 :
249 : // Must add jg if it wasn't found
250 451723980 : if (pos.first == pos.second)
251 258256457 : dofs_to_add.push_back(jg);
252 :
253 : // pos.first is now a valid lower bound for any
254 : // remaining element j DOFs. (That's why we sorted them.)
255 : // Use it for the next search
256 40643666 : low = pos.first;
257 : }
258 :
259 : // Add to the sparsity pattern
260 51692653 : if (!dofs_to_add.empty())
261 : {
262 7947152 : const std::size_t old_size = row->size();
263 :
264 39346947 : row->insert (row->end(),
265 : dofs_to_add.begin(),
266 11920728 : dofs_to_add.end());
267 :
268 : SparsityPattern::sort_row
269 39346947 : (row->begin(), row->begin()+old_size,
270 : row->end());
271 : }
272 : }
273 : } // End dofs-of-var-i loop
274 : } // End if-dofs-of-var-j
275 11318723 : }
276 :
277 :
278 :
279 41679 : void Build::operator()(const ConstElemRange & range)
280 : {
281 : // Compute the sparsity structure of the global matrix. This can be
282 : // fed into a PetscMatrixBase to allocate exactly the number of nonzeros
283 : // necessary to store the matrix. This algorithm should be linear
284 : // in the (# of elements)*(# nodes per element)
285 43877 : sparsity_pattern.resize(dof_map.n_local_dofs());
286 :
287 : // Handle dof coupling specified by library and user coupling functors
288 : {
289 41679 : const unsigned int n_var = dof_map.n_variables();
290 :
291 46075 : std::vector<std::vector<dof_id_type> > element_dofs_i(n_var);
292 :
293 4396 : std::vector<const Elem *> coupled_neighbors;
294 3384496 : for (const auto & elem : range)
295 : {
296 : // Make some fake element iterators defining a range
297 : // pointing to only this element.
298 3342817 : Elem * const * elempp = const_cast<Elem * const *>(&elem);
299 3342817 : Elem * const * elemend = elempp+1;
300 :
301 : const MeshBase::const_element_iterator fake_elem_it =
302 : MeshBase::const_element_iterator(elempp,
303 : elemend,
304 3657670 : Predicates::NotNull<Elem * const *>());
305 :
306 : const MeshBase::const_element_iterator fake_elem_end =
307 : MeshBase::const_element_iterator(elemend,
308 : elemend,
309 6685634 : Predicates::NotNull<Elem * const *>());
310 :
311 629706 : GhostingFunctor::map_type elements_to_couple;
312 629706 : DofMap::CouplingMatricesSet temporary_coupling_matrices;
313 :
314 3342817 : dof_map.merge_ghost_functor_outputs(elements_to_couple,
315 : temporary_coupling_matrices,
316 3342817 : dof_map.coupling_functors_begin(),
317 3657671 : dof_map.coupling_functors_end(),
318 : fake_elem_it,
319 : fake_elem_end,
320 : DofObject::invalid_processor_id);
321 7889394 : for (unsigned int vi=0; vi<n_var; vi++)
322 4962925 : this->sorted_connected_dofs(elem, element_dofs_i[vi], vi);
323 :
324 7889394 : for (unsigned int vi=0; vi<n_var; vi++)
325 11561562 : for (const auto & [partner, ghost_coupling] : elements_to_couple)
326 : {
327 : // Loop over coupling matrix row variables if we have a
328 : // coupling matrix, or all variables if not.
329 7014985 : if (ghost_coupling)
330 : {
331 16 : libmesh_assert_equal_to (ghost_coupling->size(), n_var);
332 176 : ConstCouplingRow ccr(vi, *ghost_coupling);
333 :
334 396 : for (const auto & idx : ccr)
335 : {
336 220 : if (partner == elem)
337 0 : this->handle_vi_vj(element_dofs_i[vi], element_dofs_i[idx]);
338 : else
339 : {
340 40 : std::vector<dof_id_type> partner_dofs;
341 220 : this->sorted_connected_dofs(partner, partner_dofs, idx);
342 240 : this->handle_vi_vj(element_dofs_i[vi], partner_dofs);
343 : }
344 : }
345 : }
346 : else
347 : {
348 18333312 : for (unsigned int vj = 0; vj != n_var; ++vj)
349 : {
350 11318503 : if (partner == elem)
351 10348667 : this->handle_vi_vj(element_dofs_i[vi], element_dofs_i[vj]);
352 : else
353 : {
354 565556 : std::vector<dof_id_type> partner_dofs;
355 2519112 : this->sorted_connected_dofs(partner, partner_dofs, vj);
356 2801890 : this->handle_vi_vj(element_dofs_i[vi], partner_dofs);
357 : }
358 : }
359 : }
360 : } // End ghosted element loop
361 : } // End range element loop
362 37283 : } // End ghosting functor section
363 41679 : }
364 :
365 :
366 :
367 2744 : void Build::join (const SparsityPattern::Build & other)
368 : {
369 958 : libmesh_assert_equal_to (sparsity_pattern.size(), other.sparsity_pattern.size());
370 :
371 1775846 : for (dof_id_type r=0; r<dof_map.n_local_dofs(); r++)
372 : {
373 : // increment the number of on and off-processor nonzeros in this row
374 : // (note this will be an upper bound unless we need the full sparsity pattern)
375 1221381 : SparsityPattern::Row & my_row = sparsity_pattern[r];
376 1221381 : const SparsityPattern::Row & their_row = other.sparsity_pattern[r];
377 :
378 : // simple copy if I have no dofs
379 1772144 : if (my_row.empty())
380 823187 : my_row = their_row;
381 :
382 : // otherwise add their DOFs to mine, resort, and re-unique the row
383 948957 : else if (!their_row.empty()) // do nothing for the trivial case where
384 : { // their row is empty
385 83172 : my_row.insert (my_row.end(),
386 : their_row.begin(),
387 135772 : their_row.end());
388 :
389 : // We cannot use SparsityPattern::sort_row() here because it expects
390 : // the [begin,middle) [middle,end) to be non-overlapping. This is not
391 : // necessarily the case here, so use std::sort()
392 128428 : std::sort (my_row.begin(), my_row.end());
393 :
394 128428 : my_row.erase(std::unique (my_row.begin(), my_row.end()), my_row.end());
395 : }
396 : }
397 :
398 : // Move nonlocal row information to ourselves; the other thread
399 : // won't need it in the map after that.
400 32180 : for (const auto & p : other.nonlocal_pattern)
401 : {
402 : #ifndef NDEBUG
403 10583 : const dof_id_type dof_id = p.first;
404 :
405 10583 : processor_id_type dbg_proc_id = 0;
406 11127 : while (dof_id >= dof_map.end_dof(dbg_proc_id))
407 544 : dbg_proc_id++;
408 10583 : libmesh_assert (dbg_proc_id != this->processor_id());
409 : #endif
410 :
411 29436 : const SparsityPattern::Row & their_row = p.second;
412 :
413 : // We should have no empty values in a map
414 10583 : libmesh_assert (!their_row.empty());
415 :
416 29436 : if (auto my_it = nonlocal_pattern.find(p.first);
417 10583 : my_it == nonlocal_pattern.end())
418 : {
419 : // nonlocal_pattern[it->first].swap(their_row);
420 24581 : nonlocal_pattern[p.first] = their_row;
421 : }
422 : else
423 : {
424 4855 : SparsityPattern::Row & my_row = my_it->second;
425 :
426 3097 : my_row.insert (my_row.end(),
427 : their_row.begin(),
428 7032 : their_row.end());
429 :
430 : // We cannot use SparsityPattern::sort_row() here because it expects
431 : // the [begin,middle) [middle,end) to be non-overlapping. This is not
432 : // necessarily the case here, so use std::sort()
433 4855 : std::sort (my_row.begin(), my_row.end());
434 :
435 4855 : my_row.erase(std::unique (my_row.begin(), my_row.end()), my_row.end());
436 : }
437 : }
438 :
439 : // Combine the other thread's hashed_dof_sets with ours.
440 1786 : hashed_dof_sets.insert(other.hashed_dof_sets.begin(),
441 : other.hashed_dof_sets.end());
442 2744 : }
443 :
444 :
445 :
446 38935 : void Build::parallel_sync ()
447 : {
448 1240 : parallel_object_only();
449 1240 : libmesh_assert(this->comm().verify(need_full_sparsity_pattern));
450 :
451 38935 : const auto n_dofs_on_proc = dof_map.n_local_dofs();
452 1240 : const auto local_first_dof = dof_map.first_dof();
453 :
454 : // The data to send
455 2480 : std::map<processor_id_type, std::vector<dof_id_type>> ids_to_send;
456 2480 : std::map<processor_id_type, std::vector<Row>> rows_to_send;
457 :
458 : // Loop over the nonlocal rows and transform them into the new datastructure
459 1240 : NonlocalGraph::iterator it = nonlocal_pattern.begin();
460 929078 : while (it != nonlocal_pattern.end())
461 : {
462 890143 : const auto dof_id = it->first;
463 890143 : auto & row = it->second;
464 :
465 890143 : processor_id_type proc_id = 0;
466 3650246 : while (dof_id >= dof_map.end_dof(proc_id))
467 2727011 : proc_id++;
468 :
469 890143 : ids_to_send[proc_id].push_back(dof_id);
470 :
471 : // Note this invalidates the data in nonlocal_pattern
472 890143 : rows_to_send[proc_id].push_back(std::move(row));
473 :
474 : // Might as well remove it since it's invalidated anyway
475 862197 : it = nonlocal_pattern.erase(it);
476 : }
477 :
478 2480 : std::map<processor_id_type, std::vector<dof_id_type>> received_ids_map;
479 :
480 : auto ids_action_functor =
481 : [& received_ids_map]
482 : (processor_id_type pid,
483 58167 : const std::vector<dof_id_type> & received_ids)
484 : {
485 58167 : received_ids_map.emplace(pid, received_ids);
486 39629 : };
487 :
488 38935 : Parallel::push_parallel_vector_data(this->comm(), ids_to_send,
489 : ids_action_functor);
490 :
491 : auto rows_action_functor =
492 57473 : [this,
493 : & received_ids_map,
494 : local_first_dof]
495 : (processor_id_type pid,
496 1032834 : const std::vector<Row> & received_rows)
497 : {
498 58861 : const std::vector<dof_id_type> & received_ids = libmesh_map_find(received_ids_map, pid);
499 :
500 1388 : std::size_t n_rows = received_rows.size();
501 694 : libmesh_assert_equal_to(n_rows, received_ids.size());
502 :
503 949004 : for (auto i : IntRange<std::size_t>(0, n_rows))
504 : {
505 890143 : const auto r = received_ids[i];
506 27942 : libmesh_assert(dof_map.local_index(r));
507 :
508 890143 : const auto my_r = r - local_first_dof;
509 :
510 55888 : auto & their_row = received_rows[i];
511 :
512 55888 : auto & my_row = sparsity_pattern[my_r];
513 :
514 : // They wouldn't have sent an empty row
515 27942 : libmesh_assert(!their_row.empty());
516 :
517 : // We can end up with an empty row on a dof that touches our
518 : // inactive elements but not our active ones
519 890143 : if (my_row.empty())
520 : {
521 4695 : my_row.assign (their_row.begin(), their_row.end());
522 : }
523 : else
524 : {
525 857704 : my_row.insert (my_row.end(),
526 : their_row.begin(),
527 110968 : their_row.end());
528 :
529 : // We cannot use SparsityPattern::sort_row() here because it expects
530 : // the [begin,middle) [middle,end) to be non-overlapping. This is not
531 : // necessarily the case here, so use std::sort()
532 885448 : std::sort (my_row.begin(), my_row.end());
533 :
534 885448 : my_row.erase(std::unique (my_row.begin(), my_row.end()), my_row.end());
535 : }
536 :
537 : }
538 39083 : };
539 :
540 38935 : Parallel::push_parallel_vector_data(this->comm(), rows_to_send,
541 : rows_action_functor);
542 :
543 : // We should have sent everything at this point.
544 1240 : libmesh_assert (nonlocal_pattern.empty());
545 :
546 : // assert these are empty because std::vector::resize will only append the specified element value
547 : // if the new size is greater than the current size. Elements whose indices are less than the
548 : // current size are untouched
549 1240 : libmesh_assert(n_nz.empty());
550 1240 : libmesh_assert(n_oz.empty());
551 38935 : n_nz.resize (n_dofs_on_proc, 0);
552 38935 : n_oz.resize (n_dofs_on_proc, 0);
553 :
554 38935 : const dof_id_type first_dof_on_proc = dof_map.first_dof();
555 1240 : const dof_id_type end_dof_on_proc = dof_map.end_dof();
556 :
557 8600638 : for (dof_id_type i=0; i<n_dofs_on_proc; i++)
558 : {
559 : // Get the row of the sparsity pattern
560 2214687 : SparsityPattern::Row & row = sparsity_pattern[i];
561 :
562 353410254 : for (const auto & df : row)
563 344848551 : if ((df < first_dof_on_proc) || (df >= end_dof_on_proc))
564 50805849 : n_oz[i]++;
565 : else
566 324128163 : n_nz[i]++;
567 :
568 765938 : libmesh_assert(n_nz[i] <= n_dofs_on_proc);
569 :
570 : // If we're not building a full sparsity pattern, then we want
571 : // to avoid overcounting these entries as much as possible.
572 8561703 : if (!need_full_sparsity_pattern)
573 765938 : row.clear();
574 : }
575 38935 : }
576 :
577 :
578 0 : void Build::apply_extra_sparsity_object(SparsityPattern::AugmentSparsityPattern & asp)
579 : {
580 0 : asp.augment_sparsity_pattern (sparsity_pattern, n_nz, n_oz);
581 0 : }
582 :
583 :
584 0 : std::size_t Build::n_nonzeros() const
585 : {
586 : // At some point I'll remember that "C++17" compilers don't always
587 : // come with complete C++17 standard libraries.
588 : // std::size_t total_nonzeros = std::reduce(n_nz.begin(), n_nz.end(), std::size_t(0));
589 : // total_nonzeros += std::reduce(n_oz.begin(), n_oz.end(), std::size_t(0));
590 :
591 0 : std::size_t total_nonzeros = 0;
592 0 : for (auto nnzi : n_nz)
593 0 : total_nonzeros += nnzi;
594 0 : for (auto nozi : n_oz)
595 0 : total_nonzeros += nozi;
596 :
597 0 : this->comm().sum(total_nonzeros);
598 0 : return total_nonzeros;
599 : }
600 :
601 :
602 : } // namespace SparsityPattern
603 : } // namespace libMesh
|
