Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #ifdef MOOSE_KOKKOS_ENABLED
11 : #include "KokkosMaterialPropertyStorage.h"
12 : #endif
13 :
14 : #include "FEProblemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialPropertyStorage.h"
17 : #include "MooseEnum.h"
18 : #include "Factory.h"
19 : #include "MooseUtils.h"
20 : #include "DisplacedProblem.h"
21 : #include "SystemBase.h"
22 : #include "MaterialData.h"
23 : #include "ComputeUserObjectsThread.h"
24 : #include "ComputeNodalUserObjectsThread.h"
25 : #include "ComputeThreadedGeneralUserObjectsThread.h"
26 : #include "ComputeMaterialsObjectThread.h"
27 : #include "ProjectMaterialProperties.h"
28 : #include "ComputeIndicatorThread.h"
29 : #include "ComputeMarkerThread.h"
30 : #include "ComputeInitialConditionThread.h"
31 : #include "ComputeFVInitialConditionThread.h"
32 : #include "ComputeBoundaryInitialConditionThread.h"
33 : #include "MaxQpsThread.h"
34 : #include "ActionWarehouse.h"
35 : #include "Conversion.h"
36 : #include "Material.h"
37 : #include "FunctorMaterial.h"
38 : #include "ConstantIC.h"
39 : #include "Parser.h"
40 : #include "ElementH1Error.h"
41 : #include "Function.h"
42 : #include "Convergence.h"
43 : #include "NonlinearSystem.h"
44 : #include "LinearSystem.h"
45 : #include "SolverSystem.h"
46 : #include "Distribution.h"
47 : #include "Sampler.h"
48 : #include "FVAdvectedInterpolationMethod.h"
49 : #include "FVFaceInterpolationMethod.h"
50 : #include "FVInterpolationMethod.h"
51 : #include "PetscSupport.h"
52 : #include "RandomInterface.h"
53 : #include "RandomData.h"
54 : #include "MooseEigenSystem.h"
55 : #include "MooseParsedFunction.h"
56 : #include "MeshChangedInterface.h"
57 : #include "MeshDisplacedInterface.h"
58 : #include "ComputeJacobianBlocksThread.h"
59 : #include "ScalarInitialCondition.h"
60 : #include "FVInitialConditionTempl.h"
61 : #include "ElementPostprocessor.h"
62 : #include "NodalPostprocessor.h"
63 : #include "SidePostprocessor.h"
64 : #include "InternalSidePostprocessor.h"
65 : #include "InterfacePostprocessor.h"
66 : #include "GeneralPostprocessor.h"
67 : #include "ElementVectorPostprocessor.h"
68 : #include "NodalVectorPostprocessor.h"
69 : #include "SideVectorPostprocessor.h"
70 : #include "InternalSideVectorPostprocessor.h"
71 : #include "GeneralVectorPostprocessor.h"
72 : #include "Positions.h"
73 : #include "Indicator.h"
74 : #include "Marker.h"
75 : #include "MultiApp.h"
76 : #include "MultiAppTransfer.h"
77 : #include "TransientMultiApp.h"
78 : #include "ElementUserObject.h"
79 : #include "DomainUserObject.h"
80 : #include "NodalUserObject.h"
81 : #include "SideUserObject.h"
82 : #include "InternalSideUserObject.h"
83 : #include "InterfaceUserObject.h"
84 : #include "GeneralUserObject.h"
85 : #include "ThreadedGeneralUserObject.h"
86 : #include "InternalSideIndicatorBase.h"
87 : #include "Transfer.h"
88 : #include "MultiAppTransfer.h"
89 : #include "MultiMooseEnum.h"
90 : #include "Predictor.h"
91 : #include "Assembly.h"
92 : #include "Control.h"
93 : #include "XFEMInterface.h"
94 : #include "ConsoleUtils.h"
95 : #include "NonlocalKernel.h"
96 : #include "NonlocalIntegratedBC.h"
97 : #include "ShapeElementUserObject.h"
98 : #include "ShapeSideUserObject.h"
99 : #include "MooseVariableFE.h"
100 : #include "MooseVariableScalar.h"
101 : #include "InputParameterWarehouse.h"
102 : #include "TimeIntegrator.h"
103 : #include "LineSearch.h"
104 : #include "FloatingPointExceptionGuard.h"
105 : #include "MaxVarNDofsPerElem.h"
106 : #include "MaxVarNDofsPerNode.h"
107 : #include "FVKernel.h"
108 : #include "LinearFVKernel.h"
109 : #include "FVTimeKernel.h"
110 : #include "MooseVariableFV.h"
111 : #include "MooseLinearVariableFV.h"
112 : #include "FVBoundaryCondition.h"
113 : #include "LinearFVBoundaryCondition.h"
114 : #include "FVInterfaceKernel.h"
115 : #include "Reporter.h"
116 : #include "ADUtils.h"
117 : #include "Executioner.h"
118 : #include "VariadicTable.h"
119 : #include "BoundaryNodeIntegrityCheckThread.h"
120 : #include "BoundaryElemIntegrityCheckThread.h"
121 : #include "NodalBCBase.h"
122 : #include "MortarUserObject.h"
123 : #include "MortarUserObjectThread.h"
124 : #include "RedistributeProperties.h"
125 : #include "Checkpoint.h"
126 : #include "MortarInterfaceWarehouse.h"
127 : #include "AutomaticMortarGeneration.h"
128 :
129 : #include "libmesh/exodusII_io.h"
130 : #include "libmesh/quadrature.h"
131 : #include "libmesh/coupling_matrix.h"
132 : #include "libmesh/nonlinear_solver.h"
133 : #include "libmesh/sparse_matrix.h"
134 : #include "libmesh/string_to_enum.h"
135 : #include "libmesh/fe_interface.h"
136 : #include "libmesh/enum_norm_type.h"
137 : #include "libmesh/petsc_solver_exception.h"
138 :
139 : #include "metaphysicl/dualnumber.h"
140 :
141 : // C++
142 : #include <cstring> // for "Jacobian" exception test
143 :
144 : using namespace libMesh;
145 :
146 : // Anonymous namespace for helper function
147 : namespace
148 : {
149 : /**
150 : * Method for sorting the MooseVariableFEBases based on variable numbers
151 : */
152 : bool
153 60 : sortMooseVariables(const MooseVariableFEBase * a, const MooseVariableFEBase * b)
154 : {
155 60 : return a->number() < b->number();
156 : }
157 : } // namespace
158 :
159 : Threads::spin_mutex get_function_mutex;
160 :
161 : InputParameters
162 190101 : FEProblemBase::validParams()
163 : {
164 190101 : InputParameters params = SubProblem::validParams();
165 760404 : params.addParam<unsigned int>("null_space_dimension", 0, "The dimension of the nullspace");
166 570303 : params.addParam<unsigned int>(
167 380202 : "transpose_null_space_dimension", 0, "The dimension of the transpose nullspace");
168 570303 : params.addParam<unsigned int>(
169 380202 : "near_null_space_dimension", 0, "The dimension of the near nullspace");
170 570303 : params.addParam<bool>("solve",
171 380202 : true,
172 : "Whether or not to actually solve the Nonlinear system. "
173 : "This is handy in the case that all you want to do is "
174 : "execute AuxKernels, Transfers, etc. without actually "
175 : "solving anything");
176 570303 : params.addParam<bool>("use_nonlinear",
177 380202 : true,
178 : "Determines whether to use a Nonlinear vs a "
179 : "Eigenvalue system (Automatically determined based "
180 : "on executioner)");
181 760404 : params.addParam<bool>("error_on_jacobian_nonzero_reallocation",
182 : "This causes PETSc to error if it had to reallocate memory in the Jacobian "
183 : "matrix due to not having enough nonzeros");
184 570303 : params.addParam<bool>("ignore_zeros_in_jacobian",
185 380202 : false,
186 : "Do not explicitly store zero values in "
187 : "the Jacobian matrix if true");
188 570303 : params.addParam<bool>("force_restart",
189 380202 : false,
190 : "EXPERIMENTAL: If true, a sub_app may use a "
191 : "restart file instead of using of using the master "
192 : "backup file");
193 950505 : params.addDeprecatedParam<bool>("skip_additional_restart_data",
194 380202 : false,
195 : "True to skip additional data in equation system for restart.",
196 : "This parameter is no longer used, as we do not load additional "
197 : "vectors by default with restart");
198 570303 : params.addParam<bool>("skip_nl_system_check",
199 380202 : false,
200 : "True to skip the NonlinearSystem check for work to do (e.g. Make sure "
201 : "that there are variables to solve for).");
202 570303 : params.addParam<bool>("allow_initial_conditions_with_restart",
203 380202 : false,
204 : "True to allow the user to specify initial conditions when restarting. "
205 : "Initial conditions can override any restarted field");
206 :
207 380202 : auto coverage_check_description = [](std::string scope, std::string list_param_name)
208 : {
209 760404 : return "Controls, if and how a " + scope +
210 : " subdomain coverage check is performed. "
211 : "With 'TRUE' or 'ON' all subdomains are checked (the default). Setting 'FALSE' or 'OFF' "
212 : "will disable the check for all subdomains. "
213 : "To exclude a predefined set of subdomains 'SKIP_LIST' is to "
214 760404 : "be used, while the subdomains to skip are to be defined in the parameter '" +
215 760404 : list_param_name +
216 : "'. To limit the check to a list of subdomains, 'ONLY_LIST' is to "
217 1140606 : "be used (again, using the parameter '" +
218 760404 : list_param_name + "').";
219 : };
220 :
221 950505 : params.addParam<std::vector<SubdomainName>>(
222 : "block",
223 : {"ANY_BLOCK_ID"},
224 : "List of subdomains for kernel coverage and material coverage checks. Setting this parameter "
225 : "is equivalent to setting 'kernel_coverage_block_list' and 'material_coverage_block_list' as "
226 : "well as using 'ONLY_LIST' as the coverage check mode.");
227 :
228 760404 : MooseEnum kernel_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
229 190101 : params.addParam<MooseEnum>("kernel_coverage_check",
230 : kernel_coverage_check_modes,
231 950505 : coverage_check_description("kernel", "kernel_coverage_block_list"));
232 760404 : params.addParam<std::vector<SubdomainName>>(
233 : "kernel_coverage_block_list",
234 : {},
235 : "List of subdomains for kernel coverage check. The meaning of this list is controlled by the "
236 : "parameter 'kernel_coverage_check' (whether this is the list of subdomains to be checked, "
237 : "not to be checked or not taken into account).");
238 570303 : params.addParam<bool>(
239 : "boundary_restricted_node_integrity_check",
240 380202 : true,
241 : "Set to false to disable checking of boundary restricted nodal object variable dependencies, "
242 : "e.g. are the variable dependencies defined on the selected boundaries?");
243 570303 : params.addParam<bool>("boundary_restricted_elem_integrity_check",
244 380202 : true,
245 : "Set to false to disable checking of boundary restricted elemental object "
246 : "variable dependencies, e.g. are the variable dependencies defined on the "
247 : "selected boundaries?");
248 760404 : MooseEnum material_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
249 190101 : params.addParam<MooseEnum>(
250 : "material_coverage_check",
251 : material_coverage_check_modes,
252 950505 : coverage_check_description("material", "material_coverage_block_list"));
253 760404 : params.addParam<std::vector<SubdomainName>>(
254 : "material_coverage_block_list",
255 : {},
256 : "List of subdomains for material coverage check. The meaning of this list is controlled by "
257 : "the parameter 'material_coverage_check' (whether this is the list of subdomains to be "
258 : "checked, not to be checked or not taken into account).");
259 :
260 570303 : params.addParam<bool>("fv_bcs_integrity_check",
261 380202 : true,
262 : "Set to false to disable checking of overlapping Dirichlet and Flux BCs "
263 : "and/or multiple DirichletBCs per sideset");
264 :
265 570303 : params.addParam<bool>(
266 380202 : "material_dependency_check", true, "Set to false to disable material dependency check");
267 570303 : params.addParam<bool>("parallel_barrier_messaging",
268 380202 : false,
269 : "Displays messaging from parallel "
270 : "barrier notifications when executing "
271 : "or transferring to/from Multiapps "
272 : "(default: false)");
273 :
274 760404 : MooseEnum verbosity("false true extra", "false");
275 760404 : params.addParam<MooseEnum>("verbose_setup",
276 : verbosity,
277 : "Set to 'true' to have the problem report on any object created. Set "
278 : "to 'extra' to also display all parameters.");
279 570303 : params.addParam<bool>("verbose_multiapps",
280 380202 : false,
281 : "Set to True to enable verbose screen printing related to MultiApps");
282 570303 : params.addParam<bool>(
283 : "verbose_restore",
284 380202 : false,
285 : "Set to True to enable verbose screen printing related to solution restoration");
286 :
287 760404 : params.addParam<FileNameNoExtension>("restart_file_base",
288 : "File base name used for restart (e.g. "
289 : "<path>/<filebase> or <path>/LATEST to "
290 : "grab the latest file available)");
291 :
292 760404 : params.addParam<std::vector<std::vector<TagName>>>(
293 : "extra_tag_vectors",
294 : {},
295 : "Extra vectors to add to the system that can be filled by objects which compute residuals "
296 : "and Jacobians (Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
297 : "nonlinear system the extra tag vectors should be added for");
298 :
299 760404 : params.addParam<std::vector<std::vector<TagName>>>(
300 : "not_zeroed_tag_vectors",
301 : {},
302 : "Extra vector tags which the sytem will not zero when other vector tags are zeroed. "
303 : "The outer index is for which nonlinear system the extra tag vectors should be added for");
304 :
305 760404 : params.addParam<std::vector<std::vector<TagName>>>(
306 : "extra_tag_matrices",
307 : {},
308 : "Extra matrices to add to the system that can be filled "
309 : "by objects which compute residuals and Jacobians "
310 : "(Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
311 : "nonlinear system the extra tag vectors should be added for");
312 :
313 760404 : params.addParam<std::vector<TagName>>(
314 : "extra_tag_solutions",
315 : {},
316 : "Extra solution vectors to add to the system that can be used by "
317 : "objects for coupling variable values stored in them.");
318 :
319 570303 : params.addParam<bool>("previous_nl_solution_required",
320 380202 : false,
321 : "True to indicate that this calculation requires a solution vector for "
322 : "storing the previous nonlinear iteration.");
323 :
324 570303 : params.addParam<std::vector<NonlinearSystemName>>(
325 760404 : "nl_sys_names", std::vector<NonlinearSystemName>{"nl0"}, "The nonlinear system names");
326 :
327 760404 : params.addParam<std::vector<LinearSystemName>>("linear_sys_names", {}, "The linear system names");
328 :
329 570303 : params.addParam<bool>("check_uo_aux_state",
330 380202 : false,
331 : "True to turn on a check that no state presents during the evaluation of "
332 : "user objects and aux kernels");
333 :
334 190101 : params.addPrivateParam<MooseMesh *>("mesh");
335 :
336 570303 : params.declareControllable("solve");
337 :
338 570303 : params.addParam<bool>(
339 : "allow_invalid_solution",
340 380202 : false,
341 : "Set to true to allow convergence even though the solution has been marked as 'invalid'");
342 570303 : params.addParam<bool>("show_invalid_solution_console",
343 380202 : true,
344 : "Set to true to show the invalid solution occurrence summary in console");
345 570303 : params.addParam<bool>("immediately_print_invalid_solution",
346 380202 : false,
347 : "Whether or not to report invalid solution warnings at the time the "
348 : "warning is produced instead of after the calculation");
349 :
350 570303 : params.addParam<bool>(
351 : "identify_variable_groups_in_nl",
352 380202 : true,
353 : "Whether to identify variable groups in nonlinear systems. This affects dof ordering");
354 :
355 570303 : params.addParam<bool>(
356 : "regard_general_exceptions_as_errors",
357 380202 : false,
358 : "If we catch an exception during residual/Jacobian evaluaton for which we don't have "
359 : "specific handling, immediately error instead of allowing the time step to be cut");
360 :
361 570303 : params.addParam<bool>("use_hash_table_matrix_assembly",
362 380202 : false,
363 : "Whether to assemble matrices using hash tables instead of preallocating "
364 : "matrix memory. This can be a good option if the sparsity pattern changes "
365 : "throughout the course of the simulation.");
366 760404 : params.addParam<bool>(
367 : "restore_original_nonzero_pattern",
368 : "Whether we should reset matrix memory for every Jacobian evaluation. This option is useful "
369 : "if the sparsity pattern is constantly changing and you are using hash table assembly or if "
370 : "you wish to continually restore the matrix to the originally preallocated sparsity pattern "
371 : "computed by relationship managers.");
372 :
373 760404 : params.addParamNamesToGroup(
374 : "skip_nl_system_check kernel_coverage_check kernel_coverage_block_list "
375 : "boundary_restricted_node_integrity_check "
376 : "boundary_restricted_elem_integrity_check material_coverage_check "
377 : "material_coverage_block_list fv_bcs_integrity_check "
378 : "material_dependency_check check_uo_aux_state error_on_jacobian_nonzero_reallocation",
379 : "Simulation checks");
380 760404 : params.addParamNamesToGroup("use_nonlinear previous_nl_solution_required nl_sys_names "
381 : "ignore_zeros_in_jacobian identify_variable_groups_in_nl "
382 : "use_hash_table_matrix_assembly restore_original_nonzero_pattern",
383 : "Nonlinear system(s)");
384 760404 : params.addParamNamesToGroup(
385 : "restart_file_base force_restart allow_initial_conditions_with_restart", "Restart");
386 760404 : params.addParamNamesToGroup(
387 : "verbose_setup verbose_multiapps verbose_restore parallel_barrier_messaging", "Verbosity");
388 760404 : params.addParamNamesToGroup(
389 : "null_space_dimension transpose_null_space_dimension near_null_space_dimension",
390 : "Null space removal");
391 760404 : params.addParamNamesToGroup(
392 : "extra_tag_vectors extra_tag_matrices extra_tag_solutions not_zeroed_tag_vectors",
393 : "Contribution to tagged field data");
394 570303 : params.addParamNamesToGroup(
395 : "allow_invalid_solution show_invalid_solution_console immediately_print_invalid_solution",
396 : "Solution validity control");
397 :
398 380202 : return params;
399 570303 : }
400 :
401 61643 : FEProblemBase::FEProblemBase(const InputParameters & parameters)
402 : : SubProblem(parameters),
403 : Restartable(this, "FEProblemBase"),
404 246572 : _mesh(*getCheckedPointerParam<MooseMesh *>("mesh")),
405 123286 : _req(declareManagedRestartableDataWithContext<RestartableEquationSystems>(
406 : "equation_systems", nullptr, _mesh)),
407 61643 : _initialized(false),
408 123286 : _solve(getParam<bool>("solve")),
409 61643 : _transient(false),
410 123286 : _time(declareRestartableData<Real>("time")),
411 123286 : _time_old(declareRestartableData<Real>("time_old")),
412 123286 : _t_step(declareRecoverableData<int>("t_step")),
413 123286 : _dt(declareRestartableData<Real>("dt")),
414 123286 : _dt_old(declareRestartableData<Real>("dt_old")),
415 61643 : _need_to_add_default_nonlinear_convergence(false),
416 61643 : _need_to_add_default_multiapp_fixed_point_convergence(false),
417 61643 : _need_to_add_default_steady_state_convergence(false),
418 123286 : _linear_sys_names(getParam<std::vector<LinearSystemName>>("linear_sys_names")),
419 61643 : _num_linear_sys(_linear_sys_names.size()),
420 123286 : _linear_systems(_num_linear_sys, nullptr),
421 61643 : _current_linear_sys(nullptr),
422 123286 : _using_default_nl(!isParamSetByUser("nl_sys_names")),
423 182969 : _nl_sys_names(!_using_default_nl || (_using_default_nl && !_linear_sys_names.size())
424 61643 : ? getParam<std::vector<NonlinearSystemName>>("nl_sys_names")
425 : : std::vector<NonlinearSystemName>()),
426 61643 : _num_nl_sys(_nl_sys_names.size()),
427 123286 : _nl(_num_nl_sys, nullptr),
428 61643 : _current_nl_sys(nullptr),
429 123286 : _solver_systems(_num_nl_sys + _num_linear_sys, nullptr),
430 61643 : _aux(nullptr),
431 61643 : _coupling(Moose::COUPLING_DIAG),
432 : #ifdef MOOSE_KOKKOS_ENABLED
433 46593 : _kokkos_assembly(*this),
434 : #endif
435 61643 : _mesh_divisions(/*threaded=*/true),
436 61643 : _material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
437 61643 : "material_props", &_mesh, _material_prop_registry, *this)),
438 61643 : _bnd_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
439 61643 : "bnd_material_props", &_mesh, _material_prop_registry, *this)),
440 61643 : _neighbor_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
441 61643 : "neighbor_material_props", &_mesh, _material_prop_registry, *this)),
442 : #ifdef MOOSE_KOKKOS_ENABLED
443 46593 : _kokkos_material_props(
444 46593 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
445 46593 : "kokkos_material_props", &_mesh, _material_prop_registry, *this)),
446 46593 : _kokkos_bnd_material_props(
447 46593 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
448 46593 : "kokkos_bnd_material_props", &_mesh, _material_prop_registry, *this)),
449 46593 : _kokkos_neighbor_material_props(
450 46593 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
451 46593 : "kokkos_neighbor_material_props", &_mesh, _material_prop_registry, *this)),
452 : #endif
453 61643 : _reporter_data(_app),
454 61643 : _multi_apps(_app.getExecuteOnEnum()),
455 61643 : _transient_multi_apps(_app.getExecuteOnEnum()),
456 61643 : _transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
457 61643 : _to_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
458 61643 : _from_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
459 61643 : _between_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
460 : #ifdef LIBMESH_ENABLE_AMR
461 61643 : _adaptivity(*this),
462 61643 : _cycles_completed(0),
463 : #endif
464 61643 : _displaced_mesh(nullptr),
465 61643 : _geometric_search_data(*this, _mesh),
466 61643 : _mortar_data(std::make_unique<MortarInterfaceWarehouse>(*this)),
467 61643 : _reinit_displaced_elem(false),
468 61643 : _reinit_displaced_face(false),
469 61643 : _reinit_displaced_neighbor(false),
470 61643 : _input_file_saved(false),
471 61643 : _has_dampers(false),
472 61643 : _has_constraints(false),
473 61643 : _snesmf_reuse_base(true),
474 61643 : _skip_exception_check(false),
475 61643 : _snesmf_reuse_base_set_by_user(false),
476 61643 : _has_initialized_stateful(false),
477 61643 : _const_jacobian(false),
478 61643 : _has_jacobian(false),
479 61643 : _needs_old_newton_iter(false),
480 123286 : _previous_nl_solution_required(getParam<bool>("previous_nl_solution_required")),
481 123286 : _previous_multiapp_fp_nl_solution_required(_num_nl_sys + _num_linear_sys, false),
482 61643 : _previous_multiapp_fp_aux_solution_required(false),
483 61643 : _has_nonlocal_coupling(false),
484 61643 : _calculate_jacobian_in_uo(false),
485 61643 : _kernel_coverage_check(
486 123286 : getParam<MooseEnum>("kernel_coverage_check").getEnum<CoverageCheckMode>()),
487 123286 : _kernel_coverage_blocks(getParam<std::vector<SubdomainName>>("kernel_coverage_block_list")),
488 61643 : _boundary_restricted_node_integrity_check(
489 123286 : getParam<bool>("boundary_restricted_node_integrity_check")),
490 61643 : _boundary_restricted_elem_integrity_check(
491 123286 : getParam<bool>("boundary_restricted_elem_integrity_check")),
492 61643 : _material_coverage_check(
493 123286 : getParam<MooseEnum>("material_coverage_check").getEnum<CoverageCheckMode>()),
494 123286 : _material_coverage_blocks(getParam<std::vector<SubdomainName>>("material_coverage_block_list")),
495 123286 : _fv_bcs_integrity_check(getParam<bool>("fv_bcs_integrity_check")),
496 123286 : _material_dependency_check(getParam<bool>("material_dependency_check")),
497 123286 : _uo_aux_state_check(getParam<bool>("check_uo_aux_state")),
498 : #ifndef NDEBUG
499 : _check_residual_for_nans(false),
500 : #endif
501 61643 : _max_qps(std::numeric_limits<unsigned int>::max()),
502 61643 : _max_scalar_order(INVALID_ORDER),
503 61643 : _has_time_integrator(false),
504 61643 : _has_exception(false),
505 123286 : _parallel_barrier_messaging(getParam<bool>("parallel_barrier_messaging")),
506 123286 : _verbose_setup(getParam<MooseEnum>("verbose_setup")),
507 123286 : _verbose_multiapps(getParam<bool>("verbose_multiapps")),
508 123286 : _verbose_restore(getParam<bool>("verbose_restore")),
509 61643 : _current_execute_on_flag(EXEC_NONE),
510 61643 : _control_warehouse(_app.getExecuteOnEnum(), /*threaded=*/false),
511 61643 : _is_petsc_options_inserted(false),
512 61643 : _line_search(nullptr),
513 61643 : _using_ad_mat_props(false),
514 61643 : _current_ic_state(0),
515 123286 : _use_hash_table_matrix_assembly(getParam<bool>("use_hash_table_matrix_assembly")),
516 61643 : _error_on_jacobian_nonzero_reallocation(
517 123286 : isParamValid("error_on_jacobian_nonzero_reallocation")
518 123813 : ? getParam<bool>("error_on_jacobian_nonzero_reallocation")
519 61116 : : _app.errorOnJacobianNonzeroReallocation()),
520 123286 : _restore_original_nonzero_pattern(isParamValid("restore_original_nonzero_pattern")
521 123286 : ? getParam<bool>("restore_original_nonzero_pattern")
522 61643 : : _use_hash_table_matrix_assembly),
523 123286 : _ignore_zeros_in_jacobian(getParam<bool>("ignore_zeros_in_jacobian")),
524 61643 : _preserve_matrix_sparsity_pattern(true),
525 123286 : _force_restart(getParam<bool>("force_restart")),
526 123286 : _allow_ics_during_restart(getParam<bool>("allow_initial_conditions_with_restart")),
527 123286 : _skip_nl_system_check(getParam<bool>("skip_nl_system_check")),
528 61643 : _fail_next_system_convergence_check(false),
529 123286 : _allow_invalid_solution(getParam<bool>("allow_invalid_solution")),
530 123286 : _show_invalid_solution_console(getParam<bool>("show_invalid_solution_console")),
531 123286 : _immediately_print_invalid_solution(getParam<bool>("immediately_print_invalid_solution")),
532 61643 : _started_initial_setup(false),
533 61643 : _has_internal_edge_residual_objects(false),
534 61643 : _u_dot_requested(false),
535 61643 : _u_dotdot_requested(false),
536 61643 : _u_dot_old_requested(false),
537 61643 : _u_dotdot_old_requested(false),
538 61643 : _has_mortar(false),
539 61643 : _num_grid_steps(0),
540 61643 : _print_execution_on(),
541 123286 : _identify_variable_groups_in_nl(getParam<bool>("identify_variable_groups_in_nl")),
542 61643 : _regard_general_exceptions_as_errors(getParam<bool>("regard_general_exceptions_as_errors")),
543 1171217 : _requires_nonlocal_coupling(false)
544 : {
545 : auto checkCoverageCheckConflict =
546 123286 : [this](const std::string & coverage_check,
547 : const CoverageCheckMode & coverage_check_mode,
548 : const std::vector<SubdomainName> & coverage_blocks) -> void
549 : {
550 123286 : if (coverage_check_mode != CoverageCheckMode::FALSE &&
551 118883 : coverage_check_mode != CoverageCheckMode::OFF)
552 118865 : if (coverage_blocks.size() > 1)
553 0 : if (std::find(coverage_blocks.begin(), coverage_blocks.end(), "ANY_BLOCK_ID") !=
554 0 : coverage_blocks.end())
555 0 : paramError(coverage_check,
556 : "The list of blocks used for ",
557 : coverage_check,
558 : " cannot contain 'ANY_BLOCK_ID' along with other blocks. ");
559 123286 : };
560 :
561 61643 : checkCoverageCheckConflict(
562 61643 : "kernel_coverage_check", _kernel_coverage_check, _kernel_coverage_blocks);
563 61643 : checkCoverageCheckConflict(
564 61643 : "material_coverage_check", _material_coverage_check, _material_coverage_blocks);
565 :
566 : // Initialize static do_derivatives member. We initialize this to true so that all the
567 : // default AD things that we setup early in the simulation actually get their derivative
568 : // vectors initalized. We will toggle this to false when doing residual evaluations
569 61643 : ADReal::do_derivatives = true;
570 :
571 : // Disable refinement/coarsening in EquationSystems::reinit because we already do this ourselves
572 61643 : es().disable_refine_in_reinit();
573 :
574 61643 : _solver_params.reserve(_num_nl_sys + _num_linear_sys);
575 : // Default constructor fine for nonlinear because it will be populated later by framework
576 : // executioner/solve object parameters
577 61643 : _solver_params.resize(_num_nl_sys);
578 122525 : for (const auto i : index_range(_nl_sys_names))
579 : {
580 60882 : const auto & name = _nl_sys_names[i];
581 60882 : _nl_sys_name_to_num[name] = i;
582 60882 : _solver_sys_name_to_num[name] = i;
583 60882 : _solver_sys_names.push_back(name);
584 : }
585 :
586 62675 : for (const auto i : index_range(_linear_sys_names))
587 : {
588 1032 : const auto & name = _linear_sys_names[i];
589 1032 : _linear_sys_name_to_num[name] = i;
590 1032 : _solver_sys_name_to_num[name] = i + _num_nl_sys;
591 1032 : _solver_sys_names.push_back(name);
592 : // Unlike for nonlinear these are basically dummy parameters
593 1032 : _solver_params.push_back(makeLinearSolverParams());
594 : }
595 :
596 61643 : _nonlocal_cm.resize(numSolverSystems());
597 61643 : _cm.resize(numSolverSystems());
598 :
599 61643 : _time = 0.0;
600 61643 : _time_old = 0.0;
601 61643 : _t_step = 0;
602 61643 : _dt = 0;
603 61643 : _dt_old = _dt;
604 :
605 61643 : unsigned int n_threads = libMesh::n_threads();
606 :
607 61643 : _real_zero.resize(n_threads, 0.);
608 61643 : _scalar_zero.resize(n_threads);
609 61643 : _zero.resize(n_threads);
610 61643 : _phi_zero.resize(n_threads);
611 61643 : _ad_zero.resize(n_threads);
612 61643 : _grad_zero.resize(n_threads);
613 61643 : _ad_grad_zero.resize(n_threads);
614 61643 : _grad_phi_zero.resize(n_threads);
615 61643 : _second_zero.resize(n_threads);
616 61643 : _ad_second_zero.resize(n_threads);
617 61643 : _second_phi_zero.resize(n_threads);
618 61643 : _point_zero.resize(n_threads);
619 61643 : _vector_zero.resize(n_threads);
620 61643 : _vector_curl_zero.resize(n_threads);
621 61643 : _uo_jacobian_moose_vars.resize(n_threads);
622 :
623 61643 : _has_active_material_properties.resize(n_threads, 0);
624 :
625 61643 : _block_mat_side_cache.resize(n_threads);
626 61643 : _bnd_mat_side_cache.resize(n_threads);
627 61643 : _interface_mat_side_cache.resize(n_threads);
628 :
629 123286 : es().parameters.set<FEProblemBase *>("_fe_problem_base") = this;
630 :
631 184929 : if (isParamValid("restart_file_base"))
632 : {
633 934 : std::string restart_file_base = getParam<FileNameNoExtension>("restart_file_base");
634 :
635 : // This check reverts to old behavior of providing "restart_file_base=" to mean
636 : // don't restart... BISON currently relies on this. It could probably be removed.
637 : // The new MooseUtils::convertLatestCheckpoint will error out if a checkpoint file
638 : // is not found, which I think makes sense. Which means, without this, if you
639 : // set "restart_file_base=", you'll get a "No checkpoint file found" error
640 467 : if (restart_file_base.size())
641 : {
642 467 : restart_file_base = MooseUtils::convertLatestCheckpoint(restart_file_base);
643 467 : setRestartFile(restart_file_base);
644 : }
645 467 : }
646 :
647 : // // Generally speaking, the mesh is prepared for use, and consequently remote elements are deleted
648 : // // well before our Problem(s) are constructed. Historically, in MooseMesh we have a bunch of
649 : // // needs_prepare type flags that make it so we never call prepare_for_use (and consequently
650 : // // delete_remote_elements) again. So the below line, historically, has had no impact. HOWEVER:
651 : // // I've added some code in SetupMeshCompleteAction for deleting remote elements post
652 : // // EquationSystems::init. If I execute that code without default ghosting, then I get > 40 MOOSE
653 : // // test failures, so we clearly have some simulations that are not yet covered properly by
654 : // // relationship managers. Until that is resolved, I am going to retain default geometric ghosting
655 : // if (!_default_ghosting)
656 : // _mesh.getMesh().remove_ghosting_functor(_mesh.getMesh().default_ghosting());
657 :
658 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
659 : // Main app should hold the default database to handle system petsc options
660 61643 : if (!_app.isUltimateMaster())
661 12088 : LibmeshPetscCall(PetscOptionsCreate(&_petsc_option_data_base));
662 : #endif
663 :
664 61643 : if (!_solve)
665 : {
666 : // If we are not solving, we do not care about seeing unused petsc options
667 49098 : Moose::PetscSupport::setSinglePetscOption("-options_left", "0");
668 : // We don't want petscSetOptions being called in solve and clearing the option that was just set
669 16366 : _is_petsc_options_inserted = true;
670 : }
671 61643 : }
672 :
673 : const MooseMesh &
674 0 : FEProblemBase::mesh(bool use_displaced) const
675 : {
676 0 : if (use_displaced && !_displaced_problem)
677 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
678 : "Regular mesh will be returned");
679 0 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
680 : }
681 :
682 : MooseMesh &
683 359630 : FEProblemBase::mesh(bool use_displaced)
684 : {
685 359630 : if (use_displaced && !_displaced_problem)
686 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
687 : "Regular mesh will be returned");
688 359630 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
689 : }
690 :
691 : void
692 61643 : FEProblemBase::createTagVectors()
693 : {
694 : // add vectors and their tags to system
695 123286 : auto & vectors = getParam<std::vector<std::vector<TagName>>>("extra_tag_vectors");
696 62359 : for (const auto sys_num : index_range(vectors))
697 1704 : for (auto & vector : vectors[sys_num])
698 : {
699 988 : auto tag = addVectorTag(vector);
700 988 : _solver_systems[sys_num]->addVector(tag, false, libMesh::GHOSTED);
701 : }
702 :
703 123286 : auto & not_zeroed_vectors = getParam<std::vector<std::vector<TagName>>>("not_zeroed_tag_vectors");
704 61654 : for (const auto sys_num : index_range(not_zeroed_vectors))
705 22 : for (auto & vector : not_zeroed_vectors[sys_num])
706 : {
707 11 : auto tag = addVectorTag(vector);
708 11 : _solver_systems[sys_num]->addVector(tag, false, GHOSTED);
709 11 : addNotZeroedVectorTag(tag);
710 : }
711 61643 : }
712 :
713 : void
714 60760 : FEProblemBase::createTagMatrices(CreateTaggedMatrixKey)
715 : {
716 121520 : auto & matrices = getParam<std::vector<std::vector<TagName>>>("extra_tag_matrices");
717 61006 : for (const auto sys_num : index_range(matrices))
718 692 : for (auto & matrix : matrices[sys_num])
719 : {
720 446 : auto tag = addMatrixTag(matrix);
721 446 : _solver_systems[sys_num]->addMatrix(tag);
722 : }
723 :
724 121767 : for (auto & sys : _solver_systems)
725 61007 : sys->sizeVariableMatrixData();
726 60760 : _aux->sizeVariableMatrixData();
727 60760 : }
728 :
729 : void
730 61643 : FEProblemBase::createTagSolutions()
731 : {
732 184954 : for (auto & vector : getParam<std::vector<TagName>>("extra_tag_solutions"))
733 : {
734 25 : auto tag = addVectorTag(vector, Moose::VECTOR_TAG_SOLUTION);
735 50 : for (auto & sys : _solver_systems)
736 25 : sys->addVector(tag, false, libMesh::GHOSTED);
737 25 : _aux->addVector(tag, false, libMesh::GHOSTED);
738 : }
739 :
740 61643 : if (_previous_nl_solution_required)
741 : {
742 : // We'll populate the zeroth state of the nonlinear iterations with the current solution for
743 : // ease of use in doing things like copying solutions backwards. We're just storing pointers in
744 : // the solution states containers so populating the zeroth state does not cost us the memory of
745 : // a new vector
746 83 : needSolutionState(1, Moose::SolutionIterationType::Nonlinear);
747 : }
748 :
749 61643 : auto tag = addVectorTag(Moose::SOLUTION_TAG, Moose::VECTOR_TAG_SOLUTION);
750 123557 : for (auto & sys : _solver_systems)
751 61914 : sys->associateVectorToTag(*sys->system().current_local_solution.get(), tag);
752 61643 : _aux->associateVectorToTag(*_aux->system().current_local_solution.get(), tag);
753 61643 : }
754 :
755 : void
756 135 : FEProblemBase::needSolutionState(unsigned int state, Moose::SolutionIterationType iteration_type)
757 : {
758 270 : for (auto & sys : _solver_systems)
759 135 : sys->needSolutionState(state, iteration_type);
760 135 : _aux->needSolutionState(state, iteration_type);
761 135 : }
762 :
763 : bool
764 48 : FEProblemBase::hasSolutionState(unsigned int state,
765 : Moose::SolutionIterationType iteration_type) const
766 : {
767 48 : bool has_solution_state = false;
768 107 : for (auto & sys : _solver_systems)
769 59 : has_solution_state |= sys->hasSolutionState(state, iteration_type);
770 48 : has_solution_state |= _aux->hasSolutionState(state, iteration_type);
771 48 : return has_solution_state;
772 : }
773 :
774 : void
775 61643 : FEProblemBase::newAssemblyArray(std::vector<std::shared_ptr<SolverSystem>> & solver_systems)
776 : {
777 61643 : unsigned int n_threads = libMesh::n_threads();
778 :
779 61643 : _assembly.resize(n_threads);
780 130097 : for (const auto i : make_range(n_threads))
781 : {
782 68454 : _assembly[i].resize(solver_systems.size());
783 137204 : for (const auto j : index_range(solver_systems))
784 68750 : _assembly[i][j] = std::make_unique<Assembly>(*solver_systems[j], i);
785 : }
786 61643 : }
787 :
788 : void
789 58855 : FEProblemBase::initNullSpaceVectors(const InputParameters & parameters,
790 : std::vector<std::shared_ptr<NonlinearSystemBase>> & nls)
791 : {
792 294275 : TIME_SECTION("initNullSpaceVectors", 5, "Initializing Null Space Vectors");
793 :
794 58855 : unsigned int dimNullSpace = parameters.get<unsigned int>("null_space_dimension");
795 : unsigned int dimTransposeNullSpace =
796 58855 : parameters.get<unsigned int>("transpose_null_space_dimension");
797 58855 : unsigned int dimNearNullSpace = parameters.get<unsigned int>("near_null_space_dimension");
798 58879 : for (unsigned int i = 0; i < dimNullSpace; ++i)
799 : {
800 24 : std::ostringstream oss;
801 24 : oss << "_" << i;
802 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
803 : // builder might march over all nodes
804 48 : for (auto & nl : nls)
805 24 : nl->addVector("NullSpace" + oss.str(), false, libMesh::GHOSTED);
806 24 : }
807 117710 : _subspace_dim["NullSpace"] = dimNullSpace;
808 58867 : for (unsigned int i = 0; i < dimTransposeNullSpace; ++i)
809 : {
810 12 : std::ostringstream oss;
811 12 : oss << "_" << i;
812 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
813 : // builder might march over all nodes
814 24 : for (auto & nl : nls)
815 12 : nl->addVector("TransposeNullSpace" + oss.str(), false, libMesh::GHOSTED);
816 12 : }
817 117710 : _subspace_dim["TransposeNullSpace"] = dimTransposeNullSpace;
818 58855 : for (unsigned int i = 0; i < dimNearNullSpace; ++i)
819 : {
820 0 : std::ostringstream oss;
821 0 : oss << "_" << i;
822 : // do not project, since this will be recomputed, but make it ghosted, since the near-nullspace
823 : // builder might march over all semilocal nodes
824 0 : for (auto & nl : nls)
825 0 : nl->addVector("NearNullSpace" + oss.str(), false, libMesh::GHOSTED);
826 0 : }
827 117710 : _subspace_dim["NearNullSpace"] = dimNearNullSpace;
828 58855 : }
829 :
830 175689 : FEProblemBase::~FEProblemBase()
831 : {
832 : // Flush the Console stream, the underlying call to Console::mooseConsole
833 : // relies on a call to Output::checkInterval that has references to
834 : // _time, etc. If it is not flushed here memory problems arise if you have
835 : // an unflushed stream and start destructing things.
836 58563 : _console << std::flush;
837 :
838 58563 : unsigned int n_threads = libMesh::n_threads();
839 122661 : for (unsigned int i = 0; i < n_threads; i++)
840 : {
841 64098 : _zero[i].release();
842 64098 : _phi_zero[i].release();
843 64098 : _scalar_zero[i].release();
844 64098 : _grad_zero[i].release();
845 64098 : _grad_phi_zero[i].release();
846 64098 : _second_zero[i].release();
847 64098 : _second_phi_zero[i].release();
848 64098 : _vector_zero[i].release();
849 64098 : _vector_curl_zero[i].release();
850 64098 : _ad_zero[i].release();
851 64098 : _ad_grad_zero[i].release();
852 64098 : _ad_second_zero[i].release();
853 : }
854 :
855 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
856 58563 : if (!_app.isUltimateMaster())
857 : {
858 11324 : auto ierr = PetscOptionsDestroy(&_petsc_option_data_base);
859 : // Don't throw on destruction
860 11324 : CHKERRABORT(this->comm().get(), ierr);
861 : }
862 : #endif
863 58563 : }
864 :
865 : void
866 0 : FEProblemBase::setCoordSystem(const std::vector<SubdomainName> & blocks,
867 : const MultiMooseEnum & coord_sys)
868 : {
869 0 : TIME_SECTION("setCoordSystem", 5, "Setting Coordinate System");
870 0 : _mesh.setCoordSystem(blocks, coord_sys);
871 0 : }
872 :
873 : void
874 0 : FEProblemBase::setAxisymmetricCoordAxis(const MooseEnum & rz_coord_axis)
875 : {
876 0 : _mesh.setAxisymmetricCoordAxis(rz_coord_axis);
877 0 : }
878 :
879 : const ConstElemRange &
880 1383 : FEProblemBase::getEvaluableElementRange()
881 : {
882 1383 : if (!_evaluable_local_elem_range)
883 : {
884 677 : std::vector<const DofMap *> dof_maps(es().n_systems());
885 2031 : for (const auto i : make_range(es().n_systems()))
886 : {
887 1354 : const auto & sys = es().get_system(i);
888 1354 : dof_maps[i] = &sys.get_dof_map();
889 : }
890 : _evaluable_local_elem_range =
891 1354 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
892 2031 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
893 677 : }
894 1383 : return *_evaluable_local_elem_range;
895 : }
896 :
897 : const ConstElemRange &
898 208 : FEProblemBase::getNonlinearEvaluableElementRange()
899 : {
900 208 : if (!_nl_evaluable_local_elem_range)
901 : {
902 208 : std::vector<const DofMap *> dof_maps(_nl.size());
903 416 : for (const auto i : index_range(dof_maps))
904 208 : dof_maps[i] = &_nl[i]->dofMap();
905 : _nl_evaluable_local_elem_range =
906 416 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
907 624 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
908 208 : }
909 :
910 208 : return *_nl_evaluable_local_elem_range;
911 : }
912 :
913 : void
914 59414 : FEProblemBase::initialSetup()
915 : {
916 297070 : TIME_SECTION("initialSetup", 2, "Performing Initial Setup");
917 :
918 59414 : SubProblem::initialSetup();
919 :
920 59414 : if (_app.isRecovering() + _app.isRestarting() + bool(_app.getExReaderForRestart()) > 1)
921 0 : mooseError("Checkpoint recovery and restart and exodus restart are all mutually exclusive.");
922 :
923 59414 : if (_skip_exception_check)
924 9 : mooseWarning("MOOSE may fail to catch an exception when the \"skip_exception_check\" parameter "
925 : "is used. If you receive a terse MPI error during execution, remove this "
926 : "parameter and rerun your simulation");
927 :
928 : // set state flag indicating that we are in or beyond initialSetup.
929 : // This can be used to throw errors in methods that _must_ be called at construction time.
930 59414 : _started_initial_setup = true;
931 59414 : setCurrentExecuteOnFlag(EXEC_INITIAL);
932 :
933 : // Setup the solution states (current, old, etc) in each system based on
934 : // its default and the states requested of each of its variables
935 119075 : for (const auto i : index_range(_solver_systems))
936 : {
937 59661 : _solver_systems[i]->initSolutionState();
938 59661 : if (getDisplacedProblem())
939 2013 : getDisplacedProblem()->solverSys(i).initSolutionState();
940 : }
941 59414 : _aux->initSolutionState();
942 59414 : if (getDisplacedProblem())
943 2013 : getDisplacedProblem()->auxSys().initSolutionState();
944 :
945 : // always execute to get the max number of DoF per element and node needed to initialize phi_zero
946 : // variables
947 59414 : dof_id_type global_max_var_n_dofs_per_elem = 0;
948 119075 : for (const auto i : index_range(_solver_systems))
949 : {
950 59661 : auto & sys = *_solver_systems[i];
951 : dof_id_type max_var_n_dofs_per_elem;
952 : dof_id_type max_var_n_dofs_per_node;
953 : {
954 298305 : TIME_SECTION("computingMaxDofs", 3, "Computing Max Dofs Per Element");
955 :
956 59661 : MaxVarNDofsPerElem mvndpe(*this, sys);
957 59661 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mvndpe);
958 59661 : max_var_n_dofs_per_elem = mvndpe.max();
959 59661 : _communicator.max(max_var_n_dofs_per_elem);
960 :
961 59661 : MaxVarNDofsPerNode mvndpn(*this, sys);
962 59661 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), mvndpn);
963 59661 : max_var_n_dofs_per_node = mvndpn.max();
964 59661 : _communicator.max(max_var_n_dofs_per_node);
965 59661 : global_max_var_n_dofs_per_elem =
966 59661 : std::max(global_max_var_n_dofs_per_elem, max_var_n_dofs_per_elem);
967 59661 : }
968 :
969 : {
970 298305 : TIME_SECTION("assignMaxDofs", 5, "Assigning Maximum Dofs Per Elem");
971 :
972 59661 : sys.assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
973 59661 : auto displaced_problem = getDisplacedProblem();
974 59661 : if (displaced_problem)
975 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
976 :
977 59661 : sys.assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
978 59661 : if (displaced_problem)
979 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
980 59661 : }
981 : }
982 :
983 : {
984 297070 : TIME_SECTION("resizingVarValues", 5, "Resizing Variable Values");
985 :
986 124799 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
987 : {
988 130770 : _phi_zero[tid].resize(global_max_var_n_dofs_per_elem, std::vector<Real>(getMaxQps(), 0.));
989 130770 : _grad_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
990 130770 : std::vector<RealGradient>(getMaxQps(), RealGradient(0.)));
991 130770 : _second_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
992 130770 : std::vector<RealTensor>(getMaxQps(), RealTensor(0.)));
993 : }
994 59414 : }
995 :
996 : // Set up stateful material property redistribution, if we suspect
997 : // it may be necessary later.
998 59414 : addAnyRedistributers();
999 :
1000 59414 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1001 : {
1002 : // Only load all of the vectors if we're recovering
1003 4781 : _req.set().setLoadAllVectors(_app.isRecovering());
1004 :
1005 : // This forces stateful material property loading to be an exact one-to-one match
1006 4781 : if (_app.isRecovering())
1007 : {
1008 17012 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1009 12759 : props->setRecovering();
1010 :
1011 : #ifdef MOOSE_KOKKOS_ENABLED
1012 16872 : for (auto props :
1013 21090 : {&_kokkos_material_props, &_kokkos_bnd_material_props, &_kokkos_neighbor_material_props})
1014 12654 : props->setRecovering();
1015 : #endif
1016 : }
1017 :
1018 23905 : TIME_SECTION("restore", 3, "Restoring from backup");
1019 :
1020 : // We could have a cached backup when this app is a sub-app and has been given a Backup
1021 4781 : if (!_app.hasInitialBackup())
1022 3735 : _app.restore(_app.restartFolderBase(_app.getRestartRecoverFileBase()), _app.isRestarting());
1023 : else
1024 1046 : _app.restoreFromInitialBackup(_app.isRestarting());
1025 :
1026 : /**
1027 : * If this is a restart run, the user may want to override the start time, which we already set
1028 : * in the constructor. "_time" however will have been "restored" from the restart file. We need
1029 : * to honor the original request of the developer now that the restore has been completed.
1030 : */
1031 4748 : if (_app.isRestarting())
1032 : {
1033 495 : if (_app.hasStartTime())
1034 173 : _time = _time_old = _app.getStartTime();
1035 : else
1036 322 : _time_old = _time;
1037 : }
1038 4748 : }
1039 : else
1040 : {
1041 54633 : libMesh::ExodusII_IO * reader = _app.getExReaderForRestart();
1042 :
1043 54633 : if (reader)
1044 : {
1045 1890 : TIME_SECTION("copyingFromExodus", 3, "Copying Variables From Exodus");
1046 :
1047 764 : for (auto & sys : _solver_systems)
1048 389 : sys->copyVars(*reader);
1049 375 : _aux->copyVars(*reader);
1050 375 : }
1051 : else
1052 : {
1053 54255 : if (_solver_systems[0]->hasVarCopy() || _aux->hasVarCopy())
1054 0 : mooseError("Need Exodus reader to restart variables but the reader is not available\n"
1055 : "Use either FileMesh with an Exodus mesh file or FileMeshGenerator with an "
1056 : "Exodus mesh file and with use_for_exodus_restart equal to true");
1057 : }
1058 : }
1059 :
1060 : // Perform output related setups
1061 59378 : _app.getOutputWarehouse().initialSetup();
1062 :
1063 : // Flush all output to _console that occur during construction and initialization of objects
1064 59332 : _app.getOutputWarehouse().mooseConsole();
1065 :
1066 : // Build Refinement and Coarsening maps for stateful material projections if necessary
1067 61585 : if ((_adaptivity.isOn() || _num_grid_steps) &&
1068 2253 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1069 2190 : _neighbor_material_props.hasStatefulProperties()))
1070 : {
1071 63 : if (_has_internal_edge_residual_objects)
1072 6 : mooseError("Stateful neighbor material properties do not work with mesh adaptivity");
1073 :
1074 57 : _mesh.buildRefinementAndCoarseningMaps(_assembly[0][0].get());
1075 : }
1076 :
1077 59326 : if (!_app.isRecovering())
1078 : {
1079 : /**
1080 : * If we are not recovering but we are doing restart (_app.getExodusFileRestart() == true) with
1081 : * additional uniform refinements. We have to delay the refinement until this point
1082 : * in time so that the equation systems are initialized and projections can be performed.
1083 : */
1084 55073 : if (_mesh.uniformRefineLevel() > 0 && _app.getExodusFileRestart())
1085 : {
1086 10 : if (!_app.isUltimateMaster())
1087 0 : mooseError(
1088 : "Doing extra refinements when restarting is NOT supported for sub-apps of a MultiApp");
1089 :
1090 10 : adaptivity().uniformRefineWithProjection();
1091 : }
1092 : }
1093 :
1094 59326 : unsigned int n_threads = libMesh::n_threads();
1095 :
1096 : // Convergence initial setup
1097 : {
1098 296630 : TIME_SECTION("convergenceInitialSetup", 5, "Initializing Convergence objects");
1099 :
1100 124572 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1101 65261 : _convergences.initialSetup(tid);
1102 59311 : }
1103 :
1104 : // UserObject initialSetup
1105 59311 : std::set<std::string> depend_objects_ic = _ics.getDependObjects();
1106 59311 : std::set<std::string> depend_objects_aux = _aux->getDependObjects();
1107 :
1108 59311 : std::map<int, std::vector<UserObjectBase *>> group_userobjs;
1109 :
1110 : // This replaces all prior updateDependObjects calls on the old user object warehouses.
1111 59311 : TheWarehouse::Query uo_query = theWarehouse().query().condition<AttribSystem>("UserObject");
1112 59311 : std::vector<UserObjectBase *> userobjs;
1113 59311 : uo_query.queryInto(userobjs);
1114 59311 : groupUserObjects(
1115 59311 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1116 :
1117 130182 : for (auto obj : userobjs)
1118 212613 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1119 :
1120 : #ifdef MOOSE_KOKKOS_ENABLED
1121 : {
1122 : TheWarehouse::Query uo_query =
1123 44877 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject");
1124 44877 : std::vector<UserObjectBase *> userobjs;
1125 44877 : uo_query.queryInto(userobjs);
1126 44877 : groupUserObjects(
1127 44877 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1128 :
1129 46102 : for (auto obj : userobjs)
1130 3675 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1131 44877 : }
1132 : #endif
1133 :
1134 89187 : for (auto & [group, objs] : group_userobjs)
1135 101919 : for (auto obj : objs)
1136 72043 : obj->initialSetup();
1137 :
1138 : // check if jacobian calculation is done in userobject
1139 124371 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1140 65140 : checkUserObjectJacobianRequirement(tid);
1141 :
1142 : // Check whether nonlocal coupling is required or not
1143 59231 : checkNonlocalCoupling();
1144 59231 : if (_requires_nonlocal_coupling)
1145 63 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
1146 :
1147 : {
1148 296155 : TIME_SECTION("initializingFunctions", 5, "Initializing Functions");
1149 :
1150 : // Call the initialSetup methods for functions
1151 124339 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1152 : {
1153 65132 : reinitScalars(tid); // initialize scalars so they are properly sized for use as input into
1154 : // ParsedFunctions
1155 65132 : _functions.initialSetup(tid);
1156 : }
1157 :
1158 : #ifdef MOOSE_KOKKOS_ENABLED
1159 44808 : _kokkos_functions.initialSetup();
1160 : #endif
1161 59207 : }
1162 :
1163 : {
1164 296035 : TIME_SECTION("initializingRandomObjects", 5, "Initializing Random Objects");
1165 :
1166 : // Random interface objects
1167 59531 : for (const auto & it : _random_data_objects)
1168 324 : it.second->updateSeeds(EXEC_INITIAL);
1169 59207 : }
1170 :
1171 59207 : if (!_app.isRecovering())
1172 : {
1173 54954 : computeUserObjects(EXEC_INITIAL, Moose::PRE_IC);
1174 :
1175 : {
1176 274770 : TIME_SECTION("ICinitialSetup", 5, "Setting Up Initial Conditions");
1177 :
1178 115756 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1179 60808 : _ics.initialSetup(tid);
1180 :
1181 54948 : _scalar_ics.initialSetup();
1182 54948 : }
1183 :
1184 54948 : projectSolution();
1185 : }
1186 :
1187 : // Materials
1188 59195 : if (_all_materials.hasActiveObjects(0))
1189 : {
1190 41845 : TIME_SECTION("materialInitialSetup", 3, "Setting Up Materials");
1191 :
1192 17465 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1193 : {
1194 : // Sort the Material objects, these will be actually computed by MOOSE in reinit methods.
1195 9114 : _materials.sort(tid);
1196 9111 : _interface_materials.sort(tid);
1197 :
1198 : // Call initialSetup on all material objects
1199 9111 : _all_materials.initialSetup(tid);
1200 :
1201 : // Discrete materials may insert additional dependencies on materials during the initial
1202 : // setup. Therefore we resolve the dependencies once more, now with the additional
1203 : // dependencies due to discrete materials.
1204 9096 : if (_discrete_materials.hasActiveObjects())
1205 : {
1206 56 : _materials.sort(tid);
1207 56 : _interface_materials.sort(tid);
1208 : }
1209 : }
1210 :
1211 : #ifdef MOOSE_KOKKOS_ENABLED
1212 6291 : _kokkos_materials.sort(0, true);
1213 : #endif
1214 :
1215 : {
1216 41745 : TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");
1217 :
1218 8349 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1219 :
1220 16023 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1221 7674 : _neighbor_material_props.hasStatefulProperties())
1222 675 : _has_initialized_stateful = true;
1223 : #ifdef MOOSE_KOKKOS_ENABLED
1224 6289 : if (_kokkos_material_props.hasStatefulProperties() ||
1225 12361 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1226 6072 : _kokkos_neighbor_material_props.hasStatefulProperties())
1227 217 : _has_initialized_stateful = true;
1228 : #endif
1229 8349 : }
1230 8349 : }
1231 :
1232 : // setRestartInPlace() is set because the property maps have now been setup and we can
1233 : // dataLoad() them directly in place
1234 : // setRecovering() is set because from now on we require a one-to-one mapping of
1235 : // stateful properties because we shouldn't be declaring any more
1236 236700 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1237 : {
1238 177525 : props->setRestartInPlace();
1239 177525 : props->setRecovering();
1240 : }
1241 :
1242 124240 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1243 : {
1244 65065 : _internal_side_indicators.initialSetup(tid);
1245 65065 : _indicators.initialSetup(tid);
1246 65065 : _markers.sort(tid);
1247 65065 : _markers.initialSetup(tid);
1248 : }
1249 :
1250 : #ifdef LIBMESH_ENABLE_AMR
1251 :
1252 59175 : if (!_app.isRecovering())
1253 : {
1254 54922 : unsigned int n = adaptivity().getInitialSteps();
1255 54922 : if (n && !_app.isUltimateMaster() && _app.isRestarting())
1256 0 : mooseError("Cannot perform initial adaptivity during restart on sub-apps of a MultiApp!");
1257 :
1258 54922 : initialAdaptMesh();
1259 : }
1260 :
1261 : #endif // LIBMESH_ENABLE_AMR
1262 :
1263 59172 : if (!_app.isRecovering() && !_app.isRestarting())
1264 : {
1265 : // During initial setup the solution is copied to the older solution states (old, older, etc)
1266 54424 : copySolutionsBackwards();
1267 :
1268 : // Check if there are old state initial conditions
1269 54424 : auto ics = _ics.getActiveObjects();
1270 54424 : auto fv_ics = _fv_ics.getActiveObjects();
1271 54424 : auto scalar_ics = _scalar_ics.getActiveObjects();
1272 54424 : unsigned short ic_state_max = 0;
1273 :
1274 163272 : auto findMax = [&ic_state_max](const auto & obj_list)
1275 : {
1276 193504 : for (auto ic : obj_list.getActiveObjects())
1277 30232 : ic_state_max = std::max(ic_state_max, ic->getState());
1278 217696 : };
1279 54424 : findMax(_ics);
1280 54424 : findMax(_fv_ics);
1281 54424 : findMax(_scalar_ics);
1282 :
1283 : // if there are old state ICs, compute them and write to old states accordingly
1284 54424 : if (ic_state_max > 0)
1285 : {
1286 : // state 0 copy (we'll overwrite current state when evaluating ICs and need to restore it once
1287 : // we're done with the old/older state ICs)
1288 0 : std::vector<std::unique_ptr<NumericVector<Real>>> state0_sys_buffers(_solver_systems.size());
1289 0 : std::unique_ptr<NumericVector<Real>> state0_aux_buffer;
1290 :
1291 : // save state 0
1292 0 : for (const auto i : index_range(_solver_systems))
1293 0 : state0_sys_buffers[i] = _solver_systems[i]->solutionState(0).clone();
1294 :
1295 0 : state0_aux_buffer = _aux->solutionState(0).clone();
1296 :
1297 : // compute old state ICs
1298 0 : for (_current_ic_state = 1; _current_ic_state <= ic_state_max; _current_ic_state++)
1299 : {
1300 0 : projectSolution();
1301 :
1302 0 : for (auto & sys : _solver_systems)
1303 0 : sys->solutionState(_current_ic_state) = sys->solutionState(0);
1304 :
1305 0 : _aux->solutionState(_current_ic_state) = _aux->solutionState(0);
1306 : }
1307 0 : _current_ic_state = 0;
1308 :
1309 : // recover state 0
1310 0 : for (const auto i : index_range(_solver_systems))
1311 : {
1312 0 : _solver_systems[i]->solutionState(0) = *state0_sys_buffers[i];
1313 0 : _solver_systems[i]->solutionState(0).close();
1314 0 : _solver_systems[i]->update();
1315 : }
1316 0 : _aux->solutionState(0) = *state0_aux_buffer;
1317 0 : _aux->solutionState(0).close();
1318 0 : _aux->update();
1319 0 : }
1320 54424 : }
1321 :
1322 59172 : if (!_app.isRecovering())
1323 : {
1324 54919 : if (haveXFEM())
1325 0 : updateMeshXFEM();
1326 : }
1327 :
1328 : // Call initialSetup on the solver systems
1329 118591 : for (auto & sys : _solver_systems)
1330 59419 : sys->initialSetup();
1331 :
1332 : // Auxilary variable initialSetup calls
1333 59172 : _aux->initialSetup();
1334 :
1335 59166 : if (_displaced_problem)
1336 : // initialSetup for displaced systems
1337 2013 : _displaced_problem->initialSetup();
1338 :
1339 118579 : for (auto & sys : _solver_systems)
1340 59413 : sys->setSolution(*(sys->system().current_local_solution.get()));
1341 :
1342 : // Update the nearest node searches (has to be called after the problem is all set up)
1343 : // We do this here because this sets up the Element's DoFs to ghost
1344 59166 : updateGeomSearch(GeometricSearchData::NEAREST_NODE);
1345 :
1346 59166 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
1347 59166 : if (_displaced_mesh)
1348 2013 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
1349 :
1350 : // We need to move the mesh in order to build a map between mortar secondary and primary
1351 : // interfaces. This map will then be used by the AgumentSparsityOnInterface ghosting functor to
1352 : // know which dofs we need ghosted when we call EquationSystems::reinit
1353 59166 : if (_displaced_problem && _mortar_data->hasDisplacedObjects())
1354 : {
1355 114 : _displaced_problem->updateMesh();
1356 : // if displacements were applied to the mesh, the mortar mesh should be updated too
1357 114 : updateMortarMesh();
1358 : }
1359 :
1360 : // Possibly reinit one more time to get ghosting correct
1361 59166 : reinitBecauseOfGhostingOrNewGeomObjects();
1362 :
1363 59166 : if (_displaced_mesh)
1364 2013 : _displaced_problem->updateMesh();
1365 :
1366 59166 : updateGeomSearch(); // Call all of the rest of the geometric searches
1367 :
1368 118576 : for (auto & sys : _solver_systems)
1369 : {
1370 59413 : const auto & tis = sys->getTimeIntegrators();
1371 :
1372 : {
1373 297065 : TIME_SECTION("timeIntegratorInitialSetup", 5, "Initializing Time Integrator");
1374 88776 : for (auto & ti : tis)
1375 29366 : ti->initialSetup();
1376 59410 : }
1377 : }
1378 :
1379 : // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ main-app restartable data
1380 : // has been restored
1381 :
1382 : // Call initialSetup on the MultiApps
1383 59163 : if (_multi_apps.hasObjects())
1384 : {
1385 34540 : TIME_SECTION("initialSetupMultiApps", 2, "Initializing MultiApps", false);
1386 6908 : _multi_apps.initialSetup();
1387 6902 : }
1388 :
1389 : // Call initialSetup on the transfers
1390 : {
1391 295785 : TIME_SECTION("initialSetupTransfers", 2, "Initializing Transfers");
1392 :
1393 59157 : _transfers.initialSetup();
1394 :
1395 : // Call initialSetup on the MultiAppTransfers to be executed on TO_MULTIAPP
1396 59157 : const auto & to_multi_app_objects = _to_multi_app_transfers.getActiveObjects();
1397 64185 : for (const auto & transfer : to_multi_app_objects)
1398 : {
1399 5076 : transfer->setCurrentDirection(Transfer::DIRECTION::TO_MULTIAPP);
1400 5076 : transfer->initialSetup();
1401 : }
1402 :
1403 : // Call initialSetup on the MultiAppTransfers to be executed on FROM_MULTIAPP
1404 59109 : const auto & from_multi_app_objects = _from_multi_app_transfers.getActiveObjects();
1405 65242 : for (const auto & transfer : from_multi_app_objects)
1406 : {
1407 6175 : transfer->setCurrentDirection(Transfer::DIRECTION::FROM_MULTIAPP);
1408 6175 : transfer->initialSetup();
1409 : }
1410 :
1411 : // Call initialSetup on the MultiAppTransfers to be executed on BETWEEN_MULTIAPP
1412 59067 : const auto & between_multi_app_objects = _between_multi_app_transfers.getActiveObjects();
1413 60536 : for (const auto & transfer : between_multi_app_objects)
1414 : {
1415 1469 : transfer->setCurrentDirection(Transfer::DIRECTION::BETWEEN_MULTIAPP);
1416 1469 : transfer->initialSetup();
1417 : }
1418 59067 : }
1419 :
1420 59067 : if (_boundary_restricted_node_integrity_check)
1421 : {
1422 177165 : TIME_SECTION("BoundaryRestrictedNodeIntegrityCheck", 5);
1423 :
1424 : // check that variables are defined along boundaries of boundary restricted nodal objects
1425 59055 : const auto & bnd_nodes = getCurrentAlgebraicBndNodeRange();
1426 59055 : BoundaryNodeIntegrityCheckThread bnict(*this, uo_query);
1427 59055 : Threads::parallel_reduce(bnd_nodes, bnict);
1428 :
1429 : // Nodal bcs aren't threaded
1430 59046 : const auto & node_to_elem_map = _mesh.nodeToActiveSemilocalElemMap();
1431 4603974 : for (const auto & bnode : bnd_nodes)
1432 : {
1433 4544937 : const auto boundary_id = bnode->_bnd_id;
1434 4544937 : const Node * const node = bnode->_node;
1435 :
1436 4544937 : if (node->processor_id() != this->processor_id())
1437 931821 : continue;
1438 :
1439 : // Only check vertices. Variables may not be defined on non-vertex nodes (think first order
1440 : // Lagrange on a second order mesh) and user-code can often handle that
1441 : const Elem * const an_elem =
1442 3613116 : _mesh.getMesh().elem_ptr(libmesh_map_find(node_to_elem_map, node->id()).front());
1443 3613116 : if (!an_elem->is_vertex(an_elem->get_node_index(node)))
1444 519526 : continue;
1445 :
1446 3093590 : const auto & bnd_name = _mesh.getBoundaryName(boundary_id);
1447 :
1448 6146672 : for (auto & nl : _nl)
1449 : {
1450 3053091 : const auto & nodal_bcs = nl->getNodalBCWarehouse();
1451 3053091 : if (!nodal_bcs.hasBoundaryObjects(boundary_id, 0))
1452 2234543 : continue;
1453 :
1454 818548 : const auto & bnd_objects = nodal_bcs.getBoundaryObjects(boundary_id, 0);
1455 1717580 : for (const auto & bnd_object : bnd_objects)
1456 : // Skip if this object uses geometric search because coupled variables may be defined on
1457 : // paired boundaries instead of the boundary this node is on
1458 1798082 : if (!bnd_object->requiresGeometricSearch() &&
1459 899041 : bnd_object->checkVariableBoundaryIntegrity())
1460 : {
1461 : std::set<MooseVariableFieldBase *> vars_to_omit = {
1462 : &static_cast<MooseVariableFieldBase &>(
1463 1798082 : const_cast<MooseVariableBase &>(bnd_object->variable()))};
1464 :
1465 1798082 : boundaryIntegrityCheckError(
1466 1798073 : *bnd_object, bnd_object->checkAllVariables(*node, vars_to_omit), bnd_name);
1467 899032 : }
1468 : }
1469 : }
1470 59037 : }
1471 :
1472 59049 : if (_boundary_restricted_elem_integrity_check)
1473 : {
1474 177057 : TIME_SECTION("BoundaryRestrictedElemIntegrityCheck", 5);
1475 :
1476 : // check that variables are defined along boundaries of boundary restricted elemental objects
1477 59019 : ConstBndElemRange & bnd_elems = *mesh().getBoundaryElementRange();
1478 59019 : BoundaryElemIntegrityCheckThread beict(*this, uo_query);
1479 59019 : Threads::parallel_reduce(bnd_elems, beict);
1480 59007 : }
1481 :
1482 59037 : if (!_app.isRecovering())
1483 : {
1484 54784 : execTransfers(EXEC_INITIAL);
1485 :
1486 54784 : bool converged = execMultiApps(EXEC_INITIAL);
1487 54778 : if (!converged)
1488 3 : mooseError("failed to converge initial MultiApp");
1489 :
1490 : // We'll backup the Multiapp here
1491 54775 : backupMultiApps(EXEC_INITIAL);
1492 :
1493 115346 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1494 60571 : reinitScalars(tid);
1495 :
1496 54775 : execute(EXEC_INITIAL);
1497 :
1498 : // The FEProblemBase::execute method doesn't call all the systems on EXEC_INITIAL, but it does
1499 : // set/unset the current flag. Therefore, this resets the current flag to EXEC_INITIAL so that
1500 : // subsequent calls (e.g., executeControls) have the proper flag.
1501 54700 : setCurrentExecuteOnFlag(EXEC_INITIAL);
1502 : }
1503 :
1504 : // Here we will initialize the stateful properties once more since they may have been updated
1505 : // during initialSetup by calls to computeProperties.
1506 : //
1507 : // It's really bad that we don't allow this during restart. It means that we can't add new
1508 : // stateful materials
1509 : // during restart. This is only happening because this _has_ to be below initial userobject
1510 : // execution.
1511 : // Otherwise this could be done up above... _before_ restoring restartable data... which would
1512 : // allow you to have
1513 : // this happen during restart. I honestly have no idea why this has to happen after initial user
1514 : // object computation.
1515 : // THAT is something we should fix... so I've opened this ticket: #5804
1516 58953 : if (!_app.isRecovering() && !_app.isRestarting())
1517 : {
1518 107838 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1519 53633 : _neighbor_material_props.hasStatefulProperties())
1520 : {
1521 2860 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1522 :
1523 572 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1524 572 : }
1525 : #ifdef MOOSE_KOKKOS_ENABLED
1526 40054 : if (_kokkos_material_props.hasStatefulProperties() ||
1527 79943 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1528 39889 : _kokkos_neighbor_material_props.hasStatefulProperties())
1529 : {
1530 825 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1531 :
1532 165 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1533 165 : }
1534 : #endif
1535 : }
1536 :
1537 : // Control Logic
1538 58953 : _control_warehouse.initialSetup();
1539 58953 : executeControls(EXEC_INITIAL);
1540 :
1541 : // Scalar variables need to reinited for the initial conditions to be available for output
1542 123660 : for (unsigned int tid = 0; tid < n_threads; tid++)
1543 64734 : reinitScalars(tid);
1544 :
1545 58926 : if (_displaced_mesh)
1546 2013 : _displaced_problem->syncSolutions();
1547 :
1548 : // Writes all calls to _console from initialSetup() methods
1549 58926 : _app.getOutputWarehouse().mooseConsole();
1550 :
1551 58926 : if (_requires_nonlocal_coupling)
1552 : {
1553 63 : setNonlocalCouplingMatrix();
1554 126 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1555 126 : for (auto & assembly : _assembly[tid])
1556 63 : assembly->initNonlocalCoupling();
1557 : }
1558 :
1559 : {
1560 294630 : TIME_SECTION("lineSearchInitialSetup", 5, "Initializing Line Search");
1561 :
1562 58926 : if (_line_search)
1563 0 : _line_search->initialSetup();
1564 58926 : }
1565 :
1566 : // Perform Reporter get/declare check
1567 58926 : _reporter_data.check();
1568 :
1569 : // We do this late to allow objects to get late restartable data
1570 58926 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1571 4748 : _app.finalizeRestore();
1572 :
1573 58926 : setCurrentExecuteOnFlag(EXEC_NONE);
1574 58926 : }
1575 :
1576 : void
1577 59748 : FEProblemBase::checkDuplicatePostprocessorVariableNames()
1578 : {
1579 109583 : for (const auto & pp : _reporter_data.getPostprocessorNames())
1580 49835 : if (hasScalarVariable(pp))
1581 0 : mooseError("Postprocessor \"" + pp +
1582 59748 : "\" has the same name as a scalar variable in the system.");
1583 59748 : }
1584 :
1585 : void
1586 265154 : FEProblemBase::timestepSetup()
1587 : {
1588 265154 : SubProblem::timestepSetup();
1589 :
1590 265154 : if (_t_step > 1 && _num_grid_steps)
1591 : {
1592 31 : libMesh::MeshRefinement mesh_refinement(_mesh);
1593 31 : std::unique_ptr<libMesh::MeshRefinement> displaced_mesh_refinement(nullptr);
1594 31 : if (_displaced_mesh)
1595 23 : displaced_mesh_refinement = std::make_unique<libMesh::MeshRefinement>(*_displaced_mesh);
1596 :
1597 62 : for (MooseIndex(_num_grid_steps) i = 0; i < _num_grid_steps; ++i)
1598 : {
1599 31 : if (_displaced_problem)
1600 : // If the DisplacedProblem is active, undisplace the DisplacedMesh in preparation for
1601 : // refinement. We can't safely refine the DisplacedMesh directly, since the Hilbert keys
1602 : // computed on the inconsistenly-displaced Mesh are different on different processors,
1603 : // leading to inconsistent Hilbert keys. We must do this before the undisplaced Mesh is
1604 : // coarsensed, so that the element and node numbering is still consistent. We also have to
1605 : // make sure this is done during every step of coarsening otherwise different partitions
1606 : // will be generated for the reference and displaced meshes (even for replicated)
1607 23 : _displaced_problem->undisplaceMesh();
1608 :
1609 31 : mesh_refinement.uniformly_coarsen();
1610 31 : if (_displaced_mesh)
1611 23 : displaced_mesh_refinement->uniformly_coarsen();
1612 :
1613 : // Mark this as an intermediate change because we do not yet want to reinit_systems. E.g. we
1614 : // need things to happen in the following order for the undisplaced problem:
1615 : // u1) EquationSystems::reinit_solutions. This will restrict the solution vectors and then
1616 : // contract the mesh
1617 : // u2) MooseMesh::meshChanged. This will update the node/side lists and other
1618 : // things which needs to happen after the contraction
1619 : // u3) GeometricSearchData::reinit. Once the node/side lists are updated we can perform our
1620 : // geometric searches which will aid in determining sparsity patterns
1621 : //
1622 : // We do these things for the displaced problem (if it exists)
1623 : // d1) EquationSystems::reinit. Restrict the displaced problem vector copies and then contract
1624 : // the mesh. It's safe to do a full reinit with the displaced because there are no
1625 : // matrices that sparsity pattern calculations will be conducted for
1626 : // d2) MooseMesh::meshChanged. This will update the node/side lists and other
1627 : // things which needs to happen after the contraction
1628 : // d3) UpdateDisplacedMeshThread::operator(). Re-displace the mesh using the *displaced*
1629 : // solution vector copy because we don't know the state of the reference solution vector.
1630 : // It's safe to use the displaced copy because we are outside of a non-linear solve,
1631 : // and there is no concern about differences between solution and current_local_solution
1632 : // d4) GeometricSearchData::reinit. With the node/side lists updated and the mesh
1633 : // re-displaced, we can perform our geometric searches, which will aid in determining the
1634 : // sparsity pattern of the matrix held by the libMesh::ImplicitSystem held by the
1635 : // NonlinearSystem held by this
1636 31 : meshChanged(
1637 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
1638 : }
1639 :
1640 : // u4) Now that all the geometric searches have been done (both undisplaced and displaced),
1641 : // we're ready to update the sparsity pattern
1642 31 : es().reinit_systems();
1643 31 : }
1644 :
1645 265154 : _control_warehouse.timestepSetup();
1646 265154 : if (_line_search)
1647 0 : _line_search->timestepSetup();
1648 :
1649 : // Random interface objects
1650 266534 : for (const auto & it : _random_data_objects)
1651 1380 : it.second->updateSeeds(EXEC_TIMESTEP_BEGIN);
1652 :
1653 265154 : unsigned int n_threads = libMesh::n_threads();
1654 556673 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1655 : {
1656 291519 : _all_materials.timestepSetup(tid);
1657 291519 : _functions.timestepSetup(tid);
1658 : }
1659 :
1660 : #ifdef MOOSE_KOKKOS_ENABLED
1661 194430 : _kokkos_functions.timestepSetup();
1662 : #endif
1663 :
1664 265154 : _aux->timestepSetup();
1665 533761 : for (auto & sys : _solver_systems)
1666 268607 : sys->timestepSetup();
1667 :
1668 265154 : if (_displaced_problem)
1669 : // timestepSetup for displaced systems
1670 30674 : _displaced_problem->timestepSetup();
1671 :
1672 556673 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1673 : {
1674 291519 : _internal_side_indicators.timestepSetup(tid);
1675 291519 : _indicators.timestepSetup(tid);
1676 291519 : _markers.timestepSetup(tid);
1677 : }
1678 :
1679 265154 : std::vector<UserObject *> userobjs;
1680 265154 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
1681 588496 : for (auto obj : userobjs)
1682 323342 : obj->timestepSetup();
1683 :
1684 : #ifdef MOOSE_KOKKOS_ENABLED
1685 : {
1686 194430 : std::vector<UserObjectBase *> userobjs;
1687 194430 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
1688 196609 : for (auto obj : userobjs)
1689 2179 : obj->timestepSetup();
1690 194430 : }
1691 : #endif
1692 :
1693 : // Timestep setup of output objects
1694 265154 : _app.getOutputWarehouse().timestepSetup();
1695 :
1696 265154 : if (_requires_nonlocal_coupling)
1697 97 : if (_nonlocal_kernels.hasActiveObjects() || _nonlocal_integrated_bcs.hasActiveObjects())
1698 97 : _has_nonlocal_coupling = true;
1699 265154 : }
1700 :
1701 : unsigned int
1702 767146 : FEProblemBase::getMaxQps() const
1703 : {
1704 767146 : if (_max_qps == std::numeric_limits<unsigned int>::max())
1705 0 : mooseError("Max QPS uninitialized");
1706 767146 : return _max_qps;
1707 : }
1708 :
1709 : Order
1710 52 : FEProblemBase::getMaxScalarOrder() const
1711 : {
1712 52 : return _max_scalar_order;
1713 : }
1714 :
1715 : void
1716 59231 : FEProblemBase::checkNonlocalCoupling()
1717 : {
1718 296155 : TIME_SECTION("checkNonlocalCoupling", 5, "Checking Nonlocal Coupling");
1719 :
1720 124371 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
1721 129514 : for (auto & nl : _nl)
1722 : {
1723 64374 : const auto & all_kernels = nl->getKernelWarehouse();
1724 64374 : const auto & kernels = all_kernels.getObjects(tid);
1725 147739 : for (const auto & kernel : kernels)
1726 : {
1727 : std::shared_ptr<NonlocalKernel> nonlocal_kernel =
1728 83365 : std::dynamic_pointer_cast<NonlocalKernel>(kernel);
1729 83365 : if (nonlocal_kernel)
1730 : {
1731 35 : if (_calculate_jacobian_in_uo)
1732 35 : _requires_nonlocal_coupling = true;
1733 35 : _nonlocal_kernels.addObject(kernel, tid);
1734 : }
1735 83365 : }
1736 : const MooseObjectWarehouse<IntegratedBCBase> & all_integrated_bcs =
1737 64374 : nl->getIntegratedBCWarehouse();
1738 64374 : const auto & integrated_bcs = all_integrated_bcs.getObjects(tid);
1739 73074 : for (const auto & integrated_bc : integrated_bcs)
1740 : {
1741 : std::shared_ptr<NonlocalIntegratedBC> nonlocal_integrated_bc =
1742 8700 : std::dynamic_pointer_cast<NonlocalIntegratedBC>(integrated_bc);
1743 8700 : if (nonlocal_integrated_bc)
1744 : {
1745 28 : if (_calculate_jacobian_in_uo)
1746 28 : _requires_nonlocal_coupling = true;
1747 28 : _nonlocal_integrated_bcs.addObject(integrated_bc, tid);
1748 : }
1749 8700 : }
1750 : }
1751 59231 : }
1752 :
1753 : void
1754 65140 : FEProblemBase::checkUserObjectJacobianRequirement(THREAD_ID tid)
1755 : {
1756 65140 : std::set<const MooseVariableFEBase *> uo_jacobian_moose_vars;
1757 : {
1758 65140 : std::vector<ShapeElementUserObject *> objs;
1759 65140 : theWarehouse()
1760 65140 : .query()
1761 130280 : .condition<AttribInterfaces>(Interfaces::ShapeElementUserObject)
1762 65140 : .condition<AttribThread>(tid)
1763 65140 : .queryInto(objs);
1764 :
1765 65184 : for (const auto & uo : objs)
1766 : {
1767 44 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1768 44 : const auto & mv_deps = uo->jacobianMooseVariables();
1769 44 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1770 : }
1771 65140 : }
1772 : {
1773 65140 : std::vector<ShapeSideUserObject *> objs;
1774 65140 : theWarehouse()
1775 65140 : .query()
1776 130280 : .condition<AttribInterfaces>(Interfaces::ShapeSideUserObject)
1777 65140 : .condition<AttribThread>(tid)
1778 65140 : .queryInto(objs);
1779 65196 : for (const auto & uo : objs)
1780 : {
1781 56 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1782 56 : const auto & mv_deps = uo->jacobianMooseVariables();
1783 56 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1784 : }
1785 65140 : }
1786 :
1787 65140 : _uo_jacobian_moose_vars[tid].assign(uo_jacobian_moose_vars.begin(), uo_jacobian_moose_vars.end());
1788 130280 : std::sort(
1789 130280 : _uo_jacobian_moose_vars[tid].begin(), _uo_jacobian_moose_vars[tid].end(), sortMooseVariables);
1790 65140 : }
1791 :
1792 : void
1793 63 : FEProblemBase::setVariableAllDoFMap(const std::vector<const MooseVariableFEBase *> & moose_vars)
1794 : {
1795 153 : for (unsigned int i = 0; i < moose_vars.size(); ++i)
1796 : {
1797 90 : VariableName var_name = moose_vars[i]->name();
1798 90 : auto & sys = _solver_systems[moose_vars[i]->sys().number()];
1799 90 : sys->setVariableGlobalDoFs(var_name);
1800 90 : _var_dof_map[var_name] = sys->getVariableGlobalDoFs();
1801 90 : }
1802 63 : }
1803 :
1804 : void
1805 373952951 : FEProblemBase::prepare(const Elem * elem, const THREAD_ID tid)
1806 : {
1807 750410369 : for (const auto i : index_range(_solver_systems))
1808 : {
1809 376457418 : _assembly[tid][i]->reinit(elem);
1810 376457418 : _solver_systems[i]->prepare(tid);
1811 :
1812 376457418 : if (i < _num_nl_sys)
1813 : {
1814 : // This method is called outside of residual/Jacobian callbacks during initial condition
1815 : // evaluation
1816 375889386 : if ((!_has_jacobian || !_const_jacobian) && currentlyComputingJacobian())
1817 47232174 : _assembly[tid][i]->prepareJacobianBlock();
1818 375889386 : _assembly[tid][i]->prepareResidual();
1819 375889386 : if (_has_nonlocal_coupling && currentlyComputingJacobian())
1820 8824 : _assembly[tid][i]->prepareNonlocal();
1821 : }
1822 : }
1823 373952951 : _aux->prepare(tid);
1824 :
1825 385599926 : if (_displaced_problem &&
1826 : // _reinit_displaced_neighbor applies to interface type objects which will do computations
1827 : // based on both elem and neighbor. Consequently, despite what you might think by its name, we
1828 : // must make sure we prepare the displaced elem
1829 385599926 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1830 : {
1831 7563782 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), tid);
1832 7563758 : if (_has_nonlocal_coupling)
1833 0 : _displaced_problem->prepareNonlocal(tid);
1834 : }
1835 373952927 : }
1836 :
1837 : void
1838 25659 : FEProblemBase::prepareFace(const Elem * elem, const THREAD_ID tid)
1839 : {
1840 51318 : for (auto & nl : _nl)
1841 25659 : nl->prepareFace(tid, true);
1842 25659 : _aux->prepareFace(tid, false);
1843 :
1844 25659 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1845 0 : _displaced_problem->prepareFace(_displaced_mesh->elemPtr(elem->id()), tid);
1846 25659 : }
1847 :
1848 : void
1849 0 : FEProblemBase::prepare(const Elem * elem,
1850 : unsigned int ivar,
1851 : unsigned int jvar,
1852 : const std::vector<dof_id_type> & dof_indices,
1853 : const THREAD_ID tid)
1854 : {
1855 0 : for (const auto i : index_range(_nl))
1856 : {
1857 0 : _assembly[tid][i]->reinit(elem);
1858 0 : _nl[i]->prepare(tid);
1859 : }
1860 :
1861 0 : _aux->prepare(tid);
1862 0 : const auto current_nl_sys_num = _current_nl_sys->number();
1863 0 : _assembly[tid][current_nl_sys_num]->prepareBlock(ivar, jvar, dof_indices);
1864 0 : if (_has_nonlocal_coupling)
1865 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1866 : {
1867 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1868 0 : _assembly[tid][current_nl_sys_num]->prepareBlockNonlocal(
1869 : ivar, jvar, dof_indices, jv.allDofIndices());
1870 : }
1871 :
1872 0 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1873 : {
1874 0 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), ivar, jvar, dof_indices, tid);
1875 0 : if (_has_nonlocal_coupling)
1876 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1877 : {
1878 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1879 0 : _displaced_problem->prepareBlockNonlocal(ivar, jvar, dof_indices, jv.allDofIndices(), tid);
1880 : }
1881 : }
1882 0 : }
1883 :
1884 : void
1885 405222405 : FEProblemBase::setCurrentSubdomainID(const Elem * elem, const THREAD_ID tid)
1886 : {
1887 405222405 : SubdomainID did = elem->subdomain_id();
1888 813957027 : for (const auto i : index_range(_solver_systems))
1889 : {
1890 408734622 : _assembly[tid][i]->setCurrentSubdomainID(did);
1891 421198638 : if (_displaced_problem &&
1892 421198638 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1893 7816535 : _displaced_problem->assembly(tid, i).setCurrentSubdomainID(did);
1894 : }
1895 405222405 : }
1896 :
1897 : void
1898 1454082018 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, unsigned int side, const THREAD_ID tid)
1899 : {
1900 1454082018 : SubdomainID did = elem->neighbor_ptr(side)->subdomain_id();
1901 2912854199 : for (const auto i : index_range(_nl))
1902 : {
1903 1458772181 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1904 1502358327 : if (_displaced_problem &&
1905 1502358327 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1906 25848827 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1907 : }
1908 1454082018 : }
1909 :
1910 : void
1911 15260370 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, const THREAD_ID tid)
1912 : {
1913 15260370 : SubdomainID did = elem->subdomain_id();
1914 31472052 : for (const auto i : index_range(_nl))
1915 : {
1916 16211682 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1917 16262978 : if (_displaced_problem &&
1918 16262978 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1919 51296 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1920 : }
1921 15260370 : }
1922 :
1923 : void
1924 135744 : FEProblemBase::prepareAssembly(const THREAD_ID tid)
1925 : {
1926 135744 : _assembly[tid][_current_nl_sys->number()]->prepare();
1927 135744 : if (_has_nonlocal_coupling)
1928 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
1929 :
1930 135744 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1931 : {
1932 51296 : _displaced_problem->prepareAssembly(tid);
1933 51296 : if (_has_nonlocal_coupling)
1934 0 : _displaced_problem->prepareNonlocal(tid);
1935 : }
1936 135744 : }
1937 :
1938 : void
1939 266607 : FEProblemBase::addResidual(const THREAD_ID tid)
1940 : {
1941 533214 : _assembly[tid][_current_nl_sys->number()]->addResidual(Assembly::GlobalDataKey{},
1942 266607 : currentResidualVectorTags());
1943 :
1944 266607 : if (_displaced_problem)
1945 4776 : _displaced_problem->addResidual(tid);
1946 266607 : }
1947 :
1948 : void
1949 1984495 : FEProblemBase::addResidualNeighbor(const THREAD_ID tid)
1950 : {
1951 3968990 : _assembly[tid][_current_nl_sys->number()]->addResidualNeighbor(Assembly::GlobalDataKey{},
1952 1984495 : currentResidualVectorTags());
1953 :
1954 1984495 : if (_displaced_problem)
1955 61744 : _displaced_problem->addResidualNeighbor(tid);
1956 1984495 : }
1957 :
1958 : void
1959 1966844 : FEProblemBase::addResidualLower(const THREAD_ID tid)
1960 : {
1961 3933688 : _assembly[tid][_current_nl_sys->number()]->addResidualLower(Assembly::GlobalDataKey{},
1962 1966844 : currentResidualVectorTags());
1963 :
1964 1966844 : if (_displaced_problem)
1965 61956 : _displaced_problem->addResidualLower(tid);
1966 1966844 : }
1967 :
1968 : void
1969 48499 : FEProblemBase::addResidualScalar(const THREAD_ID tid /* = 0*/)
1970 : {
1971 96998 : _assembly[tid][_current_nl_sys->number()]->addResidualScalar(Assembly::GlobalDataKey{},
1972 48499 : currentResidualVectorTags());
1973 48499 : }
1974 :
1975 : void
1976 287769609 : FEProblemBase::cacheResidual(const THREAD_ID tid)
1977 : {
1978 287769609 : SubProblem::cacheResidual(tid);
1979 287769609 : if (_displaced_problem)
1980 7593378 : _displaced_problem->cacheResidual(tid);
1981 287769609 : }
1982 :
1983 : void
1984 49429 : FEProblemBase::cacheResidualNeighbor(const THREAD_ID tid)
1985 : {
1986 49429 : SubProblem::cacheResidualNeighbor(tid);
1987 49429 : if (_displaced_problem)
1988 32 : _displaced_problem->cacheResidualNeighbor(tid);
1989 49429 : }
1990 :
1991 : void
1992 16758770 : FEProblemBase::addCachedResidual(const THREAD_ID tid)
1993 : {
1994 16758770 : SubProblem::addCachedResidual(tid);
1995 16758770 : if (_displaced_problem)
1996 492503 : _displaced_problem->addCachedResidual(tid);
1997 16758770 : }
1998 :
1999 : void
2000 11165 : FEProblemBase::addCachedResidualDirectly(NumericVector<Number> & residual, const THREAD_ID tid)
2001 : {
2002 11165 : if (_current_nl_sys->hasVector(_current_nl_sys->timeVectorTag()))
2003 29943 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2004 29943 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->timeVectorTag()));
2005 :
2006 11165 : if (_current_nl_sys->hasVector(_current_nl_sys->nonTimeVectorTag()))
2007 33495 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2008 33495 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->nonTimeVectorTag()));
2009 :
2010 11165 : std::vector<VectorTag> extra_residual_vector_tags;
2011 11165 : extra_residual_vector_tags.reserve(currentResidualVectorTags().size());
2012 11165 : const auto time_tag = _current_nl_sys->timeVectorTag();
2013 11165 : const auto non_time_tag = _current_nl_sys->nonTimeVectorTag();
2014 43548 : for (const auto & vector_tag : currentResidualVectorTags())
2015 32383 : if (vector_tag._id != time_tag && vector_tag._id != non_time_tag)
2016 11237 : extra_residual_vector_tags.push_back(vector_tag);
2017 :
2018 : // Flush extra vector tag caches (e.g. from extra_vector_tags on NodalConstraints)
2019 : // to their respective system vectors after the standard TIME/NONTIME caches above.
2020 : // Without this, NodalConstraint contributions to extra vector tags are silently
2021 : // discarded by the blanket clearCachedResiduals.
2022 11165 : _assembly[tid][_current_nl_sys->number()]->addCachedResiduals(Assembly::GlobalDataKey{},
2023 : extra_residual_vector_tags);
2024 :
2025 : // We do this because by adding the cached residual directly, we cannot ensure that all of the
2026 : // cached residuals are emptied after only the two add calls above
2027 11165 : _assembly[tid][_current_nl_sys->number()]->clearCachedResiduals(Assembly::GlobalDataKey{});
2028 :
2029 11165 : if (_displaced_problem)
2030 35 : _displaced_problem->addCachedResidualDirectly(residual, tid);
2031 11165 : }
2032 :
2033 : void
2034 0 : FEProblemBase::setResidual(NumericVector<Number> & residual, const THREAD_ID tid)
2035 : {
2036 0 : _assembly[tid][_current_nl_sys->number()]->setResidual(
2037 : residual,
2038 0 : Assembly::GlobalDataKey{},
2039 0 : getVectorTag(_nl[_current_nl_sys->number()]->residualVectorTag()));
2040 0 : if (_displaced_problem)
2041 0 : _displaced_problem->setResidual(residual, tid);
2042 0 : }
2043 :
2044 : void
2045 0 : FEProblemBase::setResidualNeighbor(NumericVector<Number> & residual, const THREAD_ID tid)
2046 : {
2047 0 : _assembly[tid][_current_nl_sys->number()]->setResidualNeighbor(
2048 0 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->residualVectorTag()));
2049 0 : if (_displaced_problem)
2050 0 : _displaced_problem->setResidualNeighbor(residual, tid);
2051 0 : }
2052 :
2053 : void
2054 37415 : FEProblemBase::addJacobian(const THREAD_ID tid)
2055 : {
2056 37415 : _assembly[tid][_current_nl_sys->number()]->addJacobian(Assembly::GlobalDataKey{});
2057 37415 : if (_has_nonlocal_coupling)
2058 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianNonlocal(Assembly::GlobalDataKey{});
2059 37415 : if (_displaced_problem)
2060 : {
2061 200 : _displaced_problem->addJacobian(tid);
2062 200 : if (_has_nonlocal_coupling)
2063 0 : _displaced_problem->addJacobianNonlocal(tid);
2064 : }
2065 37415 : }
2066 :
2067 : void
2068 7852 : FEProblemBase::addJacobianNeighbor(const THREAD_ID tid)
2069 : {
2070 7852 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighbor(Assembly::GlobalDataKey{});
2071 7852 : if (_displaced_problem)
2072 44 : _displaced_problem->addJacobianNeighbor(tid);
2073 7852 : }
2074 :
2075 : void
2076 109320 : FEProblemBase::addJacobianNeighborLowerD(const THREAD_ID tid)
2077 : {
2078 109320 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborLowerD(Assembly::GlobalDataKey{});
2079 109320 : if (_displaced_problem)
2080 3072 : _displaced_problem->addJacobianNeighborLowerD(tid);
2081 109320 : }
2082 :
2083 : void
2084 4696 : FEProblemBase::addJacobianLowerD(const THREAD_ID tid)
2085 : {
2086 4696 : _assembly[tid][_current_nl_sys->number()]->addJacobianLowerD(Assembly::GlobalDataKey{});
2087 4696 : if (_displaced_problem)
2088 192 : _displaced_problem->addJacobianLowerD(tid);
2089 4696 : }
2090 :
2091 : void
2092 11616 : FEProblemBase::addJacobianScalar(const THREAD_ID tid /* = 0*/)
2093 : {
2094 11616 : _assembly[tid][_current_nl_sys->number()]->addJacobianScalar(Assembly::GlobalDataKey{});
2095 11616 : }
2096 :
2097 : void
2098 30054 : FEProblemBase::addJacobianOffDiagScalar(unsigned int ivar, const THREAD_ID tid /* = 0*/)
2099 : {
2100 60108 : _assembly[tid][_current_nl_sys->number()]->addJacobianOffDiagScalar(ivar,
2101 30054 : Assembly::GlobalDataKey{});
2102 30054 : }
2103 :
2104 : void
2105 46666576 : FEProblemBase::cacheJacobian(const THREAD_ID tid)
2106 : {
2107 46666576 : SubProblem::cacheJacobian(tid);
2108 46666576 : if (_displaced_problem)
2109 1544138 : _displaced_problem->cacheJacobian(tid);
2110 46666576 : }
2111 :
2112 : void
2113 9300 : FEProblemBase::cacheJacobianNeighbor(const THREAD_ID tid)
2114 : {
2115 9300 : SubProblem::cacheJacobianNeighbor(tid);
2116 9300 : if (_displaced_problem)
2117 0 : _displaced_problem->cacheJacobianNeighbor(tid);
2118 9300 : }
2119 :
2120 : void
2121 2853007 : FEProblemBase::addCachedJacobian(const THREAD_ID tid)
2122 : {
2123 2853007 : SubProblem::addCachedJacobian(tid);
2124 2853004 : if (_displaced_problem)
2125 95805 : _displaced_problem->addCachedJacobian(tid);
2126 2853004 : }
2127 :
2128 : void
2129 70832 : FEProblemBase::addJacobianBlockTags(SparseMatrix<Number> & jacobian,
2130 : unsigned int ivar,
2131 : unsigned int jvar,
2132 : const DofMap & dof_map,
2133 : std::vector<dof_id_type> & dof_indices,
2134 : const std::set<TagID> & tags,
2135 : const THREAD_ID tid)
2136 : {
2137 141664 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockTags(
2138 70832 : jacobian, ivar, jvar, dof_map, dof_indices, Assembly::GlobalDataKey{}, tags);
2139 :
2140 70832 : if (_has_nonlocal_coupling)
2141 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2142 : {
2143 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2144 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockNonlocalTags(
2145 : jacobian,
2146 : ivar,
2147 : jvar,
2148 : dof_map,
2149 : dof_indices,
2150 : jv.allDofIndices(),
2151 0 : Assembly::GlobalDataKey{},
2152 : tags);
2153 : }
2154 :
2155 70832 : if (_displaced_problem)
2156 : {
2157 0 : _displaced_problem->addJacobianBlockTags(jacobian, ivar, jvar, dof_map, dof_indices, tags, tid);
2158 0 : if (_has_nonlocal_coupling)
2159 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2160 : {
2161 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2162 0 : _displaced_problem->addJacobianBlockNonlocal(
2163 : jacobian, ivar, jvar, dof_map, dof_indices, jv.allDofIndices(), tags, tid);
2164 : }
2165 : }
2166 70832 : }
2167 :
2168 : void
2169 768 : FEProblemBase::addJacobianNeighbor(SparseMatrix<Number> & jacobian,
2170 : unsigned int ivar,
2171 : unsigned int jvar,
2172 : const DofMap & dof_map,
2173 : std::vector<dof_id_type> & dof_indices,
2174 : std::vector<dof_id_type> & neighbor_dof_indices,
2175 : const std::set<TagID> & tags,
2176 : const THREAD_ID tid)
2177 : {
2178 1536 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborTags(jacobian,
2179 : ivar,
2180 : jvar,
2181 : dof_map,
2182 : dof_indices,
2183 : neighbor_dof_indices,
2184 768 : Assembly::GlobalDataKey{},
2185 : tags);
2186 768 : if (_displaced_problem)
2187 0 : _displaced_problem->addJacobianNeighbor(
2188 : jacobian, ivar, jvar, dof_map, dof_indices, neighbor_dof_indices, tags, tid);
2189 768 : }
2190 :
2191 : void
2192 125643374 : FEProblemBase::prepareShapes(unsigned int var, const THREAD_ID tid)
2193 : {
2194 125643374 : _assembly[tid][_current_nl_sys->number()]->copyShapes(var);
2195 125643374 : }
2196 :
2197 : void
2198 572483 : FEProblemBase::prepareFaceShapes(unsigned int var, const THREAD_ID tid)
2199 : {
2200 572483 : _assembly[tid][_current_nl_sys->number()]->copyFaceShapes(var);
2201 572483 : }
2202 :
2203 : void
2204 185289 : FEProblemBase::prepareNeighborShapes(unsigned int var, const THREAD_ID tid)
2205 : {
2206 185289 : _assembly[tid][_current_nl_sys->number()]->copyNeighborShapes(var);
2207 185289 : }
2208 :
2209 : void
2210 874615 : FEProblemBase::addGhostedElem(dof_id_type elem_id)
2211 : {
2212 874615 : if (_mesh.elemPtr(elem_id)->processor_id() != processor_id())
2213 205677 : _ghosted_elems.insert(elem_id);
2214 874615 : }
2215 :
2216 : void
2217 29378 : FEProblemBase::addGhostedBoundary(BoundaryID boundary_id)
2218 : {
2219 29378 : _mesh.addGhostedBoundary(boundary_id);
2220 29378 : if (_displaced_problem)
2221 26702 : _displaced_mesh->addGhostedBoundary(boundary_id);
2222 29378 : }
2223 :
2224 : void
2225 66879 : FEProblemBase::ghostGhostedBoundaries()
2226 : {
2227 334395 : TIME_SECTION("ghostGhostedBoundaries", 3, "Ghosting Ghosted Boundaries");
2228 :
2229 66879 : _mesh.ghostGhostedBoundaries();
2230 :
2231 66879 : if (_displaced_problem)
2232 2591 : _displaced_mesh->ghostGhostedBoundaries();
2233 66879 : }
2234 :
2235 : void
2236 0 : FEProblemBase::sizeZeroes(unsigned int /*size*/, const THREAD_ID /*tid*/)
2237 : {
2238 0 : mooseDoOnce(mooseWarning(
2239 : "This function is deprecated and no longer performs any function. Please do not call it."));
2240 0 : }
2241 :
2242 : bool
2243 304022 : FEProblemBase::reinitDirac(const Elem * elem, const THREAD_ID tid)
2244 : {
2245 304022 : std::vector<Point> & points = _dirac_kernel_info.getPoints()[elem].first;
2246 :
2247 304022 : unsigned int n_points = points.size();
2248 :
2249 304022 : if (n_points)
2250 : {
2251 299046 : if (n_points > _max_qps)
2252 : {
2253 0 : _max_qps = n_points;
2254 :
2255 : /**
2256 : * The maximum number of qps can rise if several Dirac points are added to a single element.
2257 : * In that case we need to resize the zeros to compensate.
2258 : */
2259 0 : unsigned int max_qpts = getMaxQps();
2260 0 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
2261 : {
2262 : // the highest available order in libMesh is 43
2263 0 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
2264 0 : _zero[tid].resize(max_qpts, 0);
2265 0 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
2266 0 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
2267 0 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
2268 0 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
2269 : }
2270 : }
2271 :
2272 598092 : for (const auto i : index_range(_nl))
2273 : {
2274 299046 : _assembly[tid][i]->reinitAtPhysical(elem, points);
2275 299046 : _nl[i]->prepare(tid);
2276 : }
2277 299046 : _aux->prepare(tid);
2278 :
2279 299046 : reinitElem(elem, tid);
2280 : }
2281 :
2282 304022 : _assembly[tid][_current_nl_sys->number()]->prepare();
2283 304022 : if (_has_nonlocal_coupling)
2284 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
2285 :
2286 304022 : bool have_points = n_points > 0;
2287 304022 : if (_displaced_problem && (_reinit_displaced_elem))
2288 : {
2289 4976 : have_points |= _displaced_problem->reinitDirac(_displaced_mesh->elemPtr(elem->id()), tid);
2290 4976 : if (_has_nonlocal_coupling)
2291 0 : _displaced_problem->prepareNonlocal(tid);
2292 : }
2293 :
2294 304022 : return have_points;
2295 : }
2296 :
2297 : void
2298 374199079 : FEProblemBase::reinitElem(const Elem * elem, const THREAD_ID tid)
2299 : {
2300 750902625 : for (auto & sys : _solver_systems)
2301 376703546 : sys->reinitElem(elem, tid);
2302 374199079 : _aux->reinitElem(elem, tid);
2303 :
2304 374199079 : if (_displaced_problem && _reinit_displaced_elem)
2305 4426155 : _displaced_problem->reinitElem(_displaced_mesh->elemPtr(elem->id()), tid);
2306 374199079 : }
2307 :
2308 : void
2309 58460 : FEProblemBase::reinitElemPhys(const Elem * const elem,
2310 : const std::vector<Point> & phys_points_in_elem,
2311 : const THREAD_ID tid)
2312 : {
2313 : mooseAssert(_mesh.queryElemPtr(elem->id()) == elem,
2314 : "Are you calling this method with a displaced mesh element?");
2315 :
2316 116920 : for (const auto i : index_range(_solver_systems))
2317 : {
2318 58460 : _assembly[tid][i]->reinitAtPhysical(elem, phys_points_in_elem);
2319 58460 : _solver_systems[i]->prepare(tid);
2320 58460 : _assembly[tid][i]->prepare();
2321 58460 : if (_has_nonlocal_coupling)
2322 0 : _assembly[tid][i]->prepareNonlocal();
2323 : }
2324 58460 : _aux->prepare(tid);
2325 :
2326 58460 : reinitElem(elem, tid);
2327 58460 : }
2328 :
2329 : void
2330 0 : FEProblemBase::reinitElemFace(const Elem * const elem,
2331 : const unsigned int side,
2332 : const BoundaryID,
2333 : const THREAD_ID tid)
2334 : {
2335 0 : mooseDeprecated(
2336 : "reinitElemFace with a BoundaryID argument is deprecated because the boundary id was never "
2337 : "used. Please call reinitElemFace without the BoundaryID argument instead");
2338 :
2339 0 : reinitElemFace(elem, side, tid);
2340 0 : }
2341 :
2342 : void
2343 5113915 : FEProblemBase::reinitElemFace(const Elem * const elem, const unsigned int side, const THREAD_ID tid)
2344 : {
2345 10228178 : for (const auto i : index_range(_solver_systems))
2346 : {
2347 5114263 : _assembly[tid][i]->reinit(elem, side);
2348 5114263 : _solver_systems[i]->reinitElemFace(elem, side, tid);
2349 : }
2350 5113915 : _aux->reinitElemFace(elem, side, tid);
2351 :
2352 5113915 : if (_displaced_problem && _reinit_displaced_face)
2353 82248 : _displaced_problem->reinitElemFace(_displaced_mesh->elemPtr(elem->id()), side, tid);
2354 5113915 : }
2355 :
2356 : void
2357 268532 : FEProblemBase::reinitLowerDElem(const Elem * lower_d_elem,
2358 : const THREAD_ID tid,
2359 : const std::vector<Point> * const pts,
2360 : const std::vector<Real> * const weights)
2361 : {
2362 268532 : SubProblem::reinitLowerDElem(lower_d_elem, tid, pts, weights);
2363 :
2364 268532 : if (_displaced_problem && _displaced_mesh)
2365 960 : _displaced_problem->reinitLowerDElem(
2366 960 : _displaced_mesh->elemPtr(lower_d_elem->id()), tid, pts, weights);
2367 268532 : }
2368 :
2369 : void
2370 25951306 : FEProblemBase::reinitNode(const Node * node, const THREAD_ID tid)
2371 : {
2372 25951306 : if (_displaced_problem && _reinit_displaced_elem)
2373 1001711 : _displaced_problem->reinitNode(&_displaced_mesh->nodeRef(node->id()), tid);
2374 :
2375 51905528 : for (const auto i : index_range(_nl))
2376 : {
2377 25954222 : _assembly[tid][i]->reinit(node);
2378 25954222 : _nl[i]->reinitNode(node, tid);
2379 : }
2380 25951306 : _aux->reinitNode(node, tid);
2381 25951306 : }
2382 :
2383 : void
2384 65064900 : FEProblemBase::reinitNodeFace(const Node * node, BoundaryID bnd_id, const THREAD_ID tid)
2385 : {
2386 65064900 : if (_displaced_problem && _reinit_displaced_face)
2387 3388984 : _displaced_problem->reinitNodeFace(&_displaced_mesh->nodeRef(node->id()), bnd_id, tid);
2388 :
2389 130919060 : for (const auto i : index_range(_nl))
2390 : {
2391 65854160 : _assembly[tid][i]->reinit(node);
2392 65854160 : _nl[i]->reinitNodeFace(node, bnd_id, tid);
2393 : }
2394 65064900 : _aux->reinitNodeFace(node, bnd_id, tid);
2395 65064900 : }
2396 :
2397 : void
2398 5093 : FEProblemBase::reinitNodes(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2399 : {
2400 5093 : if (_displaced_problem && _reinit_displaced_elem)
2401 0 : _displaced_problem->reinitNodes(nodes, tid);
2402 :
2403 10186 : for (auto & nl : _nl)
2404 5093 : nl->reinitNodes(nodes, tid);
2405 5093 : _aux->reinitNodes(nodes, tid);
2406 5093 : }
2407 :
2408 : void
2409 999 : FEProblemBase::reinitNodesNeighbor(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2410 : {
2411 999 : if (_displaced_problem && _reinit_displaced_elem)
2412 0 : _displaced_problem->reinitNodesNeighbor(nodes, tid);
2413 :
2414 1998 : for (auto & nl : _nl)
2415 999 : nl->reinitNodesNeighbor(nodes, tid);
2416 999 : _aux->reinitNodesNeighbor(nodes, tid);
2417 999 : }
2418 :
2419 : void
2420 8408365 : FEProblemBase::reinitScalars(const THREAD_ID tid, bool reinit_for_derivative_reordering /*=false*/)
2421 : {
2422 42041825 : TIME_SECTION("reinitScalars", 3, "Reinitializing Scalar Variables");
2423 :
2424 8408365 : if (_displaced_problem && _reinit_displaced_elem)
2425 113553 : _displaced_problem->reinitScalars(tid, reinit_for_derivative_reordering);
2426 :
2427 17045427 : for (auto & nl : _nl)
2428 8637062 : nl->reinitScalars(tid, reinit_for_derivative_reordering);
2429 8408365 : _aux->reinitScalars(tid, reinit_for_derivative_reordering);
2430 :
2431 : // This is called outside of residual/Jacobian call-backs
2432 17050749 : for (auto & assembly : _assembly[tid])
2433 8642384 : assembly->prepareScalar();
2434 8408365 : }
2435 :
2436 : void
2437 185261 : FEProblemBase::reinitOffDiagScalars(const THREAD_ID tid)
2438 : {
2439 185261 : _assembly[tid][_current_nl_sys->number()]->prepareOffDiagScalar();
2440 185261 : if (_displaced_problem)
2441 60 : _displaced_problem->reinitOffDiagScalars(tid);
2442 185261 : }
2443 :
2444 : void
2445 3642430 : FEProblemBase::reinitNeighbor(const Elem * elem, unsigned int side, const THREAD_ID tid)
2446 : {
2447 3642430 : setNeighborSubdomainID(elem, side, tid);
2448 :
2449 3642430 : const Elem * neighbor = elem->neighbor_ptr(side);
2450 3642430 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(elem);
2451 :
2452 7284887 : for (const auto i : index_range(_nl))
2453 : {
2454 3642457 : _assembly[tid][i]->reinitElemAndNeighbor(elem, side, neighbor, neighbor_side);
2455 3642457 : _nl[i]->prepareNeighbor(tid);
2456 : // Called during stateful material property evaluation outside of solve
2457 3642457 : _assembly[tid][i]->prepareNeighbor();
2458 : }
2459 3642430 : _aux->prepareNeighbor(tid);
2460 :
2461 7284887 : for (auto & nl : _nl)
2462 : {
2463 3642457 : nl->reinitElemFace(elem, side, tid);
2464 3642457 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2465 : }
2466 3642430 : _aux->reinitElemFace(elem, side, tid);
2467 3642430 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2468 :
2469 3642430 : if (_displaced_problem && _reinit_displaced_neighbor)
2470 : {
2471 : // There are cases like for cohesive zone modeling without significant sliding where we cannot
2472 : // use FEInterface::inverse_map in Assembly::reinitElemAndNeighbor in the displaced problem
2473 : // because the physical points coming from the element don't actually lie on the neighbor.
2474 : // Moreover, what's the point of doing another physical point inversion in other cases? We only
2475 : // care about the reference points which we can just take from the undisplaced computation
2476 64800 : const auto & displaced_ref_pts = _assembly[tid][0]->qRuleNeighbor()->get_points();
2477 :
2478 64800 : _displaced_problem->reinitNeighbor(
2479 64800 : _displaced_mesh->elemPtr(elem->id()), side, tid, &displaced_ref_pts);
2480 : }
2481 3642430 : }
2482 :
2483 : void
2484 2069409 : FEProblemBase::reinitElemNeighborAndLowerD(const Elem * elem,
2485 : unsigned int side,
2486 : const THREAD_ID tid)
2487 : {
2488 2069409 : reinitNeighbor(elem, side, tid);
2489 :
2490 2069409 : const Elem * lower_d_elem = _mesh.getLowerDElem(elem, side);
2491 2069409 : if (lower_d_elem && _mesh.interiorLowerDBlocks().count(lower_d_elem->subdomain_id()) > 0)
2492 10320 : reinitLowerDElem(lower_d_elem, tid);
2493 : else
2494 : {
2495 : // with mesh refinement, lower-dimensional element might be defined on neighbor side
2496 2059089 : auto & neighbor = _assembly[tid][0]->neighbor();
2497 2059089 : auto & neighbor_side = _assembly[tid][0]->neighborSide();
2498 2059089 : const Elem * lower_d_elem_neighbor = _mesh.getLowerDElem(neighbor, neighbor_side);
2499 2059089 : if (lower_d_elem_neighbor &&
2500 2059089 : _mesh.interiorLowerDBlocks().count(lower_d_elem_neighbor->subdomain_id()) > 0)
2501 : {
2502 0 : auto qps = _assembly[tid][0]->qPointsFaceNeighbor().stdVector();
2503 0 : std::vector<Point> reference_points;
2504 0 : FEMap::inverse_map(
2505 0 : lower_d_elem_neighbor->dim(), lower_d_elem_neighbor, qps, reference_points);
2506 0 : reinitLowerDElem(lower_d_elem_neighbor, tid, &reference_points);
2507 0 : }
2508 : }
2509 :
2510 2069409 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
2511 64740 : _displaced_problem->reinitElemNeighborAndLowerD(
2512 64740 : _displaced_mesh->elemPtr(elem->id()), side, tid);
2513 2069409 : }
2514 :
2515 : void
2516 96312 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2517 : unsigned int neighbor_side,
2518 : const std::vector<Point> & physical_points,
2519 : const THREAD_ID tid)
2520 : {
2521 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2522 : "Are you calling this method with a displaced mesh element?");
2523 :
2524 192624 : for (const auto i : index_range(_nl))
2525 : {
2526 : // Reinits shape the functions at the physical points
2527 96312 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, neighbor_side, physical_points);
2528 :
2529 : // Sets the neighbor dof indices
2530 96312 : _nl[i]->prepareNeighbor(tid);
2531 : }
2532 96312 : _aux->prepareNeighbor(tid);
2533 :
2534 : // Resizes Re and Ke
2535 96312 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2536 :
2537 : // Compute the values of each variable at the points
2538 192624 : for (auto & nl : _nl)
2539 96312 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2540 96312 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2541 96312 : }
2542 :
2543 : void
2544 19880 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2545 : const std::vector<Point> & physical_points,
2546 : const THREAD_ID tid)
2547 : {
2548 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2549 : "Are you calling this method with a displaced mesh element?");
2550 :
2551 39760 : for (const auto i : index_range(_nl))
2552 : {
2553 : // Reinits shape the functions at the physical points
2554 19880 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, physical_points);
2555 :
2556 : // Sets the neighbor dof indices
2557 19880 : _nl[i]->prepareNeighbor(tid);
2558 : }
2559 19880 : _aux->prepareNeighbor(tid);
2560 :
2561 : // Resizes Re and Ke
2562 19880 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2563 :
2564 : // Compute the values of each variable at the points
2565 39760 : for (auto & nl : _nl)
2566 19880 : nl->reinitNeighbor(neighbor, tid);
2567 19880 : _aux->reinitNeighbor(neighbor, tid);
2568 19880 : }
2569 :
2570 : void
2571 35512 : FEProblemBase::getDiracElements(std::set<const Elem *> & elems)
2572 : {
2573 : // First add in the undisplaced elements
2574 35512 : elems = _dirac_kernel_info.getElements();
2575 :
2576 35512 : if (_displaced_problem)
2577 : {
2578 2359 : std::set<const Elem *> displaced_elements;
2579 2359 : _displaced_problem->getDiracElements(displaced_elements);
2580 :
2581 : { // Use the ids from the displaced elements to get the undisplaced elements
2582 : // and add them to the list
2583 7335 : for (const auto & elem : displaced_elements)
2584 4976 : elems.insert(_mesh.elemPtr(elem->id()));
2585 : }
2586 2359 : }
2587 35512 : }
2588 :
2589 : void
2590 3508864 : FEProblemBase::clearDiracInfo()
2591 : {
2592 3508864 : _dirac_kernel_info.clearPoints();
2593 :
2594 3508864 : if (_displaced_problem)
2595 144140 : _displaced_problem->clearDiracInfo();
2596 3508864 : }
2597 :
2598 : void
2599 5446222 : FEProblemBase::subdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2600 : {
2601 5446222 : _all_materials.subdomainSetup(subdomain, tid);
2602 : // Call the subdomain methods of the output system, these are not threaded so only call it once
2603 5446222 : if (tid == 0)
2604 4987620 : _app.getOutputWarehouse().subdomainSetup();
2605 :
2606 11032133 : for (auto & nl : _nl)
2607 5585911 : nl->subdomainSetup(subdomain, tid);
2608 :
2609 : // FIXME: call displaced_problem->subdomainSetup() ?
2610 : // When adding possibility with materials being evaluated on displaced mesh
2611 5446222 : }
2612 :
2613 : void
2614 16897394 : FEProblemBase::neighborSubdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2615 : {
2616 16897394 : _all_materials.neighborSubdomainSetup(subdomain, tid);
2617 16897394 : }
2618 :
2619 : void
2620 48919 : FEProblemBase::addFunction(const std::string & type,
2621 : const std::string & name,
2622 : InputParameters & parameters)
2623 : {
2624 : parallel_object_only();
2625 :
2626 97838 : parameters.set<SubProblem *>("_subproblem") = this;
2627 :
2628 102132 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2629 : {
2630 53319 : std::shared_ptr<Function> func = _factory.create<Function>(type, name, parameters, tid);
2631 53213 : logAdd("Function", name, type, parameters);
2632 53213 : _functions.addObject(func, tid);
2633 :
2634 53213 : if (auto * const functor = dynamic_cast<Moose::FunctorBase<Real> *>(func.get()))
2635 : {
2636 53213 : this->addFunctor(name, *functor, tid);
2637 53213 : if (_displaced_problem)
2638 1780 : _displaced_problem->addFunctor(name, *functor, tid);
2639 : }
2640 : else
2641 0 : mooseError("Unrecognized function functor type");
2642 53213 : }
2643 48813 : }
2644 :
2645 : void
2646 150805 : FEProblemBase::addConvergence(const std::string & type,
2647 : const std::string & name,
2648 : InputParameters & parameters)
2649 : {
2650 : parallel_object_only();
2651 :
2652 317865 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2653 : {
2654 167087 : std::shared_ptr<Convergence> conv = _factory.create<Convergence>(type, name, parameters, tid);
2655 167060 : _convergences.addObject(conv, tid);
2656 167060 : }
2657 150778 : }
2658 :
2659 : void
2660 59133 : FEProblemBase::addDefaultNonlinearConvergence(const InputParameters & params_to_apply)
2661 : {
2662 59133 : const std::string class_name = "DefaultNonlinearConvergence";
2663 59133 : InputParameters params = _factory.getValidParams(class_name);
2664 59133 : params.applyParameters(params_to_apply);
2665 59133 : params.applyParameters(parameters());
2666 59133 : params.set<bool>("added_as_default") = true;
2667 117487 : for (const auto & conv_name : getNonlinearConvergenceNames())
2668 58354 : addConvergence(class_name, conv_name, params);
2669 59133 : }
2670 :
2671 : void
2672 60908 : FEProblemBase::addDefaultMultiAppFixedPointConvergence(const InputParameters & params_to_apply)
2673 : {
2674 60908 : const std::string class_name = "DefaultMultiAppFixedPointConvergence";
2675 60908 : InputParameters params = _factory.getValidParams(class_name);
2676 60908 : params.applyParameters(params_to_apply);
2677 60908 : params.applyParameters(parameters());
2678 60908 : params.set<bool>("added_as_default") = true;
2679 60908 : addConvergence(class_name, getMultiAppFixedPointConvergenceName(), params);
2680 60899 : }
2681 :
2682 : void
2683 29915 : FEProblemBase::addDefaultSteadyStateConvergence(const InputParameters & params_to_apply)
2684 : {
2685 29915 : const std::string class_name = "DefaultSteadyStateConvergence";
2686 29915 : InputParameters params = _factory.getValidParams(class_name);
2687 29915 : params.applyParameters(params_to_apply);
2688 29915 : params.applyParameters(parameters());
2689 29915 : params.set<bool>("added_as_default") = true;
2690 29915 : addConvergence(class_name, getSteadyStateConvergenceName(), params);
2691 29915 : }
2692 :
2693 : bool
2694 85694 : FEProblemBase::hasFunction(const std::string & name, const THREAD_ID tid)
2695 : {
2696 85694 : return _functions.hasActiveObject(name, tid);
2697 : }
2698 :
2699 : Function &
2700 62254 : FEProblemBase::getFunction(const std::string & name, const THREAD_ID tid)
2701 : {
2702 : // This thread lock is necessary since this method will create functions
2703 : // for all threads if one is missing.
2704 62254 : Threads::spin_mutex::scoped_lock lock(get_function_mutex);
2705 :
2706 62254 : if (!hasFunction(name, tid))
2707 : {
2708 : // If we didn't find a function, it might be a default function, attempt to construct one now
2709 20330 : std::istringstream ss(name);
2710 : Real real_value;
2711 :
2712 : // First see if it's just a constant. If it is, build a ConstantFunction
2713 20330 : if (ss >> real_value && ss.eof())
2714 : {
2715 12592 : InputParameters params = _factory.getValidParams("ConstantFunction");
2716 12592 : params.set<Real>("value") = real_value;
2717 18888 : addFunction("ConstantFunction", ss.str(), params);
2718 6296 : }
2719 : else
2720 : {
2721 14034 : FunctionParserBase<Real> fp;
2722 14034 : std::string vars = "x,y,z,t,NaN,pi,e";
2723 14034 : if (fp.Parse(name, vars) == -1) // -1 for success
2724 : {
2725 : // It parsed ok, so build a MooseParsedFunction
2726 42087 : InputParameters params = _factory.getValidParams("ParsedFunction");
2727 14029 : params.set<std::string>("expression") = name;
2728 28058 : addFunction("ParsedFunction", name, params);
2729 14029 : }
2730 14034 : }
2731 :
2732 : // Try once more
2733 20330 : if (!hasFunction(name, tid))
2734 : {
2735 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_function"),
2736 : "getFunction() was called before Functions have been constructed. The requested "
2737 : "Function '" +
2738 : name + "' may exist in the input file, but Functions are not available yet.");
2739 :
2740 7 : mooseError("Unable to find function " + name);
2741 : }
2742 20327 : }
2743 :
2744 62249 : auto * const ret = dynamic_cast<Function *>(_functions.getActiveObject(name, tid).get());
2745 62249 : if (!ret)
2746 0 : mooseError("No function named ", name, " of appropriate type");
2747 :
2748 62249 : return *ret;
2749 62251 : }
2750 :
2751 : bool
2752 208927 : FEProblemBase::hasConvergence(const std::string & name, const THREAD_ID tid) const
2753 : {
2754 208927 : return _convergences.hasActiveObject(name, tid);
2755 : }
2756 :
2757 : Convergence &
2758 1128053 : FEProblemBase::getConvergence(const std::string & name, const THREAD_ID tid) const
2759 : {
2760 1128053 : auto * const ret = dynamic_cast<Convergence *>(_convergences.getActiveObject(name, tid).get());
2761 1128053 : if (!ret)
2762 0 : mooseError("The Convergence object '", name, "' does not exist.");
2763 :
2764 1128053 : return *ret;
2765 : }
2766 :
2767 : const std::vector<std::shared_ptr<Convergence>> &
2768 191827 : FEProblemBase::getConvergenceObjects(const THREAD_ID tid) const
2769 : {
2770 191827 : return _convergences.getActiveObjects(tid);
2771 : }
2772 :
2773 : void
2774 687 : FEProblemBase::addMeshDivision(const std::string & type,
2775 : const std::string & name,
2776 : InputParameters & parameters)
2777 : {
2778 : parallel_object_only();
2779 1374 : parameters.set<FEProblemBase *>("_fe_problem_base") = this;
2780 1374 : parameters.set<SubProblem *>("_subproblem") = this;
2781 1462 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2782 : {
2783 784 : std::shared_ptr<MeshDivision> func = _factory.create<MeshDivision>(type, name, parameters, tid);
2784 775 : _mesh_divisions.addObject(func, tid);
2785 775 : }
2786 678 : }
2787 :
2788 : MeshDivision &
2789 1567 : FEProblemBase::getMeshDivision(const std::string & name, const THREAD_ID tid) const
2790 : {
2791 1567 : auto * const ret = dynamic_cast<MeshDivision *>(_mesh_divisions.getActiveObject(name, tid).get());
2792 1567 : if (!ret)
2793 0 : mooseError("No MeshDivision object named ", name, " of appropriate type");
2794 1567 : return *ret;
2795 : }
2796 :
2797 : void
2798 0 : FEProblemBase::lineSearch()
2799 : {
2800 0 : _line_search->lineSearch();
2801 0 : }
2802 :
2803 : NonlinearSystem &
2804 0 : FEProblemBase::getNonlinearSystem(const unsigned int sys_num)
2805 : {
2806 0 : mooseDeprecated("FEProblemBase::getNonlinearSystem() is deprecated, please use "
2807 : "FEProblemBase::getNonlinearSystemBase() \n");
2808 :
2809 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
2810 0 : auto nl_sys = std::dynamic_pointer_cast<NonlinearSystem>(_nl[sys_num]);
2811 :
2812 0 : if (!nl_sys)
2813 0 : mooseError("This is not a NonlinearSystem");
2814 :
2815 0 : return *nl_sys;
2816 0 : }
2817 :
2818 : void
2819 2 : FEProblemBase::addDistribution(const std::string & type,
2820 : const std::string & name,
2821 : InputParameters & parameters)
2822 : {
2823 4 : parameters.set<std::string>("type") = type;
2824 2 : addObject<Distribution>(type, name, parameters, /* threaded = */ false);
2825 2 : }
2826 :
2827 : bool
2828 4 : FEProblemBase::hasDistribution(const std::string & name) const
2829 : {
2830 4 : std::vector<Distribution *> objs;
2831 4 : theWarehouse()
2832 8 : .query()
2833 4 : .condition<AttribSystem>("Distribution")
2834 4 : .condition<AttribName>(name)
2835 4 : .queryInto(objs);
2836 8 : return !objs.empty();
2837 4 : }
2838 :
2839 : Distribution &
2840 4 : FEProblemBase::getDistribution(const std::string & name)
2841 : {
2842 4 : std::vector<Distribution *> objs;
2843 4 : theWarehouse()
2844 8 : .query()
2845 4 : .condition<AttribSystem>("Distribution")
2846 4 : .condition<AttribName>(name)
2847 4 : .queryInto(objs);
2848 4 : if (objs.empty())
2849 0 : mooseError("Unable to find Distribution with name '" + name + "'");
2850 8 : return *(objs[0]);
2851 4 : }
2852 :
2853 : void
2854 275 : FEProblemBase::addSampler(const std::string & type,
2855 : const std::string & name,
2856 : InputParameters & parameters)
2857 : {
2858 275 : const auto samplers = addObject<Sampler>(type, name, parameters);
2859 569 : for (auto & sampler : samplers)
2860 303 : sampler->init();
2861 266 : }
2862 :
2863 : Sampler &
2864 266 : FEProblemBase::getSampler(const std::string & name, const THREAD_ID tid)
2865 : {
2866 266 : std::vector<Sampler *> objs;
2867 266 : theWarehouse()
2868 532 : .query()
2869 266 : .condition<AttribSystem>("Sampler")
2870 266 : .condition<AttribThread>(tid)
2871 266 : .condition<AttribName>(name)
2872 266 : .queryInto(objs);
2873 266 : if (objs.empty())
2874 0 : mooseError(
2875 0 : "Unable to find Sampler with name '" + name +
2876 : "', if you are attempting to access this object in the constructor of another object then "
2877 : "the object being retrieved must occur prior to the caller within the input file.");
2878 532 : return *(objs[0]);
2879 266 : }
2880 :
2881 : bool
2882 152996 : FEProblemBase::duplicateVariableCheck(const std::string & var_name,
2883 : const FEType & type,
2884 : bool is_aux,
2885 : const std::set<SubdomainID> * const active_subdomains)
2886 : {
2887 152996 : std::set<SubdomainID> subdomainIDs;
2888 152996 : if (active_subdomains->size() == 0)
2889 : {
2890 144896 : const auto subdomains = _mesh.meshSubdomains();
2891 144896 : subdomainIDs.insert(subdomains.begin(), subdomains.end());
2892 144896 : }
2893 : else
2894 8100 : subdomainIDs.insert(active_subdomains->begin(), active_subdomains->end());
2895 :
2896 306660 : for (auto & sys : _solver_systems)
2897 : {
2898 153676 : SystemBase * curr_sys_ptr = sys.get();
2899 153676 : SystemBase * other_sys_ptr = _aux.get();
2900 153676 : std::string error_prefix = "";
2901 153676 : if (is_aux)
2902 : {
2903 93957 : curr_sys_ptr = _aux.get();
2904 93957 : other_sys_ptr = sys.get();
2905 93957 : error_prefix = "aux";
2906 : }
2907 :
2908 153676 : if (other_sys_ptr->hasVariable(var_name))
2909 3 : mooseError("Cannot have an auxiliary variable and a solver variable with the same name: ",
2910 : var_name);
2911 :
2912 153673 : if (curr_sys_ptr->hasVariable(var_name))
2913 : {
2914 : const Variable & var =
2915 9 : curr_sys_ptr->system().variable(curr_sys_ptr->system().variable_number(var_name));
2916 :
2917 : // variable type
2918 9 : if (var.type() != type)
2919 : {
2920 12 : const auto stringifyType = [](FEType t)
2921 12 : { return Moose::stringify(t.family) + " of order " + Moose::stringify(t.order); };
2922 :
2923 6 : mooseError("Mismatching types are specified for ",
2924 : error_prefix,
2925 : "variable with name '",
2926 : var_name,
2927 : "': '",
2928 6 : stringifyType(var.type()),
2929 : "' and '",
2930 6 : stringifyType(type),
2931 : "'");
2932 : }
2933 :
2934 : // block-restriction
2935 3 : if (!(active_subdomains->size() == 0 && var.active_subdomains().size() == 0))
2936 : {
2937 3 : const auto varActiveSubdomains = var.active_subdomains();
2938 3 : std::set<SubdomainID> varSubdomainIDs;
2939 3 : if (varActiveSubdomains.size() == 0)
2940 : {
2941 0 : const auto subdomains = _mesh.meshSubdomains();
2942 0 : varSubdomainIDs.insert(subdomains.begin(), subdomains.end());
2943 0 : }
2944 : else
2945 3 : varSubdomainIDs.insert(varActiveSubdomains.begin(), varActiveSubdomains.end());
2946 :
2947 : // Is subdomainIDs a subset of varSubdomainIDs? With this we allow the case that the newly
2948 : // requested block restriction is only a subset of the existing one.
2949 3 : const auto isSubset = std::includes(varSubdomainIDs.begin(),
2950 : varSubdomainIDs.end(),
2951 : subdomainIDs.begin(),
2952 : subdomainIDs.end());
2953 :
2954 3 : if (!isSubset)
2955 : {
2956 : // helper function: make a string from a set of subdomain ids
2957 6 : const auto stringifySubdomains = [this](std::set<SubdomainID> subdomainIDs)
2958 : {
2959 6 : std::stringstream s;
2960 15 : for (auto const i : subdomainIDs)
2961 : {
2962 : // do we need to insert a comma?
2963 9 : if (s.tellp() != 0)
2964 3 : s << ", ";
2965 :
2966 : // insert subdomain name and id -or- only the id (if no name is given)
2967 9 : const auto subdomainName = _mesh.getSubdomainName(i);
2968 9 : if (subdomainName.empty())
2969 9 : s << i;
2970 : else
2971 0 : s << subdomainName << " (" << i << ")";
2972 9 : }
2973 12 : return s.str();
2974 6 : };
2975 :
2976 6 : const std::string msg = "Mismatching block-restrictions are specified for " +
2977 6 : error_prefix + "variable with name '" + var_name + "': {" +
2978 12 : stringifySubdomains(varSubdomainIDs) + "} and {" +
2979 9 : stringifySubdomains(subdomainIDs) + "}";
2980 :
2981 3 : mooseError(msg);
2982 0 : }
2983 0 : }
2984 :
2985 0 : return true;
2986 : }
2987 153664 : }
2988 :
2989 152984 : return false;
2990 152984 : }
2991 :
2992 : void
2993 59171 : FEProblemBase::addVariable(const std::string & var_type,
2994 : const std::string & var_name,
2995 : InputParameters & params)
2996 : {
2997 : parallel_object_only();
2998 :
2999 59171 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3000 59171 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3001 59171 : const auto fe_type = FEType(order, family);
3002 :
3003 : const auto active_subdomains_vector =
3004 59171 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3005 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3006 59171 : active_subdomains_vector.end());
3007 :
3008 59171 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ false, &active_subdomains))
3009 0 : return;
3010 :
3011 177504 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3012 59168 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_SOLVER;
3013 59168 : SolverSystemName sys_name = params.get<SolverSystemName>("solver_sys");
3014 :
3015 59168 : const auto solver_system_number = solverSysNum(sys_name);
3016 59168 : logAdd("Variable", var_name, var_type, params);
3017 59168 : _solver_systems[solver_system_number]->addVariable(var_type, var_name, params);
3018 59156 : if (_displaced_problem)
3019 : // MooseObjects need to be unique so change the name here
3020 3453 : _displaced_problem->addVariable(var_type, var_name, params, solver_system_number);
3021 :
3022 59156 : _solver_var_to_sys_num[var_name] = solver_system_number;
3023 :
3024 59156 : markFamilyPRefinement(params);
3025 59156 : }
3026 :
3027 : std::pair<bool, unsigned int>
3028 4544192 : FEProblemBase::determineSolverSystem(const std::string & var_name,
3029 : const bool error_if_not_found) const
3030 : {
3031 4544192 : auto map_it = _solver_var_to_sys_num.find(var_name);
3032 4544192 : const bool var_in_sys = map_it != _solver_var_to_sys_num.end();
3033 4544192 : if (var_in_sys)
3034 : mooseAssert(_solver_systems[map_it->second]->hasVariable(var_name) ||
3035 : _solver_systems[map_it->second]->hasScalarVariable(var_name),
3036 : "If the variable is in our FEProblem solver system map, then it must be in the "
3037 : "solver system we expect");
3038 3100353 : else if (error_if_not_found)
3039 : {
3040 32 : if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
3041 21 : mooseError("No solver variable named ",
3042 : var_name,
3043 : " found. Did you specify an auxiliary variable when you meant to specify a "
3044 : "solver variable?");
3045 : else
3046 11 : mooseError("Unknown variable '",
3047 : var_name,
3048 : "'. It does not exist in the solver system(s) or auxiliary system");
3049 : }
3050 :
3051 9088320 : return std::make_pair(var_in_sys, var_in_sys ? map_it->second : libMesh::invalid_uint);
3052 : }
3053 :
3054 : void
3055 158576 : FEProblemBase::setResidualObjectParamsAndLog(const std::string & ro_name,
3056 : const std::string & name,
3057 : InputParameters & parameters,
3058 : const unsigned int nl_sys_num,
3059 : const std::string & base_name,
3060 : bool & reinit_displaced)
3061 : {
3062 158576 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3063 : {
3064 1920 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3065 1920 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3066 960 : reinit_displaced = true;
3067 : }
3068 : else
3069 : {
3070 157616 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3071 : {
3072 : // We allow Kernels to request that they use_displaced_mesh,
3073 : // but then be overridden when no displacements variables are
3074 : // provided in the Mesh block. If that happened, update the value
3075 : // of use_displaced_mesh appropriately for this Kernel.
3076 105 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3077 210 : parameters.set<bool>("use_displaced_mesh") = false;
3078 : }
3079 :
3080 315232 : parameters.set<SubProblem *>("_subproblem") = this;
3081 472848 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3082 : }
3083 :
3084 158576 : logAdd(base_name, name, ro_name, parameters);
3085 158576 : }
3086 :
3087 : void
3088 65048 : FEProblemBase::setAuxKernelParamsAndLog(const std::string & ak_name,
3089 : const std::string & name,
3090 : InputParameters & parameters,
3091 : const std::string & base_name)
3092 : {
3093 65048 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3094 : {
3095 22872 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3096 22872 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3097 22872 : parameters.set<SystemBase *>("_nl_sys") = &_displaced_problem->solverSys(0);
3098 11436 : if (!parameters.get<std::vector<BoundaryName>>("boundary").empty())
3099 10990 : _reinit_displaced_face = true;
3100 : else
3101 446 : _reinit_displaced_elem = true;
3102 : }
3103 : else
3104 : {
3105 53612 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3106 : {
3107 : // We allow AuxKernels to request that they use_displaced_mesh,
3108 : // but then be overridden when no displacements variables are
3109 : // provided in the Mesh block. If that happened, update the value
3110 : // of use_displaced_mesh appropriately for this AuxKernel.
3111 795 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3112 1590 : parameters.set<bool>("use_displaced_mesh") = false;
3113 : }
3114 :
3115 107224 : parameters.set<SubProblem *>("_subproblem") = this;
3116 107224 : parameters.set<SystemBase *>("_sys") = _aux.get();
3117 160836 : parameters.set<SystemBase *>("_nl_sys") = _solver_systems[0].get();
3118 : }
3119 :
3120 65048 : logAdd(base_name, name, ak_name, parameters);
3121 65048 : }
3122 :
3123 : void
3124 76102 : FEProblemBase::addKernel(const std::string & kernel_name,
3125 : const std::string & name,
3126 : InputParameters & parameters)
3127 : {
3128 : parallel_object_only();
3129 152204 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3130 76090 : if (!isSolverSystemNonlinear(nl_sys_num))
3131 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
3132 : "supported at the moment!");
3133 76090 : setResidualObjectParamsAndLog(
3134 76090 : kernel_name, name, parameters, nl_sys_num, "Kernel", _reinit_displaced_elem);
3135 :
3136 76090 : _nl[nl_sys_num]->addKernel(kernel_name, name, parameters);
3137 75937 : }
3138 :
3139 : void
3140 431 : FEProblemBase::addHDGKernel(const std::string & kernel_name,
3141 : const std::string & name,
3142 : InputParameters & parameters)
3143 : {
3144 : parallel_object_only();
3145 862 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3146 431 : if (!isSolverSystemNonlinear(nl_sys_num))
3147 0 : mooseError("You are trying to add a HDGKernel to a linear variable/system, which is not "
3148 : "supported at the moment!");
3149 431 : setResidualObjectParamsAndLog(
3150 431 : kernel_name, name, parameters, nl_sys_num, "HDGKernel", _reinit_displaced_elem);
3151 :
3152 431 : _nl[nl_sys_num]->addHDGKernel(kernel_name, name, parameters);
3153 431 : }
3154 :
3155 : void
3156 602 : FEProblemBase::addNodalKernel(const std::string & kernel_name,
3157 : const std::string & name,
3158 : InputParameters & parameters)
3159 : {
3160 : parallel_object_only();
3161 :
3162 1204 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3163 599 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3164 : {
3165 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3166 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3167 0 : _reinit_displaced_elem = true;
3168 : }
3169 : else
3170 : {
3171 599 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3172 : {
3173 : // We allow NodalKernels to request that they use_displaced_mesh,
3174 : // but then be overridden when no displacements variables are
3175 : // provided in the Mesh block. If that happened, update the value
3176 : // of use_displaced_mesh appropriately for this NodalKernel.
3177 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3178 0 : parameters.set<bool>("use_displaced_mesh") = false;
3179 : }
3180 :
3181 1198 : parameters.set<SubProblem *>("_subproblem") = this;
3182 1797 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3183 : }
3184 599 : logAdd("NodalKernel", name, kernel_name, parameters);
3185 599 : _nl[nl_sys_num]->addNodalKernel(kernel_name, name, parameters);
3186 599 : }
3187 :
3188 : void
3189 1319 : FEProblemBase::addScalarKernel(const std::string & kernel_name,
3190 : const std::string & name,
3191 : InputParameters & parameters)
3192 : {
3193 : parallel_object_only();
3194 :
3195 2638 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3196 1319 : if (!isSolverSystemNonlinear(nl_sys_num))
3197 0 : mooseError("You are trying to add a ScalarKernel to a linear variable/system, which is not "
3198 : "supported at the moment!");
3199 :
3200 1319 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3201 : {
3202 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3203 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3204 : }
3205 : else
3206 : {
3207 1319 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3208 : {
3209 : // We allow ScalarKernels to request that they use_displaced_mesh,
3210 : // but then be overridden when no displacements variables are
3211 : // provided in the Mesh block. If that happened, update the value
3212 : // of use_displaced_mesh appropriately for this ScalarKernel.
3213 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3214 0 : parameters.set<bool>("use_displaced_mesh") = false;
3215 : }
3216 :
3217 2638 : parameters.set<SubProblem *>("_subproblem") = this;
3218 3957 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3219 : }
3220 :
3221 1319 : logAdd("ScalarKernel", name, kernel_name, parameters);
3222 1319 : _nl[nl_sys_num]->addScalarKernel(kernel_name, name, parameters);
3223 1313 : }
3224 :
3225 : void
3226 74555 : FEProblemBase::addBoundaryCondition(const std::string & bc_name,
3227 : const std::string & name,
3228 : InputParameters & parameters)
3229 : {
3230 : parallel_object_only();
3231 :
3232 149114 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3233 74547 : if (!isSolverSystemNonlinear(nl_sys_num))
3234 0 : mooseError(
3235 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
3236 : "supported at the moment!");
3237 :
3238 74547 : setResidualObjectParamsAndLog(
3239 74547 : bc_name, name, parameters, nl_sys_num, "BoundaryCondition", _reinit_displaced_face);
3240 74547 : _nl[nl_sys_num]->addBoundaryCondition(bc_name, name, parameters);
3241 74503 : }
3242 :
3243 : void
3244 1609 : FEProblemBase::addConstraint(const std::string & c_name,
3245 : const std::string & name,
3246 : InputParameters & parameters)
3247 : {
3248 : parallel_object_only();
3249 :
3250 1609 : _has_constraints = true;
3251 :
3252 1609 : auto determine_var_param_name = [¶meters, this]()
3253 : {
3254 4827 : if (parameters.isParamValid("variable"))
3255 1048 : return "variable";
3256 : else
3257 : {
3258 : // must be a mortar constraint
3259 1122 : const bool has_secondary_var = parameters.isParamValid("secondary_variable");
3260 1122 : const bool has_primary_var = parameters.isParamValid("primary_variable");
3261 561 : if (!has_secondary_var && !has_primary_var)
3262 0 : mooseError(
3263 : "Either a 'secondary_variable' or 'primary_variable' parameter must be supplied for '",
3264 0 : parameters.getObjectName(),
3265 : "'");
3266 561 : return has_secondary_var ? "secondary_variable" : "primary_variable";
3267 : }
3268 1609 : };
3269 :
3270 : const auto nl_sys_num =
3271 3218 : determineSolverSystem(parameters.varName(determine_var_param_name(), name), true).second;
3272 1606 : if (!isSolverSystemNonlinear(nl_sys_num))
3273 0 : mooseError("You are trying to add a Constraint to a linear variable/system, which is not "
3274 : "supported at the moment!");
3275 :
3276 1606 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3277 : {
3278 266 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3279 266 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3280 133 : _reinit_displaced_face = true;
3281 : }
3282 : else
3283 : {
3284 : // It might _want_ to use a displaced mesh... but we're not so set it to false
3285 1473 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3286 2946 : parameters.set<bool>("use_displaced_mesh") = false;
3287 :
3288 2946 : parameters.set<SubProblem *>("_subproblem") = this;
3289 4419 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3290 : }
3291 :
3292 1606 : logAdd("Constraint", name, c_name, parameters);
3293 1606 : _nl[nl_sys_num]->addConstraint(c_name, name, parameters);
3294 1591 : }
3295 :
3296 : void
3297 93825 : FEProblemBase::addAuxVariable(const std::string & var_type,
3298 : const std::string & var_name,
3299 : InputParameters & params)
3300 : {
3301 : parallel_object_only();
3302 :
3303 93825 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3304 93825 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3305 93825 : const auto fe_type = FEType(order, family);
3306 :
3307 : const auto active_subdomains_vector =
3308 93825 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3309 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3310 93825 : active_subdomains_vector.end());
3311 :
3312 93825 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ true, &active_subdomains))
3313 0 : return;
3314 :
3315 281448 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3316 187632 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3317 :
3318 93816 : logAdd("AuxVariable", var_name, var_type, params);
3319 93816 : _aux->addVariable(var_type, var_name, params);
3320 93816 : if (_displaced_problem)
3321 : // MooseObjects need to be unique so change the name here
3322 10166 : _displaced_problem->addAuxVariable(var_type, var_name, params);
3323 :
3324 93816 : markFamilyPRefinement(params);
3325 93816 : }
3326 :
3327 : void
3328 2612 : FEProblemBase::addElementalFieldVariable(const std::string & var_type,
3329 : const std::string & var_name,
3330 : InputParameters & params)
3331 : {
3332 2612 : addAuxVariable(var_type, var_name, params);
3333 2612 : }
3334 :
3335 : void
3336 0 : FEProblemBase::addAuxVariable(const std::string & var_name,
3337 : const FEType & type,
3338 : const std::set<SubdomainID> * const active_subdomains)
3339 : {
3340 : parallel_object_only();
3341 :
3342 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3343 :
3344 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3345 0 : return;
3346 :
3347 0 : std::string var_type;
3348 0 : if (type == FEType(0, MONOMIAL))
3349 0 : var_type = "MooseVariableConstMonomial";
3350 0 : else if (type.family == SCALAR)
3351 0 : var_type = "MooseVariableScalar";
3352 0 : else if (FEInterface::field_type(type) == TYPE_VECTOR)
3353 0 : var_type = "VectorMooseVariable";
3354 : else
3355 0 : var_type = "MooseVariable";
3356 :
3357 0 : InputParameters params = _factory.getValidParams(var_type);
3358 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3359 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3360 0 : params.set<MooseEnum>("order") = type.order.get_order();
3361 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3362 :
3363 0 : if (active_subdomains)
3364 0 : for (const SubdomainID & id : *active_subdomains)
3365 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3366 :
3367 0 : logAdd("AuxVariable", var_name, var_type, params);
3368 0 : _aux->addVariable(var_type, var_name, params);
3369 0 : if (_displaced_problem)
3370 0 : _displaced_problem->addAuxVariable("MooseVariable", var_name, params);
3371 :
3372 0 : markFamilyPRefinement(params);
3373 0 : }
3374 :
3375 : void
3376 0 : FEProblemBase::addAuxArrayVariable(const std::string & var_name,
3377 : const FEType & type,
3378 : unsigned int components,
3379 : const std::set<SubdomainID> * const active_subdomains)
3380 : {
3381 : parallel_object_only();
3382 :
3383 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3384 :
3385 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3386 0 : return;
3387 :
3388 0 : InputParameters params = _factory.getValidParams("ArrayMooseVariable");
3389 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3390 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3391 0 : params.set<MooseEnum>("order") = type.order.get_order();
3392 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3393 0 : params.set<unsigned int>("components") = components;
3394 :
3395 0 : if (active_subdomains)
3396 0 : for (const SubdomainID & id : *active_subdomains)
3397 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3398 :
3399 0 : logAdd("Variable", var_name, "ArrayMooseVariable", params);
3400 0 : _aux->addVariable("ArrayMooseVariable", var_name, params);
3401 0 : if (_displaced_problem)
3402 0 : _displaced_problem->addAuxVariable("ArrayMooseVariable", var_name, params);
3403 :
3404 0 : markFamilyPRefinement(params);
3405 0 : }
3406 :
3407 : void
3408 0 : FEProblemBase::addAuxScalarVariable(const std::string & var_name,
3409 : Order order,
3410 : Real /*scale_factor*/,
3411 : const std::set<SubdomainID> * const active_subdomains)
3412 : {
3413 : parallel_object_only();
3414 :
3415 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3416 :
3417 0 : if (order > _max_scalar_order)
3418 0 : _max_scalar_order = order;
3419 :
3420 0 : FEType type(order, SCALAR);
3421 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3422 0 : return;
3423 :
3424 0 : InputParameters params = _factory.getValidParams("MooseVariableScalar");
3425 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3426 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3427 :
3428 0 : params.set<MooseEnum>("order") = type.order.get_order();
3429 0 : params.set<MooseEnum>("family") = "SCALAR";
3430 0 : params.set<std::vector<Real>>("scaling") = std::vector<Real>{1};
3431 0 : if (active_subdomains)
3432 0 : for (const SubdomainID & id : *active_subdomains)
3433 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3434 :
3435 0 : logAdd("ScalarVariable", var_name, "MooseVariableScalar", params);
3436 0 : _aux->addVariable("MooseVariableScalar", var_name, params);
3437 0 : if (_displaced_problem)
3438 0 : _displaced_problem->addAuxVariable("MooseVariableScalar", var_name, params);
3439 0 : }
3440 :
3441 : void
3442 64427 : FEProblemBase::addAuxKernel(const std::string & kernel_name,
3443 : const std::string & name,
3444 : InputParameters & parameters)
3445 : {
3446 : parallel_object_only();
3447 :
3448 64427 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "AuxKernel");
3449 :
3450 64427 : _aux->addKernel(kernel_name, name, parameters);
3451 64316 : }
3452 :
3453 : void
3454 475 : FEProblemBase::addAuxScalarKernel(const std::string & kernel_name,
3455 : const std::string & name,
3456 : InputParameters & parameters)
3457 : {
3458 : parallel_object_only();
3459 :
3460 475 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3461 : {
3462 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3463 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3464 : }
3465 : else
3466 : {
3467 475 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3468 : {
3469 : // We allow AuxScalarKernels to request that they use_displaced_mesh,
3470 : // but then be overridden when no displacements variables are
3471 : // provided in the Mesh block. If that happened, update the value
3472 : // of use_displaced_mesh appropriately for this AuxScalarKernel.
3473 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3474 0 : parameters.set<bool>("use_displaced_mesh") = false;
3475 : }
3476 :
3477 950 : parameters.set<SubProblem *>("_subproblem") = this;
3478 1425 : parameters.set<SystemBase *>("_sys") = _aux.get();
3479 : }
3480 :
3481 475 : logAdd("AuxScalarKernel", name, kernel_name, parameters);
3482 475 : _aux->addScalarKernel(kernel_name, name, parameters);
3483 472 : }
3484 :
3485 : void
3486 871 : FEProblemBase::addDiracKernel(const std::string & kernel_name,
3487 : const std::string & name,
3488 : InputParameters & parameters)
3489 : {
3490 : parallel_object_only();
3491 :
3492 1742 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3493 868 : if (!isSolverSystemNonlinear(nl_sys_num))
3494 0 : mooseError("You are trying to add a DiracKernel to a linear variable/system, which is not "
3495 : "supported at the moment!");
3496 :
3497 868 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3498 : {
3499 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3500 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3501 12 : _reinit_displaced_elem = true;
3502 : }
3503 : else
3504 : {
3505 856 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3506 : {
3507 : // We allow DiracKernels to request that they use_displaced_mesh,
3508 : // but then be overridden when no displacements variables are
3509 : // provided in the Mesh block. If that happened, update the value
3510 : // of use_displaced_mesh appropriately for this DiracKernel.
3511 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3512 0 : parameters.set<bool>("use_displaced_mesh") = false;
3513 : }
3514 :
3515 1712 : parameters.set<SubProblem *>("_subproblem") = this;
3516 2568 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3517 : }
3518 :
3519 868 : logAdd("DiracKernel", name, kernel_name, parameters);
3520 868 : _nl[nl_sys_num]->addDiracKernel(kernel_name, name, parameters);
3521 862 : }
3522 :
3523 : // DGKernels ////
3524 :
3525 : void
3526 1298 : FEProblemBase::addDGKernel(const std::string & dg_kernel_name,
3527 : const std::string & name,
3528 : InputParameters & parameters)
3529 : {
3530 : parallel_object_only();
3531 :
3532 2596 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3533 1295 : if (!isSolverSystemNonlinear(nl_sys_num))
3534 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
3535 : "supported at the moment!");
3536 :
3537 1295 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3538 : {
3539 48 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3540 48 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3541 24 : _reinit_displaced_neighbor = true;
3542 : }
3543 : else
3544 : {
3545 1271 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3546 : {
3547 : // We allow DGKernels to request that they use_displaced_mesh,
3548 : // but then be overridden when no displacements variables are
3549 : // provided in the Mesh block. If that happened, update the value
3550 : // of use_displaced_mesh appropriately for this DGKernel.
3551 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3552 0 : parameters.set<bool>("use_displaced_mesh") = false;
3553 : }
3554 :
3555 2542 : parameters.set<SubProblem *>("_subproblem") = this;
3556 3813 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3557 : }
3558 :
3559 1295 : logAdd("DGKernel", name, dg_kernel_name, parameters);
3560 1295 : _nl[nl_sys_num]->addDGKernel(dg_kernel_name, name, parameters);
3561 :
3562 1295 : _has_internal_edge_residual_objects = true;
3563 1295 : }
3564 :
3565 : void
3566 6609 : FEProblemBase::addFVKernel(const std::string & fv_kernel_name,
3567 : const std::string & name,
3568 : InputParameters & parameters)
3569 : {
3570 6609 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3571 : // FVElementalKernels are computed in the historically finite element threaded loops. They rely
3572 : // on Assembly data like _current_elem. When we call reinit on the FEProblemBase we will only
3573 : // reinit the DisplacedProblem and its associated Assembly objects if we mark this boolean as
3574 : // true
3575 0 : _reinit_displaced_elem = true;
3576 6609 : addObject<FVKernel>(fv_kernel_name, name, parameters);
3577 6603 : }
3578 :
3579 : void
3580 5855 : FEProblemBase::addFVBC(const std::string & fv_bc_name,
3581 : const std::string & name,
3582 : InputParameters & parameters)
3583 : {
3584 5855 : addObject<FVBoundaryCondition>(fv_bc_name, name, parameters);
3585 5855 : }
3586 :
3587 : void
3588 238 : FEProblemBase::addFVInterfaceKernel(const std::string & fv_ik_name,
3589 : const std::string & name,
3590 : InputParameters & parameters)
3591 : {
3592 : /// We assume that variable1 and variable2 can live on different systems, in this case
3593 : /// the user needs to create two interface kernels with flipped variables and parameters
3594 238 : addObject<FVInterfaceKernel>(
3595 : fv_ik_name, name, parameters, /*threaded=*/true, /*variable_param_name=*/"variable1");
3596 229 : }
3597 :
3598 : void
3599 2157 : FEProblemBase::addLinearFVKernel(const std::string & kernel_name,
3600 : const std::string & name,
3601 : InputParameters & parameters)
3602 : {
3603 2157 : addObject<LinearFVKernel>(kernel_name, name, parameters);
3604 2157 : }
3605 :
3606 : void
3607 2064 : FEProblemBase::addLinearFVBC(const std::string & bc_name,
3608 : const std::string & name,
3609 : InputParameters & parameters)
3610 : {
3611 2064 : addObject<LinearFVBoundaryCondition>(bc_name, name, parameters);
3612 2064 : }
3613 :
3614 : // InterfaceKernels ////
3615 :
3616 : void
3617 800 : FEProblemBase::addInterfaceKernel(const std::string & interface_kernel_name,
3618 : const std::string & name,
3619 : InputParameters & parameters)
3620 : {
3621 : parallel_object_only();
3622 :
3623 1600 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3624 797 : if (!isSolverSystemNonlinear(nl_sys_num))
3625 0 : mooseError("You are trying to add a InterfaceKernel to a linear variable/system, which is not "
3626 : "supported at the moment!");
3627 :
3628 797 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3629 : {
3630 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3631 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3632 12 : _reinit_displaced_neighbor = true;
3633 : }
3634 : else
3635 : {
3636 785 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3637 : {
3638 : // We allow InterfaceKernels to request that they use_displaced_mesh,
3639 : // but then be overridden when no displacements variables are
3640 : // provided in the Mesh block. If that happened, update the value
3641 : // of use_displaced_mesh appropriately for this InterfaceKernel.
3642 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3643 0 : parameters.set<bool>("use_displaced_mesh") = false;
3644 : }
3645 :
3646 1570 : parameters.set<SubProblem *>("_subproblem") = this;
3647 2355 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3648 : }
3649 :
3650 797 : logAdd("InterfaceKernel", name, interface_kernel_name, parameters);
3651 797 : _nl[nl_sys_num]->addInterfaceKernel(interface_kernel_name, name, parameters);
3652 :
3653 797 : _has_internal_edge_residual_objects = true;
3654 797 : }
3655 :
3656 : void
3657 33709 : FEProblemBase::checkICRestartError(const std::string & ic_name,
3658 : const std::string & name,
3659 : const VariableName & var_name)
3660 : {
3661 33709 : if (!_allow_ics_during_restart)
3662 : {
3663 33630 : std::string restart_method = "";
3664 33630 : if (_app.isRestarting())
3665 : restart_method =
3666 0 : "a checkpoint restart, by IC object '" + ic_name + "' for variable '" + name + "'";
3667 33630 : else if (_app.getExReaderForRestart())
3668 : {
3669 3 : std::vector<std::string> restarted_vars = _app.getExReaderForRestart()->get_elem_var_names();
3670 3 : const auto nodal_vars = _app.getExReaderForRestart()->get_nodal_var_names();
3671 3 : const auto global_vars = _app.getExReaderForRestart()->get_global_var_names();
3672 3 : restarted_vars.insert(restarted_vars.end(), nodal_vars.begin(), nodal_vars.end());
3673 3 : restarted_vars.insert(restarted_vars.end(), global_vars.begin(), global_vars.end());
3674 :
3675 3 : if (std::find(restarted_vars.begin(), restarted_vars.end(), var_name) != restarted_vars.end())
3676 6 : restart_method = "an Exodus restart, by IC object '" + ic_name + "' for variable '" + name +
3677 3 : "' that is also being restarted";
3678 3 : }
3679 33630 : if (!restart_method.empty())
3680 3 : mooseError(
3681 : "Initial conditions have been specified during ",
3682 : restart_method,
3683 : ".\nThis is only allowed if you specify 'allow_initial_conditions_with_restart' to "
3684 : "the [Problem], as initial conditions can override restarted fields");
3685 33627 : }
3686 33706 : }
3687 :
3688 : void
3689 31443 : FEProblemBase::addInitialCondition(const std::string & ic_name,
3690 : const std::string & name,
3691 : InputParameters & parameters)
3692 : {
3693 : parallel_object_only();
3694 :
3695 : // before we start to mess with the initial condition, we need to check parameters for errors.
3696 31443 : parameters.checkParams(name);
3697 31440 : const std::string & var_name = parameters.get<VariableName>("variable");
3698 :
3699 : // Forbid initial conditions on a restarted problem, as they would override the restart
3700 31440 : checkICRestartError(ic_name, name, var_name);
3701 :
3702 62874 : parameters.set<SubProblem *>("_subproblem") = this;
3703 :
3704 : // field IC
3705 31437 : if (hasVariable(var_name))
3706 : {
3707 63201 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3708 : {
3709 33324 : MooseVariableFEBase & var = getVariable(
3710 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3711 66648 : parameters.set<SystemBase *>("_sys") = &var.sys();
3712 33324 : std::shared_ptr<InitialConditionBase> ic;
3713 33324 : if (dynamic_cast<MooseVariable *>(&var))
3714 31203 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3715 2121 : else if (dynamic_cast<VectorMooseVariable *>(&var))
3716 250 : ic = _factory.create<VectorInitialCondition>(ic_name, name, parameters, tid);
3717 1871 : else if (dynamic_cast<ArrayMooseVariable *>(&var))
3718 1207 : ic = _factory.create<ArrayInitialCondition>(ic_name, name, parameters, tid);
3719 664 : else if (dynamic_cast<MooseVariableFVReal *>(&var))
3720 664 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3721 0 : else if (dynamic_cast<MooseLinearVariableFVReal *>(&var))
3722 0 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3723 : else
3724 0 : mooseError("Your FE variable in initial condition ",
3725 : name,
3726 : " must be either of scalar or vector type");
3727 33303 : logAdd("IC", name, ic_name, parameters);
3728 33303 : _ics.addObject(ic, tid);
3729 33288 : }
3730 : }
3731 :
3732 : // scalar IC
3733 1524 : else if (hasScalarVariable(var_name))
3734 : {
3735 1524 : MooseVariableScalar & var = getScalarVariable(0, var_name);
3736 3048 : parameters.set<SystemBase *>("_sys") = &var.sys();
3737 : std::shared_ptr<ScalarInitialCondition> ic =
3738 1524 : _factory.create<ScalarInitialCondition>(ic_name, name, parameters);
3739 1524 : logAdd("ScalarIC", name, ic_name, parameters);
3740 1524 : _scalar_ics.addObject(ic);
3741 1524 : }
3742 :
3743 : else
3744 0 : mooseError(
3745 : "Variable '", var_name, "' requested in initial condition '", name, "' does not exist.");
3746 31401 : }
3747 :
3748 : void
3749 2269 : FEProblemBase::addFVInitialCondition(const std::string & ic_name,
3750 : const std::string & name,
3751 : InputParameters & parameters)
3752 : {
3753 : parallel_object_only();
3754 :
3755 : // before we start to mess with the initial condition, we need to check parameters for errors.
3756 2269 : parameters.checkParams(name);
3757 2269 : const std::string & var_name = parameters.get<VariableName>("variable");
3758 :
3759 : // Forbid initial conditions on a restarted problem, as they would override the restart
3760 2269 : checkICRestartError(ic_name, name, var_name);
3761 :
3762 4538 : parameters.set<SubProblem *>("_subproblem") = this;
3763 :
3764 : // field IC
3765 2269 : if (hasVariable(var_name))
3766 : {
3767 4620 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3768 : {
3769 2351 : auto & var = getVariable(
3770 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3771 4702 : parameters.set<SystemBase *>("_sys") = &var.sys();
3772 2351 : std::shared_ptr<FVInitialConditionBase> ic;
3773 2351 : if (var.isFV())
3774 2351 : ic = _factory.create<FVInitialCondition>(ic_name, name, parameters, tid);
3775 : else
3776 0 : mooseError(
3777 : "Your variable for an FVInitialCondition needs to be an a finite volume variable!");
3778 2351 : _fv_ics.addObject(ic, tid);
3779 2351 : }
3780 : }
3781 : else
3782 0 : mooseError("Variable '",
3783 : var_name,
3784 : "' requested in finite volume initial condition '",
3785 : name,
3786 : "' does not exist.");
3787 2269 : }
3788 :
3789 : void
3790 55828 : FEProblemBase::projectSolution()
3791 : {
3792 279140 : TIME_SECTION("projectSolution", 2, "Projecting Initial Solutions")
3793 :
3794 55828 : FloatingPointExceptionGuard fpe_guard(_app);
3795 :
3796 55828 : ComputeInitialConditionThread cic(*this);
3797 55828 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cic);
3798 :
3799 55822 : if (haveFV())
3800 : {
3801 : using ElemInfoRange = StoredRange<MooseMesh::const_elem_info_iterator, const ElemInfo *>;
3802 3930 : ElemInfoRange elem_info_range(_mesh.ownedElemInfoBegin(), _mesh.ownedElemInfoEnd());
3803 :
3804 3930 : ComputeFVInitialConditionThread cfvic(*this);
3805 3930 : Threads::parallel_reduce(elem_info_range, cfvic);
3806 3930 : }
3807 :
3808 : // Need to close the solution vector here so that boundary ICs take precendence
3809 110875 : for (auto & nl : _nl)
3810 55053 : nl->solution().close();
3811 55822 : _aux->solution().close();
3812 :
3813 : // now run boundary-restricted initial conditions
3814 55822 : ComputeBoundaryInitialConditionThread cbic(*this);
3815 55822 : Threads::parallel_reduce(getCurrentAlgebraicBndNodeRange(), cbic);
3816 :
3817 110875 : for (auto & nl : _nl)
3818 55053 : nl->solution().close();
3819 55822 : _aux->solution().close();
3820 :
3821 : // Also, load values into the SCALAR dofs
3822 : // Note: We assume that all SCALAR dofs are on the
3823 : // processor with highest ID
3824 55822 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3825 : {
3826 567 : const auto & ics = _scalar_ics.getActiveObjects();
3827 1561 : for (const auto & ic : ics)
3828 : {
3829 994 : MooseVariableScalar & var = ic->variable();
3830 994 : var.reinit();
3831 :
3832 994 : DenseVector<Number> vals(var.order());
3833 994 : ic->compute(vals);
3834 :
3835 994 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3836 2323 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3837 : {
3838 1329 : const auto global_index = var.dofIndices()[i];
3839 1329 : var.sys().solution().set(global_index, vals(i));
3840 1329 : var.setValue(i, vals(i));
3841 : }
3842 994 : }
3843 : }
3844 :
3845 111873 : for (auto & sys : _solver_systems)
3846 : {
3847 56051 : sys->solution().close();
3848 56051 : sys->solution().localize(*sys->system().current_local_solution, sys->dofMap().get_send_list());
3849 : }
3850 :
3851 55822 : _aux->solution().close();
3852 55822 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3853 55822 : }
3854 :
3855 : void
3856 2037 : FEProblemBase::projectInitialConditionOnCustomRange(
3857 : ConstElemRange & elem_range,
3858 : ConstBndNodeRange & bnd_nodes,
3859 : const std::optional<std::set<VariableName>> & target_vars)
3860 : {
3861 2037 : if (target_vars)
3862 : {
3863 2037 : ComputeInitialConditionThread cic(*this, &(*target_vars));
3864 2037 : Threads::parallel_reduce(elem_range, cic);
3865 : }
3866 : else
3867 : {
3868 0 : ComputeInitialConditionThread cic(*this);
3869 0 : Threads::parallel_reduce(elem_range, cic);
3870 : }
3871 :
3872 : // Need to close the solution vector here so that boundary ICs take precendence
3873 4074 : for (auto & nl : _nl)
3874 2037 : nl->solution().close();
3875 2037 : _aux->solution().close();
3876 :
3877 2037 : if (target_vars)
3878 : {
3879 2037 : ComputeBoundaryInitialConditionThread cbic(*this, &(*target_vars));
3880 2037 : Threads::parallel_reduce(bnd_nodes, cbic);
3881 2037 : }
3882 : else
3883 : {
3884 0 : ComputeBoundaryInitialConditionThread cbic(*this);
3885 0 : Threads::parallel_reduce(bnd_nodes, cbic);
3886 0 : }
3887 :
3888 4074 : for (auto & nl : _nl)
3889 2037 : nl->solution().close();
3890 2037 : _aux->solution().close();
3891 :
3892 : // Also, load values into the SCALAR dofs
3893 : // Note: We assume that all SCALAR dofs are on the
3894 : // processor with highest ID
3895 2037 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3896 : {
3897 0 : const auto & ics = _scalar_ics.getActiveObjects();
3898 0 : for (const auto & ic : ics)
3899 : {
3900 0 : MooseVariableScalar & var = ic->variable();
3901 :
3902 0 : if (target_vars && !target_vars->count(var.name()))
3903 0 : continue;
3904 :
3905 0 : var.reinit();
3906 :
3907 0 : DenseVector<Number> vals(var.order());
3908 0 : ic->compute(vals);
3909 :
3910 0 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3911 0 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3912 : {
3913 0 : const auto global_index = var.dofIndices()[i];
3914 0 : var.sys().solution().set(global_index, vals(i));
3915 0 : var.setValue(i, vals(i));
3916 : }
3917 0 : }
3918 : }
3919 :
3920 4074 : for (auto & nl : _nl)
3921 : {
3922 2037 : nl->solution().close();
3923 2037 : nl->solution().localize(*nl->system().current_local_solution, nl->dofMap().get_send_list());
3924 : }
3925 :
3926 2037 : _aux->solution().close();
3927 2037 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3928 2037 : }
3929 :
3930 : void
3931 737 : FEProblemBase::projectFunctionOnCustomRange(ConstElemRange & elem_range,
3932 : Number (*func)(const Point &,
3933 : const libMesh::Parameters &,
3934 : const std::string &,
3935 : const std::string &),
3936 : Gradient (*func_grad)(const Point &,
3937 : const libMesh::Parameters &,
3938 : const std::string &,
3939 : const std::string &),
3940 : const libMesh::Parameters & params,
3941 : const std::vector<VariableName> & target_vars)
3942 : {
3943 : mooseAssert(!Threads::in_threads,
3944 : "We're performing a projection based on data from just the thread 0 variable, so any "
3945 : "modifications to the variable solution must have been thread joined already");
3946 :
3947 737 : std::unordered_map<unsigned int, std::vector<unsigned int>> sys_to_var_nums;
3948 :
3949 1474 : for (const auto & target_var : target_vars)
3950 : {
3951 737 : const auto sn = systemNumForVariable(target_var);
3952 737 : const auto & var = getStandardVariable(0, target_var);
3953 737 : sys_to_var_nums[sn].push_back(var.number());
3954 : }
3955 :
3956 1474 : for (const auto & [sys_num, var_nums] : sys_to_var_nums)
3957 : {
3958 737 : System & libmesh_sys = getSystemBase(sys_num).system();
3959 737 : libmesh_sys.project_solution(func, func_grad, params, elem_range, var_nums);
3960 : }
3961 737 : }
3962 :
3963 : std::shared_ptr<MaterialBase>
3964 252 : FEProblemBase::getMaterial(std::string name,
3965 : Moose::MaterialDataType type,
3966 : const THREAD_ID tid,
3967 : bool no_warn)
3968 : {
3969 252 : switch (type)
3970 : {
3971 65 : case Moose::NEIGHBOR_MATERIAL_DATA:
3972 65 : name += "_neighbor";
3973 65 : break;
3974 65 : case Moose::FACE_MATERIAL_DATA:
3975 65 : name += "_face";
3976 65 : break;
3977 122 : default:
3978 122 : break;
3979 : }
3980 :
3981 252 : std::shared_ptr<MaterialBase> material = _all_materials[type].getActiveObject(name, tid);
3982 657 : if (!no_warn && material->getParam<bool>("compute") && type == Moose::BLOCK_MATERIAL_DATA)
3983 3 : mooseWarning("You are retrieving a Material object (",
3984 3 : material->name(),
3985 : "), but its compute flag is set to true. This indicates that MOOSE is "
3986 : "computing this property which may not be desired and produce un-expected "
3987 : "results.");
3988 :
3989 246 : return material;
3990 : }
3991 :
3992 : MaterialData &
3993 33126936 : FEProblemBase::getMaterialData(Moose::MaterialDataType type,
3994 : const THREAD_ID tid,
3995 : const MooseObject * object) const
3996 : {
3997 33126936 : switch (type)
3998 : {
3999 845321 : case Moose::BLOCK_MATERIAL_DATA:
4000 845321 : if (object)
4001 258579 : _material_props.addConsumer(type, object);
4002 845321 : return _material_props.getMaterialData(tid);
4003 15166520 : case Moose::NEIGHBOR_MATERIAL_DATA:
4004 15166520 : if (object)
4005 22342 : _neighbor_material_props.addConsumer(type, object);
4006 15166520 : return _neighbor_material_props.getMaterialData(tid);
4007 17115095 : case Moose::BOUNDARY_MATERIAL_DATA:
4008 : case Moose::FACE_MATERIAL_DATA:
4009 : case Moose::INTERFACE_MATERIAL_DATA:
4010 17115095 : if (object)
4011 51840 : _bnd_material_props.addConsumer(type, object);
4012 17115095 : return _bnd_material_props.getMaterialData(tid);
4013 : }
4014 :
4015 0 : mooseError("FEProblemBase::getMaterialData(): Invalid MaterialDataType ", type);
4016 : }
4017 :
4018 : const std::set<const MooseObject *> &
4019 0 : FEProblemBase::getMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
4020 : {
4021 0 : switch (type)
4022 : {
4023 0 : case Moose::BLOCK_MATERIAL_DATA:
4024 0 : return _material_props.getConsumers(type);
4025 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
4026 0 : return _neighbor_material_props.getConsumers(type);
4027 0 : case Moose::BOUNDARY_MATERIAL_DATA:
4028 : case Moose::FACE_MATERIAL_DATA:
4029 : case Moose::INTERFACE_MATERIAL_DATA:
4030 0 : return _bnd_material_props.getConsumers(type);
4031 : }
4032 :
4033 0 : mooseError("FEProblemBase::getMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
4034 : type);
4035 : }
4036 :
4037 : void
4038 0 : FEProblemBase::setPreserveMatrixSparsityPattern(bool preserve)
4039 : {
4040 0 : if (_ignore_zeros_in_jacobian && preserve)
4041 0 : paramWarning(
4042 : "ignore_zeros_in_jacobian",
4043 : "We likely cannot preserve the sparsity pattern if ignoring zeros in the Jacobian, which "
4044 : "leads to removing those entries from the Jacobian sparsity pattern");
4045 0 : _preserve_matrix_sparsity_pattern = preserve;
4046 0 : }
4047 :
4048 : bool
4049 309255 : FEProblemBase::acceptInvalidSolution() const
4050 : {
4051 618425 : return allowInvalidSolution() || // invalid solutions are always allowed
4052 618425 : !_app.solutionInvalidity().hasInvalidSolutionError(); // if not allowed, check for errors
4053 : }
4054 :
4055 : void
4056 964 : FEProblemBase::addFunctorMaterial(const std::string & functor_material_name,
4057 : const std::string & name,
4058 : InputParameters & parameters)
4059 : {
4060 : parallel_object_only();
4061 :
4062 964 : auto add_functor_materials = [&](const auto & parameters, const auto & name)
4063 : {
4064 2011 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
4065 : {
4066 : // Create the general Block/Boundary MaterialBase object
4067 1047 : std::shared_ptr<MaterialBase> material =
4068 1047 : _factory.create<MaterialBase>(functor_material_name, name, parameters, tid);
4069 2094 : logAdd("FunctorMaterial", name, functor_material_name, parameters);
4070 1047 : _all_materials.addObject(material, tid);
4071 1047 : _materials.addObject(material, tid);
4072 : }
4073 964 : };
4074 :
4075 1928 : parameters.set<SubProblem *>("_subproblem") = this;
4076 964 : add_functor_materials(parameters, name);
4077 964 : if (_displaced_problem)
4078 : {
4079 0 : auto disp_params = parameters;
4080 0 : disp_params.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4081 0 : add_functor_materials(disp_params, name + "_displaced");
4082 0 : }
4083 964 : }
4084 :
4085 : void
4086 12078 : FEProblemBase::addMaterial(const std::string & mat_name,
4087 : const std::string & name,
4088 : InputParameters & parameters)
4089 : {
4090 24156 : addMaterialHelper({&_materials}, mat_name, name, parameters);
4091 11980 : }
4092 :
4093 : void
4094 340 : FEProblemBase::addInterfaceMaterial(const std::string & mat_name,
4095 : const std::string & name,
4096 : InputParameters & parameters)
4097 : {
4098 680 : addMaterialHelper({&_interface_materials}, mat_name, name, parameters);
4099 340 : }
4100 :
4101 : void
4102 13360 : FEProblemBase::addMaterialHelper(std::vector<MaterialWarehouse *> warehouses,
4103 : const std::string & mat_name,
4104 : const std::string & name,
4105 : InputParameters & parameters)
4106 : {
4107 : parallel_object_only();
4108 :
4109 13360 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4110 : {
4111 252 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4112 126 : _reinit_displaced_elem = _reinit_displaced_face = _reinit_displaced_neighbor = true;
4113 : }
4114 : else
4115 : {
4116 13234 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
4117 : {
4118 : // We allow Materials to request that they use_displaced_mesh,
4119 : // but then be overridden when no displacements variables are
4120 : // provided in the Mesh block. If that happened, update the value
4121 : // of use_displaced_mesh appropriately for this Material.
4122 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
4123 0 : parameters.set<bool>("use_displaced_mesh") = false;
4124 : }
4125 :
4126 39702 : parameters.set<SubProblem *>("_subproblem") = this;
4127 : }
4128 :
4129 13360 : unsigned int n_threads = libMesh::n_threads();
4130 :
4131 : #ifdef MOOSE_KOKKOS_ENABLED
4132 10010 : if (parameters.isKokkosObject())
4133 942 : n_threads = 1;
4134 : #endif
4135 :
4136 27802 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4137 : {
4138 : // Create the general Block/Boundary MaterialBase object
4139 : std::shared_ptr<MaterialBase> material =
4140 14548 : _factory.create<MaterialBase>(mat_name, name, parameters, tid);
4141 14442 : logAdd("Material", name, mat_name, parameters);
4142 28884 : bool discrete = !material->getParam<bool>("compute");
4143 :
4144 : // If the object is boundary restricted or if it is a functor material we do not create the
4145 : // neighbor and face objects
4146 14442 : if (material->boundaryRestricted() || dynamic_cast<FunctorMaterial *>(material.get()))
4147 : {
4148 2970 : _all_materials.addObject(material, tid);
4149 2970 : if (discrete)
4150 4 : _discrete_materials.addObject(material, tid);
4151 : else
4152 5932 : for (auto && warehouse : warehouses)
4153 2966 : warehouse->addObject(material, tid);
4154 : }
4155 :
4156 : // Non-boundary restricted require face and neighbor objects
4157 : else
4158 : {
4159 : // TODO: we only need to do this if we have needs for face materials (e.g.
4160 : // FV, DG, etc.) - but currently we always do it. Figure out how to fix
4161 : // this.
4162 :
4163 : // The name of the object being created, this is changed multiple times as objects are
4164 : // created below
4165 11472 : std::string object_name;
4166 :
4167 : // Create a copy of the supplied parameters to the setting for "_material_data_type" isn't
4168 : // used from a previous tid loop
4169 11472 : InputParameters current_parameters = parameters;
4170 :
4171 : // face material
4172 11472 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4173 : Moose::FACE_MATERIAL_DATA;
4174 11472 : object_name = name + "_face";
4175 : std::shared_ptr<MaterialBase> face_material =
4176 11472 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4177 :
4178 : // neighbor material
4179 22944 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4180 : Moose::NEIGHBOR_MATERIAL_DATA;
4181 11472 : current_parameters.set<bool>("_neighbor") = true;
4182 11472 : object_name = name + "_neighbor";
4183 : std::shared_ptr<MaterialBase> neighbor_material =
4184 11472 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4185 :
4186 : // Store the material objects
4187 11472 : _all_materials.addObjects(material, neighbor_material, face_material, tid);
4188 :
4189 11472 : if (discrete)
4190 73 : _discrete_materials.addObjects(material, neighbor_material, face_material, tid);
4191 : else
4192 22798 : for (auto && warehouse : warehouses)
4193 11399 : warehouse->addObjects(material, neighbor_material, face_material, tid);
4194 :
4195 : // Names of all controllable parameters for this Material object
4196 11472 : const std::string & base = parameters.getBase();
4197 34416 : MooseObjectParameterName name(MooseObjectName(base, material->name()), "*");
4198 : const auto param_names =
4199 11472 : _app.getInputParameterWarehouse().getControllableParameterNames(name);
4200 :
4201 : // Connect parameters of the primary Material object to those on the face and neighbor
4202 : // objects
4203 29203 : for (const auto & p_name : param_names)
4204 : {
4205 35462 : MooseObjectParameterName primary_name(MooseObjectName(base, material->name()),
4206 35462 : p_name.parameter());
4207 35462 : MooseObjectParameterName face_name(MooseObjectName(base, face_material->name()),
4208 35462 : p_name.parameter());
4209 35462 : MooseObjectParameterName neighbor_name(MooseObjectName(base, neighbor_material->name()),
4210 35462 : p_name.parameter());
4211 17731 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4212 : primary_name, face_name, false);
4213 17731 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4214 : primary_name, neighbor_name, false);
4215 17731 : }
4216 11472 : }
4217 14442 : }
4218 13254 : }
4219 :
4220 : void
4221 5068400 : FEProblemBase::prepareMaterials(const std::unordered_set<unsigned int> & consumer_needed_mat_props,
4222 : const SubdomainID blk_id,
4223 : const THREAD_ID tid)
4224 : {
4225 5068400 : std::set<MooseVariableFEBase *> needed_moose_vars;
4226 5068400 : std::unordered_set<unsigned int> needed_mat_props;
4227 :
4228 5068400 : if (_all_materials.hasActiveBlockObjects(blk_id, tid))
4229 : {
4230 607961 : _all_materials.updateVariableDependency(needed_moose_vars, tid);
4231 607961 : _all_materials.updateBlockMatPropDependency(blk_id, needed_mat_props, tid);
4232 : }
4233 :
4234 5068400 : const auto & ids = _mesh.getSubdomainBoundaryIds(blk_id);
4235 23498318 : for (const auto id : ids)
4236 : {
4237 18429918 : _materials.updateBoundaryVariableDependency(id, needed_moose_vars, tid);
4238 18429918 : _materials.updateBoundaryMatPropDependency(id, needed_mat_props, tid);
4239 : }
4240 :
4241 5068400 : const auto & current_active_elemental_moose_variables = getActiveElementalMooseVariables(tid);
4242 5068400 : needed_moose_vars.insert(current_active_elemental_moose_variables.begin(),
4243 : current_active_elemental_moose_variables.end());
4244 :
4245 5068400 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
4246 :
4247 5068400 : setActiveElementalMooseVariables(needed_moose_vars, tid);
4248 5068400 : setActiveMaterialProperties(needed_mat_props, tid);
4249 5068400 : }
4250 :
4251 : void
4252 363148685 : FEProblemBase::reinitMaterials(SubdomainID blk_id, const THREAD_ID tid, bool swap_stateful)
4253 : {
4254 363148685 : if (hasActiveMaterialProperties(tid))
4255 : {
4256 15089197 : auto && elem = _assembly[tid][0]->elem();
4257 15089197 : unsigned int n_points = _assembly[tid][0]->qRule()->n_points();
4258 :
4259 15089197 : auto & material_data = _material_props.getMaterialData(tid);
4260 15089197 : material_data.resize(n_points);
4261 :
4262 : // Only swap if requested
4263 15089197 : if (swap_stateful)
4264 15082655 : material_data.swap(*elem);
4265 :
4266 15089197 : if (_discrete_materials.hasActiveBlockObjects(blk_id, tid))
4267 2748 : material_data.reset(_discrete_materials.getActiveBlockObjects(blk_id, tid));
4268 :
4269 15089194 : if (_materials.hasActiveBlockObjects(blk_id, tid))
4270 15070605 : material_data.reinit(_materials.getActiveBlockObjects(blk_id, tid));
4271 : }
4272 363148626 : }
4273 :
4274 : void
4275 5099676 : FEProblemBase::reinitMaterialsFace(const SubdomainID blk_id,
4276 : const THREAD_ID tid,
4277 : const bool swap_stateful,
4278 : const std::deque<MaterialBase *> * const reinit_mats)
4279 : {
4280 : // we reinit more often than needed here because we dont have a way to check whether
4281 : // we need to compute the face materials on a particular (possibly external) face
4282 5099676 : if (hasActiveMaterialProperties(tid))
4283 : {
4284 1045323 : auto && elem = _assembly[tid][0]->elem();
4285 1045323 : unsigned int side = _assembly[tid][0]->side();
4286 1045323 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4287 :
4288 1045323 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4289 1045323 : bnd_material_data.resize(n_points);
4290 :
4291 1045323 : if (swap_stateful && !bnd_material_data.isSwapped())
4292 1023849 : bnd_material_data.swap(*elem, side);
4293 :
4294 1045323 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4295 0 : bnd_material_data.reset(
4296 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4297 :
4298 1045323 : if (reinit_mats)
4299 21474 : bnd_material_data.reinit(*reinit_mats);
4300 1023849 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4301 1023849 : bnd_material_data.reinit(
4302 1023849 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4303 : }
4304 5099676 : }
4305 :
4306 : void
4307 3809451 : FEProblemBase::reinitMaterialsFaceOnBoundary(const BoundaryID boundary_id,
4308 : const SubdomainID blk_id,
4309 : const THREAD_ID tid,
4310 : const bool swap_stateful,
4311 : const std::deque<MaterialBase *> * const reinit_mats)
4312 : {
4313 4140250 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4314 330799 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4315 330799 : needInternalNeighborSideMaterial(blk_id, tid)))
4316 : {
4317 407306 : const auto * const elem = _assembly[tid][0]->elem();
4318 407306 : unsigned int side = _assembly[tid][0]->side();
4319 407306 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4320 :
4321 407306 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4322 407306 : bnd_material_data.resize(n_points);
4323 :
4324 407306 : if (swap_stateful && !bnd_material_data.isSwapped())
4325 407306 : bnd_material_data.swap(*elem, side);
4326 :
4327 407306 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4328 0 : bnd_material_data.reset(
4329 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4330 :
4331 407306 : if (reinit_mats)
4332 0 : bnd_material_data.reinit(*reinit_mats);
4333 407306 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4334 392914 : bnd_material_data.reinit(
4335 392914 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4336 : }
4337 3809451 : }
4338 :
4339 : void
4340 43901 : FEProblemBase::reinitMaterialsNeighborOnBoundary(
4341 : const BoundaryID boundary_id,
4342 : const SubdomainID blk_id,
4343 : const THREAD_ID tid,
4344 : const bool swap_stateful,
4345 : const std::deque<MaterialBase *> * const reinit_mats)
4346 : {
4347 : // Since objects don't declare whether they need the face or neighbor (side) material properties,
4348 : // we use the same criteria for skipping material property computations as for face material
4349 : // properties This could be a future optimization.
4350 48111 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4351 4210 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4352 4210 : needInternalNeighborSideMaterial(blk_id, tid)))
4353 35956 : reinitMaterialsNeighbor(blk_id, tid, swap_stateful, reinit_mats);
4354 43901 : }
4355 :
4356 : void
4357 3972745 : FEProblemBase::reinitMaterialsNeighbor(const SubdomainID blk_id,
4358 : const THREAD_ID tid,
4359 : const bool swap_stateful,
4360 : const std::deque<MaterialBase *> * const reinit_mats)
4361 : {
4362 3972745 : if (hasActiveMaterialProperties(tid))
4363 : {
4364 : // NOTE: this will not work with h-adaptivity
4365 : // lindsayad: why not?
4366 :
4367 886601 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4368 886601 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(_assembly[tid][0]->elem());
4369 :
4370 : mooseAssert(neighbor, "neighbor should be non-null");
4371 : mooseAssert(blk_id == neighbor->subdomain_id(),
4372 : "The provided blk_id " << blk_id << " and neighbor subdomain ID "
4373 : << neighbor->subdomain_id() << " do not match.");
4374 :
4375 886601 : unsigned int n_points = _assembly[tid][0]->qRuleNeighbor()->n_points();
4376 :
4377 886601 : auto & neighbor_material_data = _neighbor_material_props.getMaterialData(tid);
4378 886601 : neighbor_material_data.resize(n_points);
4379 :
4380 : // Only swap if requested
4381 886601 : if (swap_stateful)
4382 865127 : neighbor_material_data.swap(*neighbor, neighbor_side);
4383 :
4384 886601 : if (_discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4385 0 : neighbor_material_data.reset(
4386 0 : _discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4387 :
4388 886601 : if (reinit_mats)
4389 21474 : neighbor_material_data.reinit(*reinit_mats);
4390 865127 : else if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4391 864864 : neighbor_material_data.reinit(
4392 864864 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4393 : }
4394 3972745 : }
4395 :
4396 : void
4397 4063543 : FEProblemBase::reinitMaterialsBoundary(const BoundaryID boundary_id,
4398 : const THREAD_ID tid,
4399 : const bool swap_stateful,
4400 : const std::deque<MaterialBase *> * const reinit_mats)
4401 : {
4402 4063543 : if (hasActiveMaterialProperties(tid) && needBoundaryMaterialOnSide(boundary_id, tid))
4403 : {
4404 285938 : auto && elem = _assembly[tid][0]->elem();
4405 285938 : unsigned int side = _assembly[tid][0]->side();
4406 285938 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4407 :
4408 285938 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4409 285938 : bnd_material_data.resize(n_points);
4410 :
4411 285938 : if (swap_stateful && !bnd_material_data.isSwapped())
4412 238245 : bnd_material_data.swap(*elem, side);
4413 :
4414 285938 : if (_discrete_materials.hasActiveBoundaryObjects(boundary_id, tid))
4415 0 : bnd_material_data.reset(_discrete_materials.getActiveBoundaryObjects(boundary_id, tid));
4416 :
4417 285938 : if (reinit_mats)
4418 21474 : bnd_material_data.reinit(*reinit_mats);
4419 264464 : else if (_materials.hasActiveBoundaryObjects(boundary_id, tid))
4420 23154 : bnd_material_data.reinit(_materials.getActiveBoundaryObjects(boundary_id, tid));
4421 : }
4422 4063543 : }
4423 :
4424 : void
4425 46758 : FEProblemBase::reinitMaterialsInterface(BoundaryID boundary_id,
4426 : const THREAD_ID tid,
4427 : bool swap_stateful)
4428 : {
4429 46758 : if (hasActiveMaterialProperties(tid) && needInterfaceMaterialOnSide(boundary_id, tid))
4430 : {
4431 36992 : const Elem * const & elem = _assembly[tid][0]->elem();
4432 36992 : unsigned int side = _assembly[tid][0]->side();
4433 36992 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4434 :
4435 36992 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4436 36992 : bnd_material_data.resize(n_points);
4437 :
4438 36992 : if (swap_stateful && !bnd_material_data.isSwapped())
4439 35497 : bnd_material_data.swap(*elem, side);
4440 :
4441 36992 : if (_interface_materials.hasActiveBoundaryObjects(boundary_id, tid))
4442 1171 : bnd_material_data.reinit(_interface_materials.getActiveBoundaryObjects(boundary_id, tid));
4443 : }
4444 46758 : }
4445 :
4446 : void
4447 361530525 : FEProblemBase::swapBackMaterials(const THREAD_ID tid)
4448 : {
4449 361530525 : auto && elem = _assembly[tid][0]->elem();
4450 361530525 : _material_props.getMaterialData(tid).swapBack(*elem);
4451 361530525 : }
4452 :
4453 : void
4454 8655021 : FEProblemBase::swapBackMaterialsFace(const THREAD_ID tid)
4455 : {
4456 8655021 : auto && elem = _assembly[tid][0]->elem();
4457 8655021 : unsigned int side = _assembly[tid][0]->side();
4458 8655021 : _bnd_material_props.getMaterialData(tid).swapBack(*elem, side);
4459 8655021 : }
4460 :
4461 : void
4462 3630286 : FEProblemBase::swapBackMaterialsNeighbor(const THREAD_ID tid)
4463 : {
4464 : // NOTE: this will not work with h-adaptivity
4465 3630286 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4466 : unsigned int neighbor_side =
4467 3630286 : neighbor ? neighbor->which_neighbor_am_i(_assembly[tid][0]->elem()) : libMesh::invalid_uint;
4468 :
4469 3630286 : if (!neighbor)
4470 : {
4471 0 : if (haveFV())
4472 : {
4473 : // If neighbor is null, then we're on the neighbor side of a mesh boundary, e.g. we're off
4474 : // the mesh in ghost-land. If we're using the finite volume method, then variable values and
4475 : // consequently material properties have well-defined values in this ghost region outside of
4476 : // the mesh and we really do want to reinit our neighbor materials in this case. Since we're
4477 : // off in ghost land it's safe to do swaps with `MaterialPropertyStorage` using the elem and
4478 : // elem_side keys
4479 0 : neighbor = _assembly[tid][0]->elem();
4480 0 : neighbor_side = _assembly[tid][0]->side();
4481 : mooseAssert(neighbor, "We should have an appropriate value for elem coming from Assembly");
4482 : }
4483 : else
4484 0 : mooseError("neighbor is null in Assembly!");
4485 : }
4486 :
4487 3630286 : _neighbor_material_props.getMaterialData(tid).swapBack(*neighbor, neighbor_side);
4488 3630286 : }
4489 :
4490 : void
4491 963434 : FEProblemBase::logAdd(const std::string & system,
4492 : const std::string & name,
4493 : const std::string & type,
4494 : const InputParameters & params) const
4495 : {
4496 963434 : if (_verbose_setup != "false")
4497 17453 : _console << "[DBG] Adding " << system << " '" << name << "' of type " << type << std::endl;
4498 963434 : if (_verbose_setup == "extra")
4499 0 : _console << params << std::endl;
4500 963434 : }
4501 :
4502 : void
4503 133546 : FEProblemBase::addObjectParamsHelper(InputParameters & parameters,
4504 : const std::string & object_name,
4505 : const std::string & var_param_name)
4506 : {
4507 : // Due to objects like SolutionUserObject which manipulate libmesh objects
4508 : // and variables directly at the back end, we need a default option here
4509 : // which is going to be the pointer to the first solver system within this
4510 : // problem
4511 133546 : unsigned int sys_num = 0;
4512 133546 : if (parameters.isParamValid(var_param_name))
4513 : {
4514 56405 : const auto variable_name = parameters.varName(var_param_name, object_name);
4515 56405 : if (this->hasVariable(variable_name) || this->hasScalarVariable(variable_name))
4516 50757 : sys_num = getSystem(variable_name).number();
4517 56405 : }
4518 267092 : if (parameters.isParamValid("solver_sys"))
4519 : {
4520 3414 : const auto var_sys_num = sys_num;
4521 3414 : sys_num = getSystemBase(parameters.get<SolverSystemName>("solver_sys")).number();
4522 3414 : if (sys_num != var_sys_num && parameters.isParamValid(var_param_name))
4523 0 : mooseError("We dont support setting 'variable' to a variable that is not set to the same "
4524 : "system as the 'solver_sys' parameter");
4525 : }
4526 :
4527 134710 : if (_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4528 134710 : parameters.get<bool>("use_displaced_mesh"))
4529 : {
4530 1132 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4531 566 : if (sys_num == _aux->number())
4532 1104 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->systemBaseAuxiliary();
4533 : else
4534 594 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(sys_num);
4535 : }
4536 : else
4537 : {
4538 : // The object requested use_displaced_mesh, but it was overridden
4539 : // due to there being no displacements variables in the [Mesh] block.
4540 : // If that happened, update the value of use_displaced_mesh appropriately.
4541 216369 : if (!_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4542 216369 : parameters.get<bool>("use_displaced_mesh"))
4543 144 : parameters.set<bool>("use_displaced_mesh") = false;
4544 :
4545 265960 : parameters.set<SubProblem *>("_subproblem") = this;
4546 :
4547 132980 : if (sys_num == _aux->number())
4548 39885 : parameters.set<SystemBase *>("_sys") = _aux.get();
4549 : else
4550 359055 : parameters.set<SystemBase *>("_sys") = _solver_systems[sys_num].get();
4551 : }
4552 133546 : }
4553 :
4554 : void
4555 60582 : FEProblemBase::checkUserObjectNameCollision(const std::string & name,
4556 : const std::string & type) const
4557 : {
4558 60582 : if (hasUserObject(name))
4559 9 : mooseError("A ",
4560 9 : getUserObjectBase(name).typeAndName(),
4561 : " already exists. You may not add a ",
4562 : type,
4563 : " by the same name.");
4564 :
4565 : #ifdef MOOSE_KOKKOS_ENABLED
4566 45298 : if (hasKokkosUserObject(name))
4567 0 : mooseError("A ",
4568 0 : getKokkosUserObject<UserObjectBase>(name).typeAndName(),
4569 : " already exists. You may not add a ",
4570 : type,
4571 : " by the same name.");
4572 : #endif
4573 60573 : }
4574 :
4575 : void
4576 48136 : FEProblemBase::addPostprocessor(const std::string & pp_name,
4577 : const std::string & name,
4578 : InputParameters & parameters)
4579 : {
4580 48136 : checkUserObjectNameCollision(name, "Postprocessor");
4581 :
4582 48133 : addUserObject(pp_name, name, parameters);
4583 48104 : }
4584 :
4585 : void
4586 5683 : FEProblemBase::addVectorPostprocessor(const std::string & pp_name,
4587 : const std::string & name,
4588 : InputParameters & parameters)
4589 : {
4590 5683 : checkUserObjectNameCollision(name, "VectorPostprocessor");
4591 :
4592 5680 : addUserObject(pp_name, name, parameters);
4593 5650 : }
4594 :
4595 : void
4596 4681 : FEProblemBase::addReporter(const std::string & type,
4597 : const std::string & name,
4598 : InputParameters & parameters)
4599 : {
4600 4681 : checkUserObjectNameCollision(name, "Reporter");
4601 :
4602 4678 : addUserObject(type, name, parameters);
4603 4645 : }
4604 :
4605 : std::vector<std::shared_ptr<UserObject>>
4606 68457 : FEProblemBase::addUserObject(const std::string & user_object_name,
4607 : const std::string & name,
4608 : InputParameters & parameters)
4609 : {
4610 : parallel_object_only();
4611 :
4612 68457 : std::vector<std::shared_ptr<UserObject>> uos;
4613 :
4614 : // Add the _subproblem and _sys parameters depending on use_displaced_mesh
4615 68457 : addObjectParamsHelper(parameters, name);
4616 :
4617 111362 : for (const auto tid : make_range(libMesh::n_threads()))
4618 : {
4619 : // Create the UserObject
4620 : std::shared_ptr<UserObject> user_object =
4621 72183 : _factory.create<UserObject>(user_object_name, name, parameters, tid);
4622 71908 : logAdd("UserObject", name, user_object_name, parameters);
4623 71908 : uos.push_back(user_object);
4624 :
4625 71908 : if (tid != 0)
4626 3726 : user_object->setPrimaryThreadCopy(uos[0].get());
4627 :
4628 71908 : theWarehouse().add(user_object);
4629 :
4630 : // Attempt to create all the possible UserObject types
4631 71902 : auto euo = std::dynamic_pointer_cast<ElementUserObject>(user_object);
4632 71902 : auto suo = std::dynamic_pointer_cast<SideUserObject>(user_object);
4633 71902 : auto isuo = std::dynamic_pointer_cast<InternalSideUserObject>(user_object);
4634 71902 : auto iuo = std::dynamic_pointer_cast<InterfaceUserObjectBase>(user_object);
4635 71902 : auto nuo = std::dynamic_pointer_cast<NodalUserObject>(user_object);
4636 71902 : auto duo = std::dynamic_pointer_cast<DomainUserObject>(user_object);
4637 71902 : auto guo = std::dynamic_pointer_cast<GeneralUserObject>(user_object);
4638 71902 : auto tguo = std::dynamic_pointer_cast<ThreadedGeneralUserObject>(user_object);
4639 71902 : auto muo = std::dynamic_pointer_cast<MortarUserObject>(user_object);
4640 :
4641 : // Account for displaced mesh use
4642 71902 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4643 : {
4644 : // Whether to re-init or not depends on the attributes of the base classes.
4645 : // For example, InterfaceUOBase has "_current_side_elem" and "_neighbor_elem"
4646 : // so it needs to reinit on displaced neighbors and faces
4647 : // _reinit_displaced_elem -> _current_elem will be reinited
4648 : // _reinit_displaced_face -> _current_elem, lowerD if any and _current_side_elem to be
4649 : // reinited _reinit_displaced_neighbor -> _current_elem, lowerD if any and _current_neighbor
4650 : // to be reinited Note that as soon as you use materials on the displaced mesh, all three get
4651 : // turned on.
4652 615 : if (euo || nuo || duo)
4653 555 : _reinit_displaced_elem = true;
4654 615 : if (suo || duo || isuo || iuo)
4655 24 : _reinit_displaced_face = true;
4656 615 : if (iuo || duo || isuo)
4657 0 : _reinit_displaced_neighbor = true;
4658 : }
4659 :
4660 : // These objects only require one thread
4661 71902 : if ((guo && !tguo) || muo)
4662 28997 : break;
4663 332875 : }
4664 :
4665 : // Add as a Functor if it is one. We usually need to add the user object from thread 0 as the
4666 : // registered functor for all threads because when user objects are thread joined, generally only
4667 : // the primary thread copy ends up with all the data
4668 143507 : for (const auto tid : make_range(libMesh::n_threads()))
4669 : {
4670 75331 : const decltype(uos)::size_type uo_index = uos.front()->needThreadedCopy() ? tid : 0;
4671 75331 : if (const auto functor = dynamic_cast<Moose::FunctorBase<Real> *>(uos[uo_index].get()))
4672 : {
4673 58676 : this->addFunctor(name, *functor, tid);
4674 58676 : if (_displaced_problem)
4675 729 : _displaced_problem->addFunctor(name, *functor, tid);
4676 : }
4677 : }
4678 :
4679 68176 : return uos;
4680 2 : }
4681 :
4682 : void
4683 2002 : FEProblemBase::addFVInterpolationMethod(const std::string & method_type,
4684 : const std::string & name,
4685 : InputParameters & parameters)
4686 : {
4687 : parallel_object_only();
4688 :
4689 2002 : addObjectParamsHelper(parameters, name);
4690 :
4691 4004 : for (const auto tid : make_range(libMesh::n_threads()))
4692 : {
4693 2002 : auto method = _factory.create<FVInterpolationMethod>(method_type, name, parameters, tid);
4694 2002 : logAdd("FVInterpolationMethod", name, method_type, parameters);
4695 2002 : theWarehouse().add(method);
4696 2002 : }
4697 2002 : }
4698 :
4699 : const UserObject &
4700 285092 : FEProblemBase::getUserObjectBase(const std::string & name, const THREAD_ID tid /* = 0 */) const
4701 : {
4702 285092 : std::vector<UserObject *> objs;
4703 285092 : theWarehouse()
4704 570184 : .query()
4705 285092 : .condition<AttribSystem>("UserObject")
4706 285092 : .condition<AttribThread>(tid)
4707 285092 : .condition<AttribName>(name)
4708 285092 : .queryInto(objs);
4709 285092 : if (objs.empty())
4710 : {
4711 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_user_object"),
4712 : "A UserObject getter was called before UserObjects have been constructed. The "
4713 : "requested UserObject '" +
4714 : name + "' may exist in the input file, but UserObjects are not available yet.");
4715 :
4716 0 : mooseError("Unable to find user object with name '" + name + "'");
4717 : }
4718 : mooseAssert(objs.size() == 1, "Should only find one UO");
4719 570184 : return *(objs[0]);
4720 285092 : }
4721 :
4722 : const Positions &
4723 1141 : FEProblemBase::getPositionsObject(const std::string & name) const
4724 : {
4725 1141 : std::vector<Positions *> objs;
4726 1141 : theWarehouse()
4727 2282 : .query()
4728 1141 : .condition<AttribSystem>("UserObject")
4729 1141 : .condition<AttribName>(name)
4730 1141 : .queryInto(objs);
4731 1141 : if (objs.empty())
4732 0 : mooseError("Unable to find Positions object with name '" + name + "'");
4733 : mooseAssert(objs.size() == 1, "Should only find one Positions");
4734 2282 : return *(objs[0]);
4735 1141 : }
4736 :
4737 : bool
4738 88528 : FEProblemBase::hasUserObject(const std::string & name) const
4739 : {
4740 88528 : std::vector<UserObject *> objs;
4741 88528 : theWarehouse()
4742 88528 : .query()
4743 88528 : .condition<AttribSystem>("UserObject")
4744 177056 : .condition<AttribThread>(0)
4745 88528 : .condition<AttribName>(name)
4746 88528 : .queryInto(objs);
4747 177056 : return !objs.empty();
4748 88528 : }
4749 :
4750 : const FVInterpolationMethod &
4751 571 : FEProblemBase::getFVInterpolationMethod(const InterpolationMethodName & name,
4752 : const THREAD_ID tid) const
4753 : {
4754 571 : std::vector<FVInterpolationMethod *> methods;
4755 571 : theWarehouse()
4756 1142 : .query()
4757 571 : .condition<AttribSystem>("FVInterpolationMethod")
4758 571 : .condition<AttribThread>(tid)
4759 571 : .condition<AttribName>(name)
4760 571 : .queryInto(methods);
4761 :
4762 571 : if (methods.empty())
4763 0 : mooseError("Unable to find FVInterpolationMethod with name '", name, "'");
4764 :
4765 : mooseAssert(methods.size() == 1, "Expected a single FVInterpolationMethod per thread");
4766 1142 : return *(methods[0]);
4767 571 : }
4768 :
4769 : const FVFaceInterpolationMethod &
4770 0 : FEProblemBase::getFVFaceInterpolationMethod(const InterpolationMethodName & name,
4771 : const THREAD_ID tid) const
4772 : {
4773 0 : const auto & method = getFVInterpolationMethod(name, tid);
4774 0 : const auto * face_method = dynamic_cast<const FVFaceInterpolationMethod *>(&method);
4775 :
4776 0 : if (!face_method)
4777 0 : mooseError("FVInterpolationMethod '",
4778 : name,
4779 : "' (",
4780 0 : method.type(),
4781 : ") is not a scalar face interpolation method.");
4782 :
4783 0 : return *face_method;
4784 : }
4785 :
4786 : const FVAdvectedInterpolationMethod &
4787 571 : FEProblemBase::getFVAdvectedInterpolationMethod(const InterpolationMethodName & name,
4788 : const THREAD_ID tid) const
4789 : {
4790 571 : const auto & method = getFVInterpolationMethod(name, tid);
4791 571 : const auto * advected_method = dynamic_cast<const FVAdvectedInterpolationMethod *>(&method);
4792 :
4793 571 : if (!advected_method)
4794 0 : mooseError("FVInterpolationMethod '",
4795 : name,
4796 : "' (",
4797 0 : method.type(),
4798 : ") is not an advected interpolation method.");
4799 :
4800 571 : return *advected_method;
4801 : }
4802 :
4803 : bool
4804 0 : FEProblemBase::hasFVInterpolationMethod(const InterpolationMethodName & name) const
4805 : {
4806 0 : std::vector<FVInterpolationMethod *> methods;
4807 0 : theWarehouse()
4808 0 : .query()
4809 0 : .condition<AttribSystem>("FVInterpolationMethod")
4810 0 : .condition<AttribThread>(0)
4811 0 : .condition<AttribName>(name)
4812 0 : .queryInto(methods);
4813 0 : return !methods.empty();
4814 0 : }
4815 :
4816 : bool
4817 308 : FEProblemBase::hasPostprocessorValueByName(const PostprocessorName & name) const
4818 : {
4819 308 : return _reporter_data.hasReporterValue<PostprocessorValue>(PostprocessorReporterName(name));
4820 : }
4821 :
4822 : const Postprocessor &
4823 2 : FEProblemBase::getPostprocessorObjectByName(const PostprocessorName & object_name,
4824 : const THREAD_ID tid) const
4825 : {
4826 2 : std::vector<Postprocessor *> objs;
4827 2 : theWarehouse()
4828 2 : .query()
4829 4 : .condition<AttribInterfaces>(Interfaces::Postprocessor)
4830 2 : .condition<AttribThread>(tid)
4831 2 : .condition<AttribName>(object_name)
4832 2 : .queryInto(objs);
4833 :
4834 2 : if (objs.empty())
4835 0 : mooseError("Unable to find Postprocessor with name '", object_name, "'");
4836 : mooseAssert(objs.size() == 1,
4837 : "We shouldn't find more than one postprocessor object for a given name");
4838 4 : return *(objs[0]);
4839 2 : }
4840 :
4841 : const PostprocessorValue &
4842 814402 : FEProblemBase::getPostprocessorValueByName(const PostprocessorName & name,
4843 : std::size_t t_index) const
4844 : {
4845 1628804 : return _reporter_data.getReporterValue<PostprocessorValue>(PostprocessorReporterName(name),
4846 1628804 : t_index);
4847 : }
4848 :
4849 : void
4850 566187 : FEProblemBase::setPostprocessorValueByName(const PostprocessorName & name,
4851 : const PostprocessorValue & value,
4852 : std::size_t t_index)
4853 : {
4854 566187 : _reporter_data.setReporterValue<PostprocessorValue>(
4855 1132374 : PostprocessorReporterName(name), value, t_index);
4856 566187 : }
4857 :
4858 : bool
4859 52 : FEProblemBase::hasPostprocessor(const std::string & name) const
4860 : {
4861 52 : mooseDeprecated("FEProblemBase::hasPostprocssor is being removed; use "
4862 : "hasPostprocessorValueByName instead.");
4863 52 : return hasPostprocessorValueByName(name);
4864 : }
4865 :
4866 : const VectorPostprocessorValue &
4867 50 : FEProblemBase::getVectorPostprocessorValueByName(const std::string & object_name,
4868 : const std::string & vector_name,
4869 : std::size_t t_index) const
4870 : {
4871 50 : return _reporter_data.getReporterValue<VectorPostprocessorValue>(
4872 100 : VectorPostprocessorReporterName(object_name, vector_name), t_index);
4873 : }
4874 :
4875 : void
4876 18 : FEProblemBase::setVectorPostprocessorValueByName(const std::string & object_name,
4877 : const std::string & vector_name,
4878 : const VectorPostprocessorValue & value,
4879 : std::size_t t_index)
4880 : {
4881 18 : _reporter_data.setReporterValue<VectorPostprocessorValue>(
4882 36 : VectorPostprocessorReporterName(object_name, vector_name), value, t_index);
4883 18 : }
4884 :
4885 : const VectorPostprocessor &
4886 9655 : FEProblemBase::getVectorPostprocessorObjectByName(const std::string & object_name,
4887 : const THREAD_ID tid) const
4888 : {
4889 9655 : std::vector<VectorPostprocessor *> objs;
4890 9655 : theWarehouse()
4891 9655 : .query()
4892 19310 : .condition<AttribInterfaces>(Interfaces::VectorPostprocessor)
4893 9655 : .condition<AttribThread>(tid)
4894 9655 : .condition<AttribName>(object_name)
4895 9655 : .queryInto(objs);
4896 :
4897 9655 : if (objs.empty())
4898 : {
4899 : mooseAssert(
4900 : getMooseApp().actionWarehouse().isTaskComplete("add_vector_postprocessor"),
4901 : "A VectorPostprocessor getter was called before VectorPostprocessors have been "
4902 : "constructed. The requested VectorPostprocessor '" +
4903 : object_name +
4904 : "' may exist in the input file, but VectorPostprocessors are not available yet.");
4905 :
4906 0 : mooseError("Unable to find VectorPostprocessor with name '", object_name, "'");
4907 : }
4908 : mooseAssert(objs.size() == 1,
4909 : "We shouldn't find more than one vector postprocessor object for a given name");
4910 19310 : return *(objs[0]);
4911 9655 : }
4912 :
4913 : void
4914 70 : FEProblemBase::parentOutputPositionChanged()
4915 : {
4916 1960 : for (const auto & it : _multi_apps)
4917 : {
4918 1890 : const auto & objects = it.second.getActiveObjects();
4919 1909 : for (const auto & obj : objects)
4920 19 : obj->parentOutputPositionChanged();
4921 : }
4922 70 : }
4923 :
4924 : void
4925 0 : FEProblemBase::computeIndicatorsAndMarkers()
4926 : {
4927 0 : computeIndicators();
4928 0 : computeMarkers();
4929 0 : }
4930 :
4931 : void
4932 220587 : FEProblemBase::computeIndicators()
4933 : {
4934 : // Initialize indicator aux variable fields
4935 220587 : if (_indicators.hasActiveObjects() || _internal_side_indicators.hasActiveObjects())
4936 : {
4937 11875 : TIME_SECTION("computeIndicators", 1, "Computing Indicators");
4938 :
4939 : // Internal side indicators may lead to creating a much larger sparsity pattern than dictated by
4940 : // the actual finite element scheme (e.g. CFEM)
4941 2375 : const auto old_do_derivatives = ADReal::do_derivatives;
4942 2375 : ADReal::do_derivatives = false;
4943 :
4944 2375 : std::vector<std::string> fields;
4945 :
4946 : // Indicator Fields
4947 2375 : const auto & indicators = _indicators.getActiveObjects();
4948 2521 : for (const auto & indicator : indicators)
4949 146 : fields.push_back(indicator->name());
4950 :
4951 : // InternalSideIndicator Fields
4952 2375 : const auto & internal_indicators = _internal_side_indicators.getActiveObjects();
4953 4706 : for (const auto & internal_indicator : internal_indicators)
4954 2331 : fields.push_back(internal_indicator->name());
4955 :
4956 2375 : _aux->zeroVariables(fields);
4957 :
4958 : // compute Indicators
4959 2375 : ComputeIndicatorThread cit(*this);
4960 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cit);
4961 2375 : _aux->solution().close();
4962 2375 : _aux->update();
4963 :
4964 2375 : ComputeIndicatorThread finalize_cit(*this, true);
4965 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), finalize_cit);
4966 2375 : _aux->solution().close();
4967 2375 : _aux->update();
4968 :
4969 2375 : ADReal::do_derivatives = old_do_derivatives;
4970 2375 : }
4971 220587 : }
4972 :
4973 : void
4974 220609 : FEProblemBase::computeMarkers()
4975 : {
4976 220609 : if (_markers.hasActiveObjects())
4977 : {
4978 32385 : TIME_SECTION("computeMarkers", 1, "Computing Markers");
4979 :
4980 6477 : std::vector<std::string> fields;
4981 :
4982 : // Marker Fields
4983 6477 : const auto & markers = _markers.getActiveObjects();
4984 13933 : for (const auto & marker : markers)
4985 7456 : fields.push_back(marker->name());
4986 :
4987 6477 : _aux->zeroVariables(fields);
4988 :
4989 6477 : _adaptivity.updateErrorVectors();
4990 :
4991 13579 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
4992 : {
4993 7105 : const auto & markers = _markers.getActiveObjects(tid);
4994 15281 : for (const auto & marker : markers)
4995 8179 : marker->markerSetup();
4996 : }
4997 :
4998 6474 : ComputeMarkerThread cmt(*this);
4999 6474 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cmt);
5000 :
5001 6474 : _aux->solution().close();
5002 6474 : _aux->update();
5003 6474 : }
5004 220606 : }
5005 :
5006 : const ExecFlagType &
5007 6156085 : FEProblemBase::getCurrentExecuteOnFlag() const
5008 : {
5009 6156085 : return _current_execute_on_flag;
5010 : }
5011 :
5012 : void
5013 6902968 : FEProblemBase::setCurrentExecuteOnFlag(const ExecFlagType & flag)
5014 : {
5015 6902968 : _current_execute_on_flag = flag;
5016 6902968 : }
5017 :
5018 : void
5019 72 : FEProblemBase::executeAllObjects(const ExecFlagType & /*exec_type*/)
5020 : {
5021 72 : }
5022 :
5023 : void
5024 1760570 : FEProblemBase::customSetup(const ExecFlagType & exec_type)
5025 : {
5026 1760570 : SubProblem::customSetup(exec_type);
5027 :
5028 1760570 : if (_line_search)
5029 0 : _line_search->customSetup(exec_type);
5030 :
5031 1760570 : unsigned int n_threads = libMesh::n_threads();
5032 3697459 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5033 : {
5034 1936889 : _all_materials.customSetup(exec_type, tid);
5035 1936889 : _functions.customSetup(exec_type, tid);
5036 : }
5037 :
5038 : #ifdef MOOSE_KOKKOS_ENABLED
5039 1289041 : _kokkos_functions.customSetup(exec_type);
5040 : #endif
5041 :
5042 1760570 : _aux->customSetup(exec_type);
5043 3547937 : for (auto & nl : _nl)
5044 1787367 : nl->customSetup(exec_type);
5045 :
5046 1760570 : if (_displaced_problem)
5047 145157 : _displaced_problem->customSetup(exec_type);
5048 :
5049 3697459 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5050 : {
5051 1936889 : _internal_side_indicators.customSetup(exec_type, tid);
5052 1936889 : _indicators.customSetup(exec_type, tid);
5053 1936889 : _markers.customSetup(exec_type, tid);
5054 : }
5055 :
5056 1760570 : std::vector<UserObject *> userobjs;
5057 1760570 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
5058 4328242 : for (auto obj : userobjs)
5059 2567672 : obj->customSetup(exec_type);
5060 :
5061 : #ifdef MOOSE_KOKKOS_ENABLED
5062 : {
5063 1289041 : std::vector<UserObjectBase *> userobjs;
5064 1289041 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
5065 1302484 : for (auto obj : userobjs)
5066 13443 : obj->customSetup(exec_type);
5067 1289041 : }
5068 : #endif
5069 :
5070 1760570 : _app.getOutputWarehouse().customSetup(exec_type);
5071 1760570 : }
5072 :
5073 : void
5074 2128894 : FEProblemBase::execute(const ExecFlagType & exec_type)
5075 : {
5076 : // Set the current flag
5077 2128894 : setCurrentExecuteOnFlag(exec_type);
5078 :
5079 2128894 : if (exec_type != EXEC_INITIAL)
5080 2073971 : executeControls(exec_type);
5081 :
5082 : // intentially call this after executing controls because the setups may rely on the controls
5083 : // FIXME: we skip the following flags because they have dedicated setup functions in
5084 : // SetupInterface and it may not be appropriate to call them here.
5085 3891917 : if (!(exec_type == EXEC_INITIAL || exec_type == EXEC_TIMESTEP_BEGIN ||
5086 1763041 : exec_type == EXEC_SUBDOMAIN || exec_type == EXEC_NONLINEAR || exec_type == EXEC_LINEAR))
5087 1760570 : customSetup(exec_type);
5088 :
5089 2128876 : executeSamplers(exec_type);
5090 :
5091 : // Pre-aux UserObjects
5092 2128852 : computeUserObjects(exec_type, Moose::PRE_AUX);
5093 :
5094 : // Systems (includes system time derivative and aux kernel calculations)
5095 2128852 : computeSystems(exec_type);
5096 : // With the auxiliary system solution computed, sync the displaced problem auxiliary solution
5097 : // before computation of post-aux user objects. The undisplaced auxiliary system current local
5098 : // solution is updated (via System::update) within the AuxiliarySystem class's variable
5099 : // computation methods (e.g. computeElementalVarsHelper, computeNodalVarsHelper), so it is safe to
5100 : // use it here
5101 2128819 : if (_displaced_problem)
5102 179606 : _displaced_problem->syncAuxSolution(*getAuxiliarySystem().currentSolution());
5103 :
5104 : // Post-aux UserObjects
5105 2128819 : computeUserObjects(exec_type, Moose::POST_AUX);
5106 :
5107 : // Return the current flag to None
5108 2128733 : setCurrentExecuteOnFlag(EXEC_NONE);
5109 :
5110 2128733 : if (_uo_aux_state_check && !_checking_uo_aux_state)
5111 : {
5112 : // we will only check aux variables and postprocessors
5113 : // checking more reporter data can be added in the future if needed
5114 559 : std::unique_ptr<NumericVector<Number>> x = _aux->currentSolution()->clone();
5115 559 : DenseVector<Real> pp_values = getReporterData().getAllRealReporterValues();
5116 :
5117 : // call THIS execute one more time for checking the possible states
5118 559 : _checking_uo_aux_state = true;
5119 559 : FEProblemBase::execute(exec_type);
5120 559 : _checking_uo_aux_state = false;
5121 :
5122 559 : const Real check_tol = 1e-8;
5123 :
5124 559 : const Real xnorm = x->l2_norm();
5125 559 : *x -= *_aux->currentSolution();
5126 559 : if (x->l2_norm() > check_tol * xnorm)
5127 : {
5128 3 : const auto & sys = _aux->system();
5129 3 : const unsigned int n_vars = sys.n_vars();
5130 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5131 15 : for (const auto i : make_range(n_vars))
5132 : {
5133 12 : const Real vnorm = sys.calculate_norm(*x, i, DISCRETE_L2);
5134 12 : ordered_map.emplace(vnorm, sys.variable_name(i));
5135 : }
5136 :
5137 3 : std::ostringstream oss;
5138 15 : for (const auto & [error_norm, var_name] : ordered_map)
5139 12 : oss << " {" << var_name << ", " << error_norm << "},\n";
5140 :
5141 3 : mooseError("Aux kernels, user objects appear to have states for aux variables on ",
5142 : exec_type,
5143 : ".\nVariable error norms in descending order:\n",
5144 3 : oss.str());
5145 0 : }
5146 :
5147 556 : const DenseVector<Real> new_pp_values = getReporterData().getAllRealReporterValues();
5148 556 : if (pp_values.size() != new_pp_values.size())
5149 0 : mooseError("Second execution for uo/aux state check should not change the number of "
5150 : "real reporter values");
5151 :
5152 556 : const Real ppnorm = pp_values.l2_norm();
5153 556 : pp_values -= new_pp_values;
5154 556 : if (pp_values.l2_norm() > check_tol * ppnorm)
5155 : {
5156 3 : const auto pp_names = getReporterData().getAllRealReporterFullNames();
5157 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5158 12 : for (const auto i : index_range(pp_names))
5159 9 : ordered_map.emplace(std::abs(pp_values(i)), pp_names[i]);
5160 :
5161 3 : std::ostringstream oss;
5162 12 : for (const auto & [error_norm, pp_name] : ordered_map)
5163 9 : oss << " {" << pp_name << ", " << error_norm << "},\n";
5164 :
5165 3 : mooseError("Aux kernels, user objects appear to have states for real reporter values on ",
5166 : exec_type,
5167 : ".\nErrors of real reporter values in descending order:\n",
5168 3 : oss.str());
5169 0 : }
5170 553 : }
5171 2128727 : }
5172 :
5173 : // Finalize, threadJoin, and update PP values of Elemental/Nodal/Side/InternalSideUserObjects
5174 : void
5175 1331759 : FEProblemBase::joinAndFinalize(TheWarehouse::Query query, bool isgen)
5176 : {
5177 1331759 : std::vector<UserObject *> objs;
5178 1331759 : query.queryInto(objs);
5179 1331759 : if (!isgen)
5180 : {
5181 : // join all threaded user objects (i.e. not regular general user objects) to the primary
5182 : // thread
5183 1415136 : for (auto obj : objs)
5184 368974 : if (obj->primaryThreadCopy())
5185 30641 : obj->primaryThreadCopy()->threadJoin(*obj);
5186 : }
5187 :
5188 1331759 : query.condition<AttribThread>(0).queryInto(objs);
5189 :
5190 : // finalize objects and retrieve/store any postprocessor values
5191 1891259 : for (auto obj : objs)
5192 : {
5193 559580 : if (isgen && dynamic_cast<ThreadedGeneralUserObject *>(obj))
5194 133 : continue;
5195 559447 : if (isgen)
5196 : {
5197 : // general user objects are not run in their own threaded loop object - so run them here
5198 221114 : if (shouldPrintExecution(0))
5199 724 : _console << "[DBG] Initializing, executing & finalizing general UO '" << obj->name()
5200 724 : << "' on " << _current_execute_on_flag.name() << std::endl;
5201 221114 : obj->initialize();
5202 221114 : obj->execute();
5203 : }
5204 :
5205 559397 : obj->finalize();
5206 :
5207 : // These have to be stored piecemeal (with every call to this function) because general
5208 : // postprocessors (which run last after other userobjects have been completed) might depend on
5209 : // them being stored. This wouldn't be a problem if all userobjects satisfied the dependency
5210 : // resolver interface and could be sorted appropriately with the general userobjects, but they
5211 : // don't.
5212 559382 : auto pp = dynamic_cast<const Postprocessor *>(obj);
5213 559382 : if (pp)
5214 : {
5215 490320 : _reporter_data.finalize(obj->name());
5216 490320 : setPostprocessorValueByName(obj->name(), pp->getValue());
5217 : }
5218 :
5219 559373 : auto vpp = dynamic_cast<VectorPostprocessor *>(obj);
5220 559373 : if (vpp)
5221 13514 : _reporter_data.finalize(obj->name());
5222 :
5223 : // Update Reporter data
5224 559373 : auto reporter = dynamic_cast<Reporter *>(obj);
5225 559373 : if (reporter)
5226 5061 : _reporter_data.finalize(obj->name());
5227 : }
5228 1331679 : }
5229 :
5230 : TheWarehouse::Query
5231 19817646 : FEProblemBase::getUOQuery(const std::string & system,
5232 : const ExecFlagType & type,
5233 : const Moose::AuxGroup & group) const
5234 : {
5235 : TheWarehouse::Query query =
5236 19817646 : theWarehouse().query().condition<AttribSystem>(system).condition<AttribExecOns>(type);
5237 :
5238 19817646 : if (group == Moose::PRE_IC)
5239 95544 : query.condition<AttribPreIC>(true);
5240 19722102 : else if (group == Moose::PRE_AUX)
5241 9860672 : query.condition<AttribPreAux>(type);
5242 9861430 : else if (group == Moose::POST_AUX)
5243 9861362 : query.condition<AttribPostAux>(type);
5244 :
5245 19817646 : return query;
5246 0 : }
5247 :
5248 : void
5249 19817646 : FEProblemBase::getUOExecutionGroups(TheWarehouse::Query & query,
5250 : std::set<int> & execution_groups) const
5251 : {
5252 19817646 : std::vector<UserObjectBase *> uos;
5253 19817646 : query.queryIntoUnsorted(uos);
5254 20410291 : for (const auto & uo : uos)
5255 1777935 : execution_groups.insert(uo->getParam<int>("execution_order_group"));
5256 19817646 : }
5257 :
5258 : void
5259 55071 : FEProblemBase::computeUserObjectByName(const ExecFlagType & type,
5260 : const Moose::AuxGroup & group,
5261 : const std::string & name)
5262 : {
5263 55071 : const auto old_exec_flag = _current_execute_on_flag;
5264 55071 : _current_execute_on_flag = type;
5265 :
5266 55071 : std::set<int> execution_groups;
5267 :
5268 : #ifdef MOOSE_KOKKOS_ENABLED
5269 : TheWarehouse::Query kokkos_query =
5270 39698 : getUOQuery("KokkosUserObject", type, group).condition<AttribName>(name);
5271 39698 : getUOExecutionGroups(kokkos_query, execution_groups);
5272 : #endif
5273 :
5274 55071 : TheWarehouse::Query query = getUOQuery("UserObject", type, group).condition<AttribName>(name);
5275 55071 : getUOExecutionGroups(query, execution_groups);
5276 :
5277 81022 : for (const auto execution_group : execution_groups)
5278 : {
5279 : #ifdef MOOSE_KOKKOS_ENABLED
5280 18721 : computeKokkosUserObjectsInternal(
5281 18721 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5282 : #endif
5283 :
5284 25951 : computeUserObjectsInternal(type,
5285 25951 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5286 : }
5287 :
5288 55071 : _current_execute_on_flag = old_exec_flag;
5289 55071 : }
5290 :
5291 : void
5292 11403056 : FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGroup & group)
5293 : {
5294 11403056 : std::set<int> execution_groups;
5295 :
5296 : #ifdef MOOSE_KOKKOS_ENABLED
5297 8319821 : TheWarehouse::Query kokkos_query = getUOQuery("KokkosUserObject", type, group);
5298 8319821 : getUOExecutionGroups(kokkos_query, execution_groups);
5299 : #endif
5300 :
5301 11403056 : TheWarehouse::Query query = getUOQuery("UserObject", type, group);
5302 11403056 : getUOExecutionGroups(query, execution_groups);
5303 :
5304 11664338 : for (const auto execution_group : execution_groups)
5305 : {
5306 : #ifdef MOOSE_KOKKOS_ENABLED
5307 190667 : computeKokkosUserObjectsInternal(
5308 190667 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5309 : #endif
5310 :
5311 261368 : computeUserObjectsInternal(type,
5312 261368 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5313 : }
5314 11402970 : }
5315 :
5316 : void
5317 287319 : FEProblemBase::computeUserObjectsInternal(const ExecFlagType & type, TheWarehouse::Query & query)
5318 : {
5319 : try
5320 : {
5321 1436595 : TIME_SECTION("computeUserObjects", 1, "Computing User Objects");
5322 :
5323 287319 : std::vector<GeneralUserObject *> genobjs;
5324 287319 : query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject).queryInto(genobjs);
5325 :
5326 287319 : std::vector<UserObject *> userobjs;
5327 287319 : query.clone()
5328 574638 : .condition<AttribInterfaces>(Interfaces::ElementUserObject | Interfaces::SideUserObject |
5329 : Interfaces::InternalSideUserObject |
5330 574638 : Interfaces::InterfaceUserObject | Interfaces::DomainUserObject)
5331 287319 : .queryInto(userobjs);
5332 :
5333 287319 : std::vector<UserObject *> tgobjs;
5334 287319 : query.clone()
5335 574638 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5336 287319 : .queryInto(tgobjs);
5337 :
5338 287319 : std::vector<UserObject *> nodal;
5339 287319 : query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).queryInto(nodal);
5340 :
5341 287319 : std::vector<MortarUserObject *> mortar;
5342 287319 : query.clone().condition<AttribInterfaces>(Interfaces::MortarUserObject).queryInto(mortar);
5343 :
5344 287319 : if (userobjs.empty() && genobjs.empty() && tgobjs.empty() && nodal.empty() && mortar.empty())
5345 1710 : return;
5346 :
5347 : // Start the timer here since we have at least one active user object
5348 285609 : std::string compute_uo_tag = "computeUserObjects(" + Moose::stringify(type) + ")";
5349 :
5350 : // Perform Residual/Jacobian setups
5351 285609 : if (type == EXEC_LINEAR)
5352 : {
5353 124307 : for (auto obj : userobjs)
5354 70711 : obj->residualSetup();
5355 58054 : for (auto obj : nodal)
5356 4458 : obj->residualSetup();
5357 53596 : for (auto obj : mortar)
5358 0 : obj->residualSetup();
5359 53605 : for (auto obj : tgobjs)
5360 9 : obj->residualSetup();
5361 64990 : for (auto obj : genobjs)
5362 11394 : obj->residualSetup();
5363 : }
5364 232013 : else if (type == EXEC_NONLINEAR)
5365 : {
5366 13576 : for (auto obj : userobjs)
5367 4076 : obj->jacobianSetup();
5368 9892 : for (auto obj : nodal)
5369 392 : obj->jacobianSetup();
5370 9500 : for (auto obj : mortar)
5371 0 : obj->jacobianSetup();
5372 9503 : for (auto obj : tgobjs)
5373 3 : obj->jacobianSetup();
5374 26145 : for (auto obj : genobjs)
5375 16645 : obj->jacobianSetup();
5376 : }
5377 :
5378 633868 : for (auto obj : userobjs)
5379 348259 : obj->initialize();
5380 :
5381 : // Execute Side/InternalSide/Interface/Elemental/DomainUserObjects
5382 285609 : if (!userobjs.empty())
5383 : {
5384 : // non-nodal user objects have to be run separately before the nodal user objects run
5385 : // because some nodal user objects (NodalNormal related) depend on elemental user objects
5386 : // :-(
5387 205939 : ComputeUserObjectsThread cppt(*this, query);
5388 205939 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cppt);
5389 :
5390 : // There is one instance in rattlesnake where an elemental user object's finalize depends
5391 : // on a side user object having been finalized first :-(
5392 205933 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::SideUserObject));
5393 205930 : joinAndFinalize(
5394 411860 : query.clone().condition<AttribInterfaces>(Interfaces::InternalSideUserObject));
5395 205930 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::InterfaceUserObject));
5396 205930 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::ElementUserObject));
5397 205927 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::DomainUserObject));
5398 205927 : }
5399 :
5400 : // if any elemental user object may have written to variables we need to close the aux solution
5401 633768 : for (const auto & uo : userobjs)
5402 348204 : if (auto euo = dynamic_cast<const ElementUserObject *>(uo);
5403 348204 : euo && euo->hasWritableCoupledVariables())
5404 : {
5405 33 : _aux->solution().close();
5406 33 : _aux->system().update();
5407 33 : break;
5408 : }
5409 :
5410 : // Execute NodalUserObjects
5411 : // BISON has an axial reloc elemental user object that has a finalize func that depends on a
5412 : // nodal user object's prev value. So we can't initialize this until after elemental objects
5413 : // have been finalized :-(
5414 305903 : for (auto obj : nodal)
5415 20306 : obj->initialize();
5416 285597 : if (query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).count() > 0)
5417 : {
5418 16379 : ComputeNodalUserObjectsThread cnppt(*this, query);
5419 16379 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), cnppt);
5420 16379 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject));
5421 16379 : }
5422 :
5423 : // if any nodal user object may have written to variables we need to close the aux solution
5424 305867 : for (const auto & uo : nodal)
5425 20292 : if (auto nuo = dynamic_cast<const NodalUserObject *>(uo);
5426 20292 : nuo && nuo->hasWritableCoupledVariables())
5427 : {
5428 22 : _aux->solution().close();
5429 22 : _aux->system().update();
5430 22 : break;
5431 : }
5432 :
5433 : // Execute MortarUserObjects
5434 : {
5435 285619 : for (auto obj : mortar)
5436 22 : obj->initialize();
5437 285597 : if (!mortar.empty())
5438 : {
5439 33 : auto create_and_run_mortar_functors = [this, type, &mortar](const bool displaced)
5440 : {
5441 : // go over mortar interfaces and construct functors
5442 33 : const auto & mortar_interfaces = getMortarInterfaces(displaced);
5443 55 : for (const auto & [primary_secondary_boundary_pair, mortar_generation_ptr] :
5444 88 : mortar_interfaces)
5445 : {
5446 : auto mortar_uos_to_execute =
5447 22 : getMortarUserObjects(primary_secondary_boundary_pair.first,
5448 22 : primary_secondary_boundary_pair.second,
5449 : displaced,
5450 22 : mortar);
5451 :
5452 : auto * const subproblem = displaced
5453 22 : ? static_cast<SubProblem *>(_displaced_problem.get())
5454 22 : : static_cast<SubProblem *>(this);
5455 : MortarUserObjectThread muot(mortar_uos_to_execute,
5456 22 : *mortar_generation_ptr,
5457 : *subproblem,
5458 : *this,
5459 : displaced,
5460 22 : subproblem->assembly(0, 0));
5461 :
5462 22 : muot();
5463 22 : }
5464 55 : };
5465 :
5466 22 : create_and_run_mortar_functors(false);
5467 22 : if (_displaced_problem)
5468 11 : create_and_run_mortar_functors(true);
5469 22 : }
5470 285619 : for (auto obj : mortar)
5471 22 : obj->finalize();
5472 : }
5473 :
5474 : // Execute threaded general user objects
5475 286038 : for (auto obj : tgobjs)
5476 441 : obj->initialize();
5477 285597 : std::vector<GeneralUserObject *> tguos_zero;
5478 285597 : query.clone()
5479 285597 : .condition<AttribThread>(0)
5480 571194 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5481 285597 : .queryInto(tguos_zero);
5482 285730 : for (auto obj : tguos_zero)
5483 : {
5484 133 : std::vector<GeneralUserObject *> tguos;
5485 133 : auto q = query.clone()
5486 133 : .condition<AttribName>(obj->name())
5487 133 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject);
5488 133 : q.queryInto(tguos);
5489 :
5490 133 : ComputeThreadedGeneralUserObjectsThread ctguot(*this);
5491 133 : Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(), tguos.end()), ctguot);
5492 133 : joinAndFinalize(q);
5493 133 : }
5494 :
5495 : // Execute general user objects
5496 285597 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject), true);
5497 295783 : }
5498 0 : catch (...)
5499 : {
5500 0 : handleException("computeUserObjectsInternal");
5501 0 : }
5502 : }
5503 :
5504 : void
5505 5678118 : FEProblemBase::executeControls(const ExecFlagType & exec_type)
5506 : {
5507 5678118 : if (_control_warehouse[exec_type].hasActiveObjects())
5508 : {
5509 38765 : TIME_SECTION("executeControls", 1, "Executing Controls");
5510 :
5511 7753 : DependencyResolver<std::shared_ptr<Control>> resolver;
5512 :
5513 7753 : auto controls_wh = _control_warehouse[exec_type];
5514 : // Add all of the dependencies into the resolver and sort them
5515 19427 : for (const auto & it : controls_wh.getActiveObjects())
5516 : {
5517 : // Make sure an item with no dependencies comes out too!
5518 11677 : resolver.addItem(it);
5519 :
5520 11677 : std::vector<std::string> & dependent_controls = it->getDependencies();
5521 14569 : for (const auto & depend_name : dependent_controls)
5522 : {
5523 2895 : if (controls_wh.hasActiveObject(depend_name))
5524 : {
5525 2892 : auto dep_control = controls_wh.getActiveObject(depend_name);
5526 2892 : resolver.addEdge(dep_control, it);
5527 2892 : }
5528 : else
5529 3 : mooseError("The Control \"",
5530 : depend_name,
5531 : "\" was not created, did you make a "
5532 : "spelling mistake or forget to include it "
5533 : "in your input file?");
5534 : }
5535 : }
5536 :
5537 7750 : const auto & ordered_controls = resolver.getSortedValues();
5538 :
5539 7750 : if (!ordered_controls.empty())
5540 : {
5541 : // already called by initialSetup when exec_type == EXEC_INITIAL
5542 7750 : if (exec_type != EXEC_INITIAL)
5543 6878 : _control_warehouse.setup(exec_type);
5544 :
5545 : // Run the controls in the proper order
5546 19382 : for (const auto & control : ordered_controls)
5547 11674 : control->execute();
5548 : }
5549 7708 : }
5550 5678073 : }
5551 :
5552 : void
5553 2128876 : FEProblemBase::executeSamplers(const ExecFlagType & exec_type)
5554 : {
5555 : // TODO: This should be done in a threaded loop, but this should be super quick so for now
5556 : // do a serial loop.
5557 4470654 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5558 : {
5559 2341802 : std::vector<Sampler *> objects;
5560 2341802 : theWarehouse()
5561 4683604 : .query()
5562 2341802 : .condition<AttribSystem>("Sampler")
5563 2341802 : .condition<AttribThread>(tid)
5564 2341802 : .condition<AttribExecOns>(exec_type)
5565 2341802 : .queryInto(objects);
5566 :
5567 2341802 : if (!objects.empty())
5568 : {
5569 1475 : TIME_SECTION("executeSamplers", 1, "Executing Samplers");
5570 295 : FEProblemBase::objectSetupHelper<Sampler>(objects, exec_type);
5571 295 : FEProblemBase::objectExecuteHelper<Sampler>(objects);
5572 271 : }
5573 2341778 : }
5574 2128852 : }
5575 :
5576 : void
5577 310899 : FEProblemBase::updateActiveObjects()
5578 : {
5579 1554495 : TIME_SECTION("updateActiveObjects", 5, "Updating Active Objects");
5580 :
5581 652366 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5582 : {
5583 685450 : for (auto & nl : _nl)
5584 343983 : nl->updateActive(tid);
5585 341467 : _aux->updateActive(tid);
5586 341467 : _indicators.updateActive(tid);
5587 341467 : _internal_side_indicators.updateActive(tid);
5588 341467 : _markers.updateActive(tid);
5589 341467 : _all_materials.updateActive(tid);
5590 341467 : _materials.updateActive(tid);
5591 341467 : _discrete_materials.updateActive(tid);
5592 : }
5593 :
5594 310899 : _control_warehouse.updateActive();
5595 310899 : _multi_apps.updateActive();
5596 310899 : _transient_multi_apps.updateActive();
5597 310899 : _transfers.updateActive();
5598 310899 : _to_multi_app_transfers.updateActive();
5599 310899 : _from_multi_app_transfers.updateActive();
5600 310899 : _between_multi_app_transfers.updateActive();
5601 :
5602 : #ifdef MOOSE_KOKKOS_ENABLED
5603 227466 : _kokkos_materials.updateActive();
5604 : #endif
5605 310899 : }
5606 :
5607 : void
5608 0 : FEProblemBase::reportMooseObjectDependency(MooseObject * /*a*/, MooseObject * /*b*/)
5609 : {
5610 : //<< "Object " << a->name() << " -> " << b->name() << std::endl;
5611 0 : }
5612 :
5613 : void
5614 66961 : FEProblemBase::reinitBecauseOfGhostingOrNewGeomObjects(const bool mortar_changed)
5615 : {
5616 334805 : TIME_SECTION("reinitBecauseOfGhostingOrNewGeomObjects",
5617 : 3,
5618 : "Reinitializing Because of Geometric Search Objects");
5619 :
5620 : // Need to see if _any_ processor has ghosted elems or geometry objects.
5621 66961 : bool needs_reinit = !_ghosted_elems.empty();
5622 133452 : needs_reinit = needs_reinit || !_geometric_search_data._nearest_node_locators.empty() ||
5623 66491 : (_mortar_data->hasObjects() && mortar_changed);
5624 66961 : needs_reinit =
5625 135151 : needs_reinit || (_displaced_problem &&
5626 4423 : (!_displaced_problem->geomSearchData()._nearest_node_locators.empty() ||
5627 67669 : (_mortar_data->hasDisplacedObjects() && mortar_changed)));
5628 66961 : _communicator.max(needs_reinit);
5629 :
5630 66961 : if (needs_reinit)
5631 : {
5632 : // Call reinit to get the ghosted vectors correct now that some geometric search has been done
5633 2175 : es().reinit();
5634 :
5635 2175 : if (_displaced_mesh)
5636 1607 : _displaced_problem->es().reinit();
5637 : }
5638 66961 : }
5639 :
5640 : void
5641 177 : FEProblemBase::addDamper(const std::string & damper_name,
5642 : const std::string & name,
5643 : InputParameters & parameters)
5644 : {
5645 : parallel_object_only();
5646 :
5647 : const auto nl_sys_num =
5648 177 : parameters.isParamValid("variable")
5649 588 : ? determineSolverSystem(parameters.varName("variable", name), true).second
5650 174 : : (unsigned int)0;
5651 :
5652 174 : if (!isSolverSystemNonlinear(nl_sys_num))
5653 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
5654 : "supported at the moment!");
5655 :
5656 348 : parameters.set<SubProblem *>("_subproblem") = this;
5657 348 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
5658 :
5659 174 : _has_dampers = true;
5660 174 : logAdd("Damper", name, damper_name, parameters);
5661 174 : _nl[nl_sys_num]->addDamper(damper_name, name, parameters);
5662 174 : }
5663 :
5664 : void
5665 162 : FEProblemBase::setupDampers()
5666 : {
5667 324 : for (auto & nl : _nl)
5668 162 : nl->setupDampers();
5669 162 : }
5670 :
5671 : void
5672 659 : FEProblemBase::addIndicator(const std::string & indicator_name,
5673 : const std::string & name,
5674 : InputParameters & parameters)
5675 : {
5676 : parallel_object_only();
5677 :
5678 659 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5679 : {
5680 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5681 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5682 0 : _reinit_displaced_elem = true;
5683 : }
5684 : else
5685 : {
5686 659 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5687 : {
5688 : // We allow Indicators to request that they use_displaced_mesh,
5689 : // but then be overridden when no displacements variables are
5690 : // provided in the Mesh block. If that happened, update the value
5691 : // of use_displaced_mesh appropriately for this Indicator.
5692 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5693 0 : parameters.set<bool>("use_displaced_mesh") = false;
5694 : }
5695 :
5696 1318 : parameters.set<SubProblem *>("_subproblem") = this;
5697 1977 : parameters.set<SystemBase *>("_sys") = _aux.get();
5698 : }
5699 :
5700 1382 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5701 : {
5702 : std::shared_ptr<Indicator> indicator =
5703 723 : _factory.create<Indicator>(indicator_name, name, parameters, tid);
5704 723 : logAdd("Indicator", name, indicator_name, parameters);
5705 : std::shared_ptr<InternalSideIndicatorBase> isi =
5706 723 : std::dynamic_pointer_cast<InternalSideIndicatorBase>(indicator);
5707 723 : if (isi)
5708 622 : _internal_side_indicators.addObject(isi, tid);
5709 : else
5710 101 : _indicators.addObject(indicator, tid);
5711 723 : }
5712 659 : }
5713 :
5714 : void
5715 1950 : FEProblemBase::addMarker(const std::string & marker_name,
5716 : const std::string & name,
5717 : InputParameters & parameters)
5718 : {
5719 : parallel_object_only();
5720 :
5721 1950 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5722 : {
5723 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5724 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5725 0 : _reinit_displaced_elem = true;
5726 : }
5727 : else
5728 : {
5729 1950 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5730 : {
5731 : // We allow Markers to request that they use_displaced_mesh,
5732 : // but then be overridden when no displacements variables are
5733 : // provided in the Mesh block. If that happened, update the value
5734 : // of use_displaced_mesh appropriately for this Marker.
5735 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5736 0 : parameters.set<bool>("use_displaced_mesh") = false;
5737 : }
5738 :
5739 3900 : parameters.set<SubProblem *>("_subproblem") = this;
5740 5850 : parameters.set<SystemBase *>("_sys") = _aux.get();
5741 : }
5742 :
5743 4074 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5744 : {
5745 2127 : std::shared_ptr<Marker> marker = _factory.create<Marker>(marker_name, name, parameters, tid);
5746 2124 : logAdd("Marker", name, marker_name, parameters);
5747 2124 : _markers.addObject(marker, tid);
5748 2124 : }
5749 1947 : }
5750 :
5751 : void
5752 8028 : FEProblemBase::addMultiApp(const std::string & multi_app_name,
5753 : const std::string & name,
5754 : InputParameters & parameters)
5755 : {
5756 : parallel_object_only();
5757 :
5758 16056 : parameters.set<MPI_Comm>("_mpi_comm") = _communicator.get();
5759 :
5760 8028 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5761 : {
5762 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5763 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5764 0 : _reinit_displaced_elem = true;
5765 : }
5766 : else
5767 : {
5768 8028 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5769 : {
5770 : // We allow MultiApps to request that they use_displaced_mesh,
5771 : // but then be overridden when no displacements variables are
5772 : // provided in the Mesh block. If that happened, update the value
5773 : // of use_displaced_mesh appropriately for this MultiApp.
5774 42 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5775 84 : parameters.set<bool>("use_displaced_mesh") = false;
5776 : }
5777 :
5778 16056 : parameters.set<SubProblem *>("_subproblem") = this;
5779 24084 : parameters.set<SystemBase *>("_sys") = _aux.get();
5780 : }
5781 :
5782 8028 : std::shared_ptr<MultiApp> multi_app = _factory.create<MultiApp>(multi_app_name, name, parameters);
5783 8016 : logAdd("MultiApp", name, multi_app_name, parameters);
5784 8016 : multi_app->setupPositions();
5785 :
5786 7929 : _multi_apps.addObject(multi_app);
5787 :
5788 : // Store TransientMultiApp objects in another container, this is needed for calling computeDT
5789 : std::shared_ptr<TransientMultiApp> trans_multi_app =
5790 7929 : std::dynamic_pointer_cast<TransientMultiApp>(multi_app);
5791 7929 : if (trans_multi_app)
5792 5353 : _transient_multi_apps.addObject(trans_multi_app);
5793 7929 : }
5794 :
5795 : bool
5796 112534 : FEProblemBase::hasMultiApps(ExecFlagType type) const
5797 : {
5798 112534 : return _multi_apps[type].hasActiveObjects();
5799 : }
5800 :
5801 : bool
5802 26192 : FEProblemBase::hasMultiApp(const std::string & multi_app_name) const
5803 : {
5804 26192 : return _multi_apps.hasActiveObject(multi_app_name);
5805 : }
5806 :
5807 : std::shared_ptr<MultiApp>
5808 26192 : FEProblemBase::getMultiApp(const std::string & multi_app_name) const
5809 : {
5810 26192 : if (!hasMultiApp(multi_app_name))
5811 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_multi_app"),
5812 : "A MultiApp getter was called before MultiApps have been constructed. "
5813 : "If you are attempting to access this object in the constructor of another object "
5814 : "then make sure that the MultiApp is constructed before the object using it.");
5815 :
5816 26192 : return _multi_apps.getObject(multi_app_name);
5817 : }
5818 :
5819 : void
5820 14397708 : FEProblemBase::execMultiAppTransfers(ExecFlagType type, Transfer::DIRECTION direction)
5821 : {
5822 14397708 : bool to_multiapp = direction == MultiAppTransfer::TO_MULTIAPP;
5823 14397708 : bool from_multiapp = direction == MultiAppTransfer::FROM_MULTIAPP;
5824 14397708 : std::string string_direction;
5825 14397708 : if (to_multiapp)
5826 4799290 : string_direction = " To ";
5827 9598418 : else if (from_multiapp)
5828 4799164 : string_direction = " From ";
5829 : else
5830 4799254 : string_direction = " Between ";
5831 :
5832 14397870 : const MooseObjectWarehouse<Transfer> & wh = to_multiapp ? _to_multi_app_transfers[type]
5833 23996036 : : from_multiapp ? _from_multi_app_transfers[type]
5834 33594616 : : _between_multi_app_transfers[type];
5835 :
5836 14397708 : if (wh.hasActiveObjects())
5837 : {
5838 537475 : TIME_SECTION("execMultiAppTransfers", 1, "Executing Transfers");
5839 :
5840 107495 : const auto & transfers = wh.getActiveObjects();
5841 :
5842 107495 : if (_verbose_multiapps)
5843 : {
5844 2454 : _console << COLOR_CYAN << "\nTransfers on " << Moose::stringify(type) << string_direction
5845 2454 : << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
5846 :
5847 : VariadicTable<std::string, std::string, std::string, std::string> table(
5848 4908 : {"Name", "Type", "From", "To"});
5849 :
5850 : // Build Table of Transfer Info
5851 5896 : for (const auto & transfer : transfers)
5852 : {
5853 3442 : auto multiapp_transfer = dynamic_cast<MultiAppTransfer *>(transfer.get());
5854 :
5855 3442 : table.addRow(multiapp_transfer->name(),
5856 3442 : multiapp_transfer->type(),
5857 6884 : multiapp_transfer->getFromName(),
5858 6884 : multiapp_transfer->getToName());
5859 : }
5860 :
5861 : // Print it
5862 2454 : table.print(_console);
5863 2454 : }
5864 :
5865 220031 : for (const auto & transfer : transfers)
5866 : {
5867 112633 : transfer->setCurrentDirection(direction);
5868 112633 : transfer->execute();
5869 : }
5870 :
5871 107398 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5872 :
5873 107398 : if (_verbose_multiapps)
5874 4896 : _console << COLOR_CYAN << "Transfers on " << Moose::stringify(type) << " Are Finished\n"
5875 2448 : << COLOR_DEFAULT << std::endl;
5876 107398 : }
5877 14290213 : else if (_multi_apps[type].getActiveObjects().size())
5878 : {
5879 109376 : if (_verbose_multiapps)
5880 7684 : _console << COLOR_CYAN << "\nNo Transfers on " << Moose::stringify(type) << string_direction
5881 3842 : << "MultiApps\n"
5882 3842 : << COLOR_DEFAULT << std::endl;
5883 : }
5884 14397611 : }
5885 :
5886 : std::vector<std::shared_ptr<Transfer>>
5887 12 : FEProblemBase::getTransfers(ExecFlagType type, Transfer::DIRECTION direction) const
5888 : {
5889 12 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5890 12 : return _to_multi_app_transfers[type].getActiveObjects();
5891 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5892 0 : return _from_multi_app_transfers[type].getActiveObjects();
5893 : else
5894 0 : return _between_multi_app_transfers[type].getActiveObjects();
5895 : }
5896 :
5897 : std::vector<std::shared_ptr<Transfer>>
5898 0 : FEProblemBase::getTransfers(Transfer::DIRECTION direction) const
5899 : {
5900 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5901 0 : return _to_multi_app_transfers.getActiveObjects();
5902 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5903 0 : return _from_multi_app_transfers.getActiveObjects();
5904 : else
5905 0 : return _between_multi_app_transfers.getActiveObjects();
5906 : }
5907 :
5908 : const ExecuteMooseObjectWarehouse<Transfer> &
5909 0 : FEProblemBase::getMultiAppTransferWarehouse(Transfer::DIRECTION direction) const
5910 : {
5911 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5912 0 : return _to_multi_app_transfers;
5913 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5914 0 : return _from_multi_app_transfers;
5915 : else
5916 0 : return _between_multi_app_transfers;
5917 : }
5918 :
5919 : bool
5920 4799293 : FEProblemBase::execMultiApps(ExecFlagType type, bool auto_advance)
5921 : {
5922 : // Active MultiApps
5923 : const std::vector<MooseSharedPointer<MultiApp>> & multi_apps =
5924 4799293 : _multi_apps[type].getActiveObjects();
5925 :
5926 : // Do anything that needs to be done to Apps before transfers
5927 4863539 : for (const auto & multi_app : multi_apps)
5928 64249 : multi_app->preTransfer(_dt, _time);
5929 :
5930 : // Execute Transfers _to_ MultiApps
5931 4799290 : execMultiAppTransfers(type, MultiAppTransfer::TO_MULTIAPP);
5932 :
5933 : // Execute Transfers _between_ Multiapps
5934 4799254 : execMultiAppTransfers(type, MultiAppTransfer::BETWEEN_MULTIAPP);
5935 :
5936 : // Execute MultiApps
5937 4799254 : if (multi_apps.size())
5938 : {
5939 314955 : TIME_SECTION("execMultiApps", 1, "Executing MultiApps", false);
5940 :
5941 62991 : if (_verbose_multiapps)
5942 4080 : _console << COLOR_CYAN << "\nExecuting MultiApps on " << Moose::stringify(type)
5943 2040 : << COLOR_DEFAULT << std::endl;
5944 :
5945 62991 : bool success = true;
5946 :
5947 127112 : for (const auto & multi_app : multi_apps)
5948 : {
5949 64204 : success = multi_app->solveStep(_dt, _time, auto_advance);
5950 : // no need to finish executing the subapps if one fails
5951 64195 : if (!success)
5952 74 : break;
5953 : }
5954 :
5955 62982 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5956 :
5957 62982 : _communicator.min(success);
5958 :
5959 62982 : if (!success)
5960 81 : return false;
5961 :
5962 62901 : if (_verbose_multiapps)
5963 4080 : _console << COLOR_CYAN << "Finished Executing MultiApps on " << Moose::stringify(type) << "\n"
5964 2040 : << COLOR_DEFAULT << std::endl;
5965 62982 : }
5966 :
5967 : // Execute Transfers _from_ MultiApps
5968 4799164 : execMultiAppTransfers(type, MultiAppTransfer::FROM_MULTIAPP);
5969 :
5970 : // If we made it here then everything passed
5971 4799103 : return true;
5972 : }
5973 :
5974 : void
5975 48099 : FEProblemBase::finalizeMultiApps()
5976 : {
5977 48099 : const auto & multi_apps = _multi_apps.getActiveObjects();
5978 :
5979 54943 : for (const auto & multi_app : multi_apps)
5980 6844 : multi_app->finalize();
5981 48099 : }
5982 :
5983 : void
5984 49756 : FEProblemBase::postExecute()
5985 : {
5986 49756 : const auto & multi_apps = _multi_apps.getActiveObjects();
5987 :
5988 56982 : for (const auto & multi_app : multi_apps)
5989 7226 : multi_app->postExecute();
5990 49756 : }
5991 :
5992 : void
5993 656076 : FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
5994 : {
5995 656076 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
5996 :
5997 656076 : if (multi_apps.size())
5998 26266 : for (const auto & multi_app : multi_apps)
5999 13304 : multi_app->incrementTStep(_time);
6000 656076 : }
6001 :
6002 : void
6003 36370 : FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
6004 : {
6005 36370 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6006 :
6007 36370 : if (multi_apps.size())
6008 : {
6009 8584 : if (_verbose_multiapps)
6010 382 : _console << COLOR_CYAN << "\nAdvancing MultiApps on " << type.name() << COLOR_DEFAULT
6011 382 : << std::endl;
6012 :
6013 17174 : for (const auto & multi_app : multi_apps)
6014 8590 : multi_app->finishStep(recurse_through_multiapp_levels);
6015 :
6016 8584 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6017 :
6018 8584 : if (_verbose_multiapps)
6019 382 : _console << COLOR_CYAN << "Finished Advancing MultiApps on " << type.name() << "\n"
6020 382 : << COLOR_DEFAULT << std::endl;
6021 : }
6022 36370 : }
6023 :
6024 : void
6025 1114367 : FEProblemBase::backupMultiApps(ExecFlagType type)
6026 : {
6027 1114367 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6028 :
6029 1114367 : if (multi_apps.size())
6030 : {
6031 102180 : TIME_SECTION("backupMultiApps", 5, "Backing Up MultiApp");
6032 :
6033 20436 : if (_verbose_multiapps)
6034 1041 : _console << COLOR_CYAN << "\nBacking Up MultiApps on " << type.name() << COLOR_DEFAULT
6035 1041 : << std::endl;
6036 :
6037 42061 : for (const auto & multi_app : multi_apps)
6038 21625 : multi_app->backup();
6039 :
6040 20436 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6041 :
6042 20436 : if (_verbose_multiapps)
6043 1041 : _console << COLOR_CYAN << "Finished Backing Up MultiApps on " << type.name() << "\n"
6044 1041 : << COLOR_DEFAULT << std::endl;
6045 20436 : }
6046 1114367 : }
6047 :
6048 : void
6049 104510 : FEProblemBase::restoreMultiApps(ExecFlagType type, bool force)
6050 : {
6051 104510 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6052 :
6053 104510 : if (multi_apps.size())
6054 : {
6055 43043 : if (_verbose_multiapps)
6056 : {
6057 1002 : if (force)
6058 3 : _console << COLOR_CYAN << "\nRestoring Multiapps on " << type.name()
6059 3 : << " because of solve failure!" << COLOR_DEFAULT << std::endl;
6060 : else
6061 999 : _console << COLOR_CYAN << "\nRestoring MultiApps on " << type.name() << COLOR_DEFAULT
6062 999 : << std::endl;
6063 : }
6064 :
6065 86128 : for (const auto & multi_app : multi_apps)
6066 43088 : multi_app->restore(force);
6067 :
6068 43040 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6069 :
6070 43040 : if (_verbose_multiapps)
6071 1002 : _console << COLOR_CYAN << "Finished Restoring MultiApps on " << type.name() << "\n"
6072 1002 : << COLOR_DEFAULT << std::endl;
6073 : }
6074 104507 : }
6075 :
6076 : Real
6077 678552 : FEProblemBase::computeMultiAppsDT(ExecFlagType type)
6078 : {
6079 678552 : const auto & multi_apps = _transient_multi_apps[type].getActiveObjects();
6080 :
6081 678552 : Real smallest_dt = std::numeric_limits<Real>::max();
6082 :
6083 695937 : for (const auto & multi_app : multi_apps)
6084 17385 : smallest_dt = std::min(smallest_dt, multi_app->computeDT());
6085 :
6086 678552 : return smallest_dt;
6087 : }
6088 :
6089 : void
6090 4746539 : FEProblemBase::execTransfers(ExecFlagType type)
6091 : {
6092 4746539 : if (_transfers[type].hasActiveObjects())
6093 : {
6094 0 : TIME_SECTION("execTransfers", 3, "Executing Transfers");
6095 :
6096 0 : const auto & transfers = _transfers[type].getActiveObjects();
6097 :
6098 0 : for (const auto & transfer : transfers)
6099 0 : transfer->execute();
6100 0 : }
6101 4746539 : }
6102 :
6103 : void
6104 13011 : FEProblemBase::addTransfer(const std::string & transfer_name,
6105 : const std::string & name,
6106 : InputParameters & parameters)
6107 : {
6108 : parallel_object_only();
6109 :
6110 13011 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
6111 : {
6112 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
6113 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
6114 0 : _reinit_displaced_elem = true;
6115 : }
6116 : else
6117 : {
6118 13011 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
6119 : {
6120 : // We allow Transfers to request that they use_displaced_mesh,
6121 : // but then be overridden when no displacements variables are
6122 : // provided in the Mesh block. If that happened, update the value
6123 : // of use_displaced_mesh appropriately for this Transfer.
6124 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
6125 0 : parameters.set<bool>("use_displaced_mesh") = false;
6126 : }
6127 :
6128 26022 : parameters.set<SubProblem *>("_subproblem") = this;
6129 39033 : parameters.set<SystemBase *>("_sys") = _aux.get();
6130 : }
6131 :
6132 : // Handle the "SAME_AS_MULTIAPP" execute option. The get method is used to test for the
6133 : // flag so the set by user flag is not reset, calling set with the true flag causes the set
6134 : // by user status to be reset, which should only be done if the EXEC_SAME_AS_MULTIAPP is
6135 : // being applied to the object.
6136 13011 : if (parameters.get<ExecFlagEnum>("execute_on").isValueSet(EXEC_SAME_AS_MULTIAPP))
6137 : {
6138 10884 : ExecFlagEnum & exec_enum = parameters.set<ExecFlagEnum>("execute_on", true);
6139 10884 : std::shared_ptr<MultiApp> multiapp;
6140 21768 : if (parameters.isParamValid("multi_app"))
6141 50 : multiapp = getMultiApp(parameters.get<MultiAppName>("multi_app"));
6142 : // This catches the sibling transfer case, where we want to be executing only as often as the
6143 : // receiving application. A transfer 'to' a multiapp is executed before that multiapp
6144 21668 : else if (parameters.isParamValid("to_multi_app"))
6145 5649 : multiapp = getMultiApp(parameters.get<MultiAppName>("to_multi_app"));
6146 10370 : else if (parameters.isParamValid("from_multi_app"))
6147 5179 : multiapp = getMultiApp(parameters.get<MultiAppName>("from_multi_app"));
6148 : // else do nothing because the user has provided invalid input. They should get a nice error
6149 : // about this during transfer construction. This necessitates checking for null in this next
6150 : // line, however
6151 10884 : if (multiapp)
6152 32634 : exec_enum = multiapp->getParam<ExecFlagEnum>("execute_on");
6153 10884 : }
6154 :
6155 : // Create the Transfer objects
6156 13011 : std::shared_ptr<Transfer> transfer = _factory.create<Transfer>(transfer_name, name, parameters);
6157 12954 : logAdd("Transfer", name, transfer_name, parameters);
6158 :
6159 : // Add MultiAppTransfer object
6160 : std::shared_ptr<MultiAppTransfer> multi_app_transfer =
6161 12954 : std::dynamic_pointer_cast<MultiAppTransfer>(transfer);
6162 12954 : if (multi_app_transfer)
6163 : {
6164 12954 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::TO_MULTIAPP))
6165 5169 : _to_multi_app_transfers.addObject(multi_app_transfer);
6166 12954 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::FROM_MULTIAPP))
6167 6316 : _from_multi_app_transfers.addObject(multi_app_transfer);
6168 12954 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::BETWEEN_MULTIAPP))
6169 1469 : _between_multi_app_transfers.addObject(multi_app_transfer);
6170 : }
6171 : else
6172 0 : _transfers.addObject(transfer);
6173 12954 : }
6174 :
6175 : bool
6176 1428197 : FEProblemBase::hasVariable(const std::string & var_name) const
6177 : {
6178 2330813 : for (auto & sys : _solver_systems)
6179 1433514 : if (sys->hasVariable(var_name))
6180 530898 : return true;
6181 897299 : if (_aux->hasVariable(var_name))
6182 828708 : return true;
6183 :
6184 68591 : return false;
6185 : }
6186 :
6187 : bool
6188 65 : FEProblemBase::hasSolverVariable(const std::string & var_name) const
6189 : {
6190 78 : for (auto & sys : _solver_systems)
6191 65 : if (sys->hasVariable(var_name))
6192 52 : return true;
6193 :
6194 13 : return false;
6195 : }
6196 :
6197 : const MooseVariableFieldBase &
6198 4290135 : FEProblemBase::getVariable(const THREAD_ID tid,
6199 : const std::string & var_name,
6200 : Moose::VarKindType expected_var_type,
6201 : Moose::VarFieldType expected_var_field_type) const
6202 : {
6203 12870399 : return getVariableHelper(
6204 4290135 : tid, var_name, expected_var_type, expected_var_field_type, _solver_systems, *_aux);
6205 : }
6206 :
6207 : MooseVariable &
6208 7509 : FEProblemBase::getStandardVariable(const THREAD_ID tid, const std::string & var_name)
6209 : {
6210 11358 : for (auto & sys : _solver_systems)
6211 7509 : if (sys->hasVariable(var_name))
6212 3660 : return sys->getFieldVariable<Real>(tid, var_name);
6213 3849 : if (_aux->hasVariable(var_name))
6214 3846 : return _aux->getFieldVariable<Real>(tid, var_name);
6215 :
6216 3 : mooseError("Unknown variable " + var_name);
6217 : }
6218 :
6219 : MooseVariableFieldBase &
6220 581 : FEProblemBase::getActualFieldVariable(const THREAD_ID tid, const std::string & var_name)
6221 : {
6222 771 : for (auto & sys : _solver_systems)
6223 581 : if (sys->hasVariable(var_name))
6224 391 : return sys->getActualFieldVariable<Real>(tid, var_name);
6225 190 : if (_aux->hasVariable(var_name))
6226 190 : return _aux->getActualFieldVariable<Real>(tid, var_name);
6227 :
6228 0 : mooseError("Unknown variable " + var_name);
6229 : }
6230 :
6231 : VectorMooseVariable &
6232 0 : FEProblemBase::getVectorVariable(const THREAD_ID tid, const std::string & var_name)
6233 : {
6234 0 : for (auto & sys : _solver_systems)
6235 0 : if (sys->hasVariable(var_name))
6236 0 : return sys->getFieldVariable<RealVectorValue>(tid, var_name);
6237 0 : if (_aux->hasVariable(var_name))
6238 0 : return _aux->getFieldVariable<RealVectorValue>(tid, var_name);
6239 :
6240 0 : mooseError("Unknown variable " + var_name);
6241 : }
6242 :
6243 : ArrayMooseVariable &
6244 351 : FEProblemBase::getArrayVariable(const THREAD_ID tid, const std::string & var_name)
6245 : {
6246 611 : for (auto & sys : _solver_systems)
6247 351 : if (sys->hasVariable(var_name))
6248 91 : return sys->getFieldVariable<RealEigenVector>(tid, var_name);
6249 260 : if (_aux->hasVariable(var_name))
6250 260 : return _aux->getFieldVariable<RealEigenVector>(tid, var_name);
6251 :
6252 0 : mooseError("Unknown variable " + var_name);
6253 : }
6254 :
6255 : bool
6256 178175 : FEProblemBase::hasScalarVariable(const std::string & var_name) const
6257 : {
6258 335777 : for (auto & sys : _solver_systems)
6259 178739 : if (sys->hasScalarVariable(var_name))
6260 21137 : return true;
6261 157038 : if (_aux->hasScalarVariable(var_name))
6262 10613 : return true;
6263 :
6264 146425 : return false;
6265 : }
6266 :
6267 : MooseVariableScalar &
6268 44158 : FEProblemBase::getScalarVariable(const THREAD_ID tid, const std::string & var_name)
6269 : {
6270 56499 : for (auto & sys : _solver_systems)
6271 44158 : if (sys->hasScalarVariable(var_name))
6272 31817 : return sys->getScalarVariable(tid, var_name);
6273 12341 : if (_aux->hasScalarVariable(var_name))
6274 12341 : return _aux->getScalarVariable(tid, var_name);
6275 :
6276 0 : mooseError("Unknown variable " + var_name);
6277 : }
6278 :
6279 : System &
6280 58224 : FEProblemBase::getSystem(const std::string & var_name)
6281 : {
6282 58224 : const auto [var_in_sys, sys_num] = determineSolverSystem(var_name);
6283 58224 : if (var_in_sys)
6284 39465 : return _solver_systems[sys_num]->system();
6285 18759 : else if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
6286 18759 : return _aux->system();
6287 : else
6288 0 : mooseError("Unable to find a system containing the variable " + var_name);
6289 : }
6290 :
6291 : const RestartableEquationSystems &
6292 0 : FEProblemBase::getRestartableEquationSystems() const
6293 : {
6294 0 : return _req.get();
6295 : }
6296 :
6297 : void
6298 481792 : FEProblemBase::setActiveFEVariableCoupleableMatrixTags(std::set<TagID> & mtags, const THREAD_ID tid)
6299 : {
6300 481792 : SubProblem::setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6301 :
6302 481792 : if (_displaced_problem)
6303 101844 : _displaced_problem->setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6304 481792 : }
6305 :
6306 : void
6307 6596085 : FEProblemBase::setActiveFEVariableCoupleableVectorTags(std::set<TagID> & vtags, const THREAD_ID tid)
6308 : {
6309 6596085 : SubProblem::setActiveFEVariableCoupleableVectorTags(vtags, tid);
6310 :
6311 6596085 : if (_displaced_problem)
6312 359320 : _displaced_problem->setActiveFEVariableCoupleableVectorTags(vtags, tid);
6313 6596085 : }
6314 :
6315 : void
6316 47856 : FEProblemBase::setActiveScalarVariableCoupleableMatrixTags(std::set<TagID> & mtags,
6317 : const THREAD_ID tid)
6318 : {
6319 47856 : SubProblem::setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6320 :
6321 47856 : if (_displaced_problem)
6322 0 : _displaced_problem->setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6323 47856 : }
6324 :
6325 : void
6326 47856 : FEProblemBase::setActiveScalarVariableCoupleableVectorTags(std::set<TagID> & vtags,
6327 : const THREAD_ID tid)
6328 : {
6329 47856 : SubProblem::setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6330 :
6331 47856 : if (_displaced_problem)
6332 0 : _displaced_problem->setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6333 47856 : }
6334 :
6335 : void
6336 10253847 : FEProblemBase::setActiveElementalMooseVariables(const std::set<MooseVariableFEBase *> & moose_vars,
6337 : const THREAD_ID tid)
6338 : {
6339 10253847 : SubProblem::setActiveElementalMooseVariables(moose_vars, tid);
6340 :
6341 10253847 : if (_displaced_problem)
6342 511923 : _displaced_problem->setActiveElementalMooseVariables(moose_vars, tid);
6343 10253847 : }
6344 :
6345 : void
6346 4197491 : FEProblemBase::clearActiveElementalMooseVariables(const THREAD_ID tid)
6347 : {
6348 4197491 : SubProblem::clearActiveElementalMooseVariables(tid);
6349 :
6350 4197491 : if (_displaced_problem)
6351 186525 : _displaced_problem->clearActiveElementalMooseVariables(tid);
6352 4197491 : }
6353 :
6354 : void
6355 225598 : FEProblemBase::clearActiveFEVariableCoupleableMatrixTags(const THREAD_ID tid)
6356 : {
6357 225598 : SubProblem::clearActiveFEVariableCoupleableMatrixTags(tid);
6358 :
6359 225598 : if (_displaced_problem)
6360 45849 : _displaced_problem->clearActiveFEVariableCoupleableMatrixTags(tid);
6361 225598 : }
6362 :
6363 : void
6364 225598 : FEProblemBase::clearActiveFEVariableCoupleableVectorTags(const THREAD_ID tid)
6365 : {
6366 225598 : SubProblem::clearActiveFEVariableCoupleableVectorTags(tid);
6367 :
6368 225598 : if (_displaced_problem)
6369 45849 : _displaced_problem->clearActiveFEVariableCoupleableVectorTags(tid);
6370 225598 : }
6371 :
6372 : void
6373 47856 : FEProblemBase::clearActiveScalarVariableCoupleableMatrixTags(const THREAD_ID tid)
6374 : {
6375 47856 : SubProblem::clearActiveScalarVariableCoupleableMatrixTags(tid);
6376 :
6377 47856 : if (_displaced_problem)
6378 0 : _displaced_problem->clearActiveScalarVariableCoupleableMatrixTags(tid);
6379 47856 : }
6380 :
6381 : void
6382 47856 : FEProblemBase::clearActiveScalarVariableCoupleableVectorTags(const THREAD_ID tid)
6383 : {
6384 47856 : SubProblem::clearActiveScalarVariableCoupleableVectorTags(tid);
6385 :
6386 47856 : if (_displaced_problem)
6387 0 : _displaced_problem->clearActiveScalarVariableCoupleableVectorTags(tid);
6388 47856 : }
6389 :
6390 : void
6391 5248404 : FEProblemBase::setActiveMaterialProperties(const std::unordered_set<unsigned int> & mat_prop_ids,
6392 : const THREAD_ID tid)
6393 : {
6394 : // mark active properties in every material
6395 6376287 : for (auto & mat : _all_materials.getObjects(tid))
6396 1127883 : mat->setActiveProperties(mat_prop_ids);
6397 6147866 : for (auto & mat : _all_materials[Moose::FACE_MATERIAL_DATA].getObjects(tid))
6398 899462 : mat->setActiveProperties(mat_prop_ids);
6399 6147866 : for (auto & mat : _all_materials[Moose::NEIGHBOR_MATERIAL_DATA].getObjects(tid))
6400 899462 : mat->setActiveProperties(mat_prop_ids);
6401 :
6402 5248404 : _has_active_material_properties[tid] = !mat_prop_ids.empty();
6403 5248404 : }
6404 :
6405 : bool
6406 381869537 : FEProblemBase::hasActiveMaterialProperties(const THREAD_ID tid) const
6407 : {
6408 381869537 : return _has_active_material_properties[tid];
6409 : }
6410 :
6411 : void
6412 4291776 : FEProblemBase::clearActiveMaterialProperties(const THREAD_ID tid)
6413 : {
6414 4291776 : _has_active_material_properties[tid] = 0;
6415 4291776 : }
6416 :
6417 : void
6418 59414 : FEProblemBase::addAnyRedistributers()
6419 : {
6420 : #ifdef LIBMESH_ENABLE_AMR
6421 61667 : if ((_adaptivity.isOn() || _num_grid_steps) &&
6422 2253 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
6423 2190 : _neighbor_material_props.hasStatefulProperties()))
6424 : {
6425 : // Even on a serialized Mesh, we don't keep our material
6426 : // properties serialized, so we'll rely on the callback to
6427 : // redistribute() to redistribute properties at the same time
6428 : // libMesh is redistributing elements.
6429 63 : auto add_redistributer = [this](MooseMesh & mesh,
6430 : const std::string & redistributer_name,
6431 : const bool use_displaced_mesh)
6432 : {
6433 63 : InputParameters redistribute_params = RedistributeProperties::validParams();
6434 63 : redistribute_params.set<MooseApp *>(MooseBase::app_param) = &_app;
6435 126 : redistribute_params.set<std::string>("for_whom") = this->name();
6436 189 : redistribute_params.set<MooseMesh *>("mesh") = &mesh;
6437 63 : redistribute_params.set<Moose::RelationshipManagerType>("rm_type") =
6438 : Moose::RelationshipManagerType::GEOMETRIC;
6439 126 : redistribute_params.set<bool>("use_displaced_mesh") = use_displaced_mesh;
6440 63 : redistribute_params.setHitNode(*parameters().getHitNode(), {});
6441 :
6442 : std::shared_ptr<RedistributeProperties> redistributer =
6443 63 : _factory.create<RedistributeProperties>(
6444 126 : "RedistributeProperties", redistributer_name, redistribute_params);
6445 :
6446 63 : if (_material_props.hasStatefulProperties())
6447 63 : redistributer->addMaterialPropertyStorage(_material_props);
6448 :
6449 63 : if (_bnd_material_props.hasStatefulProperties())
6450 63 : redistributer->addMaterialPropertyStorage(_bnd_material_props);
6451 :
6452 63 : if (_neighbor_material_props.hasStatefulProperties())
6453 63 : redistributer->addMaterialPropertyStorage(_neighbor_material_props);
6454 :
6455 63 : mesh.getMesh().add_ghosting_functor(redistributer);
6456 126 : };
6457 :
6458 63 : add_redistributer(_mesh, "mesh_property_redistributer", false);
6459 63 : if (_displaced_problem)
6460 0 : add_redistributer(_displaced_problem->mesh(), "displaced_mesh_property_redistributer", true);
6461 : }
6462 : #endif // LIBMESH_ENABLE_AMR
6463 59414 : }
6464 :
6465 : void
6466 61259 : FEProblemBase::updateMaxQps()
6467 : {
6468 : // Find the maximum number of quadrature points
6469 : {
6470 61259 : MaxQpsThread mqt(*this);
6471 61259 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mqt);
6472 61259 : _max_qps = mqt.max();
6473 :
6474 : // If we have more shape functions or more quadrature points on
6475 : // another processor, then we may need to handle those elements
6476 : // ourselves later after repartitioning.
6477 61259 : _communicator.max(_max_qps);
6478 : }
6479 :
6480 61259 : unsigned int max_qpts = getMaxQps();
6481 61259 : if (max_qpts > Moose::constMaxQpsPerElem)
6482 0 : mooseError("Max quadrature points per element assumptions made in some code (e.g. Coupleable ",
6483 : "and MaterialPropertyInterface classes) have been violated.\n",
6484 : "Complain to Moose developers to have constMaxQpsPerElem increased from ",
6485 : Moose::constMaxQpsPerElem,
6486 : " to ",
6487 : max_qpts);
6488 129076 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6489 : {
6490 : // the highest available order in libMesh is 43
6491 67817 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
6492 67817 : _zero[tid].resize(max_qpts, 0);
6493 67817 : _ad_zero[tid].resize(max_qpts, 0);
6494 67817 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
6495 67817 : _ad_grad_zero[tid].resize(max_qpts, ADRealGradient(0));
6496 67817 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
6497 67817 : _ad_second_zero[tid].resize(max_qpts, ADRealTensorValue(0));
6498 67817 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
6499 67817 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
6500 : }
6501 61259 : }
6502 :
6503 : void
6504 78 : FEProblemBase::bumpVolumeQRuleOrder(Order order, SubdomainID block)
6505 : {
6506 168 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6507 180 : for (const auto i : index_range(_nl))
6508 90 : _assembly[tid][i]->bumpVolumeQRuleOrder(order, block);
6509 :
6510 78 : if (_displaced_problem)
6511 0 : _displaced_problem->bumpVolumeQRuleOrder(order, block);
6512 :
6513 78 : updateMaxQps();
6514 78 : }
6515 :
6516 : void
6517 13 : FEProblemBase::bumpAllQRuleOrder(Order order, SubdomainID block)
6518 : {
6519 28 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6520 30 : for (const auto i : index_range(_nl))
6521 15 : _assembly[tid][i]->bumpAllQRuleOrder(order, block);
6522 :
6523 13 : if (_displaced_problem)
6524 0 : _displaced_problem->bumpAllQRuleOrder(order, block);
6525 :
6526 13 : updateMaxQps();
6527 13 : }
6528 :
6529 : void
6530 61168 : FEProblemBase::createQRules(QuadratureType type,
6531 : Order order,
6532 : Order volume_order,
6533 : Order face_order,
6534 : SubdomainID block,
6535 : const bool allow_negative_qweights)
6536 : {
6537 61168 : if (order == INVALID_ORDER)
6538 : {
6539 : // automatically determine the integration order
6540 60607 : order = _solver_systems[0]->getMinQuadratureOrder();
6541 60860 : for (const auto i : make_range(std::size_t(1), _solver_systems.size()))
6542 253 : if (order < _solver_systems[i]->getMinQuadratureOrder())
6543 0 : order = _solver_systems[i]->getMinQuadratureOrder();
6544 60607 : if (order < _aux->getMinQuadratureOrder())
6545 5983 : order = _aux->getMinQuadratureOrder();
6546 : }
6547 :
6548 61168 : if (volume_order == INVALID_ORDER)
6549 61016 : volume_order = order;
6550 :
6551 61168 : if (face_order == INVALID_ORDER)
6552 61016 : face_order = order;
6553 :
6554 128880 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6555 135696 : for (const auto i : index_range(_solver_systems))
6556 67984 : _assembly[tid][i]->createQRules(
6557 : type, order, volume_order, face_order, block, allow_negative_qweights);
6558 :
6559 61168 : if (_displaced_problem)
6560 2022 : _displaced_problem->createQRules(
6561 : type, order, volume_order, face_order, block, allow_negative_qweights);
6562 :
6563 61168 : updateMaxQps();
6564 61168 : }
6565 :
6566 : void
6567 19471 : FEProblemBase::setCoupling(Moose::CouplingType type)
6568 : {
6569 19471 : if (_trust_user_coupling_matrix)
6570 : {
6571 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6572 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6573 : "haven't been provided a coupling matrix!");
6574 :
6575 : // We've been told to trust the user coupling matrix, so we're going to leave things alone
6576 3 : return;
6577 : }
6578 :
6579 19468 : _coupling = type;
6580 : }
6581 :
6582 : void
6583 0 : FEProblemBase::setCouplingMatrix(CouplingMatrix * cm, const unsigned int i)
6584 : {
6585 : // TODO: Deprecate method
6586 0 : setCoupling(Moose::COUPLING_CUSTOM);
6587 0 : _cm[i].reset(cm);
6588 0 : }
6589 :
6590 : void
6591 13152 : FEProblemBase::setCouplingMatrix(std::unique_ptr<CouplingMatrix> cm, const unsigned int i)
6592 : {
6593 13152 : setCoupling(Moose::COUPLING_CUSTOM);
6594 13152 : _cm[i] = std::move(cm);
6595 13152 : }
6596 :
6597 : void
6598 3 : FEProblemBase::trustUserCouplingMatrix()
6599 : {
6600 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6601 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6602 : "haven't been provided a coupling matrix!");
6603 :
6604 3 : _trust_user_coupling_matrix = true;
6605 3 : }
6606 :
6607 : void
6608 63 : FEProblemBase::setNonlocalCouplingMatrix()
6609 : {
6610 315 : TIME_SECTION("setNonlocalCouplingMatrix", 5, "Setting Nonlocal Coupling Matrix");
6611 :
6612 63 : if (_nl.size() > 1)
6613 0 : mooseError("Nonlocal kernels are weirdly stored on the FEProblem so we don't currently support "
6614 : "multiple nonlinear systems with nonlocal kernels.");
6615 :
6616 126 : for (const auto nl_sys_num : index_range(_nl))
6617 : {
6618 63 : auto & nl = _nl[nl_sys_num];
6619 63 : auto & nonlocal_cm = _nonlocal_cm[nl_sys_num];
6620 63 : unsigned int n_vars = nl->nVariables();
6621 63 : nonlocal_cm.resize(n_vars);
6622 63 : const auto & vars = nl->getVariables(0);
6623 63 : const auto & nonlocal_kernel = _nonlocal_kernels.getObjects();
6624 63 : const auto & nonlocal_integrated_bc = _nonlocal_integrated_bcs.getObjects();
6625 189 : for (const auto & ivar : vars)
6626 : {
6627 196 : for (const auto & kernel : nonlocal_kernel)
6628 : {
6629 140 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6630 70 : if (i == kernel->variable().number())
6631 105 : for (const auto & jvar : vars)
6632 : {
6633 70 : const auto it = _var_dof_map.find(jvar->name());
6634 70 : if (it != _var_dof_map.end())
6635 : {
6636 62 : unsigned int j = jvar->number();
6637 62 : nonlocal_cm(i, j) = 1;
6638 : }
6639 : }
6640 : }
6641 182 : for (const auto & integrated_bc : nonlocal_integrated_bc)
6642 : {
6643 112 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6644 56 : if (i == integrated_bc->variable().number())
6645 84 : for (const auto & jvar : vars)
6646 : {
6647 56 : const auto it = _var_dof_map.find(jvar->name());
6648 56 : if (it != _var_dof_map.end())
6649 : {
6650 28 : unsigned int j = jvar->number();
6651 28 : nonlocal_cm(i, j) = 1;
6652 : }
6653 : }
6654 : }
6655 : }
6656 : }
6657 63 : }
6658 :
6659 : bool
6660 624 : FEProblemBase::areCoupled(const unsigned int ivar,
6661 : const unsigned int jvar,
6662 : const unsigned int nl_sys) const
6663 : {
6664 624 : return (*_cm[nl_sys])(ivar, jvar);
6665 : }
6666 :
6667 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6668 15118162 : FEProblemBase::couplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6669 : {
6670 15118162 : return _assembly[tid][nl_sys]->couplingEntries();
6671 : }
6672 :
6673 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6674 4162 : FEProblemBase::nonlocalCouplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6675 : {
6676 4162 : return _assembly[tid][nl_sys]->nonlocalCouplingEntries();
6677 : }
6678 :
6679 : void
6680 59881 : FEProblemBase::init()
6681 : {
6682 59881 : if (_initialized)
6683 0 : return;
6684 :
6685 299405 : TIME_SECTION("init", 2, "Initializing");
6686 :
6687 : // call executioner's preProblemInit so that it can do some setups before problem init
6688 59881 : _app.getExecutioner()->preProblemInit();
6689 :
6690 : // If we have AD and we are doing global AD indexing, then we should by default set the matrix
6691 : // coupling to full. If the user has told us to trust their coupling matrix, then this call will
6692 : // not do anything
6693 59881 : if (haveADObjects() && Moose::globalADIndexing())
6694 6226 : setCoupling(Moose::COUPLING_FULL);
6695 :
6696 118977 : for (const auto i : index_range(_nl))
6697 : {
6698 59096 : auto & nl = _nl[i];
6699 59096 : auto & cm = _cm[i];
6700 :
6701 59096 : unsigned int n_vars = nl->nVariables();
6702 : {
6703 295480 : TIME_SECTION("fillCouplingMatrix", 3, "Filling Coupling Matrix");
6704 :
6705 59096 : switch (_coupling)
6706 : {
6707 44273 : case Moose::COUPLING_DIAG:
6708 44273 : cm = std::make_unique<CouplingMatrix>(n_vars);
6709 80450 : for (unsigned int i = 0; i < n_vars; i++)
6710 36177 : (*cm)(i, i) = 1;
6711 44273 : break;
6712 :
6713 : // for full jacobian
6714 6358 : case Moose::COUPLING_FULL:
6715 6358 : cm = std::make_unique<CouplingMatrix>(n_vars);
6716 16247 : for (unsigned int i = 0; i < n_vars; i++)
6717 28654 : for (unsigned int j = 0; j < n_vars; j++)
6718 18765 : (*cm)(i, j) = 1;
6719 6358 : break;
6720 :
6721 8465 : case Moose::COUPLING_CUSTOM:
6722 : // do nothing, _cm was already set through couplingMatrix() call
6723 8465 : break;
6724 : }
6725 59096 : }
6726 :
6727 59096 : nl->dofMap()._dof_coupling = cm.get();
6728 :
6729 : // If there are no variables, make sure to pass a nullptr coupling
6730 : // matrix, to avoid warnings about non-nullptr yet empty
6731 : // CouplingMatrices.
6732 59096 : if (n_vars == 0)
6733 13619 : nl->dofMap()._dof_coupling = nullptr;
6734 :
6735 59096 : nl->dofMap().attach_extra_sparsity_function(&extraSparsity, nl.get());
6736 59096 : nl->dofMap().attach_extra_send_list_function(&extraSendList, nl.get());
6737 59096 : _aux->dofMap().attach_extra_send_list_function(&extraSendList, _aux.get());
6738 :
6739 59096 : if (!_skip_nl_system_check && _solve && n_vars == 0)
6740 0 : mooseError("No variables specified in nonlinear system '", nl->name(), "'.");
6741 : }
6742 :
6743 59881 : ghostGhostedBoundaries(); // We do this again right here in case new boundaries have been added
6744 :
6745 : // We may have added element/nodes to the mesh in ghostGhostedBoundaries so we need to update
6746 : // all of our mesh information. We need to make sure that mesh information is up-to-date before
6747 : // EquationSystems::init because that will call through to updateGeomSearch (for sparsity
6748 : // augmentation) and if we haven't added back boundary node information before that latter call,
6749 : // then we're screwed. We'll get things like "Unable to find closest node!"
6750 59881 : _mesh.meshChanged();
6751 59881 : if (_displaced_problem)
6752 2022 : _displaced_mesh->meshChanged();
6753 :
6754 59881 : if (_mesh.doingPRefinement())
6755 : {
6756 240 : preparePRefinement();
6757 240 : if (_displaced_problem)
6758 0 : _displaced_problem->preparePRefinement();
6759 : }
6760 :
6761 : // do not assemble system matrix for JFNK solve
6762 118977 : for (auto & nl : _nl)
6763 59096 : if (solverParams(nl->number())._type == Moose::ST_JFNK)
6764 120 : nl->turnOffJacobian();
6765 :
6766 120009 : for (auto & sys : _solver_systems)
6767 60128 : sys->preInit();
6768 59881 : _aux->preInit();
6769 :
6770 : // Build the mortar segment meshes, if they haven't been already, for a couple reasons:
6771 : // 1) Get the ghosting correct for both static and dynamic meshes
6772 : // 2) Make sure the mortar mesh is built for mortar constraints that live on the static mesh
6773 : //
6774 : // It is worth-while to note that mortar meshes that live on a dynamic mesh will be built
6775 : // during residual and Jacobian evaluation because when displacements are solution variables
6776 : // the mortar mesh will move and change during the course of a non-linear solve. We DO NOT
6777 : // redo ghosting during non-linear solve, so for purpose 1) the below call has to be made
6778 59881 : if (!_mortar_data->initialized())
6779 44604 : updateMortarMesh();
6780 :
6781 : {
6782 299405 : TIME_SECTION("EquationSystems::Init", 2, "Initializing Equation Systems");
6783 59881 : es().init();
6784 59881 : }
6785 :
6786 120009 : for (auto & sys : _solver_systems)
6787 60128 : sys->postInit();
6788 59881 : _aux->postInit();
6789 :
6790 : // Now that the equation system and the dof distribution is done, we can generate the
6791 : // finite volume-related parts if needed.
6792 59881 : if (haveFV())
6793 4129 : _mesh.setupFiniteVolumeMeshData();
6794 :
6795 120009 : for (auto & sys : _solver_systems)
6796 60128 : sys->update();
6797 59881 : _aux->update();
6798 :
6799 125876 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
6800 131222 : for (const auto i : index_range(_nl))
6801 : {
6802 : mooseAssert(
6803 : _cm[i],
6804 : "Coupling matrix not set for system "
6805 : << i
6806 : << ". This should only happen if a preconditioner was not setup for this system");
6807 65227 : _assembly[tid][i]->init(_cm[i].get());
6808 : }
6809 :
6810 59881 : if (_displaced_problem)
6811 2022 : _displaced_problem->init();
6812 :
6813 : #ifdef MOOSE_KOKKOS_ENABLED
6814 45268 : if (_has_kokkos_objects)
6815 2265 : initKokkos();
6816 : #endif
6817 :
6818 59881 : _initialized = true;
6819 59881 : }
6820 :
6821 : unsigned int
6822 11889 : FEProblemBase::nlSysNum(const NonlinearSystemName & nl_sys_name) const
6823 : {
6824 11889 : std::istringstream ss(nl_sys_name);
6825 : unsigned int nl_sys_num;
6826 11889 : if (!(ss >> nl_sys_num) || !ss.eof())
6827 11889 : nl_sys_num = libmesh_map_find(_nl_sys_name_to_num, nl_sys_name);
6828 :
6829 11889 : return nl_sys_num;
6830 11889 : }
6831 :
6832 : unsigned int
6833 78396 : FEProblemBase::linearSysNum(const LinearSystemName & linear_sys_name) const
6834 : {
6835 78396 : std::istringstream ss(linear_sys_name);
6836 : unsigned int linear_sys_num;
6837 78396 : if (!(ss >> linear_sys_num) || !ss.eof())
6838 78396 : linear_sys_num = libmesh_map_find(_linear_sys_name_to_num, linear_sys_name);
6839 :
6840 78396 : return linear_sys_num;
6841 78396 : }
6842 :
6843 : unsigned int
6844 123031 : FEProblemBase::solverSysNum(const SolverSystemName & solver_sys_name) const
6845 : {
6846 123031 : std::istringstream ss(solver_sys_name);
6847 : unsigned int solver_sys_num;
6848 123031 : if (!(ss >> solver_sys_num) || !ss.eof())
6849 : {
6850 123031 : const auto & search = _solver_sys_name_to_num.find(solver_sys_name);
6851 123031 : if (search == _solver_sys_name_to_num.end())
6852 0 : mooseError("The solver system number was requested for system '" + solver_sys_name,
6853 : "' but this system does not exist in the Problem. Systems can be added to the "
6854 : "problem using the 'nl_sys_names'/'linear_sys_names' parameter.\nSystems in the "
6855 0 : "Problem: " +
6856 0 : Moose::stringify(_solver_sys_names));
6857 123031 : solver_sys_num = search->second;
6858 : }
6859 :
6860 123031 : return solver_sys_num;
6861 123031 : }
6862 :
6863 : unsigned int
6864 1644 : FEProblemBase::systemNumForVariable(const VariableName & variable_name) const
6865 : {
6866 1746 : for (const auto & solver_sys : _solver_systems)
6867 1644 : if (solver_sys->hasVariable(variable_name))
6868 1542 : return solver_sys->number();
6869 : mooseAssert(_aux, "Should have an auxiliary system");
6870 102 : if (_aux->hasVariable(variable_name))
6871 102 : return _aux->number();
6872 :
6873 0 : mooseError("Variable '",
6874 : variable_name,
6875 : "' was not found in any solver (nonlinear/linear) or auxiliary system");
6876 : }
6877 :
6878 : void
6879 318241 : FEProblemBase::solve(const unsigned int nl_sys_num)
6880 : {
6881 1591205 : TIME_SECTION("solve", 1, "Solving", false);
6882 :
6883 318241 : setCurrentNonlinearSystem(nl_sys_num);
6884 :
6885 : // This prevents stale dof indices from lingering around and possibly leading to invalid reads
6886 : // and writes. Dof indices may be made stale through operations like mesh adaptivity
6887 318241 : clearAllDofIndices();
6888 318241 : if (_displaced_problem)
6889 32837 : _displaced_problem->clearAllDofIndices();
6890 :
6891 : // Setup the output system for printing linear/nonlinear iteration information and some solver
6892 : // settings, including setting matrix prefixes. This must occur before petscSetOptions
6893 318241 : initPetscOutputAndSomeSolverSettings();
6894 :
6895 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6896 : Moose::PetscSupport::petscSetOptions(
6897 : _petsc_options, _solver_params); // Make sure the PETSc options are setup for this app
6898 : #else
6899 : // Now this database will be the default
6900 : // Each app should have only one database
6901 318241 : if (!_app.isUltimateMaster())
6902 82839 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6903 : // We did not add PETSc options to database yet
6904 318241 : if (!_is_petsc_options_inserted)
6905 : {
6906 : // Insert options for all systems all at once
6907 38959 : Moose::PetscSupport::petscSetOptions(_petsc_options, _solver_params, this);
6908 38956 : _is_petsc_options_inserted = true;
6909 : }
6910 : #endif
6911 :
6912 : // set up DM which is required if use a field split preconditioner
6913 : // We need to setup DM every "solve()" because libMesh destroy SNES after solve()
6914 : // Do not worry, DM setup is very cheap
6915 318238 : _current_nl_sys->setupDM();
6916 :
6917 318238 : possiblyRebuildGeomSearchPatches();
6918 :
6919 : // reset flag so that residual evaluation does not get skipped
6920 : // and the next non-linear iteration does not automatically fail with
6921 : // "DIVERGED_NANORINF", when we throw an exception and stop solve
6922 318238 : _fail_next_system_convergence_check = false;
6923 :
6924 318238 : if (_solve)
6925 : {
6926 287715 : _current_nl_sys->solve();
6927 287660 : _current_nl_sys->update();
6928 : }
6929 :
6930 : // sync solutions in displaced problem
6931 318183 : if (_displaced_problem)
6932 32834 : _displaced_problem->syncSolutions();
6933 :
6934 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6935 318183 : if (!_app.isUltimateMaster())
6936 82839 : LibmeshPetscCall(PetscOptionsPop());
6937 : #endif
6938 318183 : }
6939 :
6940 : void
6941 204 : FEProblemBase::setException(const std::string & message)
6942 : {
6943 204 : _has_exception = true;
6944 204 : _exception_message = message;
6945 204 : }
6946 :
6947 : void
6948 19869190 : FEProblemBase::checkExceptionAndStopSolve(bool print_message)
6949 : {
6950 19869190 : if (_skip_exception_check)
6951 216 : return;
6952 :
6953 59606922 : TIME_SECTION("checkExceptionAndStopSolve", 5);
6954 :
6955 : // See if any processor had an exception. If it did, get back the
6956 : // processor that the exception occurred on.
6957 : unsigned int processor_id;
6958 :
6959 19868974 : _communicator.maxloc(_has_exception, processor_id);
6960 :
6961 19868974 : if (_has_exception)
6962 : {
6963 304 : _communicator.broadcast(_exception_message, processor_id);
6964 :
6965 436 : if (_current_execute_on_flag == EXEC_LINEAR || _current_execute_on_flag == EXEC_NONLINEAR ||
6966 132 : _current_execute_on_flag == EXEC_POSTCHECK)
6967 : {
6968 : // Print the message
6969 304 : if (_communicator.rank() == 0 && print_message)
6970 : {
6971 186 : _console << "\n" << _exception_message << "\n";
6972 186 : if (isTransient())
6973 : _console
6974 : << "To recover, the solution will fail and then be re-attempted with a reduced time "
6975 174 : "step.\n"
6976 174 : << std::endl;
6977 : }
6978 :
6979 : // Stop the solve -- this entails setting
6980 : // SNESSetFunctionDomainError() or directly inserting NaNs in the
6981 : // residual vector to let PETSc >= 3.6 return DIVERGED_NANORINF.
6982 304 : if (_current_nl_sys)
6983 304 : _current_nl_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
6984 :
6985 304 : if (_current_linear_sys)
6986 0 : _current_linear_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
6987 :
6988 : // and close Aux system (we MUST do this here; see #11525)
6989 304 : _aux->solution().close();
6990 :
6991 : // We've handled this exception, so we no longer have one.
6992 304 : _has_exception = false;
6993 :
6994 : // Force the next non-linear convergence check to fail (and all further residual evaluation
6995 : // to be skipped).
6996 304 : _fail_next_system_convergence_check = true;
6997 :
6998 : // Repropagate the exception, so it can be caught at a higher level, typically
6999 : // this is NonlinearSystem::computeResidual().
7000 304 : throw MooseException(_exception_message);
7001 : }
7002 : else
7003 0 : mooseError("The following parallel-communicated exception was detected during " +
7004 0 : Moose::stringify(_current_execute_on_flag) + " evaluation:\n" +
7005 0 : _exception_message +
7006 : "\nBecause this did not occur during residual evaluation, there"
7007 : " is no way to handle this, so the solution is aborting.\n");
7008 : }
7009 19868974 : }
7010 :
7011 : void
7012 3525449 : FEProblemBase::resetState()
7013 : {
7014 : // Our default state is to allow computing derivatives
7015 3525449 : ADReal::do_derivatives = true;
7016 3525449 : _current_execute_on_flag = EXEC_NONE;
7017 :
7018 : // Clear the VectorTags and MatrixTags
7019 3525449 : clearCurrentResidualVectorTags();
7020 3525449 : clearCurrentJacobianMatrixTags();
7021 :
7022 3525449 : _safe_access_tagged_vectors = true;
7023 3525449 : _safe_access_tagged_matrices = true;
7024 :
7025 3525449 : setCurrentlyComputingResidual(false);
7026 3525449 : setCurrentlyComputingJacobian(false);
7027 3525449 : setCurrentlyComputingResidualAndJacobian(false);
7028 3525449 : if (_displaced_problem)
7029 : {
7030 145293 : _displaced_problem->setCurrentlyComputingResidual(false);
7031 145293 : _displaced_problem->setCurrentlyComputingJacobian(false);
7032 145293 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(false);
7033 : }
7034 3525449 : }
7035 :
7036 : void
7037 26086 : FEProblemBase::solveLinearSystem(const unsigned int linear_sys_num,
7038 : const Moose::PetscSupport::PetscOptions * po)
7039 : {
7040 130430 : TIME_SECTION("solve", 1, "Solving", false);
7041 :
7042 26086 : setCurrentLinearSystem(linear_sys_num);
7043 :
7044 26086 : const Moose::PetscSupport::PetscOptions & options = po ? *po : _petsc_options;
7045 26086 : auto & solver_params = _solver_params[numNonlinearSystems() + linear_sys_num];
7046 :
7047 : // Set custom convergence criteria
7048 26086 : Moose::PetscSupport::petscSetDefaults(*this);
7049 :
7050 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
7051 : LibmeshPetscCall(Moose::PetscSupport::petscSetOptions(
7052 : options, solver_params)); // Make sure the PETSc options are setup for this app
7053 : #else
7054 : // Now this database will be the default
7055 : // Each app should have only one database
7056 26086 : if (!_app.isUltimateMaster())
7057 195 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
7058 :
7059 : // We did not add PETSc options to database yet
7060 26086 : if (!_is_petsc_options_inserted)
7061 : {
7062 942 : Moose::PetscSupport::petscSetOptions(options, solver_params, this);
7063 942 : _is_petsc_options_inserted = true;
7064 : }
7065 : #endif
7066 :
7067 26086 : if (_solve)
7068 26076 : _current_linear_sys->solve();
7069 :
7070 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
7071 26086 : if (!_app.isUltimateMaster())
7072 195 : LibmeshPetscCall(PetscOptionsPop());
7073 : #endif
7074 26086 : }
7075 :
7076 : bool
7077 334899 : FEProblemBase::solverSystemConverged(const unsigned int sys_num)
7078 : {
7079 334899 : if (_solve)
7080 334877 : return _solver_systems[sys_num]->converged();
7081 : else
7082 22 : return true;
7083 : }
7084 :
7085 : unsigned int
7086 4901 : FEProblemBase::nNonlinearIterations(const unsigned int nl_sys_num) const
7087 : {
7088 4901 : return _nl[nl_sys_num]->nNonlinearIterations();
7089 : }
7090 :
7091 : unsigned int
7092 4051 : FEProblemBase::nLinearIterations(const unsigned int nl_sys_num) const
7093 : {
7094 4051 : return _nl[nl_sys_num]->nLinearIterations();
7095 : }
7096 :
7097 : Real
7098 242 : FEProblemBase::finalNonlinearResidual(const unsigned int nl_sys_num) const
7099 : {
7100 242 : return _nl[nl_sys_num]->finalNonlinearResidual();
7101 : }
7102 :
7103 : bool
7104 761344 : FEProblemBase::computingPreSMOResidual(const unsigned int nl_sys_num) const
7105 : {
7106 761344 : return _nl[nl_sys_num]->computingPreSMOResidual();
7107 : }
7108 :
7109 : void
7110 54424 : FEProblemBase::copySolutionsBackwards()
7111 : {
7112 272120 : TIME_SECTION("copySolutionsBackwards", 3, "Copying Solutions Backward");
7113 :
7114 109077 : for (auto & sys : _solver_systems)
7115 54653 : sys->copySolutionsBackwards();
7116 54424 : _aux->copySolutionsBackwards();
7117 54424 : }
7118 :
7119 : void
7120 152 : FEProblemBase::skipNextForwardSolutionCopyToOld()
7121 : {
7122 324 : for (auto & sys : _solver_systems)
7123 172 : sys->skipNextSolutionToOldCopy();
7124 152 : _aux->skipNextSolutionToOldCopy();
7125 152 : }
7126 :
7127 : void
7128 229015 : FEProblemBase::advanceState()
7129 : {
7130 1145075 : TIME_SECTION("advanceState", 5, "Advancing State");
7131 :
7132 461377 : for (auto & sys : _solver_systems)
7133 232362 : sys->copyOldSolutions();
7134 229015 : _aux->copyOldSolutions();
7135 :
7136 229015 : if (_displaced_problem)
7137 : {
7138 61858 : for (const auto i : index_range(_solver_systems))
7139 30929 : _displaced_problem->solverSys(i).copyOldSolutions();
7140 30929 : _displaced_problem->auxSys().copyOldSolutions();
7141 : }
7142 :
7143 229015 : _reporter_data.copyValuesBack();
7144 :
7145 229015 : getMooseApp().getChainControlDataSystem().copyValuesBack();
7146 :
7147 229015 : if (_material_props.hasStatefulProperties())
7148 1760 : _material_props.shift();
7149 :
7150 229015 : if (_bnd_material_props.hasStatefulProperties())
7151 1612 : _bnd_material_props.shift();
7152 :
7153 229015 : if (_neighbor_material_props.hasStatefulProperties())
7154 1496 : _neighbor_material_props.shift();
7155 :
7156 : #ifdef MOOSE_KOKKOS_ENABLED
7157 168080 : if (_kokkos_material_props.hasStatefulProperties())
7158 566 : _kokkos_material_props.shift();
7159 :
7160 168080 : if (_kokkos_bnd_material_props.hasStatefulProperties())
7161 641 : _kokkos_bnd_material_props.shift();
7162 :
7163 168080 : if (_kokkos_neighbor_material_props.hasStatefulProperties())
7164 566 : _kokkos_neighbor_material_props.shift();
7165 : #endif
7166 229015 : }
7167 :
7168 : void
7169 3295 : FEProblemBase::restoreSolutions()
7170 : {
7171 16475 : TIME_SECTION("restoreSolutions", 5, "Restoring Solutions");
7172 :
7173 3295 : if (!_not_zeroed_tagged_vectors.empty())
7174 0 : paramError("not_zeroed_tag_vectors",
7175 : "There is currently no way to restore not-zeroed vectors.");
7176 :
7177 6592 : for (auto & sys : _solver_systems)
7178 : {
7179 3297 : if (_verbose_restore)
7180 305 : _console << "Restoring solutions on system " << sys->name() << "..." << std::endl;
7181 3297 : sys->restoreSolutions();
7182 : }
7183 :
7184 3295 : if (_verbose_restore)
7185 304 : _console << "Restoring solutions on Auxiliary system..." << std::endl;
7186 3295 : _aux->restoreSolutions();
7187 :
7188 3295 : if (_verbose_restore)
7189 304 : _console << "Restoring postprocessor, vector-postprocessor, and reporter data..." << std::endl;
7190 3295 : _reporter_data.restoreState(_verbose_restore);
7191 :
7192 3295 : if (_displaced_problem)
7193 132 : _displaced_problem->updateMesh();
7194 3295 : }
7195 :
7196 : void
7197 100 : FEProblemBase::saveOldSolutions()
7198 : {
7199 500 : TIME_SECTION("saveOldSolutions", 5, "Saving Old Solutions");
7200 :
7201 200 : for (auto & sys : _solver_systems)
7202 100 : sys->saveOldSolutions();
7203 100 : _aux->saveOldSolutions();
7204 100 : }
7205 :
7206 : void
7207 100 : FEProblemBase::restoreOldSolutions()
7208 : {
7209 500 : TIME_SECTION("restoreOldSolutions", 5, "Restoring Old Solutions");
7210 :
7211 200 : for (auto & sys : _solver_systems)
7212 100 : sys->restoreOldSolutions();
7213 100 : _aux->restoreOldSolutions();
7214 100 : }
7215 :
7216 : void
7217 1229232 : FEProblemBase::outputStep(ExecFlagType type)
7218 : {
7219 6146160 : TIME_SECTION("outputStep", 1, "Outputting");
7220 :
7221 1229232 : setCurrentExecuteOnFlag(type);
7222 :
7223 2472574 : for (auto & sys : _solver_systems)
7224 1243342 : sys->update();
7225 1229232 : _aux->update();
7226 :
7227 1229232 : if (_displaced_problem)
7228 126253 : _displaced_problem->syncSolutions();
7229 1229232 : _app.getOutputWarehouse().outputStep(type);
7230 :
7231 1229223 : setCurrentExecuteOnFlag(EXEC_NONE);
7232 1229223 : }
7233 :
7234 : void
7235 83513 : FEProblemBase::allowOutput(bool state)
7236 : {
7237 83513 : _app.getOutputWarehouse().allowOutput(state);
7238 83513 : }
7239 :
7240 : void
7241 23 : FEProblemBase::forceOutput()
7242 : {
7243 23 : _app.getOutputWarehouse().forceOutput();
7244 23 : }
7245 :
7246 : void
7247 337266 : FEProblemBase::initPetscOutputAndSomeSolverSettings()
7248 : {
7249 337266 : _app.getOutputWarehouse().solveSetup();
7250 337266 : Moose::PetscSupport::petscSetDefaults(*this);
7251 337266 : }
7252 :
7253 : void
7254 235805 : FEProblemBase::onTimestepBegin()
7255 : {
7256 707415 : TIME_SECTION("onTimestepBegin", 2);
7257 :
7258 474651 : for (auto & nl : _nl)
7259 238846 : nl->onTimestepBegin();
7260 235805 : }
7261 :
7262 : void
7263 309081 : FEProblemBase::onTimestepEnd()
7264 : {
7265 309081 : }
7266 :
7267 : Real
7268 4621439 : FEProblemBase::getTimeFromStateArg(const Moose::StateArg & state) const
7269 : {
7270 4621439 : if (state.iteration_type != Moose::SolutionIterationType::Time)
7271 : // If we are any iteration type other than time (e.g. nonlinear), then temporally we are still
7272 : // in the present time
7273 0 : return time();
7274 :
7275 4621439 : switch (state.state)
7276 : {
7277 4621439 : case 0:
7278 4621439 : return time();
7279 :
7280 0 : case 1:
7281 0 : return timeOld();
7282 :
7283 0 : default:
7284 0 : mooseError("Unhandled state ", state.state, " in FEProblemBase::getTimeFromStateArg");
7285 : }
7286 : }
7287 :
7288 : void
7289 30038 : FEProblemBase::addTimeIntegrator(const std::string & type,
7290 : const std::string & name,
7291 : InputParameters & parameters)
7292 : {
7293 : parallel_object_only();
7294 :
7295 90114 : parameters.set<SubProblem *>("_subproblem") = this;
7296 30038 : logAdd("TimeIntegrator", name, type, parameters);
7297 30038 : _aux->addTimeIntegrator(type, name + ":aux", parameters);
7298 60103 : for (auto & sys : _solver_systems)
7299 30065 : sys->addTimeIntegrator(type, name + ":" + sys->name(), parameters);
7300 30038 : _has_time_integrator = true;
7301 :
7302 : // add vectors to store u_dot, u_dotdot, udot_old, u_dotdot_old and
7303 : // solution vectors older than 2 time steps, if requested by the time
7304 : // integrator
7305 30038 : _aux->addDotVectors();
7306 60072 : for (auto & nl : _nl)
7307 : {
7308 30034 : nl->addDotVectors();
7309 :
7310 30034 : auto tag_udot = nl->getTimeIntegrators()[0]->uDotFactorTag();
7311 30034 : if (!nl->hasVector(tag_udot))
7312 29998 : nl->associateVectorToTag(*nl->solutionUDot(), tag_udot);
7313 30034 : auto tag_udotdot = nl->getTimeIntegrators()[0]->uDotDotFactorTag();
7314 30034 : if (!nl->hasVector(tag_udotdot) && uDotDotRequested())
7315 150 : nl->associateVectorToTag(*nl->solutionUDotDot(), tag_udotdot);
7316 : }
7317 :
7318 30038 : if (_displaced_problem)
7319 : // Time integrator does not exist when displaced problem is created.
7320 1728 : _displaced_problem->addTimeIntegrator();
7321 30038 : }
7322 :
7323 : void
7324 22 : FEProblemBase::addPredictor(const std::string & type,
7325 : const std::string & name,
7326 : InputParameters & parameters)
7327 : {
7328 : parallel_object_only();
7329 :
7330 22 : if (!numNonlinearSystems() && numLinearSystems())
7331 0 : mooseError("Vector bounds cannot be used with LinearSystems!");
7332 :
7333 44 : parameters.set<SubProblem *>("_subproblem") = this;
7334 22 : std::shared_ptr<Predictor> predictor = _factory.create<Predictor>(type, name, parameters);
7335 22 : logAdd("Predictor", name, type, parameters);
7336 :
7337 44 : for (auto & nl : _nl)
7338 22 : nl->setPredictor(predictor);
7339 22 : }
7340 :
7341 : Real
7342 62579 : FEProblemBase::computeResidualL2Norm(NonlinearSystemBase & sys)
7343 : {
7344 62579 : _current_nl_sys = &sys;
7345 62579 : computeResidual(*sys.currentSolution(), sys.RHS(), sys.number());
7346 62579 : return sys.RHS().l2_norm();
7347 : }
7348 :
7349 : Real
7350 84 : FEProblemBase::computeResidualL2Norm(LinearSystem & sys)
7351 : {
7352 84 : _current_linear_sys = &sys;
7353 :
7354 : // We assemble the current system to check the current residual
7355 84 : computeLinearSystemSys(sys.linearImplicitSystem(),
7356 84 : *sys.linearImplicitSystem().matrix,
7357 84 : *sys.linearImplicitSystem().rhs,
7358 : /*compute fresh gradients*/ true);
7359 :
7360 : // Unfortunate, but we have to allocate a new vector for the residual
7361 84 : auto residual = sys.linearImplicitSystem().rhs->clone();
7362 84 : residual->scale(-1.0);
7363 84 : residual->add_vector(*sys.currentSolution(), *sys.linearImplicitSystem().matrix);
7364 168 : return residual->l2_norm();
7365 84 : }
7366 :
7367 : Real
7368 62633 : FEProblemBase::computeResidualL2Norm()
7369 : {
7370 313165 : TIME_SECTION("computeResidualL2Norm", 2, "Computing L2 Norm of Residual");
7371 :
7372 : // We use sum the squared norms of the individual systems and then take the square root of it
7373 62633 : Real l2_norm = 0.0;
7374 125212 : for (auto sys : _nl)
7375 : {
7376 62579 : const auto norm = computeResidualL2Norm(*sys);
7377 62579 : l2_norm += norm * norm;
7378 62579 : }
7379 :
7380 62717 : for (auto sys : _linear_systems)
7381 : {
7382 84 : const auto norm = computeResidualL2Norm(*sys);
7383 84 : l2_norm += norm * norm;
7384 84 : }
7385 :
7386 125266 : return std::sqrt(l2_norm);
7387 62633 : }
7388 :
7389 : void
7390 2937979 : FEProblemBase::computeResidualSys(NonlinearImplicitSystem & sys,
7391 : const NumericVector<Number> & soln,
7392 : NumericVector<Number> & residual)
7393 : {
7394 : parallel_object_only();
7395 :
7396 8813937 : TIME_SECTION("computeResidualSys", 5);
7397 : // Reset before residual setup, calculation & execution
7398 2937979 : _app.solutionInvalidity().resetIterationOccurences();
7399 :
7400 2937979 : computeResidual(soln, residual, sys.number());
7401 2937949 : }
7402 :
7403 : void
7404 0 : FEProblemBase::computeResidual(NonlinearImplicitSystem & sys,
7405 : const NumericVector<Number> & soln,
7406 : NumericVector<Number> & residual)
7407 : {
7408 0 : mooseDeprecated("Please use computeResidualSys");
7409 :
7410 0 : computeResidualSys(sys, soln, residual);
7411 0 : }
7412 :
7413 : void
7414 2997377 : FEProblemBase::computeResidual(const NumericVector<Number> & soln,
7415 : NumericVector<Number> & residual,
7416 : const unsigned int nl_sys_num)
7417 : {
7418 2997377 : setCurrentNonlinearSystem(nl_sys_num);
7419 :
7420 : // We associate the residual tag with the given residual vector to make sure we
7421 : // don't filter it out below
7422 2997377 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7423 2997377 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7424 :
7425 : mooseAssert(_fe_vector_tags.empty(), "This should be empty indicating a clean starting state");
7426 : // We filter out tags which do not have associated vectors in the current nonlinear
7427 : // system. This is essential to be able to use system-dependent residual tags.
7428 2997377 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7429 :
7430 2997377 : computeResidualInternal(soln, residual, _fe_vector_tags);
7431 2997347 : _fe_vector_tags.clear();
7432 2997347 : }
7433 :
7434 : void
7435 9598 : FEProblemBase::computeResidualAndJacobian(const NumericVector<Number> & soln,
7436 : NumericVector<Number> & residual,
7437 : SparseMatrix<Number> & jacobian)
7438 : {
7439 : try
7440 : {
7441 : try
7442 : {
7443 : // vector tags
7444 9598 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7445 9598 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7446 :
7447 : mooseAssert(_fe_vector_tags.empty(),
7448 : "This should be empty indicating a clean starting state");
7449 : // We filter out tags which do not have associated vectors in the current nonlinear
7450 : // system. This is essential to be able to use system-dependent residual tags.
7451 9598 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7452 :
7453 9598 : setCurrentResidualVectorTags(_fe_vector_tags);
7454 :
7455 : // matrix tags
7456 : {
7457 9598 : _fe_matrix_tags.clear();
7458 :
7459 9598 : auto & tags = getMatrixTags();
7460 28794 : for (auto & tag : tags)
7461 19196 : _fe_matrix_tags.insert(tag.second);
7462 : }
7463 :
7464 9598 : _current_nl_sys->setSolution(soln);
7465 :
7466 9598 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7467 9598 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7468 :
7469 28794 : for (const auto tag : _fe_matrix_tags)
7470 19196 : if (_current_nl_sys->hasMatrix(tag))
7471 : {
7472 9598 : auto & matrix = _current_nl_sys->getMatrix(tag);
7473 9598 : matrix.zero();
7474 9598 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7475 : // PETSc algorithms require diagonal allocations regardless of whether there is non-zero
7476 : // diagonal dependence. With global AD indexing we only add non-zero
7477 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7478 123018 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7479 120004 : matrix.add(index, index, 0);
7480 : }
7481 :
7482 9598 : _aux->zeroVariablesForResidual();
7483 :
7484 9598 : unsigned int n_threads = libMesh::n_threads();
7485 :
7486 9598 : _current_execute_on_flag = EXEC_LINEAR;
7487 :
7488 : // Random interface objects
7489 9598 : for (const auto & it : _random_data_objects)
7490 0 : it.second->updateSeeds(EXEC_LINEAR);
7491 :
7492 9598 : setCurrentlyComputingResidual(true);
7493 9598 : setCurrentlyComputingJacobian(true);
7494 9598 : setCurrentlyComputingResidualAndJacobian(true);
7495 9598 : if (_displaced_problem)
7496 : {
7497 1057 : _displaced_problem->setCurrentlyComputingResidual(true);
7498 1057 : _displaced_problem->setCurrentlyComputingJacobian(true);
7499 1057 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(true);
7500 : }
7501 :
7502 9598 : execTransfers(EXEC_LINEAR);
7503 :
7504 9598 : execMultiApps(EXEC_LINEAR);
7505 :
7506 20599 : for (unsigned int tid = 0; tid < n_threads; tid++)
7507 11001 : reinitScalars(tid);
7508 :
7509 9598 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7510 :
7511 9598 : _aux->residualSetup();
7512 :
7513 9598 : if (_displaced_problem)
7514 : {
7515 1057 : computeSystems(EXEC_PRE_DISPLACE);
7516 1057 : _displaced_problem->updateMesh();
7517 1057 : if (_mortar_data->hasDisplacedObjects())
7518 1057 : updateMortarMesh();
7519 : }
7520 :
7521 20599 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7522 : {
7523 11001 : _all_materials.residualSetup(tid);
7524 11001 : _functions.residualSetup(tid);
7525 : }
7526 :
7527 : #ifdef MOOSE_KOKKOS_ENABLED
7528 8637 : _kokkos_functions.residualSetup();
7529 : #endif
7530 :
7531 9598 : computeSystems(EXEC_LINEAR);
7532 :
7533 9598 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7534 :
7535 9598 : executeControls(EXEC_LINEAR);
7536 :
7537 9598 : _app.getOutputWarehouse().residualSetup();
7538 :
7539 9598 : _safe_access_tagged_vectors = false;
7540 9598 : _safe_access_tagged_matrices = false;
7541 :
7542 9598 : _current_nl_sys->computeResidualAndJacobianTags(_fe_vector_tags, _fe_matrix_tags);
7543 :
7544 9598 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7545 9598 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7546 : }
7547 0 : catch (...)
7548 : {
7549 0 : handleException("computeResidualAndJacobian");
7550 0 : }
7551 : }
7552 0 : catch (const MooseException &)
7553 : {
7554 : // The buck stops here, we have already handled the exception by
7555 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7556 : // "diverged" reason during the next solve.
7557 0 : }
7558 0 : catch (...)
7559 : {
7560 0 : mooseError("Unexpected exception type");
7561 0 : }
7562 :
7563 9598 : resetState();
7564 9598 : _fe_vector_tags.clear();
7565 9598 : _fe_matrix_tags.clear();
7566 9598 : }
7567 :
7568 : void
7569 0 : FEProblemBase::computeResidualTag(const NumericVector<Number> & soln,
7570 : NumericVector<Number> & residual,
7571 : TagID tag)
7572 : {
7573 : try
7574 : {
7575 0 : _current_nl_sys->setSolution(soln);
7576 :
7577 0 : _current_nl_sys->associateVectorToTag(residual, tag);
7578 :
7579 0 : computeResidualTags({tag});
7580 :
7581 0 : _current_nl_sys->disassociateVectorFromTag(residual, tag);
7582 : }
7583 0 : catch (MooseException & e)
7584 : {
7585 : // If a MooseException propagates all the way to here, it means
7586 : // that it was thrown from a MOOSE system where we do not
7587 : // (currently) properly support the throwing of exceptions, and
7588 : // therefore we have no choice but to error out. It may be
7589 : // *possible* to handle exceptions from other systems, but in the
7590 : // meantime, we don't want to silently swallow any unhandled
7591 : // exceptions here.
7592 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7593 : "contact the MOOSE team for assistance.");
7594 0 : }
7595 0 : }
7596 :
7597 : void
7598 3000964 : FEProblemBase::computeResidualInternal(const NumericVector<Number> & soln,
7599 : NumericVector<Number> & residual,
7600 : const std::set<TagID> & tags)
7601 : {
7602 : parallel_object_only();
7603 :
7604 9002892 : TIME_SECTION("computeResidualInternal", 1);
7605 :
7606 : try
7607 : {
7608 3000964 : _current_nl_sys->setSolution(soln);
7609 :
7610 3000964 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7611 :
7612 3000964 : computeResidualTags(tags);
7613 :
7614 3000934 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7615 : }
7616 0 : catch (MooseException & e)
7617 : {
7618 : // If a MooseException propagates all the way to here, it means
7619 : // that it was thrown from a MOOSE system where we do not
7620 : // (currently) properly support the throwing of exceptions, and
7621 : // therefore we have no choice but to error out. It may be
7622 : // *possible* to handle exceptions from other systems, but in the
7623 : // meantime, we don't want to silently swallow any unhandled
7624 : // exceptions here.
7625 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7626 : "contact the MOOSE team for assistance.");
7627 0 : }
7628 3000934 : }
7629 :
7630 : void
7631 0 : FEProblemBase::computeResidualType(const NumericVector<Number> & soln,
7632 : NumericVector<Number> & residual,
7633 : TagID tag)
7634 : {
7635 0 : TIME_SECTION("computeResidualType", 5);
7636 :
7637 : try
7638 : {
7639 0 : _current_nl_sys->setSolution(soln);
7640 :
7641 0 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7642 :
7643 0 : computeResidualTags({tag, _current_nl_sys->residualVectorTag()});
7644 :
7645 0 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7646 : }
7647 0 : catch (MooseException & e)
7648 : {
7649 : // If a MooseException propagates all the way to here, it means
7650 : // that it was thrown from a MOOSE system where we do not
7651 : // (currently) properly support the throwing of exceptions, and
7652 : // therefore we have no choice but to error out. It may be
7653 : // *possible* to handle exceptions from other systems, but in the
7654 : // meantime, we don't want to silently swallow any unhandled
7655 : // exceptions here.
7656 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7657 : "contact the MOOSE team for assistance.");
7658 0 : }
7659 0 : }
7660 :
7661 : void
7662 3 : FEProblemBase::handleException(const std::string & calling_method)
7663 : {
7664 : auto create_exception_message =
7665 3 : [&calling_method](const std::string & exception_type, const auto & exception)
7666 : {
7667 : return std::string("A " + exception_type + " was raised during FEProblemBase::" +
7668 6 : calling_method + "\n" + std::string(exception.what()));
7669 3 : };
7670 :
7671 : try
7672 : {
7673 3 : throw;
7674 : }
7675 3 : catch (const MooseException & e)
7676 : {
7677 0 : setException(create_exception_message("MooseException", e));
7678 0 : }
7679 0 : catch (const MetaPhysicL::LogicError & e)
7680 : {
7681 0 : moose::translateMetaPhysicLError(e);
7682 0 : }
7683 3 : catch (const libMesh::PetscSolverException & e)
7684 : {
7685 : // One PETSc solver exception that we cannot currently recover from are new nonzero errors. In
7686 : // particular I have observed the following scenario in a parallel test:
7687 : // - Both processes throw because of a new nonzero during MOOSE's computeJacobianTags
7688 : // - We potentially handle the exceptions nicely here
7689 : // - When the matrix is closed in libMesh's libmesh_petsc_snes_solver, there is a new nonzero
7690 : // throw which we do not catch here in MOOSE and the simulation terminates. This only appears
7691 : // in parallel (and not all the time; a test I was examining threw with distributed mesh, but
7692 : // not with replicated). In serial there are no new throws from libmesh_petsc_snes_solver.
7693 : // So for uniformity of behavior across serial/parallel, we will choose to abort here and always
7694 : // produce a non-zero exit code
7695 6 : mooseError(create_exception_message("libMesh::PetscSolverException", e));
7696 0 : }
7697 0 : catch (const std::exception & e)
7698 : {
7699 : // This might be libMesh detecting a degenerate Jacobian or matrix
7700 0 : if (strstr(e.what(), "Jacobian") || strstr(e.what(), "singular") ||
7701 0 : strstr(e.what(), "det != 0"))
7702 0 : setException(create_exception_message("libMesh DegenerateMap", e));
7703 : else
7704 : {
7705 0 : const auto message = create_exception_message("std::exception", e);
7706 0 : if (_regard_general_exceptions_as_errors)
7707 0 : mooseError(message);
7708 : else
7709 0 : setException(message);
7710 0 : }
7711 0 : }
7712 :
7713 0 : checkExceptionAndStopSolve();
7714 0 : }
7715 :
7716 : void
7717 3036365 : FEProblemBase::computeResidualTags(const std::set<TagID> & tags)
7718 : {
7719 : parallel_object_only();
7720 :
7721 : try
7722 : {
7723 : try
7724 : {
7725 15181825 : TIME_SECTION("computeResidualTags", 5, "Computing Residual");
7726 :
7727 3036365 : ADReal::do_derivatives = false;
7728 :
7729 3036365 : setCurrentResidualVectorTags(tags);
7730 :
7731 3036365 : _aux->zeroVariablesForResidual();
7732 :
7733 3036365 : unsigned int n_threads = libMesh::n_threads();
7734 :
7735 3036365 : _current_execute_on_flag = EXEC_LINEAR;
7736 :
7737 : // Random interface objects
7738 3049247 : for (const auto & it : _random_data_objects)
7739 12882 : it.second->updateSeeds(EXEC_LINEAR);
7740 :
7741 3036365 : execTransfers(EXEC_LINEAR);
7742 :
7743 3036365 : execMultiApps(EXEC_LINEAR);
7744 :
7745 6379984 : for (unsigned int tid = 0; tid < n_threads; tid++)
7746 3343619 : reinitScalars(tid);
7747 :
7748 3036365 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7749 :
7750 3036365 : _aux->residualSetup();
7751 :
7752 3036365 : if (_displaced_problem)
7753 : {
7754 123092 : computeSystems(EXEC_PRE_DISPLACE);
7755 123092 : _displaced_problem->updateMesh();
7756 123092 : if (_mortar_data->hasDisplacedObjects())
7757 2211 : updateMortarMesh();
7758 : }
7759 :
7760 6379984 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7761 : {
7762 3343619 : _all_materials.residualSetup(tid);
7763 3343619 : _functions.residualSetup(tid);
7764 : }
7765 :
7766 : #ifdef MOOSE_KOKKOS_ENABLED
7767 2209041 : _kokkos_functions.residualSetup();
7768 : #endif
7769 :
7770 3036365 : computeSystems(EXEC_LINEAR);
7771 :
7772 3036365 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7773 :
7774 3036365 : executeControls(EXEC_LINEAR);
7775 :
7776 3036365 : _app.getOutputWarehouse().residualSetup();
7777 :
7778 3036365 : _safe_access_tagged_vectors = false;
7779 3036365 : _current_nl_sys->computeResidualTags(tags);
7780 3036335 : }
7781 0 : catch (...)
7782 : {
7783 0 : handleException("computeResidualTags");
7784 0 : }
7785 : }
7786 0 : catch (const MooseException &)
7787 : {
7788 : // The buck stops here, we have already handled the exception by
7789 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7790 : // "diverged" reason during the next solve.
7791 0 : }
7792 0 : catch (...)
7793 : {
7794 0 : mooseError("Unexpected exception type");
7795 0 : }
7796 :
7797 3036335 : resetState();
7798 3036335 : }
7799 :
7800 : void
7801 471125 : FEProblemBase::computeJacobianSys(NonlinearImplicitSystem & sys,
7802 : const NumericVector<Number> & soln,
7803 : SparseMatrix<Number> & jacobian)
7804 : {
7805 : // Reset before Jacobian setup, calculation & execution
7806 471125 : _app.solutionInvalidity().resetIterationOccurences();
7807 471125 : computeJacobian(soln, jacobian, sys.number());
7808 471112 : }
7809 :
7810 : void
7811 4467 : FEProblemBase::computeJacobianTag(const NumericVector<Number> & soln,
7812 : SparseMatrix<Number> & jacobian,
7813 : TagID tag)
7814 : {
7815 4467 : _current_nl_sys->setSolution(soln);
7816 :
7817 4467 : _current_nl_sys->associateMatrixToTag(jacobian, tag);
7818 :
7819 8934 : computeJacobianTags({tag});
7820 :
7821 4467 : _current_nl_sys->disassociateMatrixFromTag(jacobian, tag);
7822 4467 : }
7823 :
7824 : void
7825 470281 : FEProblemBase::computeJacobian(const NumericVector<Number> & soln,
7826 : SparseMatrix<Number> & jacobian,
7827 : const unsigned int nl_sys_num)
7828 : {
7829 470281 : setCurrentNonlinearSystem(nl_sys_num);
7830 :
7831 470281 : _fe_matrix_tags.clear();
7832 :
7833 470281 : auto & tags = getMatrixTags();
7834 1411105 : for (auto & tag : tags)
7835 940824 : _fe_matrix_tags.insert(tag.second);
7836 :
7837 470281 : computeJacobianInternal(soln, jacobian, _fe_matrix_tags);
7838 470268 : }
7839 :
7840 : void
7841 470281 : FEProblemBase::computeJacobianInternal(const NumericVector<Number> & soln,
7842 : SparseMatrix<Number> & jacobian,
7843 : const std::set<TagID> & tags)
7844 : {
7845 1410843 : TIME_SECTION("computeJacobianInternal", 1);
7846 :
7847 470281 : _current_nl_sys->setSolution(soln);
7848 :
7849 470281 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7850 :
7851 470281 : computeJacobianTags(tags);
7852 :
7853 470268 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7854 470268 : }
7855 :
7856 : void
7857 479529 : FEProblemBase::computeJacobianTags(const std::set<TagID> & tags)
7858 : {
7859 : try
7860 : {
7861 : try
7862 : {
7863 479529 : if (!_has_jacobian || !_const_jacobian)
7864 : {
7865 2365370 : TIME_SECTION("computeJacobianTags", 5, "Computing Jacobian");
7866 :
7867 1411892 : for (auto tag : tags)
7868 938818 : if (_current_nl_sys->hasMatrix(tag))
7869 : {
7870 474992 : auto & matrix = _current_nl_sys->getMatrix(tag);
7871 474992 : if (_restore_original_nonzero_pattern)
7872 7006 : matrix.restore_original_nonzero_pattern();
7873 : else
7874 467986 : matrix.zero();
7875 474992 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7876 : // PETSc algorithms require diagonal allocations regardless of whether there is
7877 : // non-zero diagonal dependence. With global AD indexing we only add non-zero
7878 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7879 4690523 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7880 4647264 : matrix.add(index, index, 0);
7881 : }
7882 :
7883 473074 : _aux->zeroVariablesForJacobian();
7884 :
7885 473074 : unsigned int n_threads = libMesh::n_threads();
7886 :
7887 : // Random interface objects
7888 475096 : for (const auto & it : _random_data_objects)
7889 2022 : it.second->updateSeeds(EXEC_NONLINEAR);
7890 :
7891 473074 : _current_execute_on_flag = EXEC_NONLINEAR;
7892 473074 : _currently_computing_jacobian = true;
7893 473074 : if (_displaced_problem)
7894 21147 : _displaced_problem->setCurrentlyComputingJacobian(true);
7895 :
7896 473074 : execTransfers(EXEC_NONLINEAR);
7897 473074 : execMultiApps(EXEC_NONLINEAR);
7898 :
7899 996107 : for (unsigned int tid = 0; tid < n_threads; tid++)
7900 523033 : reinitScalars(tid);
7901 :
7902 473074 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7903 :
7904 473074 : _aux->jacobianSetup();
7905 :
7906 473074 : if (_displaced_problem)
7907 : {
7908 21147 : computeSystems(EXEC_PRE_DISPLACE);
7909 21147 : _displaced_problem->updateMesh();
7910 : }
7911 :
7912 996100 : for (unsigned int tid = 0; tid < n_threads; tid++)
7913 : {
7914 523029 : _all_materials.jacobianSetup(tid);
7915 523029 : _functions.jacobianSetup(tid);
7916 : }
7917 :
7918 : #ifdef MOOSE_KOKKOS_ENABLED
7919 345126 : _kokkos_functions.jacobianSetup();
7920 : #endif
7921 :
7922 473071 : computeSystems(EXEC_NONLINEAR);
7923 :
7924 473071 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7925 :
7926 473071 : executeControls(EXEC_NONLINEAR);
7927 :
7928 473071 : _app.getOutputWarehouse().jacobianSetup();
7929 :
7930 473071 : _safe_access_tagged_matrices = false;
7931 :
7932 473071 : _current_nl_sys->computeJacobianTags(tags);
7933 :
7934 : // For explicit Euler calculations for example we often compute the Jacobian one time and
7935 : // then re-use it over and over. If we're performing automatic scaling, we don't want to
7936 : // use that kernel, diagonal-block only Jacobian for our actual matrix when performing
7937 : // solves!
7938 473061 : if (!_current_nl_sys->computingScalingJacobian())
7939 472578 : _has_jacobian = true;
7940 473064 : }
7941 : }
7942 3 : catch (...)
7943 : {
7944 3 : handleException("computeJacobianTags");
7945 0 : }
7946 : }
7947 0 : catch (const MooseException &)
7948 : {
7949 : // The buck stops here, we have already handled the exception by
7950 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7951 : // "diverged" reason during the next solve.
7952 0 : }
7953 0 : catch (...)
7954 : {
7955 0 : mooseError("Unexpected exception type");
7956 0 : }
7957 :
7958 479516 : resetState();
7959 479516 : }
7960 :
7961 : void
7962 263 : FEProblemBase::computeJacobianBlocks(std::vector<JacobianBlock *> & blocks,
7963 : const unsigned int nl_sys_num)
7964 : {
7965 789 : TIME_SECTION("computeTransientImplicitJacobian", 2);
7966 263 : setCurrentNonlinearSystem(nl_sys_num);
7967 :
7968 263 : if (_displaced_problem)
7969 : {
7970 0 : computeSystems(EXEC_PRE_DISPLACE);
7971 0 : _displaced_problem->updateMesh();
7972 : }
7973 :
7974 263 : computeSystems(EXEC_NONLINEAR);
7975 :
7976 263 : _currently_computing_jacobian = true;
7977 263 : _current_nl_sys->computeJacobianBlocks(blocks);
7978 263 : _currently_computing_jacobian = false;
7979 263 : }
7980 :
7981 : void
7982 0 : FEProblemBase::computeJacobianBlock(SparseMatrix<Number> & jacobian,
7983 : libMesh::System & precond_system,
7984 : unsigned int ivar,
7985 : unsigned int jvar)
7986 : {
7987 0 : JacobianBlock jac_block(precond_system, jacobian, ivar, jvar);
7988 0 : std::vector<JacobianBlock *> blocks = {&jac_block};
7989 : mooseAssert(_current_nl_sys, "This should be non-null");
7990 0 : computeJacobianBlocks(blocks, _current_nl_sys->number());
7991 0 : }
7992 :
7993 : void
7994 714 : FEProblemBase::computeBounds(NonlinearImplicitSystem & libmesh_dbg_var(sys),
7995 : NumericVector<Number> & lower,
7996 : NumericVector<Number> & upper)
7997 : {
7998 : try
7999 : {
8000 : try
8001 : {
8002 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8003 : "I expect these system numbers to be the same");
8004 :
8005 3570 : if (!_current_nl_sys->hasVector("lower_bound") || !_current_nl_sys->hasVector("upper_bound"))
8006 0 : return;
8007 :
8008 3570 : TIME_SECTION("computeBounds", 1, "Computing Bounds");
8009 :
8010 1428 : NumericVector<Number> & _lower = _current_nl_sys->getVector("lower_bound");
8011 1428 : NumericVector<Number> & _upper = _current_nl_sys->getVector("upper_bound");
8012 714 : _lower.swap(lower);
8013 714 : _upper.swap(upper);
8014 1492 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
8015 778 : _all_materials.residualSetup(tid);
8016 :
8017 714 : _aux->residualSetup();
8018 714 : computeSystems(EXEC_LINEAR);
8019 714 : _lower.swap(lower);
8020 714 : _upper.swap(upper);
8021 714 : }
8022 0 : catch (...)
8023 : {
8024 0 : handleException("computeBounds");
8025 0 : }
8026 : }
8027 0 : catch (MooseException & e)
8028 : {
8029 0 : mooseError("Irrecoverable exception: " + std::string(e.what()));
8030 0 : }
8031 0 : catch (...)
8032 : {
8033 0 : mooseError("Unexpected exception type");
8034 0 : }
8035 : }
8036 :
8037 : void
8038 26160 : FEProblemBase::computeLinearSystemSys(LinearImplicitSystem & sys,
8039 : SparseMatrix<Number> & system_matrix,
8040 : NumericVector<Number> & rhs,
8041 : const bool compute_gradients)
8042 : {
8043 78480 : TIME_SECTION("computeLinearSystemSys", 5);
8044 :
8045 26160 : setCurrentLinearSystem(linearSysNum(sys.name()));
8046 :
8047 26160 : _current_linear_sys->associateVectorToTag(rhs, _current_linear_sys->rightHandSideVectorTag());
8048 26160 : _current_linear_sys->associateMatrixToTag(system_matrix, _current_linear_sys->systemMatrixTag());
8049 :
8050 : // We are using the residual tag system for right hand sides so we fetch everything
8051 26160 : const auto & vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
8052 :
8053 : // We filter out tags which do not have associated vectors in the current
8054 : // system. This is essential to be able to use system-dependent vector tags.
8055 26160 : selectVectorTagsFromSystem(*_current_linear_sys, vector_tags, _linear_vector_tags);
8056 26160 : selectMatrixTagsFromSystem(*_current_linear_sys, getMatrixTags(), _linear_matrix_tags);
8057 :
8058 26160 : computeLinearSystemTags(*(_current_linear_sys->currentSolution()),
8059 26160 : _linear_vector_tags,
8060 26160 : _linear_matrix_tags,
8061 : compute_gradients);
8062 :
8063 26160 : _current_linear_sys->disassociateMatrixFromTag(system_matrix,
8064 26160 : _current_linear_sys->systemMatrixTag());
8065 26160 : _current_linear_sys->disassociateVectorFromTag(rhs,
8066 26160 : _current_linear_sys->rightHandSideVectorTag());
8067 : // We reset the tags to the default containers for further operations
8068 26160 : _current_linear_sys->associateVectorToTag(_current_linear_sys->getRightHandSideVector(),
8069 26160 : _current_linear_sys->rightHandSideVectorTag());
8070 26160 : _current_linear_sys->associateMatrixToTag(_current_linear_sys->getSystemMatrix(),
8071 26160 : _current_linear_sys->systemMatrixTag());
8072 26160 : }
8073 :
8074 : void
8075 26160 : FEProblemBase::computeLinearSystemTags(const NumericVector<Number> & soln,
8076 : const std::set<TagID> & vector_tags,
8077 : const std::set<TagID> & matrix_tags,
8078 : const bool compute_gradients)
8079 : {
8080 130800 : TIME_SECTION("computeLinearSystemTags", 5, "Computing Linear System");
8081 :
8082 26160 : _current_linear_sys->setSolution(soln);
8083 :
8084 52340 : for (auto tag : matrix_tags)
8085 : {
8086 26180 : auto & matrix = _current_linear_sys->getMatrix(tag);
8087 26180 : matrix.zero();
8088 : }
8089 :
8090 26160 : unsigned int n_threads = libMesh::n_threads();
8091 :
8092 26160 : _current_execute_on_flag = EXEC_NONLINEAR;
8093 :
8094 : // Random interface objects
8095 26160 : for (const auto & it : _random_data_objects)
8096 0 : it.second->updateSeeds(EXEC_NONLINEAR);
8097 :
8098 26160 : execTransfers(EXEC_NONLINEAR);
8099 26160 : execMultiApps(EXEC_NONLINEAR);
8100 :
8101 26160 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
8102 :
8103 26160 : _aux->jacobianSetup();
8104 :
8105 52320 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
8106 : {
8107 26160 : _functions.jacobianSetup(tid);
8108 : }
8109 :
8110 : #ifdef MOOSE_KOKKOS_ENABLED
8111 17995 : _kokkos_functions.jacobianSetup();
8112 : #endif
8113 :
8114 : try
8115 : {
8116 26160 : computeSystems(EXEC_NONLINEAR);
8117 : }
8118 0 : catch (MooseException & e)
8119 : {
8120 0 : _console << "\nA MooseException was raised during Auxiliary variable computation.\n"
8121 0 : << "The next solve will fail, the timestep will be reduced, and we will try again.\n"
8122 0 : << std::endl;
8123 :
8124 : // We know the next solve is going to fail, so there's no point in
8125 : // computing anything else after this. Plus, using incompletely
8126 : // computed AuxVariables in subsequent calculations could lead to
8127 : // other errors or unhandled exceptions being thrown.
8128 0 : return;
8129 0 : }
8130 :
8131 26160 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
8132 26160 : executeControls(EXEC_NONLINEAR);
8133 :
8134 26160 : _app.getOutputWarehouse().jacobianSetup();
8135 :
8136 26160 : _current_linear_sys->computeLinearSystemTags(vector_tags, matrix_tags, compute_gradients);
8137 :
8138 : // Reset execution flag as after this point we are no longer on LINEAR
8139 26160 : _current_execute_on_flag = EXEC_NONE;
8140 :
8141 : // These are the relevant parts of resetState()
8142 26160 : _safe_access_tagged_vectors = true;
8143 26160 : _safe_access_tagged_matrices = true;
8144 26160 : }
8145 :
8146 : void
8147 294844 : FEProblemBase::computeNearNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8148 : std::vector<NumericVector<Number> *> & sp)
8149 : {
8150 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8151 : "I expect these system numbers to be the same");
8152 :
8153 294844 : sp.clear();
8154 884532 : for (unsigned int i = 0; i < subspaceDim("NearNullSpace"); ++i)
8155 : {
8156 0 : std::stringstream postfix;
8157 0 : postfix << "_" << i;
8158 0 : std::string modename = "NearNullSpace" + postfix.str();
8159 0 : sp.push_back(&_current_nl_sys->getVector(modename));
8160 0 : }
8161 294844 : }
8162 :
8163 : void
8164 294844 : FEProblemBase::computeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8165 : std::vector<NumericVector<Number> *> & sp)
8166 : {
8167 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8168 : "I expect these system numbers to be the same");
8169 294844 : sp.clear();
8170 884598 : for (unsigned int i = 0; i < subspaceDim("NullSpace"); ++i)
8171 : {
8172 22 : std::stringstream postfix;
8173 22 : postfix << "_" << i;
8174 22 : sp.push_back(&_current_nl_sys->getVector("NullSpace" + postfix.str()));
8175 22 : }
8176 294844 : }
8177 :
8178 : void
8179 294844 : FEProblemBase::computeTransposeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8180 : std::vector<NumericVector<Number> *> & sp)
8181 : {
8182 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8183 : "I expect these system numbers to be the same");
8184 294844 : sp.clear();
8185 884565 : for (unsigned int i = 0; i < subspaceDim("TransposeNullSpace"); ++i)
8186 : {
8187 11 : std::stringstream postfix;
8188 11 : postfix << "_" << i;
8189 11 : sp.push_back(&_current_nl_sys->getVector("TransposeNullSpace" + postfix.str()));
8190 11 : }
8191 294844 : }
8192 :
8193 : void
8194 2128 : FEProblemBase::computePostCheck(NonlinearImplicitSystem & sys,
8195 : const NumericVector<Number> & old_soln,
8196 : NumericVector<Number> & search_direction,
8197 : NumericVector<Number> & new_soln,
8198 : bool & changed_search_direction,
8199 : bool & changed_new_soln)
8200 : {
8201 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8202 : "I expect these system numbers to be the same");
8203 :
8204 : // This function replaces the old PetscSupport::dampedCheck() function.
8205 : //
8206 : // 1.) Recreate code in PetscSupport::dampedCheck() for constructing
8207 : // ghosted "soln" and "update" vectors.
8208 : // 2.) Call FEProblemBase::computeDamping() with these ghost vectors.
8209 : // 3.) Recreate the code in PetscSupport::dampedCheck() to actually update
8210 : // the solution vector based on the damping, and set the "changed" flags
8211 : // appropriately.
8212 :
8213 10640 : TIME_SECTION("computePostCheck", 2, "Computing Post Check");
8214 :
8215 2128 : _current_execute_on_flag = EXEC_POSTCHECK;
8216 :
8217 : // MOOSE's FEProblemBase doesn't update the solution during the
8218 : // postcheck, but FEProblemBase-derived classes might.
8219 2128 : if (_has_dampers || shouldUpdateSolution())
8220 : {
8221 : // We need ghosted versions of new_soln and search_direction (the
8222 : // ones we get from libmesh/PETSc are PARALLEL vectors. To make
8223 : // our lives simpler, we use the same ghosting pattern as the
8224 : // system's current_local_solution to create new ghosted vectors.
8225 :
8226 : // Construct zeroed-out clones with the same ghosted dofs as the
8227 : // System's current_local_solution.
8228 : std::unique_ptr<NumericVector<Number>> ghosted_solution =
8229 1625 : sys.current_local_solution->zero_clone(),
8230 : ghosted_search_direction =
8231 1625 : sys.current_local_solution->zero_clone();
8232 :
8233 : // Copy values from input vectors into clones with ghosted values.
8234 1625 : *ghosted_solution = new_soln;
8235 1625 : *ghosted_search_direction = search_direction;
8236 :
8237 1625 : if (_has_dampers)
8238 : {
8239 : // Compute the damping coefficient using the ghosted vectors
8240 1625 : Real damping = computeDamping(*ghosted_solution, *ghosted_search_direction);
8241 :
8242 : // If some non-trivial damping was computed, update the new_soln
8243 : // vector accordingly.
8244 1625 : if (damping < 1.0)
8245 : {
8246 1223 : new_soln = old_soln;
8247 1223 : new_soln.add(-damping, search_direction);
8248 1223 : changed_new_soln = true;
8249 : }
8250 : }
8251 :
8252 1625 : if (shouldUpdateSolution())
8253 : {
8254 : // Update the ghosted copy of the new solution, if necessary.
8255 0 : if (changed_new_soln)
8256 0 : *ghosted_solution = new_soln;
8257 :
8258 0 : bool updated_solution = updateSolution(new_soln, *ghosted_solution);
8259 0 : if (updated_solution)
8260 0 : changed_new_soln = true;
8261 : }
8262 1625 : }
8263 :
8264 2128 : if (vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
8265 : {
8266 503 : _current_nl_sys->setPreviousNewtonSolution(old_soln);
8267 503 : _aux->copyCurrentIntoPreviousNL();
8268 : }
8269 :
8270 : // MOOSE doesn't change the search_direction
8271 2128 : changed_search_direction = false;
8272 :
8273 2128 : _current_execute_on_flag = EXEC_NONE;
8274 2128 : }
8275 :
8276 : Real
8277 1625 : FEProblemBase::computeDamping(const NumericVector<Number> & soln,
8278 : const NumericVector<Number> & update)
8279 : {
8280 : // Default to no damping
8281 1625 : Real damping = 1.0;
8282 :
8283 1625 : if (_has_dampers)
8284 : {
8285 8125 : TIME_SECTION("computeDamping", 1, "Computing Damping");
8286 :
8287 : // Save pointer to the current solution
8288 1625 : const NumericVector<Number> * _saved_current_solution = _current_nl_sys->currentSolution();
8289 :
8290 1625 : _current_nl_sys->setSolution(soln);
8291 : // For now, do not re-compute auxiliary variables. Doing so allows a wild solution increment
8292 : // to get to the material models, which may not be able to cope with drastically different
8293 : // values. Once more complete dependency checking is in place, auxiliary variables (and
8294 : // material properties) will be computed as needed by dampers.
8295 : // _aux.compute();
8296 1625 : damping = _current_nl_sys->computeDamping(soln, update);
8297 :
8298 : // restore saved solution
8299 1625 : _current_nl_sys->setSolution(*_saved_current_solution);
8300 1625 : }
8301 :
8302 1625 : return damping;
8303 : }
8304 :
8305 : bool
8306 289440 : FEProblemBase::shouldUpdateSolution()
8307 : {
8308 289440 : return false;
8309 : }
8310 :
8311 : bool
8312 0 : FEProblemBase::updateSolution(NumericVector<Number> & /*vec_solution*/,
8313 : NumericVector<Number> & /*ghosted_solution*/)
8314 : {
8315 0 : return false;
8316 : }
8317 :
8318 : void
8319 203 : FEProblemBase::predictorCleanup(NumericVector<Number> & /*ghosted_solution*/)
8320 : {
8321 203 : }
8322 :
8323 : void
8324 2022 : FEProblemBase::addDisplacedProblem(std::shared_ptr<DisplacedProblem> displaced_problem)
8325 : {
8326 : parallel_object_only();
8327 :
8328 2022 : _displaced_mesh = &displaced_problem->mesh();
8329 2022 : _displaced_problem = displaced_problem;
8330 2022 : }
8331 :
8332 : void
8333 119490 : FEProblemBase::updateGeomSearch(GeometricSearchData::GeometricSearchType type)
8334 : {
8335 597450 : TIME_SECTION("updateGeometricSearch", 3, "Updating Geometric Search");
8336 :
8337 119490 : _geometric_search_data.update(type);
8338 :
8339 119490 : if (_displaced_problem)
8340 4209 : _displaced_problem->updateGeomSearch(type);
8341 119490 : }
8342 :
8343 : void
8344 63263 : FEProblemBase::updateMortarMesh()
8345 : {
8346 316315 : TIME_SECTION("updateMortarMesh", 5, "Updating Mortar Mesh");
8347 :
8348 63263 : FloatingPointExceptionGuard fpe_guard(_app);
8349 :
8350 63263 : _mortar_data->update();
8351 63263 : }
8352 :
8353 : void
8354 1326 : FEProblemBase::createMortarInterface(
8355 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8356 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8357 : bool on_displaced,
8358 : bool periodic,
8359 : const bool debug,
8360 : const bool correct_edge_dropping,
8361 : const Real minimum_projection_angle)
8362 : {
8363 1326 : _has_mortar = true;
8364 :
8365 1326 : if (on_displaced)
8366 254 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8367 : primary_secondary_subdomain_pair,
8368 127 : *_displaced_problem,
8369 : on_displaced,
8370 : periodic,
8371 : debug,
8372 : correct_edge_dropping,
8373 127 : minimum_projection_angle);
8374 : else
8375 1199 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8376 : primary_secondary_subdomain_pair,
8377 : *this,
8378 : on_displaced,
8379 : periodic,
8380 : debug,
8381 : correct_edge_dropping,
8382 1199 : minimum_projection_angle);
8383 : }
8384 :
8385 : const AutomaticMortarGeneration &
8386 0 : FEProblemBase::getMortarInterface(
8387 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8388 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8389 : bool on_displaced) const
8390 : {
8391 0 : return _mortar_data->getMortarInterface(
8392 0 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8393 : }
8394 :
8395 : AutomaticMortarGeneration &
8396 165399 : FEProblemBase::getMortarInterface(
8397 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8398 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8399 : bool on_displaced)
8400 : {
8401 165399 : return _mortar_data->getMortarInterface(
8402 165399 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8403 : }
8404 :
8405 : void
8406 318238 : FEProblemBase::possiblyRebuildGeomSearchPatches()
8407 : {
8408 318238 : if (_displaced_problem) // Only need to do this if things are moving...
8409 : {
8410 164185 : TIME_SECTION("possiblyRebuildGeomSearchPatches", 5, "Rebuilding Geometric Search Patches");
8411 :
8412 32837 : switch (_mesh.getPatchUpdateStrategy())
8413 : {
8414 31742 : case Moose::Never:
8415 31742 : break;
8416 365 : case Moose::Iteration:
8417 : // Update the list of ghosted elements at the start of the time step
8418 365 : _geometric_search_data.updateGhostedElems();
8419 365 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8420 :
8421 365 : _displaced_problem->geomSearchData().updateGhostedElems();
8422 365 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8423 :
8424 : // The commands below ensure that the sparsity of the Jacobian matrix is
8425 : // augmented at the start of the time step using neighbor nodes from the end
8426 : // of the previous time step.
8427 :
8428 365 : reinitBecauseOfGhostingOrNewGeomObjects();
8429 :
8430 : // This is needed to reinitialize PETSc output
8431 365 : initPetscOutputAndSomeSolverSettings();
8432 :
8433 365 : break;
8434 :
8435 331 : case Moose::Auto:
8436 : {
8437 331 : Real max = _displaced_problem->geomSearchData().maxPatchPercentage();
8438 331 : _communicator.max(max);
8439 :
8440 : // If we haven't moved very far through the patch
8441 331 : if (max < 0.4)
8442 298 : break;
8443 : }
8444 : libmesh_fallthrough();
8445 :
8446 : // Let this fall through if things do need to be updated...
8447 : case Moose::Always:
8448 : // Flush output here to see the message before the reinitialization, which could take a
8449 : // while
8450 432 : _console << "\n\nUpdating geometric search patches\n" << std::endl;
8451 :
8452 432 : _geometric_search_data.clearNearestNodeLocators();
8453 432 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8454 :
8455 432 : _displaced_problem->geomSearchData().clearNearestNodeLocators();
8456 432 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8457 :
8458 432 : reinitBecauseOfGhostingOrNewGeomObjects();
8459 :
8460 : // This is needed to reinitialize PETSc output
8461 432 : initPetscOutputAndSomeSolverSettings();
8462 : }
8463 32837 : }
8464 318238 : }
8465 :
8466 : #ifdef LIBMESH_ENABLE_AMR
8467 : void
8468 54922 : FEProblemBase::initialAdaptMesh()
8469 : {
8470 54922 : unsigned int n = adaptivity().getInitialSteps();
8471 54922 : _cycles_completed = 0;
8472 54922 : if (n)
8473 : {
8474 629 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8475 3 : mooseError("HFEM does not support mesh adaptivity currently.");
8476 :
8477 3130 : TIME_SECTION("initialAdaptMesh", 2, "Performing Initial Adaptivity");
8478 :
8479 1506 : for (unsigned int i = 0; i < n; i++)
8480 : {
8481 1055 : computeIndicators();
8482 1055 : computeMarkers();
8483 :
8484 1055 : if (_adaptivity.initialAdaptMesh())
8485 : {
8486 880 : meshChanged(
8487 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8488 :
8489 : // reproject the initial condition
8490 880 : projectSolution();
8491 :
8492 880 : _cycles_completed++;
8493 : }
8494 : else
8495 : {
8496 175 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8497 175 : return;
8498 : }
8499 : }
8500 626 : }
8501 : }
8502 :
8503 : bool
8504 164935 : FEProblemBase::adaptMesh()
8505 : {
8506 : // reset cycle counter
8507 164935 : _cycles_completed = 0;
8508 :
8509 164935 : if (!_adaptivity.isAdaptivityDue())
8510 160380 : return false;
8511 :
8512 22775 : TIME_SECTION("adaptMesh", 3, "Adapting Mesh");
8513 :
8514 4555 : unsigned int cycles_per_step = _adaptivity.getCyclesPerStep();
8515 :
8516 4555 : bool mesh_changed = false;
8517 :
8518 8012 : for (unsigned int i = 0; i < cycles_per_step; ++i)
8519 : {
8520 4707 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8521 0 : mooseError("HFEM does not support mesh adaptivity currently.");
8522 :
8523 : // Markers were already computed once by Executioner
8524 4707 : if (_adaptivity.getRecomputeMarkersFlag() && i > 0)
8525 22 : computeMarkers();
8526 :
8527 : bool mesh_changed_this_step;
8528 4707 : mesh_changed_this_step = _adaptivity.adaptMesh();
8529 :
8530 4707 : if (mesh_changed_this_step)
8531 : {
8532 3457 : mesh_changed = true;
8533 :
8534 3457 : meshChanged(
8535 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8536 3457 : _cycles_completed++;
8537 : }
8538 : else
8539 : {
8540 : // If the mesh didn't change, we still need to update the displaced mesh
8541 : // to undo the undisplacement performed in Adaptivity::adaptMesh
8542 1250 : if (_displaced_problem)
8543 44 : _displaced_problem->updateMesh();
8544 :
8545 1250 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8546 1250 : break;
8547 : }
8548 :
8549 : // Show adaptivity progress
8550 3457 : _console << std::flush;
8551 : }
8552 :
8553 : // We're done with all intermediate changes; now get systems ready
8554 : // for real if necessary.
8555 4555 : if (mesh_changed)
8556 3316 : es().reinit_systems();
8557 :
8558 : // Execute multi-apps that need to run after adaptivity, but before the next timestep.
8559 4555 : execMultiApps(EXEC_POST_ADAPTIVITY);
8560 :
8561 4555 : return mesh_changed;
8562 4555 : }
8563 : #endif // LIBMESH_ENABLE_AMR
8564 :
8565 : void
8566 0 : FEProblemBase::initXFEM(std::shared_ptr<XFEMInterface> xfem)
8567 : {
8568 0 : _xfem = xfem;
8569 0 : _xfem->setMesh(&_mesh);
8570 0 : if (_displaced_mesh)
8571 0 : _xfem->setDisplacedMesh(_displaced_mesh);
8572 :
8573 0 : auto fill_data = [](auto & storage)
8574 : {
8575 0 : std::vector<MaterialData *> data(libMesh::n_threads());
8576 0 : for (const auto tid : make_range(libMesh::n_threads()))
8577 0 : data[tid] = &storage.getMaterialData(tid);
8578 0 : return data;
8579 0 : };
8580 0 : _xfem->setMaterialData(fill_data(_material_props));
8581 0 : _xfem->setBoundaryMaterialData(fill_data(_bnd_material_props));
8582 :
8583 0 : unsigned int n_threads = libMesh::n_threads();
8584 0 : for (unsigned int i = 0; i < n_threads; ++i)
8585 0 : for (const auto nl_sys_num : index_range(_nl))
8586 : {
8587 0 : _assembly[i][nl_sys_num]->setXFEM(_xfem);
8588 0 : if (_displaced_problem)
8589 0 : _displaced_problem->assembly(i, nl_sys_num).setXFEM(_xfem);
8590 : }
8591 0 : }
8592 :
8593 : bool
8594 0 : FEProblemBase::updateMeshXFEM()
8595 : {
8596 0 : TIME_SECTION("updateMeshXFEM", 5, "Updating XFEM");
8597 :
8598 0 : bool updated = false;
8599 0 : if (haveXFEM())
8600 : {
8601 0 : if (_xfem->updateHeal())
8602 : // XFEM exodiff tests rely on a given numbering because they cannot use map = true due to
8603 : // having coincident elements. While conceptually speaking we do not need to contract the
8604 : // mesh, we need its call to renumber_nodes_and_elements in order to preserve these tests
8605 0 : meshChanged(
8606 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8607 :
8608 0 : updated = _xfem->update(_time, _nl, *_aux);
8609 0 : if (updated)
8610 : {
8611 0 : meshChanged(
8612 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8613 0 : _xfem->initSolution(_nl, *_aux);
8614 0 : restoreSolutions();
8615 0 : _console << "\nXFEM update complete: Mesh modified" << std::endl;
8616 : }
8617 : else
8618 0 : _console << "\nXFEM update complete: Mesh not modified" << std::endl;
8619 : }
8620 0 : return updated;
8621 0 : }
8622 :
8623 : void
8624 6998 : FEProblemBase::meshChanged(const bool intermediate_change,
8625 : const bool contract_mesh,
8626 : const bool clean_refinement_flags)
8627 : {
8628 34990 : TIME_SECTION("meshChanged", 3, "Handling Mesh Changes");
8629 :
8630 13788 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8631 6790 : _neighbor_material_props.hasStatefulProperties())
8632 208 : _mesh.cacheChangedLists(); // Currently only used with adaptivity and stateful material
8633 : // properties
8634 :
8635 : // Clear these out because they corresponded to the old mesh
8636 6998 : _ghosted_elems.clear();
8637 6998 : ghostGhostedBoundaries();
8638 :
8639 : // The mesh changed. We notify the MooseMesh first, because
8640 : // callbacks (e.g. for sparsity calculations) triggered by the
8641 : // EquationSystems reinit may require up-to-date MooseMesh caches.
8642 6998 : _mesh.meshChanged();
8643 :
8644 : // If we're just going to alter the mesh again, all we need to
8645 : // handle here is AMR and projections, not full system reinit
8646 6998 : if (intermediate_change)
8647 3488 : es().reinit_solutions();
8648 : else
8649 3510 : es().reinit();
8650 :
8651 6998 : if (contract_mesh)
8652 : // Once vectors are restricted, we can delete children of coarsened elements
8653 4428 : _mesh.getMesh().contract();
8654 6998 : if (clean_refinement_flags)
8655 : {
8656 : // Finally clear refinement flags so that if someone tries to project vectors again without
8657 : // an intervening mesh refinement to clear flags they won't run into trouble
8658 4428 : MeshRefinement refinement(_mesh.getMesh());
8659 4428 : refinement.clean_refinement_flags();
8660 4428 : }
8661 :
8662 6998 : if (!intermediate_change)
8663 : {
8664 : // Since the mesh has changed, we need to make sure that we update any of our
8665 : // MOOSE-system specific data.
8666 7020 : for (auto & sys : _solver_systems)
8667 3510 : sys->reinit();
8668 3510 : _aux->reinit();
8669 : }
8670 :
8671 : // Updating MooseMesh first breaks other adaptivity code, unless we
8672 : // then *again* update the MooseMesh caches. E.g. the definition of
8673 : // "active" and "local" may have been *changed* by refinement and
8674 : // repartitioning done in EquationSystems::reinit().
8675 6998 : _mesh.meshChanged();
8676 :
8677 : // If we have finite volume variables, we will need to recompute additional elemental/face
8678 : // quantities
8679 6998 : if (haveFV() && _mesh.isFiniteVolumeInfoDirty())
8680 351 : _mesh.setupFiniteVolumeMeshData();
8681 :
8682 : // Let the meshChangedInterface notify the mesh changed event before we update the active
8683 : // semilocal nodes, because the set of ghosted elements may potentially be updated during a mesh
8684 : // changed event.
8685 124895 : for (const auto & mci : _notify_when_mesh_changes)
8686 117897 : mci->meshChanged();
8687 :
8688 : // Since the Mesh changed, update the PointLocator object used by DiracKernels.
8689 6998 : _dirac_kernel_info.updatePointLocator(_mesh);
8690 :
8691 : // Need to redo ghosting
8692 6998 : _geometric_search_data.reinit();
8693 :
8694 6998 : if (_displaced_problem)
8695 : {
8696 569 : _displaced_problem->meshChanged(contract_mesh, clean_refinement_flags);
8697 569 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8698 : }
8699 :
8700 6998 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8701 :
8702 6998 : _evaluable_local_elem_range.reset();
8703 6998 : _nl_evaluable_local_elem_range.reset();
8704 :
8705 : // Just like we reinitialized our geometric search objects, we also need to reinitialize our
8706 : // mortar meshes. Note that this needs to happen after DisplacedProblem::meshChanged because the
8707 : // mortar mesh discretization will depend necessarily on the displaced mesh being re-displaced
8708 6998 : _mortar_data->meshChanged();
8709 :
8710 : // Nonlinear systems hold the mortar mesh functors. The domains of definition of the mortar
8711 : // functors might have changed when the mesh changed.
8712 13986 : for (auto & nl_sys : _nl)
8713 6988 : nl_sys->reinitMortarFunctors();
8714 :
8715 6998 : reinitBecauseOfGhostingOrNewGeomObjects(/*mortar_changed=*/true);
8716 :
8717 : // We need to create new storage for newly active elements, and copy
8718 : // stateful properties from the old elements.
8719 7206 : if (_has_initialized_stateful &&
8720 208 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties()))
8721 : {
8722 208 : if (havePRefinement())
8723 66 : _mesh.buildPRefinementAndCoarseningMaps(_assembly[0][0].get());
8724 :
8725 : // Prolong properties onto newly refined elements' children
8726 : {
8727 : ProjectMaterialProperties pmp(
8728 208 : /* refine = */ true, *this, _material_props, _bnd_material_props, _assembly);
8729 208 : const auto & range = *_mesh.refinedElementRange();
8730 208 : Threads::parallel_reduce(range, pmp);
8731 :
8732 : // Concurrent erasure from the shared hash map is not safe while we are reading from it in
8733 : // ProjectMaterialProperties, so we handle erasure here. Moreover, erasure based on key is
8734 : // not thread safe in and of itself because it is a read-write operation. Note that we do not
8735 : // do the erasure for p-refinement because the coarse level element is the same as our active
8736 : // refined level element
8737 208 : if (!doingPRefinement())
8738 3272 : for (const auto & elem : range)
8739 : {
8740 3130 : _material_props.eraseProperty(elem);
8741 3130 : _bnd_material_props.eraseProperty(elem);
8742 3130 : _neighbor_material_props.eraseProperty(elem);
8743 : }
8744 208 : }
8745 :
8746 : // Restrict properties onto newly coarsened elements
8747 : {
8748 : ProjectMaterialProperties pmp(
8749 208 : /* refine = */ false, *this, _material_props, _bnd_material_props, _assembly);
8750 208 : const auto & range = *_mesh.coarsenedElementRange();
8751 208 : Threads::parallel_reduce(range, pmp);
8752 : // Note that we do not do the erasure for p-refinement because the coarse level element is the
8753 : // same as our active refined level element
8754 208 : if (!doingPRefinement())
8755 1322 : for (const auto & elem : range)
8756 : {
8757 1180 : auto && coarsened_children = _mesh.coarsenedElementChildren(elem);
8758 7732 : for (auto && child : coarsened_children)
8759 : {
8760 6552 : _material_props.eraseProperty(child);
8761 6552 : _bnd_material_props.eraseProperty(child);
8762 6552 : _neighbor_material_props.eraseProperty(child);
8763 : }
8764 : }
8765 208 : }
8766 : }
8767 :
8768 6998 : if (_calculate_jacobian_in_uo)
8769 0 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
8770 :
8771 6998 : _has_jacobian = false; // we have to recompute jacobian when mesh changed
8772 :
8773 : // Now for backwards compatibility with user code that overrode the old no-arg meshChanged we must
8774 : // call it here
8775 6998 : meshChanged();
8776 6998 : }
8777 :
8778 : void
8779 911344 : FEProblemBase::notifyWhenMeshChanges(MeshChangedInterface * mci)
8780 : {
8781 911344 : _notify_when_mesh_changes.push_back(mci);
8782 911344 : }
8783 :
8784 : void
8785 74911 : FEProblemBase::notifyWhenMeshDisplaces(MeshDisplacedInterface * mdi)
8786 : {
8787 74911 : _notify_when_mesh_displaces.push_back(mdi);
8788 74911 : }
8789 :
8790 : void
8791 63048 : FEProblemBase::meshDisplaced()
8792 : {
8793 91725 : for (const auto & mdi : _notify_when_mesh_displaces)
8794 28677 : mdi->meshDisplaced();
8795 63048 : }
8796 :
8797 : void
8798 9663 : FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const bool threaded)
8799 : {
8800 : ComputeMaterialsObjectThread cmt(
8801 9663 : *this, _material_props, _bnd_material_props, _neighbor_material_props, _assembly);
8802 9663 : if (threaded)
8803 9663 : Threads::parallel_reduce(elem_range, cmt);
8804 : else
8805 0 : cmt(elem_range, true);
8806 :
8807 : #ifdef MOOSE_KOKKOS_ENABLED
8808 7291 : if (_has_kokkos_objects)
8809 781 : initKokkosStatefulProps();
8810 : #endif
8811 9663 : }
8812 :
8813 : void
8814 59829 : FEProblemBase::checkProblemIntegrity()
8815 : {
8816 179487 : TIME_SECTION("checkProblemIntegrity", 5);
8817 :
8818 : // Subdomains specified by the "Problem/block" parameter
8819 119658 : const auto & subdomain_names = getParam<std::vector<SubdomainName>>("block");
8820 59829 : auto mesh_subdomains_vec = MooseMeshUtils::getSubdomainIDs(_mesh, subdomain_names);
8821 59829 : std::set<SubdomainID> mesh_subdomains(mesh_subdomains_vec.begin(), mesh_subdomains_vec.end());
8822 :
8823 : // Check kernel coverage of subdomains (blocks) in the mesh
8824 59829 : if (!_skip_nl_system_check && _solve && _kernel_coverage_check != CoverageCheckMode::FALSE &&
8825 41164 : _kernel_coverage_check != CoverageCheckMode::OFF)
8826 : {
8827 41152 : std::set<SubdomainID> blocks;
8828 41152 : if (_kernel_coverage_check == CoverageCheckMode::TRUE ||
8829 231 : _kernel_coverage_check == CoverageCheckMode::ON)
8830 40921 : blocks = mesh_subdomains;
8831 231 : else if (_kernel_coverage_check == CoverageCheckMode::SKIP_LIST)
8832 : {
8833 12 : blocks = mesh_subdomains;
8834 24 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8835 : {
8836 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8837 12 : if (id == Moose::INVALID_BLOCK_ID)
8838 0 : paramError("kernel_coverage_block_list",
8839 : "Subdomain \"",
8840 : subdomain_name,
8841 : "\" not found in mesh.");
8842 12 : blocks.erase(id);
8843 : }
8844 : }
8845 219 : else if (_kernel_coverage_check == CoverageCheckMode::ONLY_LIST)
8846 438 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8847 : {
8848 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8849 219 : if (id == Moose::INVALID_BLOCK_ID)
8850 0 : paramError("kernel_coverage_block_list",
8851 : "Subdomain \"",
8852 : subdomain_name,
8853 : "\" not found in mesh.");
8854 219 : blocks.insert(id);
8855 : }
8856 41152 : if (!blocks.empty())
8857 81494 : for (auto & nl : _nl)
8858 40354 : nl->checkKernelCoverage(blocks);
8859 41140 : }
8860 :
8861 : // Check materials
8862 : {
8863 : #ifdef LIBMESH_ENABLE_AMR
8864 62070 : if ((_adaptivity.isOn() || _num_grid_steps) &&
8865 2253 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8866 2190 : _neighbor_material_props.hasStatefulProperties()))
8867 : {
8868 63 : _console << "Using EXPERIMENTAL Stateful Material Property projection with Adaptivity!\n"
8869 63 : << std::flush;
8870 : }
8871 : #endif
8872 :
8873 59817 : std::set<SubdomainID> local_mesh_subs(mesh_subdomains);
8874 :
8875 59817 : if (_material_coverage_check != CoverageCheckMode::FALSE &&
8876 59733 : _material_coverage_check != CoverageCheckMode::OFF)
8877 : {
8878 : /**
8879 : * If a material is specified for any block in the simulation, then all blocks must
8880 : * have a material specified.
8881 : */
8882 59733 : bool check_material_coverage = false;
8883 59733 : std::set<SubdomainID> ids = _all_materials.getActiveBlocks();
8884 72252 : for (const auto & id : ids)
8885 : {
8886 12519 : local_mesh_subs.erase(id);
8887 12519 : check_material_coverage = true;
8888 : }
8889 :
8890 : // did the user limit the subdomains to be checked?
8891 59733 : if (_material_coverage_check == CoverageCheckMode::SKIP_LIST)
8892 : {
8893 24 : for (const auto & subdomain_name : _material_coverage_blocks)
8894 : {
8895 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8896 12 : if (id == Moose::INVALID_BLOCK_ID)
8897 0 : paramError("material_coverage_block_list",
8898 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8899 12 : local_mesh_subs.erase(id);
8900 : }
8901 : }
8902 59721 : else if (_material_coverage_check == CoverageCheckMode::ONLY_LIST)
8903 : {
8904 219 : std::set<SubdomainID> blocks(local_mesh_subs);
8905 438 : for (const auto & subdomain_name : _material_coverage_blocks)
8906 : {
8907 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8908 219 : if (id == Moose::INVALID_BLOCK_ID)
8909 0 : paramError("material_coverage_block_list",
8910 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8911 219 : blocks.erase(id);
8912 : }
8913 231 : for (const auto id : blocks)
8914 12 : local_mesh_subs.erase(id);
8915 219 : }
8916 :
8917 : // also exclude mortar spaces from the material check
8918 59733 : auto && mortar_subdomain_ids = _mortar_data->getMortarSubdomainIDs();
8919 61587 : for (auto subdomain_id : mortar_subdomain_ids)
8920 1854 : local_mesh_subs.erase(subdomain_id);
8921 :
8922 : // Check Material Coverage
8923 59733 : if (check_material_coverage && !local_mesh_subs.empty())
8924 : {
8925 6 : std::stringstream extra_subdomain_ids;
8926 : /// unsigned int is necessary to print SubdomainIDs in the statement below
8927 6 : std::copy(local_mesh_subs.begin(),
8928 : local_mesh_subs.end(),
8929 12 : std::ostream_iterator<unsigned int>(extra_subdomain_ids, " "));
8930 : /// vector is necessary to get the subdomain names
8931 : std::vector<SubdomainID> local_mesh_subs_vec(local_mesh_subs.begin(),
8932 6 : local_mesh_subs.end());
8933 :
8934 18 : mooseError("The following blocks from your input mesh do not contain an active material: " +
8935 12 : extra_subdomain_ids.str() +
8936 18 : "(names: " + Moose::stringify(_mesh.getSubdomainNames(local_mesh_subs_vec)) +
8937 : ")\nWhen ANY mesh block contains a Material object, "
8938 : "all blocks must contain a Material object.\n");
8939 0 : }
8940 59727 : }
8941 :
8942 : // Check material properties on blocks and boundaries
8943 59811 : checkBlockMatProps();
8944 59779 : checkBoundaryMatProps();
8945 :
8946 : // Check that material properties exist when requested by other properties on a given block
8947 59770 : const auto & materials = _all_materials.getActiveObjects();
8948 73667 : for (const auto & material : materials)
8949 13897 : material->checkStatefulSanity();
8950 :
8951 : // auto mats_to_check = _materials.getActiveBlockObjects();
8952 : // const auto & discrete_materials = _discrete_materials.getActiveBlockObjects();
8953 : // for (const auto & map_it : discrete_materials)
8954 : // for (const auto & container_element : map_it.second)
8955 : // mats_to_check[map_it.first].push_back(container_element);
8956 59770 : if (_material_dependency_check)
8957 59746 : checkDependMaterialsHelper(_all_materials.getActiveBlockObjects());
8958 59757 : }
8959 :
8960 59757 : checkUserObjects();
8961 :
8962 : // Verify that we don't have any Element type/Coordinate Type conflicts
8963 59757 : checkCoordinateSystems();
8964 :
8965 : // Coordinate transforms are only intended for use with MultiApps at this time. If you are not
8966 : // using multiapps but still require these, contact a moose developer
8967 59894 : if (_mesh.coordTransform().hasScalingOrRotationTransformation() && _app.isUltimateMaster() &&
8968 140 : !hasMultiApps())
8969 3 : mooseError("Coordinate transformation parameters, listed below, are only to be used in the "
8970 : "context of application to application field transfers at this time. The mesh is "
8971 : "not modified by these parameters within an application.\n"
8972 : "You should likely use a 'TransformGenerator' in the [Mesh] block to achieve the "
8973 : "desired mesh modification.\n\n",
8974 3 : Moose::stringify(MooseAppCoordTransform::validParams()));
8975 :
8976 : // If using displacements, verify that the order of the displacement
8977 : // variables matches the order of the elements in the displaced
8978 : // mesh.
8979 59751 : checkDisplacementOrders();
8980 :
8981 : // Check for postprocessor names with same name as a scalar variable
8982 59748 : checkDuplicatePostprocessorVariableNames();
8983 59748 : }
8984 :
8985 : void
8986 59751 : FEProblemBase::checkDisplacementOrders()
8987 : {
8988 59751 : if (_displaced_problem)
8989 : {
8990 2022 : bool mesh_has_second_order_elements = false;
8991 4044 : for (const auto & elem : as_range(_displaced_mesh->activeLocalElementsBegin(),
8992 464948 : _displaced_mesh->activeLocalElementsEnd()))
8993 : {
8994 229785 : if (elem->default_order() == SECOND)
8995 : {
8996 344 : mesh_has_second_order_elements = true;
8997 344 : break;
8998 : }
8999 2022 : }
9000 :
9001 : // We checked our local elements, so take the max over all processors.
9002 2022 : _displaced_mesh->comm().max(mesh_has_second_order_elements);
9003 :
9004 : // If the Mesh has second order elements, make sure the
9005 : // displacement variables are second-order.
9006 2022 : if (mesh_has_second_order_elements)
9007 : {
9008 : const std::vector<std::string> & displacement_variables =
9009 344 : _displaced_problem->getDisplacementVarNames();
9010 :
9011 1133 : for (const auto & var_name : displacement_variables)
9012 : {
9013 : MooseVariableFEBase & mv =
9014 792 : _displaced_problem->getVariable(/*tid=*/0,
9015 : var_name,
9016 : Moose::VarKindType::VAR_ANY,
9017 : Moose::VarFieldType::VAR_FIELD_STANDARD);
9018 792 : if (mv.order() != SECOND)
9019 3 : mooseError("Error: mesh has SECOND order elements, so all displacement variables must be "
9020 : "SECOND order.");
9021 : }
9022 : }
9023 : }
9024 59748 : }
9025 :
9026 : void
9027 59757 : FEProblemBase::checkUserObjects()
9028 : {
9029 : // Check user_objects block coverage
9030 59757 : std::set<SubdomainID> mesh_subdomains = _mesh.meshSubdomains();
9031 59757 : std::set<SubdomainID> user_objects_blocks;
9032 :
9033 : // gather names of all user_objects that were defined in the input file
9034 : // and the blocks that they are defined on
9035 59757 : std::set<std::string> names;
9036 :
9037 59757 : std::vector<UserObjectBase *> objects;
9038 59757 : theWarehouse().query().condition<AttribInterfaces>(Interfaces::UserObject).queryInto(objects);
9039 :
9040 132404 : for (const auto & obj : objects)
9041 72647 : names.insert(obj->name());
9042 :
9043 : // See if all referenced blocks are covered
9044 59757 : std::set<SubdomainID> difference;
9045 59757 : std::set_difference(user_objects_blocks.begin(),
9046 : user_objects_blocks.end(),
9047 : mesh_subdomains.begin(),
9048 : mesh_subdomains.end(),
9049 : std::inserter(difference, difference.end()));
9050 :
9051 59757 : if (!difference.empty())
9052 : {
9053 0 : std::ostringstream oss;
9054 0 : oss << "One or more UserObjects is referencing a nonexistent block:\n";
9055 0 : for (const auto & id : difference)
9056 0 : oss << id << "\n";
9057 0 : mooseError(oss.str());
9058 0 : }
9059 59757 : }
9060 :
9061 : void
9062 59746 : FEProblemBase::checkDependMaterialsHelper(
9063 : const std::map<SubdomainID, std::vector<std::shared_ptr<MaterialBase>>> & materials_map)
9064 : {
9065 72207 : for (const auto & it : materials_map)
9066 : {
9067 : /// These two sets are used to make sure that all dependent props on a block are actually supplied
9068 12469 : std::set<std::string> block_depend_props, block_supplied_props;
9069 :
9070 30921 : for (const auto & mat1 : it.second)
9071 : {
9072 18452 : auto & alldeps = mat1->getMatPropDependencies(); // includes requested stateful props
9073 20771 : for (auto & dep : alldeps)
9074 2319 : block_depend_props.insert(_material_prop_registry.getName(dep));
9075 :
9076 : // See if any of the active materials supply this property
9077 56736 : for (const auto & mat2 : it.second)
9078 : {
9079 38284 : const std::set<std::string> & supplied_props = mat2->MaterialBase::getSuppliedItems();
9080 38284 : block_supplied_props.insert(supplied_props.begin(), supplied_props.end());
9081 : }
9082 : }
9083 :
9084 : // Add zero material properties specific to this block and unrestricted
9085 12469 : block_supplied_props.insert(_zero_block_material_props[it.first].begin(),
9086 12469 : _zero_block_material_props[it.first].end());
9087 :
9088 : // Error check to make sure all properties consumed by materials are supplied on this block
9089 12469 : std::set<std::string> difference;
9090 12469 : std::set_difference(block_depend_props.begin(),
9091 : block_depend_props.end(),
9092 : block_supplied_props.begin(),
9093 : block_supplied_props.end(),
9094 : std::inserter(difference, difference.end()));
9095 :
9096 12469 : if (!difference.empty())
9097 : {
9098 8 : std::ostringstream oss;
9099 8 : oss << "One or more Material Properties were not supplied on block ";
9100 8 : const std::string & subdomain_name = _mesh.getSubdomainName(it.first);
9101 8 : if (subdomain_name.length() > 0)
9102 0 : oss << subdomain_name << " (" << it.first << ")";
9103 : else
9104 8 : oss << it.first;
9105 8 : oss << ":\n";
9106 16 : for (const auto & name : difference)
9107 8 : oss << name << "\n";
9108 8 : mooseError(oss.str());
9109 0 : }
9110 12461 : }
9111 :
9112 : // This loop checks that materials are not supplied by multiple Material objects
9113 72194 : for (const auto & it : materials_map)
9114 : {
9115 12461 : const auto & materials = it.second;
9116 12461 : std::set<std::string> inner_supplied, outer_supplied;
9117 :
9118 30893 : for (const auto & outer_mat : materials)
9119 : {
9120 : // Storage for properties for this material (outer) and all other materials (inner)
9121 18437 : outer_supplied = outer_mat->getSuppliedItems();
9122 18437 : inner_supplied.clear();
9123 :
9124 : // Property to material map for error reporting
9125 18437 : std::map<std::string, std::set<std::string>> prop_to_mat;
9126 39149 : for (const auto & name : outer_supplied)
9127 20712 : prop_to_mat[name].insert(outer_mat->name());
9128 :
9129 56697 : for (const auto & inner_mat : materials)
9130 : {
9131 38260 : if (outer_mat == inner_mat)
9132 18437 : continue;
9133 :
9134 : // Check whether these materials are an AD pair
9135 19823 : auto outer_mat_type = outer_mat->type();
9136 19823 : auto inner_mat_type = inner_mat->type();
9137 39646 : removeSubstring(outer_mat_type, "<RESIDUAL>");
9138 39646 : removeSubstring(outer_mat_type, "<JACOBIAN>");
9139 39646 : removeSubstring(inner_mat_type, "<RESIDUAL>");
9140 19823 : removeSubstring(inner_mat_type, "<JACOBIAN>");
9141 19823 : if (outer_mat_type == inner_mat_type && outer_mat_type != outer_mat->type() &&
9142 0 : inner_mat_type != inner_mat->type())
9143 0 : continue;
9144 :
9145 19823 : inner_supplied.insert(inner_mat->getSuppliedItems().begin(),
9146 19823 : inner_mat->getSuppliedItems().end());
9147 :
9148 114479 : for (const auto & inner_supplied_name : inner_supplied)
9149 94656 : prop_to_mat[inner_supplied_name].insert(inner_mat->name());
9150 19823 : }
9151 :
9152 : // Test that a property isn't supplied on multiple blocks
9153 18437 : std::set<std::string> intersection;
9154 18437 : std::set_intersection(outer_supplied.begin(),
9155 : outer_supplied.end(),
9156 : inner_supplied.begin(),
9157 : inner_supplied.end(),
9158 : std::inserter(intersection, intersection.end()));
9159 :
9160 18437 : if (!intersection.empty())
9161 : {
9162 5 : std::ostringstream oss;
9163 5 : oss << "The following material properties are declared on block " << it.first
9164 5 : << " by multiple materials:\n";
9165 10 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << "Material Property"
9166 5 : << "Material Objects\n";
9167 20 : for (const auto & outer_name : intersection)
9168 : {
9169 15 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << outer_name;
9170 45 : for (const auto & inner_name : prop_to_mat[outer_name])
9171 30 : oss << inner_name << " ";
9172 15 : oss << '\n';
9173 : }
9174 :
9175 5 : mooseError(oss.str());
9176 : break;
9177 0 : }
9178 18432 : }
9179 12456 : }
9180 59733 : }
9181 :
9182 : void
9183 59757 : FEProblemBase::checkCoordinateSystems()
9184 : {
9185 59757 : _mesh.checkCoordinateSystems();
9186 59754 : }
9187 :
9188 : void
9189 467 : FEProblemBase::setRestartFile(const std::string & file_name)
9190 : {
9191 467 : if (_app.isRecovering())
9192 : {
9193 23 : mooseInfo("Restart file ", file_name, " is NOT being used since we are performing recovery.");
9194 : }
9195 : else
9196 : {
9197 444 : _app.setRestart(true);
9198 444 : _app.setRestartRecoverFileBase(file_name);
9199 444 : mooseInfo("Using ", file_name, " for restart.");
9200 : }
9201 467 : }
9202 :
9203 : std::vector<VariableName>
9204 355433 : FEProblemBase::getVariableNames()
9205 : {
9206 355433 : std::vector<VariableName> names;
9207 :
9208 714978 : for (auto & sys : _solver_systems)
9209 : {
9210 359545 : const std::vector<VariableName> & var_names = sys->getVariableNames();
9211 359545 : names.insert(names.end(), var_names.begin(), var_names.end());
9212 : }
9213 :
9214 355433 : const std::vector<VariableName> & aux_var_names = _aux->getVariableNames();
9215 355433 : names.insert(names.end(), aux_var_names.begin(), aux_var_names.end());
9216 :
9217 355433 : return names;
9218 0 : }
9219 :
9220 : SolverParams &
9221 1610459 : FEProblemBase::solverParams(const unsigned int solver_sys_num)
9222 : {
9223 : mooseAssert(solver_sys_num < numSolverSystems(),
9224 : "Solver system number '" << solver_sys_num << "' is out of bounds. We have '"
9225 : << numSolverSystems() << "' solver systems");
9226 1610459 : return _solver_params[solver_sys_num];
9227 : }
9228 :
9229 : const SolverParams &
9230 14652 : FEProblemBase::solverParams(const unsigned int solver_sys_num) const
9231 : {
9232 14652 : return const_cast<FEProblemBase *>(this)->solverParams(solver_sys_num);
9233 : }
9234 :
9235 : void
9236 371 : FEProblemBase::registerRandomInterface(RandomInterface & random_interface, const std::string & name)
9237 : {
9238 371 : auto insert_pair = moose_try_emplace(
9239 371 : _random_data_objects, name, std::make_unique<RandomData>(*this, random_interface));
9240 :
9241 371 : auto random_data_ptr = insert_pair.first->second.get();
9242 371 : random_interface.setRandomDataPointer(random_data_ptr);
9243 371 : }
9244 :
9245 : bool
9246 1600432 : FEProblemBase::needBoundaryMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9247 : {
9248 1600432 : if (_bnd_mat_side_cache[tid].find(bnd_id) == _bnd_mat_side_cache[tid].end())
9249 : {
9250 29129 : auto & bnd_mat_side_cache = _bnd_mat_side_cache[tid][bnd_id];
9251 29129 : bnd_mat_side_cache = false;
9252 :
9253 : // Check systems
9254 29129 : if (_aux->needMaterialOnSide(bnd_id))
9255 : {
9256 531 : bnd_mat_side_cache = true;
9257 531 : return true;
9258 : }
9259 54915 : for (auto & nl : _nl)
9260 28440 : if (nl->needBoundaryMaterialOnSide(bnd_id, tid))
9261 : {
9262 2123 : bnd_mat_side_cache = true;
9263 2123 : return true;
9264 : }
9265 :
9266 : // TODO: these objects should be checked for whether they actually consume materials
9267 : // NOTE: InterfaceUO can use use boundary properties too
9268 26475 : if (theWarehouse()
9269 52950 : .query()
9270 26475 : .condition<AttribThread>(tid)
9271 26475 : .condition<AttribInterfaces>(Interfaces::SideUserObject | Interfaces::DomainUserObject |
9272 : Interfaces::InterfaceUserObject)
9273 26475 : .condition<AttribBoundaries>(bnd_id)
9274 26475 : .count() > 0)
9275 : {
9276 579 : bnd_mat_side_cache = true;
9277 579 : return true;
9278 : }
9279 : }
9280 :
9281 1597199 : return _bnd_mat_side_cache[tid][bnd_id];
9282 : }
9283 :
9284 : bool
9285 386533 : FEProblemBase::needInterfaceMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9286 : {
9287 386533 : if (_interface_mat_side_cache[tid].find(bnd_id) == _interface_mat_side_cache[tid].end())
9288 : {
9289 2812 : auto & interface_mat_side_cache = _interface_mat_side_cache[tid][bnd_id];
9290 2812 : interface_mat_side_cache = false;
9291 :
9292 : // Aux-system has not needed interface materials so far
9293 5394 : for (auto & nl : _nl)
9294 2812 : if (nl->needInterfaceMaterialOnSide(bnd_id, tid))
9295 : {
9296 230 : interface_mat_side_cache = true;
9297 230 : return true;
9298 : }
9299 :
9300 : // TODO: these objects should be checked for whether they actually consume materials
9301 2582 : if (theWarehouse()
9302 5164 : .query()
9303 2582 : .condition<AttribThread>(tid)
9304 2582 : .condition<AttribInterfaces>(Interfaces::InterfaceUserObject |
9305 : Interfaces::DomainUserObject)
9306 2582 : .condition<AttribBoundaries>(bnd_id)
9307 2582 : .count() > 0)
9308 : {
9309 79 : interface_mat_side_cache = true;
9310 79 : return true;
9311 : }
9312 2503 : else if (_interface_materials.hasActiveBoundaryObjects(bnd_id, tid))
9313 : {
9314 10 : interface_mat_side_cache = true;
9315 10 : return true;
9316 : }
9317 : }
9318 386214 : return _interface_mat_side_cache[tid][bnd_id];
9319 : }
9320 :
9321 : bool
9322 423045 : FEProblemBase::needInternalNeighborSideMaterial(SubdomainID subdomain_id, const THREAD_ID tid)
9323 : {
9324 423045 : if (_block_mat_side_cache[tid].find(subdomain_id) == _block_mat_side_cache[tid].end())
9325 : {
9326 12764 : _block_mat_side_cache[tid][subdomain_id] = false;
9327 :
9328 25047 : for (auto & nl : _nl)
9329 12723 : if (nl->needInternalNeighborSideMaterial(subdomain_id, tid))
9330 : {
9331 440 : _block_mat_side_cache[tid][subdomain_id] = true;
9332 440 : return true;
9333 : }
9334 :
9335 : // TODO: these objects should be checked for whether they actually consume materials
9336 12324 : if (theWarehouse()
9337 24648 : .query()
9338 12324 : .condition<AttribThread>(tid)
9339 12324 : .condition<AttribInterfaces>(Interfaces::InternalSideUserObject |
9340 : Interfaces::DomainUserObject)
9341 12324 : .condition<AttribSubdomains>(subdomain_id)
9342 12324 : .count() > 0)
9343 : {
9344 35 : _block_mat_side_cache[tid][subdomain_id] = true;
9345 35 : return true;
9346 : }
9347 : }
9348 :
9349 422570 : return _block_mat_side_cache[tid][subdomain_id];
9350 : }
9351 :
9352 : bool
9353 287312 : FEProblemBase::needsPreviousNewtonIteration() const
9354 : {
9355 287312 : return vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG);
9356 : }
9357 :
9358 : void
9359 76 : FEProblemBase::needsPreviousNewtonIteration(bool state)
9360 : {
9361 76 : if (state && !vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
9362 0 : mooseError("Previous nonlinear solution is required but not added through "
9363 : "Problem/previous_nl_solution_required=true");
9364 76 : }
9365 :
9366 : void
9367 52 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(bool needed,
9368 : const unsigned int solver_sys_num)
9369 : {
9370 52 : _previous_multiapp_fp_nl_solution_required[solver_sys_num] = needed;
9371 52 : }
9372 :
9373 : bool
9374 57782 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(
9375 : const unsigned int solver_sys_num) const
9376 : {
9377 57782 : return _previous_multiapp_fp_nl_solution_required[solver_sys_num];
9378 : }
9379 :
9380 : void
9381 13 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary(bool state)
9382 : {
9383 13 : _previous_multiapp_fp_aux_solution_required = state;
9384 13 : }
9385 :
9386 : bool
9387 57646 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary() const
9388 : {
9389 57646 : return _previous_multiapp_fp_aux_solution_required;
9390 : }
9391 :
9392 : bool
9393 7563758 : FEProblemBase::hasJacobian() const
9394 : {
9395 7563758 : return _has_jacobian;
9396 : }
9397 :
9398 : bool
9399 7079230 : FEProblemBase::constJacobian() const
9400 : {
9401 7079230 : return _const_jacobian;
9402 : }
9403 :
9404 : void
9405 285085 : FEProblemBase::addOutput(const std::string & object_type,
9406 : const std::string & object_name,
9407 : InputParameters & parameters)
9408 : {
9409 : parallel_object_only();
9410 :
9411 : // Get a reference to the OutputWarehouse
9412 285085 : OutputWarehouse & output_warehouse = _app.getOutputWarehouse();
9413 :
9414 : // Reject the reserved names for objects not built by MOOSE
9415 285085 : if (!parameters.get<bool>("_built_by_moose") && output_warehouse.isReservedName(object_name))
9416 6 : mooseError("The name '", object_name, "' is a reserved name for output objects");
9417 :
9418 : // Check that an object by the same name does not already exist; this must be done before the
9419 : // object is created to avoid getting misleading errors from the Parser
9420 285079 : if (output_warehouse.hasOutput(object_name))
9421 3 : mooseError("An output object named '", object_name, "' already exists");
9422 :
9423 : // Add a pointer to the FEProblemBase class
9424 570152 : parameters.addPrivateParam<FEProblemBase *>("_fe_problem_base", this);
9425 :
9426 : // --show-input should enable the display of the input file on the screen
9427 690891 : if (object_type == "Console" && _app.getParam<bool>("show_input") &&
9428 285103 : parameters.get<bool>("output_screen"))
9429 54 : parameters.set<ExecFlagEnum>("execute_input_on") = EXEC_INITIAL;
9430 :
9431 : // Apply only user-set parameters from the common [Outputs] block so that
9432 : // each output type's own defaults are not overridden by common defaults.
9433 285076 : const InputParameters * common = output_warehouse.getCommonParameters();
9434 285076 : if (common)
9435 285076 : parameters.applyCommonUserSetParameters(*common);
9436 :
9437 : // Set the correct value for the binary flag for XDA/XDR output
9438 285076 : if (object_type == "XDR")
9439 120 : parameters.set<bool>("_binary") = true;
9440 285016 : else if (object_type == "XDA")
9441 244 : parameters.set<bool>("_binary") = false;
9442 :
9443 : // Adjust the checkpoint suffix if auto recovery was enabled
9444 285076 : if (object_name == "auto_recovery_checkpoint")
9445 0 : parameters.set<std::string>("suffix") = "auto_recovery";
9446 :
9447 : // Create the object and add it to the warehouse
9448 285076 : std::shared_ptr<Output> output = _factory.create<Output>(object_type, object_name, parameters);
9449 285064 : logAdd("Output", object_name, object_type, parameters);
9450 285064 : output_warehouse.addOutput(output);
9451 285064 : }
9452 :
9453 : void
9454 22785 : FEProblemBase::haveADObjects(const bool have_ad_objects)
9455 : {
9456 22785 : _have_ad_objects = have_ad_objects;
9457 22785 : if (_displaced_problem)
9458 227 : _displaced_problem->SubProblem::haveADObjects(have_ad_objects);
9459 22785 : }
9460 :
9461 : const SystemBase &
9462 0 : FEProblemBase::getSystemBase(const unsigned int sys_num) const
9463 : {
9464 0 : if (sys_num < _solver_systems.size())
9465 0 : return *_solver_systems[sys_num];
9466 :
9467 0 : return *_aux;
9468 : }
9469 :
9470 : SystemBase &
9471 3414 : FEProblemBase::getSystemBase(const std::string & sys_name)
9472 : {
9473 3414 : if (std::find(_solver_sys_names.begin(), _solver_sys_names.end(), sys_name) !=
9474 6828 : _solver_sys_names.end())
9475 3414 : return getSystemBase(solverSysNum(sys_name));
9476 0 : else if (sys_name == "aux0")
9477 0 : return *_aux;
9478 : else
9479 0 : mooseError("System '" + sys_name + "' was requested from problem but does not exist.");
9480 : }
9481 :
9482 : SystemBase &
9483 5058 : FEProblemBase::getSystemBase(const unsigned int sys_num)
9484 : {
9485 5058 : if (sys_num < _solver_systems.size())
9486 4956 : return *_solver_systems[sys_num];
9487 :
9488 102 : return *_aux;
9489 : }
9490 :
9491 : const SystemBase &
9492 12422 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num) const
9493 : {
9494 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9495 12422 : return *_nl[sys_num];
9496 : }
9497 :
9498 : SystemBase &
9499 1256946 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num)
9500 : {
9501 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9502 1256946 : return *_nl[sys_num];
9503 : }
9504 :
9505 : const SystemBase &
9506 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num) const
9507 : {
9508 : mooseAssert(sys_num < _linear_systems.size(),
9509 : "System number greater than the number of linear systems");
9510 0 : return *_linear_systems[sys_num];
9511 : }
9512 :
9513 : SystemBase &
9514 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num)
9515 : {
9516 : mooseAssert(sys_num < _linear_systems.size(),
9517 : "System number greater than the number of linear systems");
9518 0 : return *_linear_systems[sys_num];
9519 : }
9520 :
9521 : const SystemBase &
9522 0 : FEProblemBase::systemBaseSolver(const unsigned int sys_num) const
9523 : {
9524 : mooseAssert(sys_num < _solver_systems.size(),
9525 : "System number greater than the number of solver systems");
9526 0 : return *_solver_systems[sys_num];
9527 : }
9528 :
9529 : SystemBase &
9530 6718696 : FEProblemBase::systemBaseSolver(const unsigned int sys_num)
9531 : {
9532 : mooseAssert(sys_num < _solver_systems.size(),
9533 : "System number greater than the number of solver systems");
9534 6718696 : return *_solver_systems[sys_num];
9535 : }
9536 :
9537 : const SystemBase &
9538 417 : FEProblemBase::systemBaseAuxiliary() const
9539 : {
9540 417 : return *_aux;
9541 : }
9542 :
9543 : SystemBase &
9544 8515614 : FEProblemBase::systemBaseAuxiliary()
9545 : {
9546 8515614 : return *_aux;
9547 : }
9548 :
9549 : void
9550 3883707 : FEProblemBase::computingNonlinearResid(bool computing_nonlinear_residual)
9551 : {
9552 : parallel_object_only();
9553 :
9554 3883707 : if (_displaced_problem)
9555 192294 : _displaced_problem->computingNonlinearResid(computing_nonlinear_residual);
9556 3883707 : _computing_nonlinear_residual = computing_nonlinear_residual;
9557 3883707 : }
9558 :
9559 : void
9560 9607894 : FEProblemBase::setCurrentlyComputingResidual(bool currently_computing_residual)
9561 : {
9562 9607894 : if (_displaced_problem)
9563 392534 : _displaced_problem->setCurrentlyComputingResidual(currently_computing_residual);
9564 9607894 : _currently_computing_residual = currently_computing_residual;
9565 9607894 : }
9566 :
9567 : void
9568 50 : FEProblemBase::uniformRefine()
9569 : {
9570 : // ResetDisplacedMeshThread::onNode looks up the reference mesh by ID, so we need to make sure
9571 : // we undisplace before adapting the reference mesh
9572 50 : if (_displaced_problem)
9573 34 : _displaced_problem->undisplaceMesh();
9574 :
9575 50 : Adaptivity::uniformRefine(&_mesh, 1);
9576 50 : if (_displaced_problem)
9577 34 : Adaptivity::uniformRefine(&_displaced_problem->mesh(), 1);
9578 :
9579 50 : meshChanged(
9580 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
9581 50 : }
9582 :
9583 : void
9584 59709 : FEProblemBase::automaticScaling(bool automatic_scaling)
9585 : {
9586 59709 : if (_displaced_problem)
9587 2022 : _displaced_problem->automaticScaling(automatic_scaling);
9588 :
9589 59709 : SubProblem::automaticScaling(automatic_scaling);
9590 59709 : }
9591 :
9592 : void
9593 254092 : FEProblemBase::reinitElemFaceRef(const Elem * elem,
9594 : unsigned int side,
9595 : Real tolerance,
9596 : const std::vector<Point> * const pts,
9597 : const std::vector<Real> * const weights,
9598 : const THREAD_ID tid)
9599 : {
9600 254092 : SubProblem::reinitElemFaceRef(elem, side, tolerance, pts, weights, tid);
9601 :
9602 254092 : if (_displaced_problem)
9603 20096 : _displaced_problem->reinitElemFaceRef(
9604 20096 : _displaced_mesh->elemPtr(elem->id()), side, tolerance, pts, weights, tid);
9605 254092 : }
9606 :
9607 : void
9608 254092 : FEProblemBase::reinitNeighborFaceRef(const Elem * neighbor_elem,
9609 : unsigned int neighbor_side,
9610 : Real tolerance,
9611 : const std::vector<Point> * const pts,
9612 : const std::vector<Real> * const weights,
9613 : const THREAD_ID tid)
9614 : {
9615 254092 : SubProblem::reinitNeighborFaceRef(neighbor_elem, neighbor_side, tolerance, pts, weights, tid);
9616 :
9617 254092 : if (_displaced_problem)
9618 20096 : _displaced_problem->reinitNeighborFaceRef(
9619 20096 : _displaced_mesh->elemPtr(neighbor_elem->id()), neighbor_side, tolerance, pts, weights, tid);
9620 254092 : }
9621 :
9622 : void
9623 3050121 : FEProblemBase::getFVMatsAndDependencies(
9624 : const SubdomainID blk_id,
9625 : std::vector<std::shared_ptr<MaterialBase>> & face_materials,
9626 : std::vector<std::shared_ptr<MaterialBase>> & neighbor_materials,
9627 : std::set<MooseVariableFieldBase *> & variables,
9628 : const THREAD_ID tid)
9629 : {
9630 3050121 : if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9631 : {
9632 : auto & this_face_mats =
9633 3560 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9634 7306 : for (std::shared_ptr<MaterialBase> face_mat : this_face_mats)
9635 3746 : if (face_mat->ghostable())
9636 : {
9637 3746 : face_materials.push_back(face_mat);
9638 3746 : auto & var_deps = face_mat->getMooseVariableDependencies();
9639 4118 : for (auto * var : var_deps)
9640 : {
9641 372 : if (!var->isFV())
9642 0 : mooseError(
9643 : "Ghostable materials should only have finite volume variables coupled into them.");
9644 372 : else if (face_mat->hasStatefulProperties())
9645 0 : mooseError("Finite volume materials do not currently support stateful properties.");
9646 372 : variables.insert(var);
9647 : }
9648 3746 : }
9649 : }
9650 :
9651 3050121 : if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9652 : {
9653 : auto & this_neighbor_mats =
9654 3560 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9655 7306 : for (std::shared_ptr<MaterialBase> neighbor_mat : this_neighbor_mats)
9656 3746 : if (neighbor_mat->ghostable())
9657 : {
9658 3746 : neighbor_materials.push_back(neighbor_mat);
9659 : #ifndef NDEBUG
9660 : auto & var_deps = neighbor_mat->getMooseVariableDependencies();
9661 : for (auto * var : var_deps)
9662 : {
9663 : if (!var->isFV())
9664 : mooseError(
9665 : "Ghostable materials should only have finite volume variables coupled into them.");
9666 : else if (neighbor_mat->hasStatefulProperties())
9667 : mooseError("Finite volume materials do not currently support stateful properties.");
9668 : auto pr = variables.insert(var);
9669 : mooseAssert(!pr.second,
9670 : "We should not have inserted any new variables dependencies from our "
9671 : "neighbor materials that didn't exist for our face materials");
9672 : }
9673 : #endif
9674 3746 : }
9675 : }
9676 3050121 : }
9677 :
9678 : void
9679 31850174 : FEProblemBase::resizeMaterialData(const Moose::MaterialDataType data_type,
9680 : const unsigned int nqp,
9681 : const THREAD_ID tid)
9682 : {
9683 31850174 : getMaterialData(data_type, tid).resize(nqp);
9684 31850174 : }
9685 :
9686 : void
9687 59649 : FEProblemBase::setNonlinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9688 : {
9689 59649 : if (convergence_names.size() != numNonlinearSystems())
9690 0 : paramError("nonlinear_convergence",
9691 : "There must be one convergence object per nonlinear system");
9692 59649 : _nonlinear_convergence_names = convergence_names;
9693 59649 : }
9694 :
9695 : void
9696 60989 : FEProblemBase::setMultiAppFixedPointConvergenceName(const ConvergenceName & convergence_name)
9697 : {
9698 60989 : _multiapp_fixed_point_convergence_name = convergence_name;
9699 60989 : }
9700 :
9701 : void
9702 29972 : FEProblemBase::setSteadyStateConvergenceName(const ConvergenceName & convergence_name)
9703 : {
9704 29972 : _steady_state_convergence_name = convergence_name;
9705 29972 : }
9706 :
9707 : const std::vector<ConvergenceName> &
9708 981950 : FEProblemBase::getNonlinearConvergenceNames() const
9709 : {
9710 981950 : if (_nonlinear_convergence_names)
9711 981950 : return *_nonlinear_convergence_names;
9712 0 : mooseError("The nonlinear system convergence name(s) have not been set.");
9713 : }
9714 :
9715 : bool
9716 26452 : FEProblemBase::hasLinearConvergenceObjects() const
9717 : {
9718 : // If false,this means we have not set one, not that we are querying this too early
9719 : // TODO: once there is a default linear CV object, error on the 'not set' case
9720 26452 : return _linear_convergence_names.has_value();
9721 : }
9722 :
9723 : void
9724 134 : FEProblemBase::setLinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9725 : {
9726 134 : if (convergence_names.size() != numLinearSystems())
9727 0 : paramError("linear_convergence", "There must be one convergence object per linear system");
9728 134 : _linear_convergence_names = convergence_names;
9729 134 : }
9730 :
9731 : const std::vector<ConvergenceName> &
9732 4635 : FEProblemBase::getLinearConvergenceNames() const
9733 : {
9734 4635 : if (_linear_convergence_names)
9735 4635 : return *_linear_convergence_names;
9736 0 : mooseError("The linear convergence name(s) have not been set.");
9737 : }
9738 :
9739 : const ConvergenceName &
9740 249067 : FEProblemBase::getMultiAppFixedPointConvergenceName() const
9741 : {
9742 249067 : if (_multiapp_fixed_point_convergence_name)
9743 249067 : return _multiapp_fixed_point_convergence_name.value();
9744 : else
9745 0 : mooseError("The fixed point convergence name has not been set.");
9746 : }
9747 :
9748 : const ConvergenceName &
9749 102009 : FEProblemBase::getSteadyStateConvergenceName() const
9750 : {
9751 102009 : if (_steady_state_convergence_name)
9752 102009 : return _steady_state_convergence_name.value();
9753 : else
9754 0 : mooseError("The steady convergence name has not been set.");
9755 : }
9756 :
9757 : void
9758 3046059 : FEProblemBase::residualSetup()
9759 : {
9760 3046059 : SubProblem::residualSetup();
9761 : // We need to setup all the nonlinear systems other than our current one which actually called
9762 : // this method (so we have to make sure we don't go in a circle)
9763 6177956 : for (const auto i : make_range(numNonlinearSystems()))
9764 3131897 : if (i != currentNlSysNum())
9765 85838 : _nl[i]->residualSetup();
9766 : // We don't setup the aux sys because that's been done elsewhere
9767 3046059 : if (_displaced_problem)
9768 124149 : _displaced_problem->residualSetup();
9769 3046059 : }
9770 :
9771 : void
9772 473071 : FEProblemBase::jacobianSetup()
9773 : {
9774 473071 : SubProblem::jacobianSetup();
9775 : // We need to setup all the nonlinear systems other than our current one which actually called
9776 : // this method (so we have to make sure we don't go in a circle)
9777 960900 : for (const auto i : make_range(numNonlinearSystems()))
9778 487829 : if (i != currentNlSysNum())
9779 14758 : _nl[i]->jacobianSetup();
9780 : // We don't setup the aux sys because that's been done elsewhere
9781 473071 : if (_displaced_problem)
9782 21144 : _displaced_problem->jacobianSetup();
9783 473071 : }
9784 :
9785 : MooseAppCoordTransform &
9786 95718 : FEProblemBase::coordTransform()
9787 : {
9788 95718 : return mesh().coordTransform();
9789 : }
9790 :
9791 : unsigned int
9792 476050202 : FEProblemBase::currentNlSysNum() const
9793 : {
9794 : // If we don't have nonlinear systems this should be an invalid number
9795 476050202 : unsigned int current_nl_sys_num = libMesh::invalid_uint;
9796 476050202 : if (_nl.size())
9797 476048090 : current_nl_sys_num = currentNonlinearSystem().number();
9798 :
9799 476050202 : return current_nl_sys_num;
9800 : }
9801 :
9802 : unsigned int
9803 0 : FEProblemBase::currentLinearSysNum() const
9804 : {
9805 : // If we don't have linear systems this should be an invalid number
9806 0 : unsigned int current_linear_sys_num = libMesh::invalid_uint;
9807 0 : if (_linear_systems.size())
9808 0 : current_linear_sys_num = currentLinearSystem().number();
9809 :
9810 0 : return current_linear_sys_num;
9811 : }
9812 :
9813 : bool
9814 124105040 : FEProblemBase::shouldPrintExecution(const THREAD_ID tid) const
9815 : {
9816 : // For now, only support printing from thread 0
9817 124105040 : if (tid != 0)
9818 8233345 : return false;
9819 :
9820 231524487 : if (_print_execution_on.isValueSet(_current_execute_on_flag) ||
9821 115652792 : _print_execution_on.isValueSet(EXEC_ALWAYS))
9822 311946 : return true;
9823 : else
9824 115559749 : return false;
9825 : }
9826 :
9827 : std::vector<MortarUserObject *>
9828 252918 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9829 : const BoundaryID secondary_boundary_id,
9830 : const bool displaced,
9831 : const std::vector<MortarUserObject *> & mortar_uo_superset)
9832 : {
9833 252918 : std::vector<MortarUserObject *> mortar_uos;
9834 252918 : auto * const subproblem = displaced ? static_cast<SubProblem *>(_displaced_problem.get())
9835 252918 : : static_cast<SubProblem *>(this);
9836 252940 : for (auto * const obj : mortar_uo_superset)
9837 44 : if (obj->onInterface(primary_boundary_id, secondary_boundary_id) &&
9838 22 : (&obj->getSubProblem() == subproblem))
9839 22 : mortar_uos.push_back(obj);
9840 :
9841 252918 : return mortar_uos;
9842 0 : }
9843 :
9844 : std::vector<MortarUserObject *>
9845 252896 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9846 : const BoundaryID secondary_boundary_id,
9847 : const bool displaced)
9848 : {
9849 252896 : std::vector<MortarUserObject *> mortar_uos;
9850 252896 : theWarehouse()
9851 252896 : .query()
9852 505792 : .condition<AttribInterfaces>(Interfaces::MortarUserObject)
9853 252896 : .queryInto(mortar_uos);
9854 505792 : return getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced, mortar_uos);
9855 252896 : }
9856 :
9857 : void
9858 252896 : FEProblemBase::reinitMortarUserObjects(const BoundaryID primary_boundary_id,
9859 : const BoundaryID secondary_boundary_id,
9860 : const bool displaced)
9861 : {
9862 : const auto mortar_uos =
9863 252896 : getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced);
9864 252896 : for (auto * const mortar_uo : mortar_uos)
9865 : {
9866 0 : mortar_uo->setNormals();
9867 0 : mortar_uo->reinit();
9868 : }
9869 252896 : }
9870 :
9871 : void
9872 1046 : FEProblemBase::setVerboseProblem(bool verbose)
9873 : {
9874 1046 : _verbose_setup = verbose ? "true" : "false";
9875 1046 : _verbose_multiapps = verbose;
9876 1046 : _verbose_restore = verbose;
9877 1046 : }
9878 :
9879 : void
9880 111834 : FEProblemBase::setCurrentLowerDElem(const Elem * const lower_d_elem, const THREAD_ID tid)
9881 : {
9882 111834 : SubProblem::setCurrentLowerDElem(lower_d_elem, tid);
9883 111834 : if (_displaced_problem)
9884 27619 : _displaced_problem->setCurrentLowerDElem(
9885 0 : lower_d_elem ? _displaced_mesh->elemPtr(lower_d_elem->id()) : nullptr, tid);
9886 111834 : }
9887 :
9888 : void
9889 121384622 : FEProblemBase::setCurrentBoundaryID(BoundaryID bid, const THREAD_ID tid)
9890 : {
9891 121384622 : SubProblem::setCurrentBoundaryID(bid, tid);
9892 121384622 : if (_displaced_problem)
9893 8992867 : _displaced_problem->setCurrentBoundaryID(bid, tid);
9894 121384622 : }
9895 :
9896 : void
9897 7391859 : FEProblemBase::setCurrentNonlinearSystem(const unsigned int nl_sys_num)
9898 : {
9899 : mooseAssert(nl_sys_num < _nl.size(),
9900 : "System number greater than the number of nonlinear systems");
9901 7391859 : _current_nl_sys = _nl[nl_sys_num].get();
9902 7391859 : _current_solver_sys = _current_nl_sys;
9903 7391859 : }
9904 :
9905 : void
9906 78406 : FEProblemBase::setCurrentLinearSystem(const unsigned int sys_num)
9907 : {
9908 : mooseAssert(sys_num < _linear_systems.size(),
9909 : "System number greater than the number of linear systems");
9910 78406 : _current_linear_sys = _linear_systems[sys_num].get();
9911 78406 : _current_solver_sys = _current_linear_sys;
9912 78406 : }
9913 :
9914 : void
9915 5820643 : FEProblemBase::computeSystems(const ExecFlagType & type)
9916 : {
9917 : // When performing an adjoint solve in the optimization module, the current solver system is the
9918 : // adjoint. However, the adjoint solve requires having accurate time derivative calculations for
9919 : // the forward system. The cleanest way to handle such uses is just to compute the time
9920 : // derivatives for all solver systems instead of trying to guess which ones we need and don't need
9921 11784839 : for (auto & solver_sys : _solver_systems)
9922 5964196 : solver_sys->compute(type);
9923 :
9924 5820643 : _aux->compute(type);
9925 5820610 : }
9926 :
9927 : const ConstElemRange &
9928 3923726 : FEProblemBase::getCurrentAlgebraicElementRange()
9929 : {
9930 3923726 : if (!_current_algebraic_elem_range)
9931 3923726 : return *_mesh.getActiveLocalElementRange();
9932 :
9933 0 : return *_current_algebraic_elem_range;
9934 : }
9935 : const ConstNodeRange &
9936 95467 : FEProblemBase::getCurrentAlgebraicNodeRange()
9937 : {
9938 95467 : if (!_current_algebraic_node_range)
9939 95467 : return *_mesh.getLocalNodeRange();
9940 :
9941 0 : return *_current_algebraic_node_range;
9942 : }
9943 : const ConstBndNodeRange &
9944 3474944 : FEProblemBase::getCurrentAlgebraicBndNodeRange()
9945 : {
9946 3474944 : if (!_current_algebraic_bnd_node_range)
9947 3474944 : return *_mesh.getBoundaryNodeRange();
9948 :
9949 0 : return *_current_algebraic_bnd_node_range;
9950 : }
9951 :
9952 : void
9953 0 : FEProblemBase::setCurrentAlgebraicElementRange(ConstElemRange * range)
9954 : {
9955 0 : if (!range)
9956 : {
9957 0 : _current_algebraic_elem_range = nullptr;
9958 0 : return;
9959 : }
9960 :
9961 0 : _current_algebraic_elem_range = std::make_unique<ConstElemRange>(*range);
9962 : }
9963 : void
9964 0 : FEProblemBase::setCurrentAlgebraicNodeRange(ConstNodeRange * range)
9965 : {
9966 0 : if (!range)
9967 : {
9968 0 : _current_algebraic_node_range = nullptr;
9969 0 : return;
9970 : }
9971 :
9972 0 : _current_algebraic_node_range = std::make_unique<ConstNodeRange>(*range);
9973 : }
9974 : void
9975 0 : FEProblemBase::setCurrentAlgebraicBndNodeRange(ConstBndNodeRange * range)
9976 : {
9977 0 : if (!range)
9978 : {
9979 0 : _current_algebraic_bnd_node_range = nullptr;
9980 0 : return;
9981 : }
9982 :
9983 0 : _current_algebraic_bnd_node_range = std::make_unique<ConstBndNodeRange>(*range);
9984 : }
9985 :
9986 : unsigned short
9987 63073 : FEProblemBase::getCurrentICState()
9988 : {
9989 63073 : return _current_ic_state;
9990 : }
9991 :
9992 : std::string
9993 53033 : FEProblemBase::solverTypeString(const unsigned int solver_sys_num)
9994 : {
9995 53033 : return Moose::stringify(solverParams(solver_sys_num)._type);
9996 : }
9997 :
9998 : SolverParams
9999 1032 : FEProblemBase::makeLinearSolverParams()
10000 : {
10001 1032 : SolverParams solver_params;
10002 1032 : solver_params._type = Moose::SolveType::ST_LINEAR;
10003 1032 : solver_params._line_search = Moose::LineSearchType::LS_NONE;
10004 1032 : return solver_params;
10005 : }
10006 :
10007 : const libMesh::CouplingMatrix &
10008 70972 : FEProblemBase::nonlocalCouplingMatrix(const unsigned i) const
10009 : {
10010 70972 : return _nonlocal_cm[i];
10011 : }
10012 :
10013 : bool
10014 90477388 : FEProblemBase::checkNonlocalCouplingRequirement() const
10015 : {
10016 90477388 : return _requires_nonlocal_coupling;
10017 : }
10018 :
10019 : const std::unordered_map<std::pair<BoundaryID, BoundaryID>,
10020 : std::unique_ptr<AutomaticMortarGeneration>> &
10021 116833 : FEProblemBase::getMortarInterfaces(bool on_displaced) const
10022 : {
10023 116833 : return _mortar_data->getMortarInterfaces(on_displaced);
10024 : }
|
