Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #ifdef MOOSE_KOKKOS_ENABLED
11 : #include "KokkosMaterialPropertyStorage.h"
12 : #endif
13 :
14 : #include "FEProblemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialPropertyStorage.h"
17 : #include "MooseEnum.h"
18 : #include "Factory.h"
19 : #include "MooseUtils.h"
20 : #include "DisplacedProblem.h"
21 : #include "SystemBase.h"
22 : #include "MaterialData.h"
23 : #include "ComputeUserObjectsThread.h"
24 : #include "ComputeNodalUserObjectsThread.h"
25 : #include "ComputeThreadedGeneralUserObjectsThread.h"
26 : #include "ComputeMaterialsObjectThread.h"
27 : #include "ProjectMaterialProperties.h"
28 : #include "ComputeIndicatorThread.h"
29 : #include "ComputeMarkerThread.h"
30 : #include "ComputeInitialConditionThread.h"
31 : #include "ComputeFVInitialConditionThread.h"
32 : #include "ComputeBoundaryInitialConditionThread.h"
33 : #include "MaxQpsThread.h"
34 : #include "ActionWarehouse.h"
35 : #include "Conversion.h"
36 : #include "Material.h"
37 : #include "FunctorMaterial.h"
38 : #include "ConstantIC.h"
39 : #include "Parser.h"
40 : #include "ElementH1Error.h"
41 : #include "Function.h"
42 : #include "Convergence.h"
43 : #include "NonlinearSystem.h"
44 : #include "LinearSystem.h"
45 : #include "SolverSystem.h"
46 : #include "Distribution.h"
47 : #include "Sampler.h"
48 : #include "FVAdvectedInterpolationMethod.h"
49 : #include "FVFaceInterpolationMethod.h"
50 : #include "FVInterpolationMethod.h"
51 : #include "PetscSupport.h"
52 : #include "RandomInterface.h"
53 : #include "RandomData.h"
54 : #include "MooseEigenSystem.h"
55 : #include "MooseParsedFunction.h"
56 : #include "MeshChangedInterface.h"
57 : #include "MeshDisplacedInterface.h"
58 : #include "ComputeJacobianBlocksThread.h"
59 : #include "ScalarInitialCondition.h"
60 : #include "FVInitialConditionTempl.h"
61 : #include "ElementPostprocessor.h"
62 : #include "NodalPostprocessor.h"
63 : #include "SidePostprocessor.h"
64 : #include "InternalSidePostprocessor.h"
65 : #include "InterfacePostprocessor.h"
66 : #include "GeneralPostprocessor.h"
67 : #include "ElementVectorPostprocessor.h"
68 : #include "NodalVectorPostprocessor.h"
69 : #include "SideVectorPostprocessor.h"
70 : #include "InternalSideVectorPostprocessor.h"
71 : #include "GeneralVectorPostprocessor.h"
72 : #include "Positions.h"
73 : #include "Indicator.h"
74 : #include "Marker.h"
75 : #include "MultiApp.h"
76 : #include "MultiAppTransfer.h"
77 : #include "TransientMultiApp.h"
78 : #include "ElementUserObject.h"
79 : #include "DomainUserObject.h"
80 : #include "NodalUserObject.h"
81 : #include "SideUserObject.h"
82 : #include "InternalSideUserObject.h"
83 : #include "InterfaceUserObject.h"
84 : #include "GeneralUserObject.h"
85 : #include "ThreadedGeneralUserObject.h"
86 : #include "InternalSideIndicatorBase.h"
87 : #include "Transfer.h"
88 : #include "MultiAppTransfer.h"
89 : #include "MultiMooseEnum.h"
90 : #include "Predictor.h"
91 : #include "Assembly.h"
92 : #include "Control.h"
93 : #include "XFEMInterface.h"
94 : #include "ConsoleUtils.h"
95 : #include "NonlocalKernel.h"
96 : #include "NonlocalIntegratedBC.h"
97 : #include "ShapeElementUserObject.h"
98 : #include "ShapeSideUserObject.h"
99 : #include "MooseVariableFE.h"
100 : #include "MooseVariableScalar.h"
101 : #include "InputParameterWarehouse.h"
102 : #include "TimeIntegrator.h"
103 : #include "LineSearch.h"
104 : #include "FloatingPointExceptionGuard.h"
105 : #include "MaxVarNDofsPerElem.h"
106 : #include "MaxVarNDofsPerNode.h"
107 : #include "FVKernel.h"
108 : #include "LinearFVKernel.h"
109 : #include "FVTimeKernel.h"
110 : #include "MooseVariableFV.h"
111 : #include "MooseLinearVariableFV.h"
112 : #include "FVBoundaryCondition.h"
113 : #include "LinearFVBoundaryCondition.h"
114 : #include "FVInterfaceKernel.h"
115 : #include "Reporter.h"
116 : #include "ADUtils.h"
117 : #include "Executioner.h"
118 : #include "VariadicTable.h"
119 : #include "BoundaryNodeIntegrityCheckThread.h"
120 : #include "BoundaryElemIntegrityCheckThread.h"
121 : #include "NodalBCBase.h"
122 : #include "MortarUserObject.h"
123 : #include "MortarUserObjectThread.h"
124 : #include "RedistributeProperties.h"
125 : #include "Checkpoint.h"
126 : #include "MortarInterfaceWarehouse.h"
127 : #include "AutomaticMortarGeneration.h"
128 :
129 : #include "libmesh/exodusII_io.h"
130 : #include "libmesh/quadrature.h"
131 : #include "libmesh/coupling_matrix.h"
132 : #include "libmesh/nonlinear_solver.h"
133 : #include "libmesh/sparse_matrix.h"
134 : #include "libmesh/string_to_enum.h"
135 : #include "libmesh/fe_interface.h"
136 : #include "libmesh/enum_norm_type.h"
137 : #include "libmesh/petsc_solver_exception.h"
138 :
139 : #include "metaphysicl/dualnumber.h"
140 :
141 : // C++
142 : #include <cstring> // for "Jacobian" exception test
143 :
144 : using namespace libMesh;
145 :
146 : // Anonymous namespace for helper function
147 : namespace
148 : {
149 : /**
150 : * Method for sorting the MooseVariableFEBases based on variable numbers
151 : */
152 : bool
153 61 : sortMooseVariables(const MooseVariableFEBase * a, const MooseVariableFEBase * b)
154 : {
155 61 : return a->number() < b->number();
156 : }
157 : } // namespace
158 :
159 : Threads::spin_mutex get_function_mutex;
160 :
161 : InputParameters
162 193988 : FEProblemBase::validParams()
163 : {
164 193988 : InputParameters params = SubProblem::validParams();
165 775952 : params.addParam<unsigned int>("null_space_dimension", 0, "The dimension of the nullspace");
166 581964 : params.addParam<unsigned int>(
167 387976 : "transpose_null_space_dimension", 0, "The dimension of the transpose nullspace");
168 581964 : params.addParam<unsigned int>(
169 387976 : "near_null_space_dimension", 0, "The dimension of the near nullspace");
170 581964 : params.addParam<bool>("solve",
171 387976 : true,
172 : "Whether or not to actually solve the Nonlinear system. "
173 : "This is handy in the case that all you want to do is "
174 : "execute AuxKernels, Transfers, etc. without actually "
175 : "solving anything");
176 581964 : params.addParam<bool>("use_nonlinear",
177 387976 : true,
178 : "Determines whether to use a Nonlinear vs a "
179 : "Eigenvalue system (Automatically determined based "
180 : "on executioner)");
181 775952 : params.addParam<bool>("error_on_jacobian_nonzero_reallocation",
182 : "This causes PETSc to error if it had to reallocate memory in the Jacobian "
183 : "matrix due to not having enough nonzeros");
184 581964 : params.addParam<bool>("ignore_zeros_in_jacobian",
185 387976 : false,
186 : "Do not explicitly store zero values in "
187 : "the Jacobian matrix if true");
188 581964 : params.addParam<bool>("force_restart",
189 387976 : false,
190 : "EXPERIMENTAL: If true, a sub_app may use a "
191 : "restart file instead of using of using the master "
192 : "backup file");
193 969940 : params.addDeprecatedParam<bool>("skip_additional_restart_data",
194 387976 : false,
195 : "True to skip additional data in equation system for restart.",
196 : "This parameter is no longer used, as we do not load additional "
197 : "vectors by default with restart");
198 581964 : params.addParam<bool>("skip_nl_system_check",
199 387976 : false,
200 : "True to skip the NonlinearSystem check for work to do (e.g. Make sure "
201 : "that there are variables to solve for).");
202 581964 : params.addParam<bool>("allow_initial_conditions_with_restart",
203 387976 : false,
204 : "True to allow the user to specify initial conditions when restarting. "
205 : "Initial conditions can override any restarted field");
206 :
207 387976 : auto coverage_check_description = [](std::string scope, std::string list_param_name)
208 : {
209 775952 : return "Controls, if and how a " + scope +
210 : " subdomain coverage check is performed. "
211 : "With 'TRUE' or 'ON' all subdomains are checked (the default). Setting 'FALSE' or 'OFF' "
212 : "will disable the check for all subdomains. "
213 : "To exclude a predefined set of subdomains 'SKIP_LIST' is to "
214 775952 : "be used, while the subdomains to skip are to be defined in the parameter '" +
215 775952 : list_param_name +
216 : "'. To limit the check to a list of subdomains, 'ONLY_LIST' is to "
217 1163928 : "be used (again, using the parameter '" +
218 775952 : list_param_name + "').";
219 : };
220 :
221 969940 : params.addParam<std::vector<SubdomainName>>(
222 : "block",
223 : {"ANY_BLOCK_ID"},
224 : "List of subdomains for kernel coverage and material coverage checks. Setting this parameter "
225 : "is equivalent to setting 'kernel_coverage_block_list' and 'material_coverage_block_list' as "
226 : "well as using 'ONLY_LIST' as the coverage check mode.");
227 :
228 775952 : MooseEnum kernel_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
229 193988 : params.addParam<MooseEnum>("kernel_coverage_check",
230 : kernel_coverage_check_modes,
231 969940 : coverage_check_description("kernel", "kernel_coverage_block_list"));
232 775952 : params.addParam<std::vector<SubdomainName>>(
233 : "kernel_coverage_block_list",
234 : {},
235 : "List of subdomains for kernel coverage check. The meaning of this list is controlled by the "
236 : "parameter 'kernel_coverage_check' (whether this is the list of subdomains to be checked, "
237 : "not to be checked or not taken into account).");
238 581964 : params.addParam<bool>(
239 : "boundary_restricted_node_integrity_check",
240 387976 : true,
241 : "Set to false to disable checking of boundary restricted nodal object variable dependencies, "
242 : "e.g. are the variable dependencies defined on the selected boundaries?");
243 581964 : params.addParam<bool>("boundary_restricted_elem_integrity_check",
244 387976 : true,
245 : "Set to false to disable checking of boundary restricted elemental object "
246 : "variable dependencies, e.g. are the variable dependencies defined on the "
247 : "selected boundaries?");
248 775952 : MooseEnum material_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
249 193988 : params.addParam<MooseEnum>(
250 : "material_coverage_check",
251 : material_coverage_check_modes,
252 969940 : coverage_check_description("material", "material_coverage_block_list"));
253 775952 : params.addParam<std::vector<SubdomainName>>(
254 : "material_coverage_block_list",
255 : {},
256 : "List of subdomains for material coverage check. The meaning of this list is controlled by "
257 : "the parameter 'material_coverage_check' (whether this is the list of subdomains to be "
258 : "checked, not to be checked or not taken into account).");
259 :
260 581964 : params.addParam<bool>("fv_bcs_integrity_check",
261 387976 : true,
262 : "Set to false to disable checking of overlapping Dirichlet and Flux BCs "
263 : "and/or multiple DirichletBCs per sideset");
264 :
265 581964 : params.addParam<bool>(
266 387976 : "material_dependency_check", true, "Set to false to disable material dependency check");
267 581964 : params.addParam<bool>("parallel_barrier_messaging",
268 387976 : false,
269 : "Displays messaging from parallel "
270 : "barrier notifications when executing "
271 : "or transferring to/from Multiapps "
272 : "(default: false)");
273 :
274 775952 : MooseEnum verbosity("false true extra", "false");
275 775952 : params.addParam<MooseEnum>("verbose_setup",
276 : verbosity,
277 : "Set to 'true' to have the problem report on any object created. Set "
278 : "to 'extra' to also display all parameters.");
279 581964 : params.addParam<bool>("verbose_multiapps",
280 387976 : false,
281 : "Set to True to enable verbose screen printing related to MultiApps");
282 581964 : params.addParam<bool>(
283 : "verbose_restore",
284 387976 : false,
285 : "Set to True to enable verbose screen printing related to solution restoration");
286 :
287 775952 : params.addParam<FileNameNoExtension>("restart_file_base",
288 : "File base name used for restart (e.g. "
289 : "<path>/<filebase> or <path>/LATEST to "
290 : "grab the latest file available)");
291 :
292 775952 : params.addParam<std::vector<std::vector<TagName>>>(
293 : "extra_tag_vectors",
294 : {},
295 : "Extra vectors to add to the system that can be filled by objects which compute residuals "
296 : "and Jacobians (Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
297 : "nonlinear system the extra tag vectors should be added for");
298 :
299 775952 : params.addParam<std::vector<std::vector<TagName>>>(
300 : "not_zeroed_tag_vectors",
301 : {},
302 : "Extra vector tags which the sytem will not zero when other vector tags are zeroed. "
303 : "The outer index is for which nonlinear system the extra tag vectors should be added for");
304 :
305 775952 : params.addParam<std::vector<std::vector<TagName>>>(
306 : "extra_tag_matrices",
307 : {},
308 : "Extra matrices to add to the system that can be filled "
309 : "by objects which compute residuals and Jacobians "
310 : "(Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
311 : "nonlinear system the extra tag vectors should be added for");
312 :
313 775952 : params.addParam<std::vector<TagName>>(
314 : "extra_tag_solutions",
315 : {},
316 : "Extra solution vectors to add to the system that can be used by "
317 : "objects for coupling variable values stored in them.");
318 :
319 581964 : params.addParam<bool>("previous_nl_solution_required",
320 387976 : false,
321 : "True to indicate that this calculation requires a solution vector for "
322 : "storing the previous nonlinear iteration.");
323 :
324 581964 : params.addParam<std::vector<NonlinearSystemName>>(
325 775952 : "nl_sys_names", std::vector<NonlinearSystemName>{"nl0"}, "The nonlinear system names");
326 :
327 775952 : params.addParam<std::vector<LinearSystemName>>("linear_sys_names", {}, "The linear system names");
328 :
329 581964 : params.addParam<bool>("check_uo_aux_state",
330 387976 : false,
331 : "True to turn on a check that no state presents during the evaluation of "
332 : "user objects and aux kernels");
333 :
334 193988 : params.addPrivateParam<MooseMesh *>("mesh");
335 :
336 581964 : params.declareControllable("solve");
337 :
338 581964 : params.addParam<bool>(
339 : "allow_invalid_solution",
340 387976 : false,
341 : "Set to true to allow convergence even though the solution has been marked as 'invalid'");
342 581964 : params.addParam<bool>("show_invalid_solution_console",
343 387976 : true,
344 : "Set to true to show the invalid solution occurrence summary in console");
345 581964 : params.addParam<bool>("immediately_print_invalid_solution",
346 387976 : false,
347 : "Whether or not to report invalid solution warnings at the time the "
348 : "warning is produced instead of after the calculation");
349 :
350 581964 : params.addParam<bool>(
351 : "identify_variable_groups_in_nl",
352 387976 : true,
353 : "Whether to identify variable groups in nonlinear systems. This affects dof ordering");
354 :
355 581964 : params.addParam<bool>(
356 : "regard_general_exceptions_as_errors",
357 387976 : false,
358 : "If we catch an exception during residual/Jacobian evaluaton for which we don't have "
359 : "specific handling, immediately error instead of allowing the time step to be cut");
360 :
361 581964 : params.addParam<bool>("use_hash_table_matrix_assembly",
362 387976 : false,
363 : "Whether to assemble matrices using hash tables instead of preallocating "
364 : "matrix memory. This can be a good option if the sparsity pattern changes "
365 : "throughout the course of the simulation.");
366 775952 : params.addParam<bool>(
367 : "restore_original_nonzero_pattern",
368 : "Whether we should reset matrix memory for every Jacobian evaluation. This option is useful "
369 : "if the sparsity pattern is constantly changing and you are using hash table assembly or if "
370 : "you wish to continually restore the matrix to the originally preallocated sparsity pattern "
371 : "computed by relationship managers.");
372 :
373 775952 : params.addParamNamesToGroup(
374 : "skip_nl_system_check kernel_coverage_check kernel_coverage_block_list "
375 : "boundary_restricted_node_integrity_check "
376 : "boundary_restricted_elem_integrity_check material_coverage_check "
377 : "material_coverage_block_list fv_bcs_integrity_check "
378 : "material_dependency_check check_uo_aux_state error_on_jacobian_nonzero_reallocation",
379 : "Simulation checks");
380 775952 : params.addParamNamesToGroup("use_nonlinear previous_nl_solution_required nl_sys_names "
381 : "ignore_zeros_in_jacobian identify_variable_groups_in_nl "
382 : "use_hash_table_matrix_assembly restore_original_nonzero_pattern",
383 : "Nonlinear system(s)");
384 775952 : params.addParamNamesToGroup(
385 : "restart_file_base force_restart allow_initial_conditions_with_restart", "Restart");
386 775952 : params.addParamNamesToGroup(
387 : "verbose_setup verbose_multiapps verbose_restore parallel_barrier_messaging", "Verbosity");
388 775952 : params.addParamNamesToGroup(
389 : "null_space_dimension transpose_null_space_dimension near_null_space_dimension",
390 : "Null space removal");
391 775952 : params.addParamNamesToGroup(
392 : "extra_tag_vectors extra_tag_matrices extra_tag_solutions not_zeroed_tag_vectors",
393 : "Contribution to tagged field data");
394 581964 : params.addParamNamesToGroup(
395 : "allow_invalid_solution show_invalid_solution_console immediately_print_invalid_solution",
396 : "Solution validity control");
397 :
398 387976 : return params;
399 581964 : }
400 :
401 63000 : FEProblemBase::FEProblemBase(const InputParameters & parameters)
402 : : SubProblem(parameters),
403 : Restartable(this, "FEProblemBase"),
404 252000 : _mesh(*getCheckedPointerParam<MooseMesh *>("mesh")),
405 126000 : _req(declareManagedRestartableDataWithContext<RestartableEquationSystems>(
406 : "equation_systems", nullptr, _mesh)),
407 63000 : _initialized(false),
408 126000 : _solve(getParam<bool>("solve")),
409 63000 : _transient(false),
410 126000 : _time(declareRestartableData<Real>("time")),
411 126000 : _time_old(declareRestartableData<Real>("time_old")),
412 126000 : _t_step(declareRecoverableData<int>("t_step")),
413 126000 : _dt(declareRestartableData<Real>("dt")),
414 126000 : _dt_old(declareRestartableData<Real>("dt_old")),
415 63000 : _need_to_add_default_nonlinear_convergence(false),
416 63000 : _need_to_add_default_multiapp_fixed_point_convergence(false),
417 63000 : _need_to_add_default_steady_state_convergence(false),
418 126000 : _linear_sys_names(getParam<std::vector<LinearSystemName>>("linear_sys_names")),
419 63000 : _num_linear_sys(_linear_sys_names.size()),
420 126000 : _linear_systems(_num_linear_sys, nullptr),
421 63000 : _current_linear_sys(nullptr),
422 126000 : _using_default_nl(!isParamSetByUser("nl_sys_names")),
423 186678 : _nl_sys_names(!_using_default_nl || (_using_default_nl && !_linear_sys_names.size())
424 63000 : ? getParam<std::vector<NonlinearSystemName>>("nl_sys_names")
425 : : std::vector<NonlinearSystemName>()),
426 63000 : _num_nl_sys(_nl_sys_names.size()),
427 126000 : _nl(_num_nl_sys, nullptr),
428 63000 : _current_nl_sys(nullptr),
429 126000 : _solver_systems(_num_nl_sys + _num_linear_sys, nullptr),
430 63000 : _aux(nullptr),
431 63000 : _coupling(Moose::COUPLING_DIAG),
432 : #ifdef MOOSE_KOKKOS_ENABLED
433 47681 : _kokkos_assembly(*this),
434 : #endif
435 63000 : _mesh_divisions(/*threaded=*/true),
436 63000 : _material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
437 63000 : "material_props", &_mesh, _material_prop_registry, *this)),
438 63000 : _bnd_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
439 63000 : "bnd_material_props", &_mesh, _material_prop_registry, *this)),
440 63000 : _neighbor_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
441 63000 : "neighbor_material_props", &_mesh, _material_prop_registry, *this)),
442 : #ifdef MOOSE_KOKKOS_ENABLED
443 47681 : _kokkos_material_props(
444 47681 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
445 47681 : "kokkos_material_props", &_mesh, _material_prop_registry, *this)),
446 47681 : _kokkos_bnd_material_props(
447 47681 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
448 47681 : "kokkos_bnd_material_props", &_mesh, _material_prop_registry, *this)),
449 47681 : _kokkos_neighbor_material_props(
450 47681 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
451 47681 : "kokkos_neighbor_material_props", &_mesh, _material_prop_registry, *this)),
452 : #endif
453 63000 : _reporter_data(_app),
454 63000 : _multi_apps(_app.getExecuteOnEnum()),
455 63000 : _transient_multi_apps(_app.getExecuteOnEnum()),
456 63000 : _transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
457 63000 : _to_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
458 63000 : _from_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
459 63000 : _between_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
460 : #ifdef LIBMESH_ENABLE_AMR
461 63000 : _adaptivity(*this),
462 63000 : _cycles_completed(0),
463 : #endif
464 63000 : _displaced_mesh(nullptr),
465 63000 : _geometric_search_data(*this, _mesh),
466 63000 : _mortar_data(std::make_unique<MortarInterfaceWarehouse>(*this)),
467 63000 : _reinit_displaced_elem(false),
468 63000 : _reinit_displaced_face(false),
469 63000 : _reinit_displaced_neighbor(false),
470 63000 : _input_file_saved(false),
471 63000 : _has_dampers(false),
472 63000 : _has_constraints(false),
473 63000 : _snesmf_reuse_base(true),
474 63000 : _skip_exception_check(false),
475 63000 : _snesmf_reuse_base_set_by_user(false),
476 63000 : _has_initialized_stateful(false),
477 63000 : _const_jacobian(false),
478 63000 : _has_jacobian(false),
479 63000 : _needs_old_newton_iter(false),
480 126000 : _previous_nl_solution_required(getParam<bool>("previous_nl_solution_required")),
481 126000 : _previous_multiapp_fp_nl_solution_required(_num_nl_sys + _num_linear_sys, false),
482 63000 : _previous_multiapp_fp_aux_solution_required(false),
483 63000 : _has_nonlocal_coupling(false),
484 63000 : _calculate_jacobian_in_uo(false),
485 63000 : _kernel_coverage_check(
486 126000 : getParam<MooseEnum>("kernel_coverage_check").getEnum<CoverageCheckMode>()),
487 126000 : _kernel_coverage_blocks(getParam<std::vector<SubdomainName>>("kernel_coverage_block_list")),
488 63000 : _boundary_restricted_node_integrity_check(
489 126000 : getParam<bool>("boundary_restricted_node_integrity_check")),
490 63000 : _boundary_restricted_elem_integrity_check(
491 126000 : getParam<bool>("boundary_restricted_elem_integrity_check")),
492 63000 : _material_coverage_check(
493 126000 : getParam<MooseEnum>("material_coverage_check").getEnum<CoverageCheckMode>()),
494 126000 : _material_coverage_blocks(getParam<std::vector<SubdomainName>>("material_coverage_block_list")),
495 126000 : _fv_bcs_integrity_check(getParam<bool>("fv_bcs_integrity_check")),
496 126000 : _material_dependency_check(getParam<bool>("material_dependency_check")),
497 126000 : _uo_aux_state_check(getParam<bool>("check_uo_aux_state")),
498 : #ifndef NDEBUG
499 : _check_residual_for_nans(false),
500 : #endif
501 63000 : _max_qps(std::numeric_limits<unsigned int>::max()),
502 63000 : _max_scalar_order(INVALID_ORDER),
503 63000 : _has_time_integrator(false),
504 63000 : _has_exception(false),
505 126000 : _parallel_barrier_messaging(getParam<bool>("parallel_barrier_messaging")),
506 126000 : _verbose_setup(getParam<MooseEnum>("verbose_setup")),
507 126000 : _verbose_multiapps(getParam<bool>("verbose_multiapps")),
508 126000 : _verbose_restore(getParam<bool>("verbose_restore")),
509 63000 : _current_execute_on_flag(EXEC_NONE),
510 63000 : _control_warehouse(_app.getExecuteOnEnum(), /*threaded=*/false),
511 63000 : _is_petsc_options_inserted(false),
512 63000 : _line_search(nullptr),
513 63000 : _using_ad_mat_props(false),
514 63000 : _current_ic_state(0),
515 126000 : _use_hash_table_matrix_assembly(getParam<bool>("use_hash_table_matrix_assembly")),
516 63000 : _error_on_jacobian_nonzero_reallocation(
517 126000 : isParamValid("error_on_jacobian_nonzero_reallocation")
518 126545 : ? getParam<bool>("error_on_jacobian_nonzero_reallocation")
519 62455 : : _app.errorOnJacobianNonzeroReallocation()),
520 126000 : _restore_original_nonzero_pattern(isParamValid("restore_original_nonzero_pattern")
521 126000 : ? getParam<bool>("restore_original_nonzero_pattern")
522 63000 : : _use_hash_table_matrix_assembly),
523 126000 : _ignore_zeros_in_jacobian(getParam<bool>("ignore_zeros_in_jacobian")),
524 63000 : _preserve_matrix_sparsity_pattern(true),
525 126000 : _force_restart(getParam<bool>("force_restart")),
526 126000 : _allow_ics_during_restart(getParam<bool>("allow_initial_conditions_with_restart")),
527 126000 : _skip_nl_system_check(getParam<bool>("skip_nl_system_check")),
528 63000 : _fail_next_system_convergence_check(false),
529 126000 : _allow_invalid_solution(getParam<bool>("allow_invalid_solution")),
530 126000 : _show_invalid_solution_console(getParam<bool>("show_invalid_solution_console")),
531 126000 : _immediately_print_invalid_solution(getParam<bool>("immediately_print_invalid_solution")),
532 63000 : _started_initial_setup(false),
533 63000 : _has_internal_edge_residual_objects(false),
534 63000 : _u_dot_requested(false),
535 63000 : _u_dotdot_requested(false),
536 63000 : _u_dot_old_requested(false),
537 63000 : _u_dotdot_old_requested(false),
538 63000 : _has_mortar(false),
539 63000 : _num_grid_steps(0),
540 63000 : _print_execution_on(),
541 126000 : _identify_variable_groups_in_nl(getParam<bool>("identify_variable_groups_in_nl")),
542 63000 : _regard_general_exceptions_as_errors(getParam<bool>("regard_general_exceptions_as_errors")),
543 1197000 : _requires_nonlocal_coupling(false)
544 : {
545 : auto checkCoverageCheckConflict =
546 126000 : [this](const std::string & coverage_check,
547 : const CoverageCheckMode & coverage_check_mode,
548 : const std::vector<SubdomainName> & coverage_blocks) -> void
549 : {
550 126000 : if (coverage_check_mode != CoverageCheckMode::FALSE &&
551 121574 : coverage_check_mode != CoverageCheckMode::OFF)
552 121556 : if (coverage_blocks.size() > 1)
553 0 : if (std::find(coverage_blocks.begin(), coverage_blocks.end(), "ANY_BLOCK_ID") !=
554 0 : coverage_blocks.end())
555 0 : paramError(coverage_check,
556 : "The list of blocks used for ",
557 : coverage_check,
558 : " cannot contain 'ANY_BLOCK_ID' along with other blocks. ");
559 126000 : };
560 :
561 63000 : checkCoverageCheckConflict(
562 63000 : "kernel_coverage_check", _kernel_coverage_check, _kernel_coverage_blocks);
563 63000 : checkCoverageCheckConflict(
564 63000 : "material_coverage_check", _material_coverage_check, _material_coverage_blocks);
565 :
566 : // Initialize static do_derivatives member. We initialize this to true so that all the
567 : // default AD things that we setup early in the simulation actually get their derivative
568 : // vectors initalized. We will toggle this to false when doing residual evaluations
569 63000 : ADReal::do_derivatives = true;
570 :
571 : // Disable refinement/coarsening in EquationSystems::reinit because we already do this ourselves
572 63000 : es().disable_refine_in_reinit();
573 :
574 63000 : _solver_params.reserve(_num_nl_sys + _num_linear_sys);
575 : // Default constructor fine for nonlinear because it will be populated later by framework
576 : // executioner/solve object parameters
577 63000 : _solver_params.resize(_num_nl_sys);
578 125118 : for (const auto i : index_range(_nl_sys_names))
579 : {
580 62118 : const auto & name = _nl_sys_names[i];
581 62118 : _nl_sys_name_to_num[name] = i;
582 62118 : _solver_sys_name_to_num[name] = i;
583 62118 : _solver_sys_names.push_back(name);
584 : }
585 :
586 64264 : for (const auto i : index_range(_linear_sys_names))
587 : {
588 1264 : const auto & name = _linear_sys_names[i];
589 1264 : _linear_sys_name_to_num[name] = i;
590 1264 : _solver_sys_name_to_num[name] = i + _num_nl_sys;
591 1264 : _solver_sys_names.push_back(name);
592 : // Unlike for nonlinear these are basically dummy parameters
593 1264 : _solver_params.push_back(makeLinearSolverParams());
594 : }
595 :
596 63000 : _nonlocal_cm.resize(numSolverSystems());
597 63000 : _cm.resize(numSolverSystems());
598 :
599 63000 : _time = 0.0;
600 63000 : _time_old = 0.0;
601 63000 : _t_step = 0;
602 63000 : _dt = 0;
603 63000 : _dt_old = _dt;
604 :
605 63000 : unsigned int n_threads = libMesh::n_threads();
606 :
607 63000 : _real_zero.resize(n_threads, 0.);
608 63000 : _scalar_zero.resize(n_threads);
609 63000 : _zero.resize(n_threads);
610 63000 : _phi_zero.resize(n_threads);
611 63000 : _ad_zero.resize(n_threads);
612 63000 : _grad_zero.resize(n_threads);
613 63000 : _ad_grad_zero.resize(n_threads);
614 63000 : _grad_phi_zero.resize(n_threads);
615 63000 : _second_zero.resize(n_threads);
616 63000 : _ad_second_zero.resize(n_threads);
617 63000 : _second_phi_zero.resize(n_threads);
618 63000 : _point_zero.resize(n_threads);
619 63000 : _vector_zero.resize(n_threads);
620 63000 : _vector_curl_zero.resize(n_threads);
621 63000 : _uo_jacobian_moose_vars.resize(n_threads);
622 :
623 63000 : _has_active_material_properties.resize(n_threads, 0);
624 :
625 63000 : _block_mat_side_cache.resize(n_threads);
626 63000 : _bnd_mat_side_cache.resize(n_threads);
627 63000 : _interface_mat_side_cache.resize(n_threads);
628 :
629 126000 : es().parameters.set<FEProblemBase *>("_fe_problem_base") = this;
630 :
631 189000 : if (isParamValid("restart_file_base"))
632 : {
633 938 : std::string restart_file_base = getParam<FileNameNoExtension>("restart_file_base");
634 :
635 : // This check reverts to old behavior of providing "restart_file_base=" to mean
636 : // don't restart... BISON currently relies on this. It could probably be removed.
637 : // The new MooseUtils::convertLatestCheckpoint will error out if a checkpoint file
638 : // is not found, which I think makes sense. Which means, without this, if you
639 : // set "restart_file_base=", you'll get a "No checkpoint file found" error
640 469 : if (restart_file_base.size())
641 : {
642 469 : restart_file_base = MooseUtils::convertLatestCheckpoint(restart_file_base);
643 469 : setRestartFile(restart_file_base);
644 : }
645 469 : }
646 :
647 : // // Generally speaking, the mesh is prepared for use, and consequently remote elements are deleted
648 : // // well before our Problem(s) are constructed. Historically, in MooseMesh we have a bunch of
649 : // // needs_prepare type flags that make it so we never call prepare_for_use (and consequently
650 : // // delete_remote_elements) again. So the below line, historically, has had no impact. HOWEVER:
651 : // // I've added some code in SetupMeshCompleteAction for deleting remote elements post
652 : // // EquationSystems::init. If I execute that code without default ghosting, then I get > 40 MOOSE
653 : // // test failures, so we clearly have some simulations that are not yet covered properly by
654 : // // relationship managers. Until that is resolved, I am going to retain default geometric ghosting
655 : // if (!_default_ghosting)
656 : // _mesh.getMesh().remove_ghosting_functor(_mesh.getMesh().default_ghosting());
657 :
658 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
659 : // Main app should hold the default database to handle system petsc options
660 63000 : if (!_app.isUltimateMaster())
661 12241 : LibmeshPetscCall(PetscOptionsCreate(&_petsc_option_data_base));
662 : #endif
663 :
664 63000 : if (!_solve)
665 : {
666 : // If we are not solving, we do not care about seeing unused petsc options
667 50493 : Moose::PetscSupport::setSinglePetscOption("-options_left", "0");
668 : // We don't want petscSetOptions being called in solve and clearing the option that was just set
669 16831 : _is_petsc_options_inserted = true;
670 : }
671 63000 : }
672 :
673 : const MooseMesh &
674 0 : FEProblemBase::mesh(bool use_displaced) const
675 : {
676 0 : if (use_displaced && !_displaced_problem)
677 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
678 : "Regular mesh will be returned");
679 0 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
680 : }
681 :
682 : MooseMesh &
683 359630 : FEProblemBase::mesh(bool use_displaced)
684 : {
685 359630 : if (use_displaced && !_displaced_problem)
686 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
687 : "Regular mesh will be returned");
688 359630 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
689 : }
690 :
691 : void
692 63000 : FEProblemBase::createTagVectors()
693 : {
694 : // add vectors and their tags to system
695 126000 : auto & vectors = getParam<std::vector<std::vector<TagName>>>("extra_tag_vectors");
696 63800 : for (const auto sys_num : index_range(vectors))
697 1908 : for (auto & vector : vectors[sys_num])
698 : {
699 1108 : auto tag = addVectorTag(vector);
700 1108 : _solver_systems[sys_num]->addVector(tag, false, libMesh::GHOSTED);
701 : }
702 :
703 126000 : auto & not_zeroed_vectors = getParam<std::vector<std::vector<TagName>>>("not_zeroed_tag_vectors");
704 63011 : for (const auto sys_num : index_range(not_zeroed_vectors))
705 22 : for (auto & vector : not_zeroed_vectors[sys_num])
706 : {
707 11 : auto tag = addVectorTag(vector);
708 11 : _solver_systems[sys_num]->addVector(tag, false, GHOSTED);
709 11 : addNotZeroedVectorTag(tag);
710 : }
711 63000 : }
712 :
713 : void
714 62068 : FEProblemBase::createTagMatrices(CreateTaggedMatrixKey)
715 : {
716 124136 : auto & matrices = getParam<std::vector<std::vector<TagName>>>("extra_tag_matrices");
717 62314 : for (const auto sys_num : index_range(matrices))
718 692 : for (auto & matrix : matrices[sys_num])
719 : {
720 446 : auto tag = addMatrixTag(matrix);
721 446 : _solver_systems[sys_num]->addMatrix(tag);
722 : }
723 :
724 124458 : for (auto & sys : _solver_systems)
725 62390 : sys->sizeVariableMatrixData();
726 62068 : _aux->sizeVariableMatrixData();
727 62068 : }
728 :
729 : void
730 63000 : FEProblemBase::createTagSolutions()
731 : {
732 189025 : for (auto & vector : getParam<std::vector<TagName>>("extra_tag_solutions"))
733 : {
734 25 : auto tag = addVectorTag(vector, Moose::VECTOR_TAG_SOLUTION);
735 50 : for (auto & sys : _solver_systems)
736 25 : sys->addVector(tag, false, libMesh::GHOSTED);
737 25 : _aux->addVector(tag, false, libMesh::GHOSTED);
738 : }
739 :
740 63000 : if (_previous_nl_solution_required)
741 : {
742 : // We'll populate the zeroth state of the nonlinear iterations with the current solution for
743 : // ease of use in doing things like copying solutions backwards. We're just storing pointers in
744 : // the solution states containers so populating the zeroth state does not cost us the memory of
745 : // a new vector
746 83 : needSolutionState(1, Moose::SolutionIterationType::Nonlinear);
747 : }
748 :
749 63000 : auto tag = addVectorTag(Moose::SOLUTION_TAG, Moose::VECTOR_TAG_SOLUTION);
750 126382 : for (auto & sys : _solver_systems)
751 63382 : sys->associateVectorToTag(*sys->system().current_local_solution.get(), tag);
752 63000 : _aux->associateVectorToTag(*_aux->system().current_local_solution.get(), tag);
753 63000 : }
754 :
755 : void
756 135 : FEProblemBase::needSolutionState(unsigned int state, Moose::SolutionIterationType iteration_type)
757 : {
758 270 : for (auto & sys : _solver_systems)
759 135 : sys->needSolutionState(state, iteration_type);
760 135 : _aux->needSolutionState(state, iteration_type);
761 135 : }
762 :
763 : bool
764 48 : FEProblemBase::hasSolutionState(unsigned int state,
765 : Moose::SolutionIterationType iteration_type) const
766 : {
767 48 : bool has_solution_state = false;
768 107 : for (auto & sys : _solver_systems)
769 59 : has_solution_state |= sys->hasSolutionState(state, iteration_type);
770 48 : has_solution_state |= _aux->hasSolutionState(state, iteration_type);
771 48 : return has_solution_state;
772 : }
773 :
774 : void
775 63000 : FEProblemBase::newAssemblyArray(std::vector<std::shared_ptr<SolverSystem>> & solver_systems)
776 : {
777 63000 : unsigned int n_threads = libMesh::n_threads();
778 :
779 63000 : _assembly.resize(n_threads);
780 132941 : for (const auto i : make_range(n_threads))
781 : {
782 69941 : _assembly[i].resize(solver_systems.size());
783 140295 : for (const auto j : index_range(solver_systems))
784 70354 : _assembly[i][j] = std::make_unique<Assembly>(*solver_systems[j], i);
785 : }
786 63000 : }
787 :
788 : void
789 59790 : FEProblemBase::initNullSpaceVectors(const InputParameters & parameters,
790 : std::vector<std::shared_ptr<NonlinearSystemBase>> & nls)
791 : {
792 298950 : TIME_SECTION("initNullSpaceVectors", 5, "Initializing Null Space Vectors");
793 :
794 59790 : unsigned int dimNullSpace = parameters.get<unsigned int>("null_space_dimension");
795 : unsigned int dimTransposeNullSpace =
796 59790 : parameters.get<unsigned int>("transpose_null_space_dimension");
797 59790 : unsigned int dimNearNullSpace = parameters.get<unsigned int>("near_null_space_dimension");
798 59814 : for (unsigned int i = 0; i < dimNullSpace; ++i)
799 : {
800 24 : std::ostringstream oss;
801 24 : oss << "_" << i;
802 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
803 : // builder might march over all nodes
804 48 : for (auto & nl : nls)
805 24 : nl->addVector("NullSpace" + oss.str(), false, libMesh::GHOSTED);
806 24 : }
807 119580 : _subspace_dim["NullSpace"] = dimNullSpace;
808 59802 : for (unsigned int i = 0; i < dimTransposeNullSpace; ++i)
809 : {
810 12 : std::ostringstream oss;
811 12 : oss << "_" << i;
812 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
813 : // builder might march over all nodes
814 24 : for (auto & nl : nls)
815 12 : nl->addVector("TransposeNullSpace" + oss.str(), false, libMesh::GHOSTED);
816 12 : }
817 119580 : _subspace_dim["TransposeNullSpace"] = dimTransposeNullSpace;
818 59790 : for (unsigned int i = 0; i < dimNearNullSpace; ++i)
819 : {
820 0 : std::ostringstream oss;
821 0 : oss << "_" << i;
822 : // do not project, since this will be recomputed, but make it ghosted, since the near-nullspace
823 : // builder might march over all semilocal nodes
824 0 : for (auto & nl : nls)
825 0 : nl->addVector("NearNullSpace" + oss.str(), false, libMesh::GHOSTED);
826 0 : }
827 119580 : _subspace_dim["NearNullSpace"] = dimNearNullSpace;
828 59790 : }
829 :
830 179628 : FEProblemBase::~FEProblemBase()
831 : {
832 : // Flush the Console stream, the underlying call to Console::mooseConsole
833 : // relies on a call to Output::checkInterval that has references to
834 : // _time, etc. If it is not flushed here memory problems arise if you have
835 : // an unflushed stream and start destructing things.
836 59876 : _console << std::flush;
837 :
838 59876 : unsigned int n_threads = libMesh::n_threads();
839 125366 : for (unsigned int i = 0; i < n_threads; i++)
840 : {
841 65490 : _zero[i].release();
842 65490 : _phi_zero[i].release();
843 65490 : _scalar_zero[i].release();
844 65490 : _grad_zero[i].release();
845 65490 : _grad_phi_zero[i].release();
846 65490 : _second_zero[i].release();
847 65490 : _second_phi_zero[i].release();
848 65490 : _vector_zero[i].release();
849 65490 : _vector_curl_zero[i].release();
850 65490 : _ad_zero[i].release();
851 65490 : _ad_grad_zero[i].release();
852 65490 : _ad_second_zero[i].release();
853 : }
854 :
855 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
856 59876 : if (!_app.isUltimateMaster())
857 : {
858 11477 : auto ierr = PetscOptionsDestroy(&_petsc_option_data_base);
859 : // Don't throw on destruction
860 11477 : CHKERRABORT(this->comm().get(), ierr);
861 : }
862 : #endif
863 59876 : }
864 :
865 : void
866 0 : FEProblemBase::setCoordSystem(const std::vector<SubdomainName> & blocks,
867 : const MultiMooseEnum & coord_sys)
868 : {
869 0 : TIME_SECTION("setCoordSystem", 5, "Setting Coordinate System");
870 0 : _mesh.setCoordSystem(blocks, coord_sys);
871 0 : }
872 :
873 : void
874 0 : FEProblemBase::setAxisymmetricCoordAxis(const MooseEnum & rz_coord_axis)
875 : {
876 0 : _mesh.setAxisymmetricCoordAxis(rz_coord_axis);
877 0 : }
878 :
879 : const ConstElemRange &
880 1711 : FEProblemBase::getEvaluableElementRange()
881 : {
882 1711 : if (!_evaluable_local_elem_range)
883 : {
884 749 : std::vector<const DofMap *> dof_maps(es().n_systems());
885 2247 : for (const auto i : make_range(es().n_systems()))
886 : {
887 1498 : const auto & sys = es().get_system(i);
888 1498 : dof_maps[i] = &sys.get_dof_map();
889 : }
890 : _evaluable_local_elem_range =
891 1498 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
892 2247 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
893 749 : }
894 1711 : return *_evaluable_local_elem_range;
895 : }
896 :
897 : const ConstElemRange &
898 208 : FEProblemBase::getNonlinearEvaluableElementRange()
899 : {
900 208 : if (!_nl_evaluable_local_elem_range)
901 : {
902 208 : std::vector<const DofMap *> dof_maps(_nl.size());
903 416 : for (const auto i : index_range(dof_maps))
904 208 : dof_maps[i] = &_nl[i]->dofMap();
905 : _nl_evaluable_local_elem_range =
906 416 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
907 624 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
908 208 : }
909 :
910 208 : return *_nl_evaluable_local_elem_range;
911 : }
912 :
913 : void
914 60665 : FEProblemBase::initialSetup()
915 : {
916 303325 : TIME_SECTION("initialSetup", 2, "Performing Initial Setup");
917 :
918 60665 : SubProblem::initialSetup();
919 :
920 60665 : if (_app.isRecovering() + _app.isRestarting() + bool(_app.getExReaderForRestart()) > 1)
921 0 : mooseError("Checkpoint recovery and restart and exodus restart are all mutually exclusive.");
922 :
923 60665 : if (_skip_exception_check)
924 9 : mooseWarning("MOOSE may fail to catch an exception when the \"skip_exception_check\" parameter "
925 : "is used. If you receive a terse MPI error during execution, remove this "
926 : "parameter and rerun your simulation");
927 :
928 : // set state flag indicating that we are in or beyond initialSetup.
929 : // This can be used to throw errors in methods that _must_ be called at construction time.
930 60665 : _started_initial_setup = true;
931 60665 : setCurrentExecuteOnFlag(EXEC_INITIAL);
932 :
933 : // Setup the solution states (current, old, etc) in each system based on
934 : // its default and the states requested of each of its variables
935 121652 : for (const auto i : index_range(_solver_systems))
936 : {
937 60987 : _solver_systems[i]->initSolutionState();
938 60987 : if (getDisplacedProblem())
939 2013 : getDisplacedProblem()->solverSys(i).initSolutionState();
940 : }
941 60665 : _aux->initSolutionState();
942 60665 : if (getDisplacedProblem())
943 2013 : getDisplacedProblem()->auxSys().initSolutionState();
944 :
945 : // always execute to get the max number of DoF per element and node needed to initialize phi_zero
946 : // variables
947 60665 : dof_id_type global_max_var_n_dofs_per_elem = 0;
948 121652 : for (const auto i : index_range(_solver_systems))
949 : {
950 60987 : auto & sys = *_solver_systems[i];
951 : dof_id_type max_var_n_dofs_per_elem;
952 : dof_id_type max_var_n_dofs_per_node;
953 : {
954 304935 : TIME_SECTION("computingMaxDofs", 3, "Computing Max Dofs Per Element");
955 :
956 60987 : MaxVarNDofsPerElem mvndpe(*this, sys);
957 60987 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mvndpe);
958 60987 : max_var_n_dofs_per_elem = mvndpe.max();
959 60987 : _communicator.max(max_var_n_dofs_per_elem);
960 :
961 60987 : MaxVarNDofsPerNode mvndpn(*this, sys);
962 60987 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), mvndpn);
963 60987 : max_var_n_dofs_per_node = mvndpn.max();
964 60987 : _communicator.max(max_var_n_dofs_per_node);
965 60987 : global_max_var_n_dofs_per_elem =
966 60987 : std::max(global_max_var_n_dofs_per_elem, max_var_n_dofs_per_elem);
967 60987 : }
968 :
969 : {
970 304935 : TIME_SECTION("assignMaxDofs", 5, "Assigning Maximum Dofs Per Elem");
971 :
972 60987 : sys.assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
973 60987 : auto displaced_problem = getDisplacedProblem();
974 60987 : if (displaced_problem)
975 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
976 :
977 60987 : sys.assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
978 60987 : if (displaced_problem)
979 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
980 60987 : }
981 : }
982 :
983 : {
984 303325 : TIME_SECTION("resizingVarValues", 5, "Resizing Variable Values");
985 :
986 127379 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
987 : {
988 133428 : _phi_zero[tid].resize(global_max_var_n_dofs_per_elem, std::vector<Real>(getMaxQps(), 0.));
989 133428 : _grad_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
990 133428 : std::vector<RealGradient>(getMaxQps(), RealGradient(0.)));
991 133428 : _second_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
992 133428 : std::vector<RealTensor>(getMaxQps(), RealTensor(0.)));
993 : }
994 60665 : }
995 :
996 : // Set up stateful material property redistribution, if we suspect
997 : // it may be necessary later.
998 60665 : addAnyRedistributers();
999 :
1000 60665 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1001 : {
1002 : // Only load all of the vectors if we're recovering
1003 4550 : _req.set().setLoadAllVectors(_app.isRecovering());
1004 :
1005 : // This forces stateful material property loading to be an exact one-to-one match
1006 4550 : if (_app.isRecovering())
1007 : {
1008 16072 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1009 12054 : props->setRecovering();
1010 :
1011 : #ifdef MOOSE_KOKKOS_ENABLED
1012 15932 : for (auto props :
1013 19915 : {&_kokkos_material_props, &_kokkos_bnd_material_props, &_kokkos_neighbor_material_props})
1014 11949 : props->setRecovering();
1015 : #endif
1016 : }
1017 :
1018 22750 : TIME_SECTION("restore", 3, "Restoring from backup");
1019 :
1020 : // We could have a cached backup when this app is a sub-app and has been given a Backup
1021 4550 : if (!_app.hasInitialBackup())
1022 3769 : _app.restore(_app.restartFolderBase(_app.getRestartRecoverFileBase()), _app.isRestarting());
1023 : else
1024 781 : _app.restoreFromInitialBackup(_app.isRestarting());
1025 :
1026 : /**
1027 : * If this is a restart run, the user may want to override the start time, which we already set
1028 : * in the constructor. "_time" however will have been "restored" from the restart file. We need
1029 : * to honor the original request of the developer now that the restore has been completed.
1030 : */
1031 4517 : if (_app.isRestarting())
1032 : {
1033 499 : if (_app.hasStartTime())
1034 173 : _time = _time_old = _app.getStartTime();
1035 : else
1036 326 : _time_old = _time;
1037 : }
1038 4517 : }
1039 : else
1040 : {
1041 56115 : libMesh::ExodusII_IO * reader = _app.getExReaderForRestart();
1042 :
1043 56115 : if (reader)
1044 : {
1045 1890 : TIME_SECTION("copyingFromExodus", 3, "Copying Variables From Exodus");
1046 :
1047 764 : for (auto & sys : _solver_systems)
1048 389 : sys->copyVars(*reader);
1049 375 : _aux->copyVars(*reader);
1050 375 : }
1051 : else
1052 : {
1053 55737 : if (_solver_systems[0]->hasVarCopy() || _aux->hasVarCopy())
1054 0 : mooseError("Need Exodus reader to restart variables but the reader is not available\n"
1055 : "Use either FileMesh with an Exodus mesh file or FileMeshGenerator with an "
1056 : "Exodus mesh file and with use_for_exodus_restart equal to true");
1057 : }
1058 : }
1059 :
1060 : // Perform output related setups
1061 60629 : _app.getOutputWarehouse().initialSetup();
1062 :
1063 : // Flush all output to _console that occur during construction and initialization of objects
1064 60583 : _app.getOutputWarehouse().mooseConsole();
1065 :
1066 : // Build Refinement and Coarsening maps for stateful material projections if necessary
1067 62870 : if ((_adaptivity.isOn() || _num_grid_steps) &&
1068 2287 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1069 2224 : _neighbor_material_props.hasStatefulProperties()))
1070 : {
1071 63 : if (_has_internal_edge_residual_objects)
1072 6 : mooseError("Stateful neighbor material properties do not work with mesh adaptivity");
1073 :
1074 57 : _mesh.buildRefinementAndCoarseningMaps(_assembly[0][0].get());
1075 : }
1076 :
1077 60577 : if (!_app.isRecovering())
1078 : {
1079 : /**
1080 : * If we are not recovering but we are doing restart (_app.getExodusFileRestart() == true) with
1081 : * additional uniform refinements. We have to delay the refinement until this point
1082 : * in time so that the equation systems are initialized and projections can be performed.
1083 : */
1084 56559 : if (_mesh.uniformRefineLevel() > 0 && _app.getExodusFileRestart())
1085 : {
1086 10 : if (!_app.isUltimateMaster())
1087 0 : mooseError(
1088 : "Doing extra refinements when restarting is NOT supported for sub-apps of a MultiApp");
1089 :
1090 10 : adaptivity().uniformRefineWithProjection();
1091 : }
1092 : }
1093 :
1094 60577 : unsigned int n_threads = libMesh::n_threads();
1095 :
1096 : // Convergence initial setup
1097 : {
1098 302885 : TIME_SECTION("convergenceInitialSetup", 5, "Initializing Convergence objects");
1099 :
1100 127152 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1101 66590 : _convergences.initialSetup(tid);
1102 60562 : }
1103 :
1104 : // UserObject initialSetup
1105 60562 : std::set<std::string> depend_objects_ic = _ics.getDependObjects();
1106 60562 : std::set<std::string> depend_objects_aux = _aux->getDependObjects();
1107 :
1108 60562 : std::map<int, std::vector<UserObjectBase *>> group_userobjs;
1109 :
1110 : // This replaces all prior updateDependObjects calls on the old user object warehouses.
1111 60562 : TheWarehouse::Query uo_query = theWarehouse().query().condition<AttribSystem>("UserObject");
1112 60562 : std::vector<UserObjectBase *> userobjs;
1113 60562 : uo_query.queryInto(userobjs);
1114 60562 : groupUserObjects(
1115 60562 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1116 :
1117 133886 : for (auto obj : userobjs)
1118 219972 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1119 :
1120 : #ifdef MOOSE_KOKKOS_ENABLED
1121 : {
1122 : TheWarehouse::Query uo_query =
1123 45875 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject");
1124 45875 : std::vector<UserObjectBase *> userobjs;
1125 45875 : uo_query.queryInto(userobjs);
1126 45875 : groupUserObjects(
1127 45875 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1128 :
1129 47115 : for (auto obj : userobjs)
1130 3720 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1131 45875 : }
1132 : #endif
1133 :
1134 91477 : for (auto & [group, objs] : group_userobjs)
1135 105426 : for (auto obj : objs)
1136 74511 : obj->initialSetup();
1137 :
1138 : // check if jacobian calculation is done in userobject
1139 126951 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1140 66469 : checkUserObjectJacobianRequirement(tid);
1141 :
1142 : // Check whether nonlocal coupling is required or not
1143 60482 : checkNonlocalCoupling();
1144 60482 : if (_requires_nonlocal_coupling)
1145 63 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
1146 :
1147 : {
1148 302410 : TIME_SECTION("initializingFunctions", 5, "Initializing Functions");
1149 :
1150 : // Call the initialSetup methods for functions
1151 126919 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1152 : {
1153 66461 : reinitScalars(tid); // initialize scalars so they are properly sized for use as input into
1154 : // ParsedFunctions
1155 66461 : _functions.initialSetup(tid);
1156 : }
1157 :
1158 : #ifdef MOOSE_KOKKOS_ENABLED
1159 45806 : _kokkos_functions.initialSetup();
1160 : #endif
1161 60458 : }
1162 :
1163 : {
1164 302290 : TIME_SECTION("initializingRandomObjects", 5, "Initializing Random Objects");
1165 :
1166 : // Random interface objects
1167 60782 : for (const auto & it : _random_data_objects)
1168 324 : it.second->updateSeeds(EXEC_INITIAL);
1169 60458 : }
1170 :
1171 60458 : if (!_app.isRecovering())
1172 : {
1173 56440 : computeUserObjects(EXEC_INITIAL, Moose::PRE_IC);
1174 :
1175 : {
1176 282200 : TIME_SECTION("ICinitialSetup", 5, "Setting Up Initial Conditions");
1177 :
1178 118806 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1179 : {
1180 62372 : _ics.initialSetup(tid);
1181 62366 : _fv_ics.initialSetup(tid);
1182 : }
1183 :
1184 56434 : _scalar_ics.initialSetup();
1185 56434 : }
1186 :
1187 56434 : projectSolution();
1188 : }
1189 :
1190 : // Materials
1191 60446 : if (_all_materials.hasActiveObjects(0))
1192 : {
1193 42535 : TIME_SECTION("materialInitialSetup", 3, "Setting Up Materials");
1194 :
1195 17754 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1196 : {
1197 : // Sort the Material objects, these will be actually computed by MOOSE in reinit methods.
1198 9265 : _materials.sort(tid);
1199 9262 : _interface_materials.sort(tid);
1200 :
1201 : // Call initialSetup on all material objects
1202 9262 : _all_materials.initialSetup(tid);
1203 :
1204 : // Discrete materials may insert additional dependencies on materials during the initial
1205 : // setup. Therefore we resolve the dependencies once more, now with the additional
1206 : // dependencies due to discrete materials.
1207 9247 : if (_discrete_materials.hasActiveObjects())
1208 : {
1209 56 : _materials.sort(tid);
1210 56 : _interface_materials.sort(tid);
1211 : }
1212 : }
1213 :
1214 : #ifdef MOOSE_KOKKOS_ENABLED
1215 6383 : _kokkos_materials.sort(0, true);
1216 : #endif
1217 :
1218 : {
1219 42435 : TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");
1220 :
1221 8487 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1222 :
1223 16299 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1224 7812 : _neighbor_material_props.hasStatefulProperties())
1225 675 : _has_initialized_stateful = true;
1226 : #ifdef MOOSE_KOKKOS_ENABLED
1227 6381 : if (_kokkos_material_props.hasStatefulProperties() ||
1228 12545 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1229 6164 : _kokkos_neighbor_material_props.hasStatefulProperties())
1230 217 : _has_initialized_stateful = true;
1231 : #endif
1232 8487 : }
1233 8487 : }
1234 :
1235 : // setRestartInPlace() is set because the property maps have now been setup and we can
1236 : // dataLoad() them directly in place
1237 : // setRecovering() is set because from now on we require a one-to-one mapping of
1238 : // stateful properties because we shouldn't be declaring any more
1239 241704 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1240 : {
1241 181278 : props->setRestartInPlace();
1242 181278 : props->setRecovering();
1243 : }
1244 :
1245 126820 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1246 : {
1247 66394 : _internal_side_indicators.initialSetup(tid);
1248 66394 : _indicators.initialSetup(tid);
1249 66394 : _markers.sort(tid);
1250 66394 : _markers.initialSetup(tid);
1251 : }
1252 :
1253 : #ifdef LIBMESH_ENABLE_AMR
1254 :
1255 60426 : if (!_app.isRecovering() && !_app.restoredInitialBackupMesh())
1256 : {
1257 56406 : unsigned int n = adaptivity().getInitialSteps();
1258 56406 : if (n && !_app.isUltimateMaster() && _app.isRestarting())
1259 0 : mooseError("Cannot perform initial adaptivity during restart on sub-apps of a MultiApp!");
1260 :
1261 56406 : initialAdaptMesh();
1262 : }
1263 :
1264 : #endif // LIBMESH_ENABLE_AMR
1265 :
1266 60423 : if (!_app.isRecovering() && !_app.isRestarting())
1267 : {
1268 : // During initial setup the solution is copied to the older solution states (old, older, etc)
1269 55906 : copySolutionsBackwards();
1270 :
1271 : // Check if there are old state initial conditions
1272 55906 : auto ics = _ics.getActiveObjects();
1273 55906 : auto fv_ics = _fv_ics.getActiveObjects();
1274 55906 : auto scalar_ics = _scalar_ics.getActiveObjects();
1275 55906 : unsigned short ic_state_max = 0;
1276 :
1277 167718 : auto findMax = [&ic_state_max](const auto & obj_list)
1278 : {
1279 198493 : for (auto ic : obj_list.getActiveObjects())
1280 30775 : ic_state_max = std::max(ic_state_max, ic->getState());
1281 223624 : };
1282 55906 : findMax(_ics);
1283 55906 : findMax(_fv_ics);
1284 55906 : findMax(_scalar_ics);
1285 :
1286 : // if there are old state ICs, compute them and write to old states accordingly
1287 55906 : if (ic_state_max > 0)
1288 : {
1289 : // state 0 copy (we'll overwrite current state when evaluating ICs and need to restore it once
1290 : // we're done with the old/older state ICs)
1291 0 : std::vector<std::unique_ptr<NumericVector<Real>>> state0_sys_buffers(_solver_systems.size());
1292 0 : std::unique_ptr<NumericVector<Real>> state0_aux_buffer;
1293 :
1294 : // save state 0
1295 0 : for (const auto i : index_range(_solver_systems))
1296 0 : state0_sys_buffers[i] = _solver_systems[i]->solutionState(0).clone();
1297 :
1298 0 : state0_aux_buffer = _aux->solutionState(0).clone();
1299 :
1300 : // compute old state ICs
1301 0 : for (_current_ic_state = 1; _current_ic_state <= ic_state_max; _current_ic_state++)
1302 : {
1303 0 : projectSolution();
1304 :
1305 0 : for (auto & sys : _solver_systems)
1306 0 : sys->solutionState(_current_ic_state) = sys->solutionState(0);
1307 :
1308 0 : _aux->solutionState(_current_ic_state) = _aux->solutionState(0);
1309 : }
1310 0 : _current_ic_state = 0;
1311 :
1312 : // recover state 0
1313 0 : for (const auto i : index_range(_solver_systems))
1314 : {
1315 0 : _solver_systems[i]->solutionState(0) = *state0_sys_buffers[i];
1316 0 : _solver_systems[i]->solutionState(0).close();
1317 0 : _solver_systems[i]->update();
1318 : }
1319 0 : _aux->solutionState(0) = *state0_aux_buffer;
1320 0 : _aux->solutionState(0).close();
1321 0 : _aux->update();
1322 0 : }
1323 55906 : }
1324 :
1325 60423 : if (!_app.isRecovering())
1326 : {
1327 56405 : if (haveXFEM())
1328 0 : updateMeshXFEM();
1329 : }
1330 :
1331 : // Call initialSetup on the solver systems
1332 121168 : for (auto & sys : _solver_systems)
1333 60745 : sys->initialSetup();
1334 :
1335 : // Auxilary variable initialSetup calls
1336 60423 : _aux->initialSetup();
1337 :
1338 60417 : if (_displaced_problem)
1339 : // initialSetup for displaced systems
1340 2013 : _displaced_problem->initialSetup();
1341 :
1342 121156 : for (auto & sys : _solver_systems)
1343 60739 : sys->setSolution(*(sys->system().current_local_solution.get()));
1344 :
1345 : // Update the nearest node searches (has to be called after the problem is all set up)
1346 : // We do this here because this sets up the Element's DoFs to ghost
1347 60417 : updateGeomSearch(GeometricSearchData::NEAREST_NODE);
1348 :
1349 60417 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
1350 60417 : if (_displaced_mesh)
1351 2013 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
1352 :
1353 : // We need to move the mesh in order to build a map between mortar secondary and primary
1354 : // interfaces. This map will then be used by the AgumentSparsityOnInterface ghosting functor to
1355 : // know which dofs we need ghosted when we call EquationSystems::reinit
1356 60417 : if (_displaced_problem && _mortar_data->hasDisplacedObjects())
1357 : {
1358 114 : _displaced_problem->updateMesh();
1359 : // if displacements were applied to the mesh, the mortar mesh should be updated too
1360 114 : updateMortarMesh();
1361 : }
1362 :
1363 : // Possibly reinit one more time to get ghosting correct
1364 60417 : reinitBecauseOfGhostingOrNewGeomObjects();
1365 :
1366 60417 : if (_displaced_mesh)
1367 2013 : _displaced_problem->updateMesh();
1368 :
1369 60417 : updateGeomSearch(); // Call all of the rest of the geometric searches
1370 :
1371 121153 : for (auto & sys : _solver_systems)
1372 : {
1373 60739 : const auto & tis = sys->getTimeIntegrators();
1374 :
1375 : {
1376 303695 : TIME_SECTION("timeIntegratorInitialSetup", 5, "Initializing Time Integrator");
1377 90634 : for (auto & ti : tis)
1378 29898 : ti->initialSetup();
1379 60736 : }
1380 : }
1381 :
1382 : // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ main-app restartable data
1383 : // has been restored
1384 :
1385 : // Call initialSetup on the MultiApps
1386 60414 : if (_multi_apps.hasObjects())
1387 : {
1388 35290 : TIME_SECTION("initialSetupMultiApps", 2, "Initializing MultiApps", false);
1389 7058 : _multi_apps.initialSetup();
1390 7052 : }
1391 :
1392 : // Call initialSetup on the transfers
1393 : {
1394 302040 : TIME_SECTION("initialSetupTransfers", 2, "Initializing Transfers");
1395 :
1396 60408 : _transfers.initialSetup();
1397 :
1398 : // Call initialSetup on the MultiAppTransfers to be executed on TO_MULTIAPP
1399 60408 : const auto & to_multi_app_objects = _to_multi_app_transfers.getActiveObjects();
1400 65541 : for (const auto & transfer : to_multi_app_objects)
1401 : {
1402 5181 : transfer->setCurrentDirection(Transfer::DIRECTION::TO_MULTIAPP);
1403 5181 : transfer->initialSetup();
1404 : }
1405 :
1406 : // Call initialSetup on the MultiAppTransfers to be executed on FROM_MULTIAPP
1407 60360 : const auto & from_multi_app_objects = _from_multi_app_transfers.getActiveObjects();
1408 66593 : for (const auto & transfer : from_multi_app_objects)
1409 : {
1410 6275 : transfer->setCurrentDirection(Transfer::DIRECTION::FROM_MULTIAPP);
1411 6275 : transfer->initialSetup();
1412 : }
1413 :
1414 : // Call initialSetup on the MultiAppTransfers to be executed on BETWEEN_MULTIAPP
1415 60318 : const auto & between_multi_app_objects = _between_multi_app_transfers.getActiveObjects();
1416 61788 : for (const auto & transfer : between_multi_app_objects)
1417 : {
1418 1470 : transfer->setCurrentDirection(Transfer::DIRECTION::BETWEEN_MULTIAPP);
1419 1470 : transfer->initialSetup();
1420 : }
1421 60318 : }
1422 :
1423 60318 : if (_boundary_restricted_node_integrity_check)
1424 : {
1425 180918 : TIME_SECTION("BoundaryRestrictedNodeIntegrityCheck", 5);
1426 :
1427 : // check that variables are defined along boundaries of boundary restricted nodal objects
1428 60306 : const auto & bnd_nodes = getCurrentAlgebraicBndNodeRange();
1429 60306 : BoundaryNodeIntegrityCheckThread bnict(*this, uo_query);
1430 60306 : Threads::parallel_reduce(bnd_nodes, bnict);
1431 :
1432 : // Nodal bcs aren't threaded
1433 119681 : for (auto & nl : _nl)
1434 : {
1435 59393 : const auto & nodal_bcs = nl->getNodalBCWarehouse();
1436 59393 : if (!nodal_bcs.hasBoundaryObjects())
1437 27832 : continue;
1438 :
1439 2502919 : for (const auto & bnode : bnd_nodes)
1440 : {
1441 2471367 : const auto boundary_id = bnode->_bnd_id;
1442 2471367 : const Node * const node = bnode->_node;
1443 :
1444 2471367 : if (node->processor_id() != this->processor_id())
1445 582916 : continue;
1446 :
1447 1888451 : const auto & bnd_name = _mesh.getBoundaryName(boundary_id);
1448 :
1449 : // Avoid assertion in getBoundaryObjects that we have boundary objects for this boundary ID
1450 1888451 : if (!nodal_bcs.hasBoundaryObjects(boundary_id))
1451 968837 : continue;
1452 :
1453 919614 : const auto & bnd_objects = nodal_bcs.getBoundaryObjects(boundary_id);
1454 1930854 : for (const auto & bnd_object : bnd_objects)
1455 : {
1456 1011249 : const auto & bnd_variable = bnd_object->variable();
1457 : // Skip if this object uses geometric search because coupled variables may be defined on
1458 : // paired boundaries instead of the boundary this node is on. Also skip if this boundary
1459 : // condition isn't applicable to the current node, e.g. if the node doesn't have any
1460 : // degrees of freedom for the boundary condition's variable
1461 1011249 : if (!bnd_object->requiresGeometricSearch() &&
1462 2022498 : bnd_object->checkVariableBoundaryIntegrity() &&
1463 1011249 : node->n_dofs(nl->number(), bnd_variable.number()))
1464 : {
1465 : std::set<MooseVariableFieldBase *> vars_to_omit = {
1466 : &static_cast<MooseVariableFieldBase &>(
1467 1931030 : const_cast<MooseVariableBase &>(bnd_variable))};
1468 :
1469 1931030 : boundaryIntegrityCheckError(
1470 1931021 : *bnd_object, bnd_object->checkAllVariables(*node, vars_to_omit), bnd_name);
1471 965506 : }
1472 : }
1473 : }
1474 : }
1475 60288 : }
1476 :
1477 60300 : if (_boundary_restricted_elem_integrity_check)
1478 : {
1479 180810 : TIME_SECTION("BoundaryRestrictedElemIntegrityCheck", 5);
1480 :
1481 : // check that variables are defined along boundaries of boundary restricted elemental objects
1482 60270 : ConstBndElemRange & bnd_elems = *mesh().getBoundaryElementRange();
1483 60270 : BoundaryElemIntegrityCheckThread beict(*this, uo_query);
1484 60270 : Threads::parallel_reduce(bnd_elems, beict);
1485 60258 : }
1486 :
1487 60288 : if (!_app.isRecovering())
1488 : {
1489 56270 : execTransfers(EXEC_INITIAL);
1490 :
1491 56270 : bool converged = execMultiApps(EXEC_INITIAL);
1492 56264 : if (!converged)
1493 3 : mooseError("failed to converge initial MultiApp");
1494 :
1495 : // We'll backup the Multiapp here
1496 56261 : backupMultiApps(EXEC_INITIAL);
1497 :
1498 118396 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1499 62135 : reinitScalars(tid);
1500 :
1501 56261 : execute(EXEC_INITIAL);
1502 :
1503 : // The FEProblemBase::execute method doesn't call all the systems on EXEC_INITIAL, but it does
1504 : // set/unset the current flag. Therefore, this resets the current flag to EXEC_INITIAL so that
1505 : // subsequent calls (e.g., executeControls) have the proper flag.
1506 56183 : setCurrentExecuteOnFlag(EXEC_INITIAL);
1507 : }
1508 :
1509 : // Here we will initialize the stateful properties once more since they may have been updated
1510 : // during initialSetup by calls to computeProperties.
1511 : //
1512 : // It's really bad that we don't allow this during restart. It means that we can't add new
1513 : // stateful materials
1514 : // during restart. This is only happening because this _has_ to be below initial userobject
1515 : // execution.
1516 : // Otherwise this could be done up above... _before_ restoring restartable data... which would
1517 : // allow you to have
1518 : // this happen during restart. I honestly have no idea why this has to happen after initial user
1519 : // object computation.
1520 : // THAT is something we should fix... so I've opened this ticket: #5804
1521 60201 : if (!_app.isRecovering() && !_app.isRestarting())
1522 : {
1523 110796 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1524 55112 : _neighbor_material_props.hasStatefulProperties())
1525 : {
1526 2860 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1527 :
1528 572 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1529 572 : }
1530 : #ifdef MOOSE_KOKKOS_ENABLED
1531 41281 : if (_kokkos_material_props.hasStatefulProperties() ||
1532 82397 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1533 41116 : _kokkos_neighbor_material_props.hasStatefulProperties())
1534 : {
1535 825 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1536 :
1537 165 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1538 165 : }
1539 : #endif
1540 : }
1541 :
1542 : // Control Logic
1543 60201 : _control_warehouse.initialSetup();
1544 60201 : executeControls(EXEC_INITIAL);
1545 :
1546 : // Scalar variables need to reinited for the initial conditions to be available for output
1547 126233 : for (unsigned int tid = 0; tid < n_threads; tid++)
1548 66059 : reinitScalars(tid);
1549 :
1550 60174 : if (_displaced_mesh)
1551 2013 : _displaced_problem->syncSolutions();
1552 :
1553 : // Writes all calls to _console from initialSetup() methods
1554 60174 : _app.getOutputWarehouse().mooseConsole();
1555 :
1556 60174 : if (_requires_nonlocal_coupling)
1557 : {
1558 63 : setNonlocalCouplingMatrix();
1559 126 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1560 126 : for (auto & assembly : _assembly[tid])
1561 63 : assembly->initNonlocalCoupling();
1562 : }
1563 :
1564 : {
1565 300870 : TIME_SECTION("lineSearchInitialSetup", 5, "Initializing Line Search");
1566 :
1567 60174 : if (_line_search)
1568 0 : _line_search->initialSetup();
1569 60174 : }
1570 :
1571 : // Perform Reporter get/declare check
1572 60174 : _reporter_data.check();
1573 :
1574 : // We do this late to allow objects to get late restartable data
1575 60174 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1576 4517 : _app.finalizeRestore();
1577 :
1578 60174 : setCurrentExecuteOnFlag(EXEC_NONE);
1579 60174 : }
1580 :
1581 : void
1582 61026 : FEProblemBase::checkDuplicatePostprocessorVariableNames()
1583 : {
1584 113149 : for (const auto & pp : _reporter_data.getPostprocessorNames())
1585 52123 : if (hasScalarVariable(pp))
1586 0 : mooseError("Postprocessor \"" + pp +
1587 61026 : "\" has the same name as a scalar variable in the system.");
1588 61026 : }
1589 :
1590 : void
1591 270290 : FEProblemBase::timestepSetup()
1592 : {
1593 270290 : SubProblem::timestepSetup();
1594 :
1595 270290 : if (_t_step > 1 && _num_grid_steps)
1596 : {
1597 31 : libMesh::MeshRefinement mesh_refinement(_mesh);
1598 31 : std::unique_ptr<libMesh::MeshRefinement> displaced_mesh_refinement(nullptr);
1599 31 : if (_displaced_mesh)
1600 23 : displaced_mesh_refinement = std::make_unique<libMesh::MeshRefinement>(*_displaced_mesh);
1601 :
1602 62 : for (MooseIndex(_num_grid_steps) i = 0; i < _num_grid_steps; ++i)
1603 : {
1604 31 : if (_displaced_problem)
1605 : // If the DisplacedProblem is active, undisplace the DisplacedMesh in preparation for
1606 : // refinement. We can't safely refine the DisplacedMesh directly, since the Hilbert keys
1607 : // computed on the inconsistenly-displaced Mesh are different on different processors,
1608 : // leading to inconsistent Hilbert keys. We must do this before the undisplaced Mesh is
1609 : // coarsensed, so that the element and node numbering is still consistent. We also have to
1610 : // make sure this is done during every step of coarsening otherwise different partitions
1611 : // will be generated for the reference and displaced meshes (even for replicated)
1612 23 : _displaced_problem->undisplaceMesh();
1613 :
1614 31 : mesh_refinement.uniformly_coarsen();
1615 31 : if (_displaced_mesh)
1616 23 : displaced_mesh_refinement->uniformly_coarsen();
1617 :
1618 : // Mark this as an intermediate change because we do not yet want to reinit_systems. E.g. we
1619 : // need things to happen in the following order for the undisplaced problem:
1620 : // u1) EquationSystems::reinit_solutions. This will restrict the solution vectors and then
1621 : // contract the mesh
1622 : // u2) MooseMesh::meshChanged. This will update the node/side lists and other
1623 : // things which needs to happen after the contraction
1624 : // u3) GeometricSearchData::reinit. Once the node/side lists are updated we can perform our
1625 : // geometric searches which will aid in determining sparsity patterns
1626 : //
1627 : // We do these things for the displaced problem (if it exists)
1628 : // d1) EquationSystems::reinit. Restrict the displaced problem vector copies and then contract
1629 : // the mesh. It's safe to do a full reinit with the displaced because there are no
1630 : // matrices that sparsity pattern calculations will be conducted for
1631 : // d2) MooseMesh::meshChanged. This will update the node/side lists and other
1632 : // things which needs to happen after the contraction
1633 : // d3) UpdateDisplacedMeshThread::operator(). Re-displace the mesh using the *displaced*
1634 : // solution vector copy because we don't know the state of the reference solution vector.
1635 : // It's safe to use the displaced copy because we are outside of a non-linear solve,
1636 : // and there is no concern about differences between solution and current_local_solution
1637 : // d4) GeometricSearchData::reinit. With the node/side lists updated and the mesh
1638 : // re-displaced, we can perform our geometric searches, which will aid in determining the
1639 : // sparsity pattern of the matrix held by the libMesh::ImplicitSystem held by the
1640 : // NonlinearSystem held by this
1641 31 : meshChanged(
1642 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
1643 : }
1644 :
1645 : // u4) Now that all the geometric searches have been done (both undisplaced and displaced),
1646 : // we're ready to update the sparsity pattern
1647 31 : es().reinit_systems();
1648 31 : }
1649 :
1650 270290 : _control_warehouse.timestepSetup();
1651 270290 : if (_line_search)
1652 0 : _line_search->timestepSetup();
1653 :
1654 : // Random interface objects
1655 271670 : for (const auto & it : _random_data_objects)
1656 1380 : it.second->updateSeeds(EXEC_TIMESTEP_BEGIN);
1657 :
1658 270290 : unsigned int n_threads = libMesh::n_threads();
1659 567330 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1660 : {
1661 297040 : _all_materials.timestepSetup(tid);
1662 297040 : _functions.timestepSetup(tid);
1663 : }
1664 :
1665 : #ifdef MOOSE_KOKKOS_ENABLED
1666 198318 : _kokkos_functions.timestepSetup();
1667 : #endif
1668 :
1669 270290 : _aux->timestepSetup();
1670 544104 : for (auto & sys : _solver_systems)
1671 273814 : sys->timestepSetup();
1672 :
1673 270290 : if (_displaced_problem)
1674 : // timestepSetup for displaced systems
1675 30674 : _displaced_problem->timestepSetup();
1676 :
1677 567330 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1678 : {
1679 297040 : _internal_side_indicators.timestepSetup(tid);
1680 297040 : _indicators.timestepSetup(tid);
1681 297040 : _markers.timestepSetup(tid);
1682 : }
1683 :
1684 270290 : std::vector<UserObject *> userobjs;
1685 270290 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
1686 606099 : for (auto obj : userobjs)
1687 335809 : obj->timestepSetup();
1688 :
1689 : #ifdef MOOSE_KOKKOS_ENABLED
1690 : {
1691 198318 : std::vector<UserObjectBase *> userobjs;
1692 198318 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
1693 200510 : for (auto obj : userobjs)
1694 2192 : obj->timestepSetup();
1695 198318 : }
1696 : #endif
1697 :
1698 : // Timestep setup of output objects
1699 270290 : _app.getOutputWarehouse().timestepSetup();
1700 :
1701 270290 : if (_requires_nonlocal_coupling)
1702 97 : if (_nonlocal_kernels.hasActiveObjects() || _nonlocal_integrated_bcs.hasActiveObjects())
1703 97 : _has_nonlocal_coupling = true;
1704 270290 : }
1705 :
1706 : unsigned int
1707 758959 : FEProblemBase::getMaxQps() const
1708 : {
1709 758959 : if (_max_qps == std::numeric_limits<unsigned int>::max())
1710 0 : mooseError("Max QPS uninitialized");
1711 758959 : return _max_qps;
1712 : }
1713 :
1714 : Order
1715 52 : FEProblemBase::getMaxScalarOrder() const
1716 : {
1717 52 : return _max_scalar_order;
1718 : }
1719 :
1720 : void
1721 60482 : FEProblemBase::checkNonlocalCoupling()
1722 : {
1723 302410 : TIME_SECTION("checkNonlocalCoupling", 5, "Checking Nonlocal Coupling");
1724 :
1725 126951 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
1726 132057 : for (auto & nl : _nl)
1727 : {
1728 65588 : const auto & all_kernels = nl->getKernelWarehouse();
1729 65588 : const auto & kernels = all_kernels.getObjects(tid);
1730 150752 : for (const auto & kernel : kernels)
1731 : {
1732 : std::shared_ptr<NonlocalKernel> nonlocal_kernel =
1733 85164 : std::dynamic_pointer_cast<NonlocalKernel>(kernel);
1734 85164 : if (nonlocal_kernel)
1735 : {
1736 35 : if (_calculate_jacobian_in_uo)
1737 35 : _requires_nonlocal_coupling = true;
1738 35 : _nonlocal_kernels.addObject(kernel, tid);
1739 : }
1740 85164 : }
1741 : const MooseObjectWarehouse<IntegratedBCBase> & all_integrated_bcs =
1742 65588 : nl->getIntegratedBCWarehouse();
1743 65588 : const auto & integrated_bcs = all_integrated_bcs.getObjects(tid);
1744 74477 : for (const auto & integrated_bc : integrated_bcs)
1745 : {
1746 : std::shared_ptr<NonlocalIntegratedBC> nonlocal_integrated_bc =
1747 8889 : std::dynamic_pointer_cast<NonlocalIntegratedBC>(integrated_bc);
1748 8889 : if (nonlocal_integrated_bc)
1749 : {
1750 28 : if (_calculate_jacobian_in_uo)
1751 28 : _requires_nonlocal_coupling = true;
1752 28 : _nonlocal_integrated_bcs.addObject(integrated_bc, tid);
1753 : }
1754 8889 : }
1755 : }
1756 60482 : }
1757 :
1758 : void
1759 66469 : FEProblemBase::checkUserObjectJacobianRequirement(THREAD_ID tid)
1760 : {
1761 66469 : std::set<const MooseVariableFEBase *> uo_jacobian_moose_vars;
1762 : {
1763 66469 : std::vector<ShapeElementUserObject *> objs;
1764 66469 : theWarehouse()
1765 66469 : .query()
1766 132938 : .condition<AttribInterfaces>(Interfaces::ShapeElementUserObject)
1767 66469 : .condition<AttribThread>(tid)
1768 66469 : .queryInto(objs);
1769 :
1770 66513 : for (const auto & uo : objs)
1771 : {
1772 44 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1773 44 : const auto & mv_deps = uo->jacobianMooseVariables();
1774 44 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1775 : }
1776 66469 : }
1777 : {
1778 66469 : std::vector<ShapeSideUserObject *> objs;
1779 66469 : theWarehouse()
1780 66469 : .query()
1781 132938 : .condition<AttribInterfaces>(Interfaces::ShapeSideUserObject)
1782 66469 : .condition<AttribThread>(tid)
1783 66469 : .queryInto(objs);
1784 66525 : for (const auto & uo : objs)
1785 : {
1786 56 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1787 56 : const auto & mv_deps = uo->jacobianMooseVariables();
1788 56 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1789 : }
1790 66469 : }
1791 :
1792 66469 : _uo_jacobian_moose_vars[tid].assign(uo_jacobian_moose_vars.begin(), uo_jacobian_moose_vars.end());
1793 132938 : std::sort(
1794 132938 : _uo_jacobian_moose_vars[tid].begin(), _uo_jacobian_moose_vars[tid].end(), sortMooseVariables);
1795 66469 : }
1796 :
1797 : void
1798 63 : FEProblemBase::setVariableAllDoFMap(const std::vector<const MooseVariableFEBase *> & moose_vars)
1799 : {
1800 153 : for (unsigned int i = 0; i < moose_vars.size(); ++i)
1801 : {
1802 90 : VariableName var_name = moose_vars[i]->name();
1803 90 : auto & sys = _solver_systems[moose_vars[i]->sys().number()];
1804 90 : sys->setVariableGlobalDoFs(var_name);
1805 90 : _var_dof_map[var_name] = sys->getVariableGlobalDoFs();
1806 90 : }
1807 63 : }
1808 :
1809 : void
1810 374504932 : FEProblemBase::prepare(const Elem * elem, const THREAD_ID tid)
1811 : {
1812 751794915 : for (const auto i : index_range(_solver_systems))
1813 : {
1814 377289983 : _assembly[tid][i]->reinit(elem);
1815 377289983 : _solver_systems[i]->prepare(tid);
1816 :
1817 377289983 : if (i < _num_nl_sys)
1818 : {
1819 : // This method is called outside of residual/Jacobian callbacks during initial condition
1820 : // evaluation
1821 376693853 : if ((!_has_jacobian || !_const_jacobian) && currentlyComputingJacobian())
1822 47239634 : _assembly[tid][i]->prepareJacobianBlock();
1823 376693853 : _assembly[tid][i]->prepareResidual();
1824 376693853 : if (_has_nonlocal_coupling && currentlyComputingJacobian())
1825 8824 : _assembly[tid][i]->prepareNonlocal();
1826 : }
1827 : }
1828 374504932 : _aux->prepare(tid);
1829 :
1830 386151907 : if (_displaced_problem &&
1831 : // _reinit_displaced_neighbor applies to interface type objects which will do computations
1832 : // based on both elem and neighbor. Consequently, despite what you might think by its name, we
1833 : // must make sure we prepare the displaced elem
1834 386151907 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1835 : {
1836 7563782 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), tid);
1837 7563758 : if (_has_nonlocal_coupling)
1838 0 : _displaced_problem->prepareNonlocal(tid);
1839 : }
1840 374504908 : }
1841 :
1842 : void
1843 25443 : FEProblemBase::prepareFace(const Elem * elem, const THREAD_ID tid)
1844 : {
1845 50886 : for (auto & nl : _nl)
1846 25443 : nl->prepareFace(tid, true);
1847 25443 : _aux->prepareFace(tid, false);
1848 :
1849 25443 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1850 0 : _displaced_problem->prepareFace(_displaced_mesh->elemPtr(elem->id()), tid);
1851 25443 : }
1852 :
1853 : void
1854 0 : FEProblemBase::prepare(const Elem * elem,
1855 : unsigned int ivar,
1856 : unsigned int jvar,
1857 : const std::vector<dof_id_type> & dof_indices,
1858 : const THREAD_ID tid)
1859 : {
1860 0 : for (const auto i : index_range(_nl))
1861 : {
1862 0 : _assembly[tid][i]->reinit(elem);
1863 0 : _nl[i]->prepare(tid);
1864 : }
1865 :
1866 0 : _aux->prepare(tid);
1867 0 : const auto current_nl_sys_num = _current_nl_sys->number();
1868 0 : _assembly[tid][current_nl_sys_num]->prepareBlock(ivar, jvar, dof_indices);
1869 0 : if (_has_nonlocal_coupling)
1870 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1871 : {
1872 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1873 0 : _assembly[tid][current_nl_sys_num]->prepareBlockNonlocal(
1874 : ivar, jvar, dof_indices, jv.allDofIndices());
1875 : }
1876 :
1877 0 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1878 : {
1879 0 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), ivar, jvar, dof_indices, tid);
1880 0 : if (_has_nonlocal_coupling)
1881 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1882 : {
1883 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1884 0 : _displaced_problem->prepareBlockNonlocal(ivar, jvar, dof_indices, jv.allDofIndices(), tid);
1885 : }
1886 : }
1887 0 : }
1888 :
1889 : void
1890 405842296 : FEProblemBase::setCurrentSubdomainID(const Elem * elem, const THREAD_ID tid)
1891 : {
1892 405842296 : SubdomainID did = elem->subdomain_id();
1893 815480900 : for (const auto i : index_range(_solver_systems))
1894 : {
1895 409638604 : _assembly[tid][i]->setCurrentSubdomainID(did);
1896 422102620 : if (_displaced_problem &&
1897 422102620 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1898 7816535 : _displaced_problem->assembly(tid, i).setCurrentSubdomainID(did);
1899 : }
1900 405842296 : }
1901 :
1902 : void
1903 1455715347 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, unsigned int side, const THREAD_ID tid)
1904 : {
1905 1455715347 : SubdomainID did = elem->neighbor_ptr(side)->subdomain_id();
1906 2916598829 : for (const auto i : index_range(_nl))
1907 : {
1908 1460883482 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1909 1504469628 : if (_displaced_problem &&
1910 1504469628 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1911 25848827 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1912 : }
1913 1455715347 : }
1914 :
1915 : void
1916 15261613 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, const THREAD_ID tid)
1917 : {
1918 15261613 : SubdomainID did = elem->subdomain_id();
1919 31474538 : for (const auto i : index_range(_nl))
1920 : {
1921 16212925 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1922 16264221 : if (_displaced_problem &&
1923 16264221 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1924 51296 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1925 : }
1926 15261613 : }
1927 :
1928 : void
1929 136662 : FEProblemBase::prepareAssembly(const THREAD_ID tid)
1930 : {
1931 136662 : _assembly[tid][_current_nl_sys->number()]->prepare();
1932 136662 : if (_has_nonlocal_coupling)
1933 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
1934 :
1935 136662 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1936 : {
1937 51296 : _displaced_problem->prepareAssembly(tid);
1938 51296 : if (_has_nonlocal_coupling)
1939 0 : _displaced_problem->prepareNonlocal(tid);
1940 : }
1941 136662 : }
1942 :
1943 : void
1944 266607 : FEProblemBase::addResidual(const THREAD_ID tid)
1945 : {
1946 533214 : _assembly[tid][_current_nl_sys->number()]->addResidual(Assembly::GlobalDataKey{},
1947 266607 : currentResidualVectorTags());
1948 :
1949 266607 : if (_displaced_problem)
1950 4776 : _displaced_problem->addResidual(tid);
1951 266607 : }
1952 :
1953 : void
1954 1984063 : FEProblemBase::addResidualNeighbor(const THREAD_ID tid)
1955 : {
1956 3968126 : _assembly[tid][_current_nl_sys->number()]->addResidualNeighbor(Assembly::GlobalDataKey{},
1957 1984063 : currentResidualVectorTags());
1958 :
1959 1984063 : if (_displaced_problem)
1960 61744 : _displaced_problem->addResidualNeighbor(tid);
1961 1984063 : }
1962 :
1963 : void
1964 1966412 : FEProblemBase::addResidualLower(const THREAD_ID tid)
1965 : {
1966 3932824 : _assembly[tid][_current_nl_sys->number()]->addResidualLower(Assembly::GlobalDataKey{},
1967 1966412 : currentResidualVectorTags());
1968 :
1969 1966412 : if (_displaced_problem)
1970 61956 : _displaced_problem->addResidualLower(tid);
1971 1966412 : }
1972 :
1973 : void
1974 48497 : FEProblemBase::addResidualScalar(const THREAD_ID tid /* = 0*/)
1975 : {
1976 96994 : _assembly[tid][_current_nl_sys->number()]->addResidualScalar(Assembly::GlobalDataKey{},
1977 48497 : currentResidualVectorTags());
1978 48497 : }
1979 :
1980 : void
1981 288126382 : FEProblemBase::cacheResidual(const THREAD_ID tid)
1982 : {
1983 288126382 : SubProblem::cacheResidual(tid);
1984 288126382 : if (_displaced_problem)
1985 7593378 : _displaced_problem->cacheResidual(tid);
1986 288126382 : }
1987 :
1988 : void
1989 50077 : FEProblemBase::cacheResidualNeighbor(const THREAD_ID tid)
1990 : {
1991 50077 : SubProblem::cacheResidualNeighbor(tid);
1992 50077 : if (_displaced_problem)
1993 32 : _displaced_problem->cacheResidualNeighbor(tid);
1994 50077 : }
1995 :
1996 : void
1997 16990759 : FEProblemBase::addCachedResidual(const THREAD_ID tid)
1998 : {
1999 16990759 : SubProblem::addCachedResidual(tid);
2000 16990759 : if (_displaced_problem)
2001 495533 : _displaced_problem->addCachedResidual(tid);
2002 16990759 : }
2003 :
2004 : void
2005 11328 : FEProblemBase::addCachedResidualDirectly(NumericVector<Number> & residual, const THREAD_ID tid)
2006 : {
2007 11328 : if (_current_nl_sys->hasVector(_current_nl_sys->timeVectorTag()))
2008 30252 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2009 30252 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->timeVectorTag()));
2010 :
2011 11328 : if (_current_nl_sys->hasVector(_current_nl_sys->nonTimeVectorTag()))
2012 33984 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2013 33984 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->nonTimeVectorTag()));
2014 :
2015 11328 : std::vector<VectorTag> extra_residual_vector_tags;
2016 11328 : extra_residual_vector_tags.reserve(currentResidualVectorTags().size());
2017 11328 : const auto time_tag = _current_nl_sys->timeVectorTag();
2018 11328 : const auto non_time_tag = _current_nl_sys->nonTimeVectorTag();
2019 44140 : for (const auto & vector_tag : currentResidualVectorTags())
2020 32812 : if (vector_tag._id != time_tag && vector_tag._id != non_time_tag)
2021 11400 : extra_residual_vector_tags.push_back(vector_tag);
2022 :
2023 : // Flush extra vector tag caches (e.g. from extra_vector_tags on NodalConstraints)
2024 : // to their respective system vectors after the standard TIME/NONTIME caches above.
2025 : // Without this, NodalConstraint contributions to extra vector tags are silently
2026 : // discarded by the blanket clearCachedResiduals.
2027 11328 : _assembly[tid][_current_nl_sys->number()]->addCachedResiduals(Assembly::GlobalDataKey{},
2028 : extra_residual_vector_tags);
2029 :
2030 : // We do this because by adding the cached residual directly, we cannot ensure that all of the
2031 : // cached residuals are emptied after only the two add calls above
2032 11328 : _assembly[tid][_current_nl_sys->number()]->clearCachedResiduals(Assembly::GlobalDataKey{});
2033 :
2034 11328 : if (_displaced_problem)
2035 35 : _displaced_problem->addCachedResidualDirectly(residual, tid);
2036 11328 : }
2037 :
2038 : void
2039 0 : FEProblemBase::setResidual(NumericVector<Number> & residual, const THREAD_ID tid)
2040 : {
2041 0 : _assembly[tid][_current_nl_sys->number()]->setResidual(
2042 : residual,
2043 0 : Assembly::GlobalDataKey{},
2044 0 : getVectorTag(_nl[_current_nl_sys->number()]->residualVectorTag()));
2045 0 : if (_displaced_problem)
2046 0 : _displaced_problem->setResidual(residual, tid);
2047 0 : }
2048 :
2049 : void
2050 0 : FEProblemBase::setResidualNeighbor(NumericVector<Number> & residual, const THREAD_ID tid)
2051 : {
2052 0 : _assembly[tid][_current_nl_sys->number()]->setResidualNeighbor(
2053 0 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->residualVectorTag()));
2054 0 : if (_displaced_problem)
2055 0 : _displaced_problem->setResidualNeighbor(residual, tid);
2056 0 : }
2057 :
2058 : void
2059 37415 : FEProblemBase::addJacobian(const THREAD_ID tid)
2060 : {
2061 37415 : _assembly[tid][_current_nl_sys->number()]->addJacobian(Assembly::GlobalDataKey{});
2062 37415 : if (_has_nonlocal_coupling)
2063 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianNonlocal(Assembly::GlobalDataKey{});
2064 37415 : if (_displaced_problem)
2065 : {
2066 200 : _displaced_problem->addJacobian(tid);
2067 200 : if (_has_nonlocal_coupling)
2068 0 : _displaced_problem->addJacobianNonlocal(tid);
2069 : }
2070 37415 : }
2071 :
2072 : void
2073 7852 : FEProblemBase::addJacobianNeighbor(const THREAD_ID tid)
2074 : {
2075 7852 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighbor(Assembly::GlobalDataKey{});
2076 7852 : if (_displaced_problem)
2077 44 : _displaced_problem->addJacobianNeighbor(tid);
2078 7852 : }
2079 :
2080 : void
2081 109104 : FEProblemBase::addJacobianNeighborLowerD(const THREAD_ID tid)
2082 : {
2083 109104 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborLowerD(Assembly::GlobalDataKey{});
2084 109104 : if (_displaced_problem)
2085 3072 : _displaced_problem->addJacobianNeighborLowerD(tid);
2086 109104 : }
2087 :
2088 : void
2089 4696 : FEProblemBase::addJacobianLowerD(const THREAD_ID tid)
2090 : {
2091 4696 : _assembly[tid][_current_nl_sys->number()]->addJacobianLowerD(Assembly::GlobalDataKey{});
2092 4696 : if (_displaced_problem)
2093 192 : _displaced_problem->addJacobianLowerD(tid);
2094 4696 : }
2095 :
2096 : void
2097 11616 : FEProblemBase::addJacobianScalar(const THREAD_ID tid /* = 0*/)
2098 : {
2099 11616 : _assembly[tid][_current_nl_sys->number()]->addJacobianScalar(Assembly::GlobalDataKey{});
2100 11616 : }
2101 :
2102 : void
2103 30054 : FEProblemBase::addJacobianOffDiagScalar(unsigned int ivar, const THREAD_ID tid /* = 0*/)
2104 : {
2105 60108 : _assembly[tid][_current_nl_sys->number()]->addJacobianOffDiagScalar(ivar,
2106 30054 : Assembly::GlobalDataKey{});
2107 30054 : }
2108 :
2109 : void
2110 46631072 : FEProblemBase::cacheJacobian(const THREAD_ID tid)
2111 : {
2112 46631072 : SubProblem::cacheJacobian(tid);
2113 46631072 : if (_displaced_problem)
2114 1544138 : _displaced_problem->cacheJacobian(tid);
2115 46631072 : }
2116 :
2117 : void
2118 9409 : FEProblemBase::cacheJacobianNeighbor(const THREAD_ID tid)
2119 : {
2120 9409 : SubProblem::cacheJacobianNeighbor(tid);
2121 9409 : if (_displaced_problem)
2122 0 : _displaced_problem->cacheJacobianNeighbor(tid);
2123 9409 : }
2124 :
2125 : void
2126 2884119 : FEProblemBase::addCachedJacobian(const THREAD_ID tid)
2127 : {
2128 2884119 : SubProblem::addCachedJacobian(tid);
2129 2884116 : if (_displaced_problem)
2130 96451 : _displaced_problem->addCachedJacobian(tid);
2131 2884116 : }
2132 :
2133 : void
2134 70832 : FEProblemBase::addJacobianBlockTags(SparseMatrix<Number> & jacobian,
2135 : unsigned int ivar,
2136 : unsigned int jvar,
2137 : const DofMap & dof_map,
2138 : std::vector<dof_id_type> & dof_indices,
2139 : const std::set<TagID> & tags,
2140 : const THREAD_ID tid)
2141 : {
2142 141664 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockTags(
2143 70832 : jacobian, ivar, jvar, dof_map, dof_indices, Assembly::GlobalDataKey{}, tags);
2144 :
2145 70832 : if (_has_nonlocal_coupling)
2146 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2147 : {
2148 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2149 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockNonlocalTags(
2150 : jacobian,
2151 : ivar,
2152 : jvar,
2153 : dof_map,
2154 : dof_indices,
2155 : jv.allDofIndices(),
2156 0 : Assembly::GlobalDataKey{},
2157 : tags);
2158 : }
2159 :
2160 70832 : if (_displaced_problem)
2161 : {
2162 0 : _displaced_problem->addJacobianBlockTags(jacobian, ivar, jvar, dof_map, dof_indices, tags, tid);
2163 0 : if (_has_nonlocal_coupling)
2164 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2165 : {
2166 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2167 0 : _displaced_problem->addJacobianBlockNonlocal(
2168 : jacobian, ivar, jvar, dof_map, dof_indices, jv.allDofIndices(), tags, tid);
2169 : }
2170 : }
2171 70832 : }
2172 :
2173 : void
2174 768 : FEProblemBase::addJacobianNeighbor(SparseMatrix<Number> & jacobian,
2175 : unsigned int ivar,
2176 : unsigned int jvar,
2177 : const DofMap & dof_map,
2178 : std::vector<dof_id_type> & dof_indices,
2179 : std::vector<dof_id_type> & neighbor_dof_indices,
2180 : const std::set<TagID> & tags,
2181 : const THREAD_ID tid)
2182 : {
2183 1536 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborTags(jacobian,
2184 : ivar,
2185 : jvar,
2186 : dof_map,
2187 : dof_indices,
2188 : neighbor_dof_indices,
2189 768 : Assembly::GlobalDataKey{},
2190 : tags);
2191 768 : if (_displaced_problem)
2192 0 : _displaced_problem->addJacobianNeighbor(
2193 : jacobian, ivar, jvar, dof_map, dof_indices, neighbor_dof_indices, tags, tid);
2194 768 : }
2195 :
2196 : void
2197 125422384 : FEProblemBase::prepareShapes(unsigned int var, const THREAD_ID tid)
2198 : {
2199 125422384 : _assembly[tid][_current_nl_sys->number()]->copyShapes(var);
2200 125422384 : }
2201 :
2202 : void
2203 602570 : FEProblemBase::prepareFaceShapes(unsigned int var, const THREAD_ID tid)
2204 : {
2205 602570 : _assembly[tid][_current_nl_sys->number()]->copyFaceShapes(var);
2206 602570 : }
2207 :
2208 : void
2209 184534 : FEProblemBase::prepareNeighborShapes(unsigned int var, const THREAD_ID tid)
2210 : {
2211 184534 : _assembly[tid][_current_nl_sys->number()]->copyNeighborShapes(var);
2212 184534 : }
2213 :
2214 : void
2215 874909 : FEProblemBase::addGhostedElem(dof_id_type elem_id)
2216 : {
2217 874909 : if (_mesh.elemPtr(elem_id)->processor_id() != processor_id())
2218 205736 : _ghosted_elems.insert(elem_id);
2219 874909 : }
2220 :
2221 : void
2222 29442 : FEProblemBase::addGhostedBoundary(BoundaryID boundary_id)
2223 : {
2224 29442 : _mesh.addGhostedBoundary(boundary_id);
2225 29442 : if (_displaced_problem)
2226 26702 : _displaced_mesh->addGhostedBoundary(boundary_id);
2227 29442 : }
2228 :
2229 : void
2230 68203 : FEProblemBase::ghostGhostedBoundaries()
2231 : {
2232 341015 : TIME_SECTION("ghostGhostedBoundaries", 3, "Ghosting Ghosted Boundaries");
2233 :
2234 68203 : _mesh.ghostGhostedBoundaries();
2235 :
2236 68203 : if (_displaced_problem)
2237 2591 : _displaced_mesh->ghostGhostedBoundaries();
2238 68203 : }
2239 :
2240 : void
2241 0 : FEProblemBase::sizeZeroes(unsigned int /*size*/, const THREAD_ID /*tid*/)
2242 : {
2243 0 : mooseDoOnce(mooseWarning(
2244 : "This function is deprecated and no longer performs any function. Please do not call it."));
2245 0 : }
2246 :
2247 : bool
2248 304022 : FEProblemBase::reinitDirac(const Elem * elem, const THREAD_ID tid)
2249 : {
2250 304022 : std::vector<Point> & points = _dirac_kernel_info.getPoints()[elem].first;
2251 :
2252 304022 : unsigned int n_points = points.size();
2253 :
2254 304022 : if (n_points)
2255 : {
2256 299046 : if (n_points > _max_qps)
2257 : {
2258 0 : _max_qps = n_points;
2259 :
2260 : /**
2261 : * The maximum number of qps can rise if several Dirac points are added to a single element.
2262 : * In that case we need to resize the zeros to compensate.
2263 : */
2264 0 : unsigned int max_qpts = getMaxQps();
2265 0 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
2266 : {
2267 : // the highest available order in libMesh is 43
2268 0 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
2269 0 : _zero[tid].resize(max_qpts, 0);
2270 0 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
2271 0 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
2272 0 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
2273 0 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
2274 : }
2275 : }
2276 :
2277 598092 : for (const auto i : index_range(_nl))
2278 : {
2279 299046 : _assembly[tid][i]->reinitAtPhysical(elem, points);
2280 299046 : _nl[i]->prepare(tid);
2281 : }
2282 299046 : _aux->prepare(tid);
2283 :
2284 299046 : reinitElem(elem, tid);
2285 : }
2286 :
2287 304022 : _assembly[tid][_current_nl_sys->number()]->prepare();
2288 304022 : if (_has_nonlocal_coupling)
2289 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
2290 :
2291 304022 : bool have_points = n_points > 0;
2292 304022 : if (_displaced_problem && (_reinit_displaced_elem))
2293 : {
2294 4976 : have_points |= _displaced_problem->reinitDirac(_displaced_mesh->elemPtr(elem->id()), tid);
2295 4976 : if (_has_nonlocal_coupling)
2296 0 : _displaced_problem->prepareNonlocal(tid);
2297 : }
2298 :
2299 304022 : return have_points;
2300 : }
2301 :
2302 : void
2303 374751118 : FEProblemBase::reinitElem(const Elem * elem, const THREAD_ID tid)
2304 : {
2305 752287287 : for (auto & sys : _solver_systems)
2306 377536169 : sys->reinitElem(elem, tid);
2307 374751118 : _aux->reinitElem(elem, tid);
2308 :
2309 374751118 : if (_displaced_problem && _reinit_displaced_elem)
2310 4426155 : _displaced_problem->reinitElem(_displaced_mesh->elemPtr(elem->id()), tid);
2311 374751118 : }
2312 :
2313 : void
2314 58516 : FEProblemBase::reinitElemPhys(const Elem * const elem,
2315 : const std::vector<Point> & phys_points_in_elem,
2316 : const THREAD_ID tid)
2317 : {
2318 : mooseAssert(_mesh.queryElemPtr(elem->id()) == elem,
2319 : "Are you calling this method with a displaced mesh element?");
2320 :
2321 117032 : for (const auto i : index_range(_solver_systems))
2322 : {
2323 58516 : _assembly[tid][i]->reinitAtPhysical(elem, phys_points_in_elem);
2324 58516 : _solver_systems[i]->prepare(tid);
2325 58516 : _assembly[tid][i]->prepare();
2326 58516 : if (_has_nonlocal_coupling)
2327 0 : _assembly[tid][i]->prepareNonlocal();
2328 : }
2329 58516 : _aux->prepare(tid);
2330 :
2331 58516 : reinitElem(elem, tid);
2332 58516 : }
2333 :
2334 : void
2335 0 : FEProblemBase::reinitElemFace(const Elem * const elem,
2336 : const unsigned int side,
2337 : const BoundaryID,
2338 : const THREAD_ID tid)
2339 : {
2340 0 : mooseDeprecated(
2341 : "reinitElemFace with a BoundaryID argument is deprecated because the boundary id was never "
2342 : "used. Please call reinitElemFace without the BoundaryID argument instead");
2343 :
2344 0 : reinitElemFace(elem, side, tid);
2345 0 : }
2346 :
2347 : void
2348 5204893 : FEProblemBase::reinitElemFace(const Elem * const elem, const unsigned int side, const THREAD_ID tid)
2349 : {
2350 10410134 : for (const auto i : index_range(_solver_systems))
2351 : {
2352 5205241 : _assembly[tid][i]->reinit(elem, side);
2353 5205241 : _solver_systems[i]->reinitElemFace(elem, side, tid);
2354 : }
2355 5204893 : _aux->reinitElemFace(elem, side, tid);
2356 :
2357 5204893 : if (_displaced_problem && _reinit_displaced_face)
2358 82248 : _displaced_problem->reinitElemFace(_displaced_mesh->elemPtr(elem->id()), side, tid);
2359 5204893 : }
2360 :
2361 : void
2362 578443 : FEProblemBase::reinitLowerDElem(const Elem * lower_d_elem,
2363 : const THREAD_ID tid,
2364 : const std::vector<Point> * const pts,
2365 : const std::vector<Real> * const weights)
2366 : {
2367 578443 : SubProblem::reinitLowerDElem(lower_d_elem, tid, pts, weights);
2368 :
2369 578443 : if (_displaced_problem && _displaced_mesh)
2370 960 : _displaced_problem->reinitLowerDElem(
2371 960 : _displaced_mesh->elemPtr(lower_d_elem->id()), tid, pts, weights);
2372 578443 : }
2373 :
2374 : void
2375 26043812 : FEProblemBase::reinitNode(const Node * node, const THREAD_ID tid)
2376 : {
2377 26043812 : if (_displaced_problem && _reinit_displaced_elem)
2378 1001711 : _displaced_problem->reinitNode(&_displaced_mesh->nodeRef(node->id()), tid);
2379 :
2380 52090540 : for (const auto i : index_range(_nl))
2381 : {
2382 26046728 : _assembly[tid][i]->reinit(node);
2383 26046728 : _nl[i]->reinitNode(node, tid);
2384 : }
2385 26043812 : _aux->reinitNode(node, tid);
2386 26043812 : }
2387 :
2388 : void
2389 65194184 : FEProblemBase::reinitNodeFace(const Node * node, BoundaryID bnd_id, const THREAD_ID tid)
2390 : {
2391 65194184 : if (_displaced_problem && _reinit_displaced_face)
2392 3388984 : _displaced_problem->reinitNodeFace(&_displaced_mesh->nodeRef(node->id()), bnd_id, tid);
2393 :
2394 131183436 : for (const auto i : index_range(_nl))
2395 : {
2396 65989252 : _assembly[tid][i]->reinit(node);
2397 65989252 : _nl[i]->reinitNodeFace(node, bnd_id, tid);
2398 : }
2399 65194184 : _aux->reinitNodeFace(node, bnd_id, tid);
2400 65194184 : }
2401 :
2402 : void
2403 5093 : FEProblemBase::reinitNodes(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2404 : {
2405 5093 : if (_displaced_problem && _reinit_displaced_elem)
2406 0 : _displaced_problem->reinitNodes(nodes, tid);
2407 :
2408 10186 : for (auto & nl : _nl)
2409 5093 : nl->reinitNodes(nodes, tid);
2410 5093 : _aux->reinitNodes(nodes, tid);
2411 5093 : }
2412 :
2413 : void
2414 999 : FEProblemBase::reinitNodesNeighbor(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2415 : {
2416 999 : if (_displaced_problem && _reinit_displaced_elem)
2417 0 : _displaced_problem->reinitNodesNeighbor(nodes, tid);
2418 :
2419 1998 : for (auto & nl : _nl)
2420 999 : nl->reinitNodesNeighbor(nodes, tid);
2421 999 : _aux->reinitNodesNeighbor(nodes, tid);
2422 999 : }
2423 :
2424 : void
2425 8442621 : FEProblemBase::reinitScalars(const THREAD_ID tid, bool reinit_for_derivative_reordering /*=false*/)
2426 : {
2427 42213105 : TIME_SECTION("reinitScalars", 3, "Reinitializing Scalar Variables");
2428 :
2429 8442621 : if (_displaced_problem && _reinit_displaced_elem)
2430 113553 : _displaced_problem->reinitScalars(tid, reinit_for_derivative_reordering);
2431 :
2432 17124695 : for (auto & nl : _nl)
2433 8682074 : nl->reinitScalars(tid, reinit_for_derivative_reordering);
2434 8442621 : _aux->reinitScalars(tid, reinit_for_derivative_reordering);
2435 :
2436 : // This is called outside of residual/Jacobian call-backs
2437 17131899 : for (auto & assembly : _assembly[tid])
2438 8689278 : assembly->prepareScalar();
2439 8442621 : }
2440 :
2441 : void
2442 185594 : FEProblemBase::reinitOffDiagScalars(const THREAD_ID tid)
2443 : {
2444 185594 : _assembly[tid][_current_nl_sys->number()]->prepareOffDiagScalar();
2445 185594 : if (_displaced_problem)
2446 60 : _displaced_problem->reinitOffDiagScalars(tid);
2447 185594 : }
2448 :
2449 : void
2450 3641566 : FEProblemBase::reinitNeighbor(const Elem * elem, unsigned int side, const THREAD_ID tid)
2451 : {
2452 3641566 : setNeighborSubdomainID(elem, side, tid);
2453 :
2454 3641566 : const Elem * neighbor = elem->neighbor_ptr(side);
2455 3641566 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(elem);
2456 :
2457 7283159 : for (const auto i : index_range(_nl))
2458 : {
2459 3641593 : _assembly[tid][i]->reinitElemAndNeighbor(elem, side, neighbor, neighbor_side);
2460 3641593 : _nl[i]->prepareNeighbor(tid);
2461 : // Called during stateful material property evaluation outside of solve
2462 3641593 : _assembly[tid][i]->prepareNeighbor();
2463 : }
2464 3641566 : _aux->prepareNeighbor(tid);
2465 :
2466 7283159 : for (auto & nl : _nl)
2467 : {
2468 3641593 : nl->reinitElemFace(elem, side, tid);
2469 3641593 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2470 : }
2471 3641566 : _aux->reinitElemFace(elem, side, tid);
2472 3641566 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2473 :
2474 3641566 : if (_displaced_problem && _reinit_displaced_neighbor)
2475 : {
2476 : // There are cases like for cohesive zone modeling without significant sliding where we cannot
2477 : // use FEInterface::inverse_map in Assembly::reinitElemAndNeighbor in the displaced problem
2478 : // because the physical points coming from the element don't actually lie on the neighbor.
2479 : // Moreover, what's the point of doing another physical point inversion in other cases? We only
2480 : // care about the reference points which we can just take from the undisplaced computation
2481 64800 : const auto & displaced_ref_pts = _assembly[tid][0]->qRuleNeighbor()->get_points();
2482 :
2483 64800 : _displaced_problem->reinitNeighbor(
2484 64800 : _displaced_mesh->elemPtr(elem->id()), side, tid, &displaced_ref_pts);
2485 : }
2486 3641566 : }
2487 :
2488 : void
2489 2068761 : FEProblemBase::reinitElemNeighborAndLowerD(const Elem * elem,
2490 : unsigned int side,
2491 : const THREAD_ID tid)
2492 : {
2493 2068761 : reinitNeighbor(elem, side, tid);
2494 :
2495 2068761 : const Elem * lower_d_elem = _mesh.getLowerDElem(elem, side);
2496 2068761 : if (lower_d_elem && _mesh.interiorLowerDBlocks().count(lower_d_elem->subdomain_id()) > 0)
2497 10320 : reinitLowerDElem(lower_d_elem, tid);
2498 : else
2499 : {
2500 : // with mesh refinement, lower-dimensional element might be defined on neighbor side
2501 2058441 : auto & neighbor = _assembly[tid][0]->neighbor();
2502 2058441 : auto & neighbor_side = _assembly[tid][0]->neighborSide();
2503 2058441 : const Elem * lower_d_elem_neighbor = _mesh.getLowerDElem(neighbor, neighbor_side);
2504 2058441 : if (lower_d_elem_neighbor &&
2505 2058441 : _mesh.interiorLowerDBlocks().count(lower_d_elem_neighbor->subdomain_id()) > 0)
2506 : {
2507 0 : auto qps = _assembly[tid][0]->qPointsFaceNeighbor().stdVector();
2508 0 : std::vector<Point> reference_points;
2509 0 : FEMap::inverse_map(
2510 0 : lower_d_elem_neighbor->dim(), lower_d_elem_neighbor, qps, reference_points);
2511 0 : reinitLowerDElem(lower_d_elem_neighbor, tid, &reference_points);
2512 0 : }
2513 : }
2514 :
2515 2068761 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
2516 64740 : _displaced_problem->reinitElemNeighborAndLowerD(
2517 64740 : _displaced_mesh->elemPtr(elem->id()), side, tid);
2518 2068761 : }
2519 :
2520 : void
2521 97230 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2522 : unsigned int neighbor_side,
2523 : const std::vector<Point> & physical_points,
2524 : const THREAD_ID tid)
2525 : {
2526 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2527 : "Are you calling this method with a displaced mesh element?");
2528 :
2529 194460 : for (const auto i : index_range(_nl))
2530 : {
2531 : // Reinits shape the functions at the physical points
2532 97230 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, neighbor_side, physical_points);
2533 :
2534 : // Sets the neighbor dof indices
2535 97230 : _nl[i]->prepareNeighbor(tid);
2536 : }
2537 97230 : _aux->prepareNeighbor(tid);
2538 :
2539 : // Resizes Re and Ke
2540 97230 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2541 :
2542 : // Compute the values of each variable at the points
2543 194460 : for (auto & nl : _nl)
2544 97230 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2545 97230 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2546 97230 : }
2547 :
2548 : void
2549 19880 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2550 : const std::vector<Point> & physical_points,
2551 : const THREAD_ID tid)
2552 : {
2553 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2554 : "Are you calling this method with a displaced mesh element?");
2555 :
2556 39760 : for (const auto i : index_range(_nl))
2557 : {
2558 : // Reinits shape the functions at the physical points
2559 19880 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, physical_points);
2560 :
2561 : // Sets the neighbor dof indices
2562 19880 : _nl[i]->prepareNeighbor(tid);
2563 : }
2564 19880 : _aux->prepareNeighbor(tid);
2565 :
2566 : // Resizes Re and Ke
2567 19880 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2568 :
2569 : // Compute the values of each variable at the points
2570 39760 : for (auto & nl : _nl)
2571 19880 : nl->reinitNeighbor(neighbor, tid);
2572 19880 : _aux->reinitNeighbor(neighbor, tid);
2573 19880 : }
2574 :
2575 : void
2576 35512 : FEProblemBase::getDiracElements(std::set<const Elem *> & elems)
2577 : {
2578 : // First add in the undisplaced elements
2579 35512 : elems = _dirac_kernel_info.getElements();
2580 :
2581 35512 : if (_displaced_problem)
2582 : {
2583 2359 : std::set<const Elem *> displaced_elements;
2584 2359 : _displaced_problem->getDiracElements(displaced_elements);
2585 :
2586 : { // Use the ids from the displaced elements to get the undisplaced elements
2587 : // and add them to the list
2588 7335 : for (const auto & elem : displaced_elements)
2589 4976 : elems.insert(_mesh.elemPtr(elem->id()));
2590 : }
2591 2359 : }
2592 35512 : }
2593 :
2594 : void
2595 3521443 : FEProblemBase::clearDiracInfo()
2596 : {
2597 3521443 : _dirac_kernel_info.clearPoints();
2598 :
2599 3521443 : if (_displaced_problem)
2600 144140 : _displaced_problem->clearDiracInfo();
2601 3521443 : }
2602 :
2603 : void
2604 5097952 : FEProblemBase::subdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2605 : {
2606 5097952 : _all_materials.subdomainSetup(subdomain, tid);
2607 : // Call the subdomain methods of the output system, these are not threaded so only call it once
2608 5097952 : if (tid == 0)
2609 5085226 : _app.getOutputWarehouse().subdomainSetup();
2610 :
2611 10330977 : for (auto & nl : _nl)
2612 5233025 : nl->subdomainSetup(subdomain, tid);
2613 :
2614 : // FIXME: call displaced_problem->subdomainSetup() ?
2615 : // When adding possibility with materials being evaluated on displaced mesh
2616 5097952 : }
2617 :
2618 : void
2619 16558770 : FEProblemBase::neighborSubdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2620 : {
2621 16558770 : _all_materials.neighborSubdomainSetup(subdomain, tid);
2622 16558770 : }
2623 :
2624 : void
2625 49984 : FEProblemBase::addFunction(const std::string & type,
2626 : const std::string & name,
2627 : InputParameters & parameters)
2628 : {
2629 : parallel_object_only();
2630 :
2631 99968 : parameters.set<SubProblem *>("_subproblem") = this;
2632 :
2633 104312 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2634 : {
2635 54434 : std::shared_ptr<Function> func = _factory.create<Function>(type, name, parameters, tid);
2636 54328 : logAdd("Function", name, type, parameters);
2637 54328 : _functions.addObject(func, tid);
2638 :
2639 54328 : if (auto * const functor = dynamic_cast<Moose::FunctorBase<Real> *>(func.get()))
2640 : {
2641 54328 : this->addFunctor(name, *functor, tid);
2642 54328 : if (_displaced_problem)
2643 1780 : _displaced_problem->addFunctor(name, *functor, tid);
2644 : }
2645 : else
2646 0 : mooseError("Unrecognized function functor type");
2647 54328 : }
2648 49878 : }
2649 :
2650 : void
2651 153726 : FEProblemBase::addConvergence(const std::string & type,
2652 : const std::string & name,
2653 : InputParameters & parameters)
2654 : {
2655 : parallel_object_only();
2656 :
2657 323943 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2658 : {
2659 170244 : std::shared_ptr<Convergence> conv = _factory.create<Convergence>(type, name, parameters, tid);
2660 170217 : _convergences.addObject(conv, tid);
2661 170217 : }
2662 153699 : }
2663 :
2664 : void
2665 60123 : FEProblemBase::addDefaultNonlinearConvergence(const InputParameters & params_to_apply)
2666 : {
2667 60123 : const std::string class_name = "DefaultNonlinearConvergence";
2668 60123 : InputParameters params = _factory.getValidParams(class_name);
2669 60123 : params.applyParameters(params_to_apply);
2670 60123 : params.applyParameters(parameters());
2671 60123 : params.set<bool>("added_as_default") = true;
2672 119298 : for (const auto & conv_name : getNonlinearConvergenceNames())
2673 59175 : addConvergence(class_name, conv_name, params);
2674 60123 : }
2675 :
2676 : void
2677 62220 : FEProblemBase::addDefaultMultiAppFixedPointConvergence(const InputParameters & params_to_apply)
2678 : {
2679 62220 : const std::string class_name = "DefaultMultiAppFixedPointConvergence";
2680 62220 : InputParameters params = _factory.getValidParams(class_name);
2681 62220 : params.applyParameters(params_to_apply);
2682 62220 : params.applyParameters(parameters());
2683 62220 : params.set<bool>("added_as_default") = true;
2684 62220 : addConvergence(class_name, getMultiAppFixedPointConvergenceName(), params);
2685 62211 : }
2686 :
2687 : void
2688 30418 : FEProblemBase::addDefaultSteadyStateConvergence(const InputParameters & params_to_apply)
2689 : {
2690 30418 : const std::string class_name = "DefaultSteadyStateConvergence";
2691 30418 : InputParameters params = _factory.getValidParams(class_name);
2692 30418 : params.applyParameters(params_to_apply);
2693 30418 : params.applyParameters(parameters());
2694 30418 : params.set<bool>("added_as_default") = true;
2695 30418 : addConvergence(class_name, getSteadyStateConvergenceName(), params);
2696 30418 : }
2697 :
2698 : bool
2699 87406 : FEProblemBase::hasFunction(const std::string & name, const THREAD_ID tid)
2700 : {
2701 87406 : return _functions.hasActiveObject(name, tid);
2702 : }
2703 :
2704 : Function &
2705 63558 : FEProblemBase::getFunction(const std::string & name, const THREAD_ID tid)
2706 : {
2707 : // This thread lock is necessary since this method will create functions
2708 : // for all threads if one is missing.
2709 63558 : Threads::spin_mutex::scoped_lock lock(get_function_mutex);
2710 :
2711 63558 : if (!hasFunction(name, tid))
2712 : {
2713 : // If we didn't find a function, it might be a default function, attempt to construct one now
2714 20498 : std::istringstream ss(name);
2715 : Real real_value;
2716 :
2717 : // First see if it's just a constant. If it is, build a ConstantFunction
2718 20498 : if (ss >> real_value && ss.eof())
2719 : {
2720 12740 : InputParameters params = _factory.getValidParams("ConstantFunction");
2721 12740 : params.set<Real>("value") = real_value;
2722 19110 : addFunction("ConstantFunction", ss.str(), params);
2723 6370 : }
2724 : else
2725 : {
2726 14128 : FunctionParserBase<Real> fp;
2727 14128 : std::string vars = "x,y,z,t,NaN,pi,e";
2728 14128 : if (fp.Parse(name, vars) == -1) // -1 for success
2729 : {
2730 : // It parsed ok, so build a MooseParsedFunction
2731 42369 : InputParameters params = _factory.getValidParams("ParsedFunction");
2732 14123 : params.set<std::string>("expression") = name;
2733 28246 : addFunction("ParsedFunction", name, params);
2734 14123 : }
2735 14128 : }
2736 :
2737 : // Try once more
2738 20498 : if (!hasFunction(name, tid))
2739 : {
2740 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_function"),
2741 : "getFunction() was called before Functions have been constructed. The requested "
2742 : "Function '" +
2743 : name + "' may exist in the input file, but Functions are not available yet.");
2744 :
2745 7 : mooseError("Unable to find function " + name);
2746 : }
2747 20495 : }
2748 :
2749 63553 : auto * const ret = dynamic_cast<Function *>(_functions.getActiveObject(name, tid).get());
2750 63553 : if (!ret)
2751 0 : mooseError("No function named ", name, " of appropriate type");
2752 :
2753 63553 : return *ret;
2754 63555 : }
2755 :
2756 : bool
2757 212816 : FEProblemBase::hasConvergence(const std::string & name, const THREAD_ID tid) const
2758 : {
2759 212816 : return _convergences.hasActiveObject(name, tid);
2760 : }
2761 :
2762 : Convergence &
2763 1135420 : FEProblemBase::getConvergence(const std::string & name, const THREAD_ID tid) const
2764 : {
2765 1135420 : auto * const ret = dynamic_cast<Convergence *>(_convergences.getActiveObject(name, tid).get());
2766 1135420 : if (!ret)
2767 0 : mooseError("The Convergence object '", name, "' does not exist.");
2768 :
2769 1135420 : return *ret;
2770 : }
2771 :
2772 : const std::vector<std::shared_ptr<Convergence>> &
2773 195394 : FEProblemBase::getConvergenceObjects(const THREAD_ID tid) const
2774 : {
2775 195394 : return _convergences.getActiveObjects(tid);
2776 : }
2777 :
2778 : void
2779 687 : FEProblemBase::addMeshDivision(const std::string & type,
2780 : const std::string & name,
2781 : InputParameters & parameters)
2782 : {
2783 : parallel_object_only();
2784 1374 : parameters.set<FEProblemBase *>("_fe_problem_base") = this;
2785 1374 : parameters.set<SubProblem *>("_subproblem") = this;
2786 1462 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2787 : {
2788 784 : std::shared_ptr<MeshDivision> func = _factory.create<MeshDivision>(type, name, parameters, tid);
2789 775 : _mesh_divisions.addObject(func, tid);
2790 775 : }
2791 678 : }
2792 :
2793 : MeshDivision &
2794 1567 : FEProblemBase::getMeshDivision(const std::string & name, const THREAD_ID tid) const
2795 : {
2796 1567 : auto * const ret = dynamic_cast<MeshDivision *>(_mesh_divisions.getActiveObject(name, tid).get());
2797 1567 : if (!ret)
2798 0 : mooseError("No MeshDivision object named ", name, " of appropriate type");
2799 1567 : return *ret;
2800 : }
2801 :
2802 : void
2803 0 : FEProblemBase::lineSearch()
2804 : {
2805 0 : _line_search->lineSearch();
2806 0 : }
2807 :
2808 : NonlinearSystem &
2809 0 : FEProblemBase::getNonlinearSystem(const unsigned int sys_num)
2810 : {
2811 0 : mooseDeprecated("FEProblemBase::getNonlinearSystem() is deprecated, please use "
2812 : "FEProblemBase::getNonlinearSystemBase() \n");
2813 :
2814 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
2815 0 : auto nl_sys = std::dynamic_pointer_cast<NonlinearSystem>(_nl[sys_num]);
2816 :
2817 0 : if (!nl_sys)
2818 0 : mooseError("This is not a NonlinearSystem");
2819 :
2820 0 : return *nl_sys;
2821 0 : }
2822 :
2823 : void
2824 2 : FEProblemBase::addDistribution(const std::string & type,
2825 : const std::string & name,
2826 : InputParameters & parameters)
2827 : {
2828 4 : parameters.set<std::string>("type") = type;
2829 2 : addObject<Distribution>(type, name, parameters, /* threaded = */ false);
2830 2 : }
2831 :
2832 : bool
2833 4 : FEProblemBase::hasDistribution(const std::string & name) const
2834 : {
2835 4 : std::vector<Distribution *> objs;
2836 4 : theWarehouse()
2837 8 : .query()
2838 4 : .condition<AttribSystem>("Distribution")
2839 4 : .condition<AttribName>(name)
2840 4 : .queryInto(objs);
2841 8 : return !objs.empty();
2842 4 : }
2843 :
2844 : Distribution &
2845 4 : FEProblemBase::getDistribution(const std::string & name)
2846 : {
2847 4 : std::vector<Distribution *> objs;
2848 4 : theWarehouse()
2849 8 : .query()
2850 4 : .condition<AttribSystem>("Distribution")
2851 4 : .condition<AttribName>(name)
2852 4 : .queryInto(objs);
2853 4 : if (objs.empty())
2854 : {
2855 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_distribution"),
2856 : "A Distribution getter was called before Distributions have been constructed. "
2857 : "If you are attempting to access this object in the constructor of another object "
2858 : "then make sure that the Distribution is constructed before the object using it.");
2859 0 : mooseError("Unable to find Distribution with name '" + name + "'");
2860 : }
2861 8 : return *(objs[0]);
2862 4 : }
2863 :
2864 : void
2865 275 : FEProblemBase::addSampler(const std::string & type,
2866 : const std::string & name,
2867 : InputParameters & parameters)
2868 : {
2869 275 : const auto samplers = addObject<Sampler>(type, name, parameters);
2870 569 : for (auto & sampler : samplers)
2871 303 : sampler->init();
2872 266 : }
2873 :
2874 : Sampler &
2875 266 : FEProblemBase::getSampler(const std::string & name, const THREAD_ID tid)
2876 : {
2877 266 : std::vector<Sampler *> objs;
2878 266 : theWarehouse()
2879 532 : .query()
2880 266 : .condition<AttribSystem>("Sampler")
2881 266 : .condition<AttribThread>(tid)
2882 266 : .condition<AttribName>(name)
2883 266 : .queryInto(objs);
2884 266 : if (objs.empty())
2885 : {
2886 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_sampler"),
2887 : "A Sampler getter was called before Samplers have been constructed. "
2888 : "If you are attempting to access this object in the constructor of another object "
2889 : "then make sure that the Sampler is constructed before the object using it.");
2890 :
2891 0 : mooseError(
2892 0 : "Unable to find Sampler with name '" + name +
2893 : "', if you are attempting to access this object in the constructor of another object then "
2894 : "make sure that the Sampler is constructed before the object using it.");
2895 : }
2896 532 : return *(objs[0]);
2897 266 : }
2898 :
2899 : bool
2900 155035 : FEProblemBase::duplicateVariableCheck(const std::string & var_name,
2901 : const FEType & type,
2902 : bool is_aux,
2903 : const std::set<SubdomainID> * const active_subdomains)
2904 : {
2905 155035 : std::set<SubdomainID> subdomainIDs;
2906 155035 : if (active_subdomains->size() == 0)
2907 : {
2908 146588 : const auto subdomains = _mesh.meshSubdomains();
2909 146588 : subdomainIDs.insert(subdomains.begin(), subdomains.end());
2910 146588 : }
2911 : else
2912 8447 : subdomainIDs.insert(active_subdomains->begin(), active_subdomains->end());
2913 :
2914 311061 : for (auto & sys : _solver_systems)
2915 : {
2916 156038 : SystemBase * curr_sys_ptr = sys.get();
2917 156038 : SystemBase * other_sys_ptr = _aux.get();
2918 156038 : std::string error_prefix = "";
2919 156038 : if (is_aux)
2920 : {
2921 94457 : curr_sys_ptr = _aux.get();
2922 94457 : other_sys_ptr = sys.get();
2923 94457 : error_prefix = "aux";
2924 : }
2925 :
2926 156038 : if (other_sys_ptr->hasVariable(var_name))
2927 3 : mooseError("Cannot have an auxiliary variable and a solver variable with the same name: ",
2928 : var_name);
2929 :
2930 156035 : if (curr_sys_ptr->hasVariable(var_name))
2931 : {
2932 : const Variable & var =
2933 9 : curr_sys_ptr->system().variable(curr_sys_ptr->system().variable_number(var_name));
2934 :
2935 : // variable type
2936 9 : if (var.type() != type)
2937 : {
2938 12 : const auto stringifyType = [](FEType t)
2939 12 : { return Moose::stringify(t.family) + " of order " + Moose::stringify(t.order); };
2940 :
2941 6 : mooseError("Mismatching types are specified for ",
2942 : error_prefix,
2943 : "variable with name '",
2944 : var_name,
2945 : "': '",
2946 6 : stringifyType(var.type()),
2947 : "' and '",
2948 6 : stringifyType(type),
2949 : "'");
2950 : }
2951 :
2952 : // block-restriction
2953 3 : if (!(active_subdomains->size() == 0 && var.active_subdomains().size() == 0))
2954 : {
2955 3 : const auto varActiveSubdomains = var.active_subdomains();
2956 3 : std::set<SubdomainID> varSubdomainIDs;
2957 3 : if (varActiveSubdomains.size() == 0)
2958 : {
2959 0 : const auto subdomains = _mesh.meshSubdomains();
2960 0 : varSubdomainIDs.insert(subdomains.begin(), subdomains.end());
2961 0 : }
2962 : else
2963 3 : varSubdomainIDs.insert(varActiveSubdomains.begin(), varActiveSubdomains.end());
2964 :
2965 : // Is subdomainIDs a subset of varSubdomainIDs? With this we allow the case that the newly
2966 : // requested block restriction is only a subset of the existing one.
2967 3 : const auto isSubset = std::includes(varSubdomainIDs.begin(),
2968 : varSubdomainIDs.end(),
2969 : subdomainIDs.begin(),
2970 : subdomainIDs.end());
2971 :
2972 3 : if (!isSubset)
2973 : {
2974 : // helper function: make a string from a set of subdomain ids
2975 6 : const auto stringifySubdomains = [this](std::set<SubdomainID> subdomainIDs)
2976 : {
2977 6 : std::stringstream s;
2978 15 : for (auto const i : subdomainIDs)
2979 : {
2980 : // do we need to insert a comma?
2981 9 : if (s.tellp() != 0)
2982 3 : s << ", ";
2983 :
2984 : // insert subdomain name and id -or- only the id (if no name is given)
2985 9 : const auto subdomainName = _mesh.getSubdomainName(i);
2986 9 : if (subdomainName.empty())
2987 9 : s << i;
2988 : else
2989 0 : s << subdomainName << " (" << i << ")";
2990 9 : }
2991 12 : return s.str();
2992 6 : };
2993 :
2994 6 : const std::string msg = "Mismatching block-restrictions are specified for " +
2995 6 : error_prefix + "variable with name '" + var_name + "': {" +
2996 12 : stringifySubdomains(varSubdomainIDs) + "} and {" +
2997 9 : stringifySubdomains(subdomainIDs) + "}";
2998 :
2999 3 : mooseError(msg);
3000 0 : }
3001 0 : }
3002 :
3003 0 : return true;
3004 : }
3005 156026 : }
3006 :
3007 155023 : return false;
3008 155023 : }
3009 :
3010 : void
3011 60718 : FEProblemBase::addVariable(const std::string & var_type,
3012 : const std::string & var_name,
3013 : InputParameters & params)
3014 : {
3015 : parallel_object_only();
3016 :
3017 60718 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3018 60718 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3019 60718 : const auto fe_type = FEType(order, family);
3020 :
3021 : const auto active_subdomains_vector =
3022 60718 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3023 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3024 60718 : active_subdomains_vector.end());
3025 :
3026 60718 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ false, &active_subdomains))
3027 0 : return;
3028 :
3029 182145 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3030 60715 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_SOLVER;
3031 60715 : SolverSystemName sys_name = params.get<SolverSystemName>("solver_sys");
3032 :
3033 60715 : const auto solver_system_number = solverSysNum(sys_name);
3034 60715 : logAdd("Variable", var_name, var_type, params);
3035 60715 : _solver_systems[solver_system_number]->addVariable(var_type, var_name, params);
3036 60703 : if (_displaced_problem)
3037 : // MooseObjects need to be unique so change the name here
3038 3453 : _displaced_problem->addVariable(var_type, var_name, params, solver_system_number);
3039 :
3040 60703 : _solver_var_to_sys_num[var_name] = solver_system_number;
3041 :
3042 60703 : markFamilyPRefinement(params);
3043 60703 : if (_displaced_problem)
3044 3453 : _displaced_problem->markFamilyPRefinement(params);
3045 60703 : }
3046 :
3047 : std::pair<bool, unsigned int>
3048 4570788 : FEProblemBase::determineSolverSystem(const std::string & var_name,
3049 : const bool error_if_not_found) const
3050 : {
3051 4570788 : auto map_it = _solver_var_to_sys_num.find(var_name);
3052 4570788 : const bool var_in_sys = map_it != _solver_var_to_sys_num.end();
3053 4570788 : if (var_in_sys)
3054 : mooseAssert(_solver_systems[map_it->second]->hasVariable(var_name) ||
3055 : _solver_systems[map_it->second]->hasScalarVariable(var_name),
3056 : "If the variable is in our FEProblem solver system map, then it must be in the "
3057 : "solver system we expect");
3058 3106808 : else if (error_if_not_found)
3059 : {
3060 32 : if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
3061 21 : mooseError("No solver variable named ",
3062 : var_name,
3063 : " found. Did you specify an auxiliary variable when you meant to specify a "
3064 : "solver variable?");
3065 : else
3066 11 : mooseError("Unknown variable '",
3067 : var_name,
3068 : "'. It does not exist in the solver system(s) or auxiliary system");
3069 : }
3070 :
3071 9141512 : return std::make_pair(var_in_sys, var_in_sys ? map_it->second : libMesh::invalid_uint);
3072 : }
3073 :
3074 : void
3075 161304 : FEProblemBase::setResidualObjectParamsAndLog(const std::string & ro_name,
3076 : const std::string & name,
3077 : InputParameters & parameters,
3078 : const unsigned int nl_sys_num,
3079 : const std::string & base_name,
3080 : bool & reinit_displaced)
3081 : {
3082 161304 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3083 : {
3084 1920 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3085 1920 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3086 960 : reinit_displaced = true;
3087 : }
3088 : else
3089 : {
3090 160344 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3091 : {
3092 : // We allow Kernels to request that they use_displaced_mesh,
3093 : // but then be overridden when no displacements variables are
3094 : // provided in the Mesh block. If that happened, update the value
3095 : // of use_displaced_mesh appropriately for this Kernel.
3096 105 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3097 210 : parameters.set<bool>("use_displaced_mesh") = false;
3098 : }
3099 :
3100 320688 : parameters.set<SubProblem *>("_subproblem") = this;
3101 481032 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3102 : }
3103 :
3104 161304 : logAdd(base_name, name, ro_name, parameters);
3105 161304 : }
3106 :
3107 : void
3108 65417 : FEProblemBase::setAuxKernelParamsAndLog(const std::string & ak_name,
3109 : const std::string & name,
3110 : InputParameters & parameters,
3111 : const std::string & base_name)
3112 : {
3113 65417 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3114 : {
3115 22872 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3116 22872 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3117 22872 : parameters.set<SystemBase *>("_nl_sys") = &_displaced_problem->solverSys(0);
3118 11436 : if (!parameters.get<std::vector<BoundaryName>>("boundary").empty())
3119 10990 : _reinit_displaced_face = true;
3120 : else
3121 446 : _reinit_displaced_elem = true;
3122 : }
3123 : else
3124 : {
3125 53981 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3126 : {
3127 : // We allow AuxKernels to request that they use_displaced_mesh,
3128 : // but then be overridden when no displacements variables are
3129 : // provided in the Mesh block. If that happened, update the value
3130 : // of use_displaced_mesh appropriately for this AuxKernel.
3131 795 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3132 1590 : parameters.set<bool>("use_displaced_mesh") = false;
3133 : }
3134 :
3135 107962 : parameters.set<SubProblem *>("_subproblem") = this;
3136 107962 : parameters.set<SystemBase *>("_sys") = _aux.get();
3137 161943 : parameters.set<SystemBase *>("_nl_sys") = _solver_systems[0].get();
3138 : }
3139 :
3140 65417 : logAdd(base_name, name, ak_name, parameters);
3141 65417 : }
3142 :
3143 : void
3144 77776 : FEProblemBase::addKernel(const std::string & kernel_name,
3145 : const std::string & name,
3146 : InputParameters & parameters)
3147 : {
3148 : parallel_object_only();
3149 155552 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3150 77764 : if (!isSolverSystemNonlinear(nl_sys_num))
3151 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
3152 : "supported at the moment!");
3153 77764 : setResidualObjectParamsAndLog(
3154 77764 : kernel_name, name, parameters, nl_sys_num, "Kernel", _reinit_displaced_elem);
3155 :
3156 77764 : _nl[nl_sys_num]->addKernel(kernel_name, name, parameters);
3157 77611 : }
3158 :
3159 : void
3160 431 : FEProblemBase::addHDGKernel(const std::string & kernel_name,
3161 : const std::string & name,
3162 : InputParameters & parameters)
3163 : {
3164 : parallel_object_only();
3165 862 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3166 431 : if (!isSolverSystemNonlinear(nl_sys_num))
3167 0 : mooseError("You are trying to add a HDGKernel to a linear variable/system, which is not "
3168 : "supported at the moment!");
3169 431 : setResidualObjectParamsAndLog(
3170 431 : kernel_name, name, parameters, nl_sys_num, "HDGKernel", _reinit_displaced_elem);
3171 :
3172 431 : _nl[nl_sys_num]->addHDGKernel(kernel_name, name, parameters);
3173 431 : }
3174 :
3175 : void
3176 602 : FEProblemBase::addNodalKernel(const std::string & kernel_name,
3177 : const std::string & name,
3178 : InputParameters & parameters)
3179 : {
3180 : parallel_object_only();
3181 :
3182 1204 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3183 599 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3184 : {
3185 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3186 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3187 0 : _reinit_displaced_elem = true;
3188 : }
3189 : else
3190 : {
3191 599 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3192 : {
3193 : // We allow NodalKernels to request that they use_displaced_mesh,
3194 : // but then be overridden when no displacements variables are
3195 : // provided in the Mesh block. If that happened, update the value
3196 : // of use_displaced_mesh appropriately for this NodalKernel.
3197 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3198 0 : parameters.set<bool>("use_displaced_mesh") = false;
3199 : }
3200 :
3201 1198 : parameters.set<SubProblem *>("_subproblem") = this;
3202 1797 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3203 : }
3204 599 : logAdd("NodalKernel", name, kernel_name, parameters);
3205 599 : _nl[nl_sys_num]->addNodalKernel(kernel_name, name, parameters);
3206 599 : }
3207 :
3208 : void
3209 1319 : FEProblemBase::addScalarKernel(const std::string & kernel_name,
3210 : const std::string & name,
3211 : InputParameters & parameters)
3212 : {
3213 : parallel_object_only();
3214 :
3215 2638 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3216 1319 : if (!isSolverSystemNonlinear(nl_sys_num))
3217 0 : mooseError("You are trying to add a ScalarKernel to a linear variable/system, which is not "
3218 : "supported at the moment!");
3219 :
3220 1319 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3221 : {
3222 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3223 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3224 : }
3225 : else
3226 : {
3227 1319 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3228 : {
3229 : // We allow ScalarKernels to request that they use_displaced_mesh,
3230 : // but then be overridden when no displacements variables are
3231 : // provided in the Mesh block. If that happened, update the value
3232 : // of use_displaced_mesh appropriately for this ScalarKernel.
3233 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3234 0 : parameters.set<bool>("use_displaced_mesh") = false;
3235 : }
3236 :
3237 2638 : parameters.set<SubProblem *>("_subproblem") = this;
3238 3957 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3239 : }
3240 :
3241 1319 : logAdd("ScalarKernel", name, kernel_name, parameters);
3242 1319 : _nl[nl_sys_num]->addScalarKernel(kernel_name, name, parameters);
3243 1313 : }
3244 :
3245 : void
3246 75564 : FEProblemBase::addBoundaryCondition(const std::string & bc_name,
3247 : const std::string & name,
3248 : InputParameters & parameters)
3249 : {
3250 : parallel_object_only();
3251 :
3252 151132 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3253 75556 : if (!isSolverSystemNonlinear(nl_sys_num))
3254 0 : mooseError(
3255 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
3256 : "supported at the moment!");
3257 :
3258 75556 : setResidualObjectParamsAndLog(
3259 75556 : bc_name, name, parameters, nl_sys_num, "BoundaryCondition", _reinit_displaced_face);
3260 75556 : _nl[nl_sys_num]->addBoundaryCondition(bc_name, name, parameters);
3261 75512 : }
3262 :
3263 : void
3264 1783 : FEProblemBase::addConstraint(const std::string & c_name,
3265 : const std::string & name,
3266 : InputParameters & parameters)
3267 : {
3268 : parallel_object_only();
3269 :
3270 1783 : _has_constraints = true;
3271 :
3272 1783 : auto determine_var_param_name = [¶meters, this]()
3273 : {
3274 5349 : if (parameters.isParamValid("variable"))
3275 1222 : return "variable";
3276 : else
3277 : {
3278 : // must be a mortar constraint
3279 1122 : const bool has_secondary_var = parameters.isParamValid("secondary_variable");
3280 1122 : const bool has_primary_var = parameters.isParamValid("primary_variable");
3281 561 : if (!has_secondary_var && !has_primary_var)
3282 0 : mooseError(
3283 : "Either a 'secondary_variable' or 'primary_variable' parameter must be supplied for '",
3284 0 : parameters.getObjectName(),
3285 : "'");
3286 561 : return has_secondary_var ? "secondary_variable" : "primary_variable";
3287 : }
3288 1783 : };
3289 :
3290 : const auto nl_sys_num =
3291 3566 : determineSolverSystem(parameters.varName(determine_var_param_name(), name), true).second;
3292 1780 : if (!isSolverSystemNonlinear(nl_sys_num))
3293 0 : mooseError("You are trying to add a Constraint to a linear variable/system, which is not "
3294 : "supported at the moment!");
3295 :
3296 1780 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3297 : {
3298 266 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3299 266 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3300 133 : _reinit_displaced_face = true;
3301 : }
3302 : else
3303 : {
3304 : // It might _want_ to use a displaced mesh... but we're not so set it to false
3305 1647 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3306 3294 : parameters.set<bool>("use_displaced_mesh") = false;
3307 :
3308 3294 : parameters.set<SubProblem *>("_subproblem") = this;
3309 4941 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3310 : }
3311 :
3312 1780 : logAdd("Constraint", name, c_name, parameters);
3313 1780 : _nl[nl_sys_num]->addConstraint(c_name, name, parameters);
3314 1756 : }
3315 :
3316 : void
3317 94317 : FEProblemBase::addAuxVariable(const std::string & var_type,
3318 : const std::string & var_name,
3319 : InputParameters & params)
3320 : {
3321 : parallel_object_only();
3322 :
3323 94317 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3324 94317 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3325 94317 : const auto fe_type = FEType(order, family);
3326 :
3327 : const auto active_subdomains_vector =
3328 94317 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3329 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3330 94317 : active_subdomains_vector.end());
3331 :
3332 94317 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ true, &active_subdomains))
3333 0 : return;
3334 :
3335 282924 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3336 188616 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3337 :
3338 94308 : logAdd("AuxVariable", var_name, var_type, params);
3339 94308 : _aux->addVariable(var_type, var_name, params);
3340 94308 : if (_displaced_problem)
3341 : // MooseObjects need to be unique so change the name here
3342 10166 : _displaced_problem->addAuxVariable(var_type, var_name, params);
3343 :
3344 94308 : markFamilyPRefinement(params);
3345 94308 : if (_displaced_problem)
3346 10166 : _displaced_problem->markFamilyPRefinement(params);
3347 94308 : }
3348 :
3349 : void
3350 2646 : FEProblemBase::addElementalFieldVariable(const std::string & var_type,
3351 : const std::string & var_name,
3352 : InputParameters & params)
3353 : {
3354 2646 : addAuxVariable(var_type, var_name, params);
3355 2646 : }
3356 :
3357 : void
3358 0 : FEProblemBase::addAuxVariable(const std::string & var_name,
3359 : const FEType & type,
3360 : const std::set<SubdomainID> * const active_subdomains)
3361 : {
3362 : parallel_object_only();
3363 :
3364 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3365 :
3366 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3367 0 : return;
3368 :
3369 0 : std::string var_type;
3370 0 : if (type == FEType(0, MONOMIAL))
3371 0 : var_type = "MooseVariableConstMonomial";
3372 0 : else if (type.family == SCALAR)
3373 0 : var_type = "MooseVariableScalar";
3374 0 : else if (FEInterface::field_type(type) == TYPE_VECTOR)
3375 0 : var_type = "VectorMooseVariable";
3376 : else
3377 0 : var_type = "MooseVariable";
3378 :
3379 0 : InputParameters params = _factory.getValidParams(var_type);
3380 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3381 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3382 0 : params.set<MooseEnum>("order") = type.order.get_order();
3383 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3384 :
3385 0 : if (active_subdomains)
3386 0 : for (const SubdomainID & id : *active_subdomains)
3387 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3388 :
3389 0 : logAdd("AuxVariable", var_name, var_type, params);
3390 0 : _aux->addVariable(var_type, var_name, params);
3391 0 : if (_displaced_problem)
3392 0 : _displaced_problem->addAuxVariable("MooseVariable", var_name, params);
3393 :
3394 0 : markFamilyPRefinement(params);
3395 0 : if (_displaced_problem)
3396 0 : _displaced_problem->markFamilyPRefinement(params);
3397 0 : }
3398 :
3399 : void
3400 0 : FEProblemBase::addAuxArrayVariable(const std::string & var_name,
3401 : const FEType & type,
3402 : unsigned int components,
3403 : const std::set<SubdomainID> * const active_subdomains)
3404 : {
3405 : parallel_object_only();
3406 :
3407 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3408 :
3409 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3410 0 : return;
3411 :
3412 0 : InputParameters params = _factory.getValidParams("ArrayMooseVariable");
3413 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3414 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3415 0 : params.set<MooseEnum>("order") = type.order.get_order();
3416 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3417 0 : params.set<unsigned int>("components") = components;
3418 :
3419 0 : if (active_subdomains)
3420 0 : for (const SubdomainID & id : *active_subdomains)
3421 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3422 :
3423 0 : logAdd("Variable", var_name, "ArrayMooseVariable", params);
3424 0 : _aux->addVariable("ArrayMooseVariable", var_name, params);
3425 0 : if (_displaced_problem)
3426 0 : _displaced_problem->addAuxVariable("ArrayMooseVariable", var_name, params);
3427 :
3428 0 : markFamilyPRefinement(params);
3429 0 : if (_displaced_problem)
3430 0 : _displaced_problem->markFamilyPRefinement(params);
3431 0 : }
3432 :
3433 : void
3434 0 : FEProblemBase::addAuxScalarVariable(const std::string & var_name,
3435 : Order order,
3436 : Real /*scale_factor*/,
3437 : const std::set<SubdomainID> * const active_subdomains)
3438 : {
3439 : parallel_object_only();
3440 :
3441 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3442 :
3443 0 : if (order > _max_scalar_order)
3444 0 : _max_scalar_order = order;
3445 :
3446 0 : FEType type(order, SCALAR);
3447 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3448 0 : return;
3449 :
3450 0 : InputParameters params = _factory.getValidParams("MooseVariableScalar");
3451 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3452 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3453 :
3454 0 : params.set<MooseEnum>("order") = type.order.get_order();
3455 0 : params.set<MooseEnum>("family") = "SCALAR";
3456 0 : params.set<std::vector<Real>>("scaling") = std::vector<Real>{1};
3457 0 : if (active_subdomains)
3458 0 : for (const SubdomainID & id : *active_subdomains)
3459 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3460 :
3461 0 : logAdd("ScalarVariable", var_name, "MooseVariableScalar", params);
3462 0 : _aux->addVariable("MooseVariableScalar", var_name, params);
3463 0 : if (_displaced_problem)
3464 0 : _displaced_problem->addAuxVariable("MooseVariableScalar", var_name, params);
3465 0 : }
3466 :
3467 : void
3468 64781 : FEProblemBase::addAuxKernel(const std::string & kernel_name,
3469 : const std::string & name,
3470 : InputParameters & parameters)
3471 : {
3472 : parallel_object_only();
3473 :
3474 64781 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "AuxKernel");
3475 :
3476 64781 : _aux->addKernel(kernel_name, name, parameters);
3477 64667 : }
3478 :
3479 : void
3480 475 : FEProblemBase::addAuxScalarKernel(const std::string & kernel_name,
3481 : const std::string & name,
3482 : InputParameters & parameters)
3483 : {
3484 : parallel_object_only();
3485 :
3486 475 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3487 : {
3488 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3489 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3490 : }
3491 : else
3492 : {
3493 475 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3494 : {
3495 : // We allow AuxScalarKernels to request that they use_displaced_mesh,
3496 : // but then be overridden when no displacements variables are
3497 : // provided in the Mesh block. If that happened, update the value
3498 : // of use_displaced_mesh appropriately for this AuxScalarKernel.
3499 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3500 0 : parameters.set<bool>("use_displaced_mesh") = false;
3501 : }
3502 :
3503 950 : parameters.set<SubProblem *>("_subproblem") = this;
3504 1425 : parameters.set<SystemBase *>("_sys") = _aux.get();
3505 : }
3506 :
3507 475 : logAdd("AuxScalarKernel", name, kernel_name, parameters);
3508 475 : _aux->addScalarKernel(kernel_name, name, parameters);
3509 472 : }
3510 :
3511 : void
3512 871 : FEProblemBase::addDiracKernel(const std::string & kernel_name,
3513 : const std::string & name,
3514 : InputParameters & parameters)
3515 : {
3516 : parallel_object_only();
3517 :
3518 1742 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3519 868 : if (!isSolverSystemNonlinear(nl_sys_num))
3520 0 : mooseError("You are trying to add a DiracKernel to a linear variable/system, which is not "
3521 : "supported at the moment!");
3522 :
3523 868 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3524 : {
3525 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3526 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3527 12 : _reinit_displaced_elem = true;
3528 : }
3529 : else
3530 : {
3531 856 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3532 : {
3533 : // We allow DiracKernels to request that they use_displaced_mesh,
3534 : // but then be overridden when no displacements variables are
3535 : // provided in the Mesh block. If that happened, update the value
3536 : // of use_displaced_mesh appropriately for this DiracKernel.
3537 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3538 0 : parameters.set<bool>("use_displaced_mesh") = false;
3539 : }
3540 :
3541 1712 : parameters.set<SubProblem *>("_subproblem") = this;
3542 2568 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3543 : }
3544 :
3545 868 : logAdd("DiracKernel", name, kernel_name, parameters);
3546 868 : _nl[nl_sys_num]->addDiracKernel(kernel_name, name, parameters);
3547 862 : }
3548 :
3549 : // DGKernels ////
3550 :
3551 : void
3552 1250 : FEProblemBase::addDGKernel(const std::string & dg_kernel_name,
3553 : const std::string & name,
3554 : InputParameters & parameters)
3555 : {
3556 : parallel_object_only();
3557 :
3558 2500 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3559 1247 : if (!isSolverSystemNonlinear(nl_sys_num))
3560 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
3561 : "supported at the moment!");
3562 :
3563 1247 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3564 : {
3565 48 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3566 48 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3567 24 : _reinit_displaced_neighbor = true;
3568 : }
3569 : else
3570 : {
3571 1223 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3572 : {
3573 : // We allow DGKernels to request that they use_displaced_mesh,
3574 : // but then be overridden when no displacements variables are
3575 : // provided in the Mesh block. If that happened, update the value
3576 : // of use_displaced_mesh appropriately for this DGKernel.
3577 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3578 0 : parameters.set<bool>("use_displaced_mesh") = false;
3579 : }
3580 :
3581 2446 : parameters.set<SubProblem *>("_subproblem") = this;
3582 3669 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3583 : }
3584 :
3585 1247 : logAdd("DGKernel", name, dg_kernel_name, parameters);
3586 1247 : _nl[nl_sys_num]->addDGKernel(dg_kernel_name, name, parameters);
3587 :
3588 1247 : _has_internal_edge_residual_objects = true;
3589 1247 : }
3590 :
3591 : void
3592 6643 : FEProblemBase::addFVKernel(const std::string & fv_kernel_name,
3593 : const std::string & name,
3594 : InputParameters & parameters)
3595 : {
3596 6643 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3597 : // FVElementalKernels are computed in the historically finite element threaded loops. They rely
3598 : // on Assembly data like _current_elem. When we call reinit on the FEProblemBase we will only
3599 : // reinit the DisplacedProblem and its associated Assembly objects if we mark this boolean as
3600 : // true
3601 0 : _reinit_displaced_elem = true;
3602 6643 : addObject<FVKernel>(fv_kernel_name, name, parameters);
3603 6637 : }
3604 :
3605 : void
3606 5891 : FEProblemBase::addFVBC(const std::string & fv_bc_name,
3607 : const std::string & name,
3608 : InputParameters & parameters)
3609 : {
3610 5891 : addObject<FVBoundaryCondition>(fv_bc_name, name, parameters);
3611 5891 : }
3612 :
3613 : void
3614 246 : FEProblemBase::addFVInterfaceKernel(const std::string & fv_ik_name,
3615 : const std::string & name,
3616 : InputParameters & parameters)
3617 : {
3618 : /// We assume that variable1 and variable2 can live on different systems, in this case
3619 : /// the user needs to create two interface kernels with flipped variables and parameters
3620 246 : addObject<FVInterfaceKernel>(
3621 : fv_ik_name, name, parameters, /*threaded=*/true, /*variable_param_name=*/"variable1");
3622 237 : }
3623 :
3624 : void
3625 2202 : FEProblemBase::addLinearFVKernel(const std::string & kernel_name,
3626 : const std::string & name,
3627 : InputParameters & parameters)
3628 : {
3629 2202 : addObject<LinearFVKernel>(kernel_name, name, parameters);
3630 2202 : }
3631 :
3632 : void
3633 2079 : FEProblemBase::addLinearFVBC(const std::string & bc_name,
3634 : const std::string & name,
3635 : InputParameters & parameters)
3636 : {
3637 2079 : addObject<LinearFVBoundaryCondition>(bc_name, name, parameters);
3638 2079 : }
3639 :
3640 : // InterfaceKernels ////
3641 :
3642 : void
3643 800 : FEProblemBase::addInterfaceKernel(const std::string & interface_kernel_name,
3644 : const std::string & name,
3645 : InputParameters & parameters)
3646 : {
3647 : parallel_object_only();
3648 :
3649 1600 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3650 797 : if (!isSolverSystemNonlinear(nl_sys_num))
3651 0 : mooseError("You are trying to add a InterfaceKernel to a linear variable/system, which is not "
3652 : "supported at the moment!");
3653 :
3654 797 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3655 : {
3656 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3657 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3658 12 : _reinit_displaced_neighbor = true;
3659 : }
3660 : else
3661 : {
3662 785 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3663 : {
3664 : // We allow InterfaceKernels to request that they use_displaced_mesh,
3665 : // but then be overridden when no displacements variables are
3666 : // provided in the Mesh block. If that happened, update the value
3667 : // of use_displaced_mesh appropriately for this InterfaceKernel.
3668 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3669 0 : parameters.set<bool>("use_displaced_mesh") = false;
3670 : }
3671 :
3672 1570 : parameters.set<SubProblem *>("_subproblem") = this;
3673 2355 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3674 : }
3675 :
3676 797 : logAdd("InterfaceKernel", name, interface_kernel_name, parameters);
3677 797 : _nl[nl_sys_num]->addInterfaceKernel(interface_kernel_name, name, parameters);
3678 :
3679 797 : _has_internal_edge_residual_objects = true;
3680 797 : }
3681 :
3682 : void
3683 33887 : FEProblemBase::checkICRestartError(const std::string & ic_name,
3684 : const std::string & name,
3685 : const VariableName & var_name)
3686 : {
3687 33887 : if (!_allow_ics_during_restart)
3688 : {
3689 33808 : std::string restart_method = "";
3690 33808 : if (_app.isRestarting())
3691 : restart_method =
3692 0 : "a checkpoint restart, by IC object '" + ic_name + "' for variable '" + name + "'";
3693 33808 : else if (_app.getExReaderForRestart())
3694 : {
3695 3 : std::vector<std::string> restarted_vars = _app.getExReaderForRestart()->get_elem_var_names();
3696 3 : const auto nodal_vars = _app.getExReaderForRestart()->get_nodal_var_names();
3697 3 : const auto global_vars = _app.getExReaderForRestart()->get_global_var_names();
3698 3 : restarted_vars.insert(restarted_vars.end(), nodal_vars.begin(), nodal_vars.end());
3699 3 : restarted_vars.insert(restarted_vars.end(), global_vars.begin(), global_vars.end());
3700 :
3701 3 : if (std::find(restarted_vars.begin(), restarted_vars.end(), var_name) != restarted_vars.end())
3702 6 : restart_method = "an Exodus restart, by IC object '" + ic_name + "' for variable '" + name +
3703 3 : "' that is also being restarted";
3704 3 : }
3705 33808 : if (!restart_method.empty())
3706 3 : mooseError(
3707 : "Initial conditions have been specified during ",
3708 : restart_method,
3709 : ".\nThis is only allowed if you specify 'allow_initial_conditions_with_restart' to "
3710 : "the [Problem], as initial conditions can override restarted fields");
3711 33805 : }
3712 33884 : }
3713 :
3714 : void
3715 31536 : FEProblemBase::addInitialCondition(const std::string & ic_name,
3716 : const std::string & name,
3717 : InputParameters & parameters)
3718 : {
3719 : parallel_object_only();
3720 :
3721 : // before we start to mess with the initial condition, we need to check parameters for errors.
3722 31536 : parameters.checkParams(name);
3723 31533 : const std::string & var_name = parameters.get<VariableName>("variable");
3724 :
3725 : // Forbid initial conditions on a restarted problem, as they would override the restart
3726 31533 : checkICRestartError(ic_name, name, var_name);
3727 :
3728 63060 : parameters.set<SubProblem *>("_subproblem") = this;
3729 :
3730 : // field IC
3731 31530 : if (hasVariable(var_name))
3732 : {
3733 63396 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3734 : {
3735 33426 : MooseVariableFEBase & var = getVariable(
3736 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3737 66852 : parameters.set<SystemBase *>("_sys") = &var.sys();
3738 33426 : std::shared_ptr<InitialConditionBase> ic;
3739 33426 : if (dynamic_cast<MooseVariable *>(&var))
3740 31305 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3741 2121 : else if (dynamic_cast<VectorMooseVariable *>(&var))
3742 250 : ic = _factory.create<VectorInitialCondition>(ic_name, name, parameters, tid);
3743 1871 : else if (dynamic_cast<ArrayMooseVariable *>(&var))
3744 1207 : ic = _factory.create<ArrayInitialCondition>(ic_name, name, parameters, tid);
3745 664 : else if (dynamic_cast<MooseVariableFVReal *>(&var))
3746 664 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3747 0 : else if (dynamic_cast<MooseLinearVariableFVReal *>(&var))
3748 0 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3749 : else
3750 0 : mooseError("Your FE variable in initial condition ",
3751 : name,
3752 : " must be either of scalar or vector type");
3753 33405 : logAdd("IC", name, ic_name, parameters);
3754 33405 : _ics.addObject(ic, tid);
3755 33390 : }
3756 : }
3757 :
3758 : // scalar IC
3759 1524 : else if (hasScalarVariable(var_name))
3760 : {
3761 1524 : MooseVariableScalar & var = getScalarVariable(0, var_name);
3762 3048 : parameters.set<SystemBase *>("_sys") = &var.sys();
3763 : std::shared_ptr<ScalarInitialCondition> ic =
3764 1524 : _factory.create<ScalarInitialCondition>(ic_name, name, parameters);
3765 1524 : logAdd("ScalarIC", name, ic_name, parameters);
3766 1524 : _scalar_ics.addObject(ic);
3767 1524 : }
3768 :
3769 : else
3770 0 : mooseError(
3771 : "Variable '", var_name, "' requested in initial condition '", name, "' does not exist.");
3772 31494 : }
3773 :
3774 : void
3775 2354 : FEProblemBase::addFVInitialCondition(const std::string & ic_name,
3776 : const std::string & name,
3777 : InputParameters & parameters)
3778 : {
3779 : parallel_object_only();
3780 :
3781 : // before we start to mess with the initial condition, we need to check parameters for errors.
3782 2354 : parameters.checkParams(name);
3783 2354 : const std::string & var_name = parameters.get<VariableName>("variable");
3784 :
3785 : // Forbid initial conditions on a restarted problem, as they would override the restart
3786 2354 : checkICRestartError(ic_name, name, var_name);
3787 :
3788 4708 : parameters.set<SubProblem *>("_subproblem") = this;
3789 :
3790 : // field IC
3791 2354 : if (hasVariable(var_name))
3792 : {
3793 4790 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3794 : {
3795 2436 : auto & var = getVariable(
3796 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3797 4872 : parameters.set<SystemBase *>("_sys") = &var.sys();
3798 2436 : std::shared_ptr<FVInitialConditionBase> ic;
3799 2436 : if (var.isFV())
3800 2436 : ic = _factory.create<FVInitialCondition>(ic_name, name, parameters, tid);
3801 : else
3802 0 : mooseError(
3803 : "Your variable for an FVInitialCondition needs to be an a finite volume variable!");
3804 2436 : _fv_ics.addObject(ic, tid);
3805 2436 : }
3806 : }
3807 : else
3808 0 : mooseError("Variable '",
3809 : var_name,
3810 : "' requested in finite volume initial condition '",
3811 : name,
3812 : "' does not exist.");
3813 2354 : }
3814 :
3815 : void
3816 57334 : FEProblemBase::projectSolution()
3817 : {
3818 286670 : TIME_SECTION("projectSolution", 2, "Projecting Initial Solutions")
3819 :
3820 57334 : FloatingPointExceptionGuard fpe_guard(_app);
3821 :
3822 57334 : ComputeInitialConditionThread cic(*this);
3823 57334 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cic);
3824 :
3825 57328 : if (haveFV())
3826 : {
3827 : using ElemInfoRange = StoredRange<MooseMesh::const_elem_info_iterator, const ElemInfo *>;
3828 4139 : ElemInfoRange elem_info_range(_mesh.ownedElemInfoBegin(), _mesh.ownedElemInfoEnd());
3829 :
3830 4139 : ComputeFVInitialConditionThread cfvic(*this);
3831 4139 : Threads::parallel_reduce(elem_info_range, cfvic);
3832 4139 : }
3833 :
3834 : // Need to close the solution vector here so that boundary ICs take precendence
3835 113767 : for (auto & nl : _nl)
3836 56439 : nl->solution().close();
3837 57328 : _aux->solution().close();
3838 :
3839 : // now run boundary-restricted initial conditions
3840 57328 : ComputeBoundaryInitialConditionThread cbic(*this);
3841 57328 : Threads::parallel_reduce(getCurrentAlgebraicBndNodeRange(), cbic);
3842 :
3843 113767 : for (auto & nl : _nl)
3844 56439 : nl->solution().close();
3845 57328 : _aux->solution().close();
3846 :
3847 : // Also, load values into the SCALAR dofs
3848 : // Note: We assume that all SCALAR dofs are on the
3849 : // processor with highest ID
3850 57328 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3851 : {
3852 567 : const auto & ics = _scalar_ics.getActiveObjects();
3853 1561 : for (const auto & ic : ics)
3854 : {
3855 994 : MooseVariableScalar & var = ic->variable();
3856 994 : var.reinit();
3857 :
3858 994 : DenseVector<Number> vals(var.order());
3859 994 : ic->compute(vals);
3860 :
3861 994 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3862 2323 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3863 : {
3864 1329 : const auto global_index = var.dofIndices()[i];
3865 1329 : var.sys().solution().set(global_index, vals(i));
3866 1329 : var.setValue(i, vals(i));
3867 : }
3868 994 : }
3869 : }
3870 :
3871 114957 : for (auto & sys : _solver_systems)
3872 : {
3873 57629 : sys->solution().close();
3874 57629 : sys->solution().localize(*sys->system().current_local_solution, sys->dofMap().get_send_list());
3875 : }
3876 :
3877 57328 : _aux->solution().close();
3878 57328 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3879 57328 : }
3880 :
3881 : void
3882 2041 : FEProblemBase::projectInitialConditionOnCustomRange(
3883 : ConstElemRange & elem_range,
3884 : ConstBndNodeRange & bnd_nodes,
3885 : const std::optional<std::set<VariableName>> & target_vars)
3886 : {
3887 2041 : if (target_vars)
3888 : {
3889 2041 : ComputeInitialConditionThread cic(*this, &(*target_vars));
3890 2041 : Threads::parallel_reduce(elem_range, cic);
3891 : }
3892 : else
3893 : {
3894 0 : ComputeInitialConditionThread cic(*this);
3895 0 : Threads::parallel_reduce(elem_range, cic);
3896 : }
3897 :
3898 : // Need to close the solution vector here so that boundary ICs take precendence
3899 4082 : for (auto & nl : _nl)
3900 2041 : nl->solution().close();
3901 2041 : _aux->solution().close();
3902 :
3903 2041 : if (target_vars)
3904 : {
3905 2041 : ComputeBoundaryInitialConditionThread cbic(*this, &(*target_vars));
3906 2041 : Threads::parallel_reduce(bnd_nodes, cbic);
3907 2041 : }
3908 : else
3909 : {
3910 0 : ComputeBoundaryInitialConditionThread cbic(*this);
3911 0 : Threads::parallel_reduce(bnd_nodes, cbic);
3912 0 : }
3913 :
3914 4082 : for (auto & nl : _nl)
3915 2041 : nl->solution().close();
3916 2041 : _aux->solution().close();
3917 :
3918 : // Also, load values into the SCALAR dofs
3919 : // Note: We assume that all SCALAR dofs are on the
3920 : // processor with highest ID
3921 2041 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3922 : {
3923 0 : const auto & ics = _scalar_ics.getActiveObjects();
3924 0 : for (const auto & ic : ics)
3925 : {
3926 0 : MooseVariableScalar & var = ic->variable();
3927 :
3928 0 : if (target_vars && !target_vars->count(var.name()))
3929 0 : continue;
3930 :
3931 0 : var.reinit();
3932 :
3933 0 : DenseVector<Number> vals(var.order());
3934 0 : ic->compute(vals);
3935 :
3936 0 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3937 0 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3938 : {
3939 0 : const auto global_index = var.dofIndices()[i];
3940 0 : var.sys().solution().set(global_index, vals(i));
3941 0 : var.setValue(i, vals(i));
3942 : }
3943 0 : }
3944 : }
3945 :
3946 4082 : for (auto & nl : _nl)
3947 : {
3948 2041 : nl->solution().close();
3949 2041 : nl->solution().localize(*nl->system().current_local_solution, nl->dofMap().get_send_list());
3950 : }
3951 :
3952 2041 : _aux->solution().close();
3953 2041 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3954 2041 : }
3955 :
3956 : void
3957 737 : FEProblemBase::projectFunctionOnCustomRange(ConstElemRange & elem_range,
3958 : Number (*func)(const Point &,
3959 : const libMesh::Parameters &,
3960 : const std::string &,
3961 : const std::string &),
3962 : Gradient (*func_grad)(const Point &,
3963 : const libMesh::Parameters &,
3964 : const std::string &,
3965 : const std::string &),
3966 : const libMesh::Parameters & params,
3967 : const std::vector<VariableName> & target_vars)
3968 : {
3969 : mooseAssert(!Threads::in_threads,
3970 : "We're performing a projection based on data from just the thread 0 variable, so any "
3971 : "modifications to the variable solution must have been thread joined already");
3972 :
3973 737 : std::unordered_map<unsigned int, std::vector<unsigned int>> sys_to_var_nums;
3974 :
3975 1474 : for (const auto & target_var : target_vars)
3976 : {
3977 737 : const auto sn = systemNumForVariable(target_var);
3978 737 : const auto & var = getStandardVariable(0, target_var);
3979 737 : sys_to_var_nums[sn].push_back(var.number());
3980 : }
3981 :
3982 1474 : for (const auto & [sys_num, var_nums] : sys_to_var_nums)
3983 : {
3984 737 : System & libmesh_sys = getSystemBase(sys_num).system();
3985 737 : libmesh_sys.project_solution(func, func_grad, params, elem_range, var_nums);
3986 : }
3987 737 : }
3988 :
3989 : std::shared_ptr<MaterialBase>
3990 252 : FEProblemBase::getMaterial(std::string name,
3991 : Moose::MaterialDataType type,
3992 : const THREAD_ID tid,
3993 : bool no_warn)
3994 : {
3995 252 : switch (type)
3996 : {
3997 65 : case Moose::NEIGHBOR_MATERIAL_DATA:
3998 65 : name += "_neighbor";
3999 65 : break;
4000 65 : case Moose::FACE_MATERIAL_DATA:
4001 65 : name += "_face";
4002 65 : break;
4003 122 : default:
4004 122 : break;
4005 : }
4006 :
4007 252 : std::shared_ptr<MaterialBase> material = _all_materials[type].getActiveObject(name, tid);
4008 657 : if (!no_warn && material->getParam<bool>("compute") && type == Moose::BLOCK_MATERIAL_DATA)
4009 3 : mooseWarning("You are retrieving a Material object (",
4010 3 : material->name(),
4011 : "), but its compute flag is set to true. This indicates that MOOSE is "
4012 : "computing this property which may not be desired and produce un-expected "
4013 : "results.");
4014 :
4015 246 : return material;
4016 : }
4017 :
4018 : MaterialData &
4019 33147707 : FEProblemBase::getMaterialData(Moose::MaterialDataType type,
4020 : const THREAD_ID tid,
4021 : const MooseObject * object) const
4022 : {
4023 33147707 : switch (type)
4024 : {
4025 859581 : case Moose::BLOCK_MATERIAL_DATA:
4026 859581 : if (object)
4027 263599 : _material_props.addConsumer(type, object);
4028 859581 : return _material_props.getMaterialData(tid);
4029 15167119 : case Moose::NEIGHBOR_MATERIAL_DATA:
4030 15167119 : if (object)
4031 22616 : _neighbor_material_props.addConsumer(type, object);
4032 15167119 : return _neighbor_material_props.getMaterialData(tid);
4033 17121007 : case Moose::BOUNDARY_MATERIAL_DATA:
4034 : case Moose::FACE_MATERIAL_DATA:
4035 : case Moose::INTERFACE_MATERIAL_DATA:
4036 17121007 : if (object)
4037 52452 : _bnd_material_props.addConsumer(type, object);
4038 17121007 : return _bnd_material_props.getMaterialData(tid);
4039 : }
4040 :
4041 0 : mooseError("FEProblemBase::getMaterialData(): Invalid MaterialDataType ", type);
4042 : }
4043 :
4044 : const std::set<const MooseObject *> &
4045 0 : FEProblemBase::getMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
4046 : {
4047 0 : switch (type)
4048 : {
4049 0 : case Moose::BLOCK_MATERIAL_DATA:
4050 0 : return _material_props.getConsumers(type);
4051 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
4052 0 : return _neighbor_material_props.getConsumers(type);
4053 0 : case Moose::BOUNDARY_MATERIAL_DATA:
4054 : case Moose::FACE_MATERIAL_DATA:
4055 : case Moose::INTERFACE_MATERIAL_DATA:
4056 0 : return _bnd_material_props.getConsumers(type);
4057 : }
4058 :
4059 0 : mooseError("FEProblemBase::getMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
4060 : type);
4061 : }
4062 :
4063 : void
4064 0 : FEProblemBase::setPreserveMatrixSparsityPattern(bool preserve)
4065 : {
4066 0 : if (_ignore_zeros_in_jacobian && preserve)
4067 0 : paramWarning(
4068 : "ignore_zeros_in_jacobian",
4069 : "We likely cannot preserve the sparsity pattern if ignoring zeros in the Jacobian, which "
4070 : "leads to removing those entries from the Jacobian sparsity pattern");
4071 0 : _preserve_matrix_sparsity_pattern = preserve;
4072 0 : }
4073 :
4074 : bool
4075 310648 : FEProblemBase::acceptInvalidSolution() const
4076 : {
4077 621211 : return allowInvalidSolution() || // invalid solutions are always allowed
4078 621211 : !_app.solutionInvalidity().hasInvalidSolutionError(); // if not allowed, check for errors
4079 : }
4080 :
4081 : void
4082 1105 : FEProblemBase::addFunctorMaterial(const std::string & functor_material_name,
4083 : const std::string & name,
4084 : InputParameters & parameters)
4085 : {
4086 : parallel_object_only();
4087 :
4088 1105 : auto add_functor_materials = [&](const auto & parameters, const auto & name)
4089 : {
4090 2305 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
4091 : {
4092 : // Create the general Block/Boundary MaterialBase object
4093 1200 : std::shared_ptr<MaterialBase> material =
4094 1200 : _factory.create<MaterialBase>(functor_material_name, name, parameters, tid);
4095 2400 : logAdd("FunctorMaterial", name, functor_material_name, parameters);
4096 1200 : _all_materials.addObject(material, tid);
4097 1200 : _materials.addObject(material, tid);
4098 : }
4099 1105 : };
4100 :
4101 2210 : parameters.set<SubProblem *>("_subproblem") = this;
4102 1105 : add_functor_materials(parameters, name);
4103 1105 : if (_displaced_problem)
4104 : {
4105 0 : auto disp_params = parameters;
4106 0 : disp_params.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4107 0 : add_functor_materials(disp_params, name + "_displaced");
4108 0 : }
4109 1105 : }
4110 :
4111 : void
4112 12241 : FEProblemBase::addMaterial(const std::string & mat_name,
4113 : const std::string & name,
4114 : InputParameters & parameters)
4115 : {
4116 24482 : addMaterialHelper({&_materials}, mat_name, name, parameters);
4117 12143 : }
4118 :
4119 : void
4120 340 : FEProblemBase::addInterfaceMaterial(const std::string & mat_name,
4121 : const std::string & name,
4122 : InputParameters & parameters)
4123 : {
4124 680 : addMaterialHelper({&_interface_materials}, mat_name, name, parameters);
4125 340 : }
4126 :
4127 : void
4128 13523 : FEProblemBase::addMaterialHelper(std::vector<MaterialWarehouse *> warehouses,
4129 : const std::string & mat_name,
4130 : const std::string & name,
4131 : InputParameters & parameters)
4132 : {
4133 : parallel_object_only();
4134 :
4135 13523 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4136 : {
4137 252 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4138 126 : _reinit_displaced_elem = _reinit_displaced_face = _reinit_displaced_neighbor = true;
4139 : }
4140 : else
4141 : {
4142 13397 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
4143 : {
4144 : // We allow Materials to request that they use_displaced_mesh,
4145 : // but then be overridden when no displacements variables are
4146 : // provided in the Mesh block. If that happened, update the value
4147 : // of use_displaced_mesh appropriately for this Material.
4148 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
4149 0 : parameters.set<bool>("use_displaced_mesh") = false;
4150 : }
4151 :
4152 40191 : parameters.set<SubProblem *>("_subproblem") = this;
4153 : }
4154 :
4155 13523 : unsigned int n_threads = libMesh::n_threads();
4156 :
4157 : #ifdef MOOSE_KOKKOS_ENABLED
4158 10086 : if (parameters.isKokkosObject())
4159 942 : n_threads = 1;
4160 : #endif
4161 :
4162 28140 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4163 : {
4164 : // Create the general Block/Boundary MaterialBase object
4165 : std::shared_ptr<MaterialBase> material =
4166 14723 : _factory.create<MaterialBase>(mat_name, name, parameters, tid);
4167 14617 : logAdd("Material", name, mat_name, parameters);
4168 29234 : bool discrete = !material->getParam<bool>("compute");
4169 :
4170 : // If the object is boundary restricted or if it is a functor material we do not create the
4171 : // neighbor and face objects
4172 14617 : if (material->boundaryRestricted() || dynamic_cast<FunctorMaterial *>(material.get()))
4173 : {
4174 2984 : _all_materials.addObject(material, tid);
4175 2984 : if (discrete)
4176 4 : _discrete_materials.addObject(material, tid);
4177 : else
4178 5960 : for (auto && warehouse : warehouses)
4179 2980 : warehouse->addObject(material, tid);
4180 : }
4181 :
4182 : // Non-boundary restricted require face and neighbor objects
4183 : else
4184 : {
4185 : // TODO: we only need to do this if we have needs for face materials (e.g.
4186 : // FV, DG, etc.) - but currently we always do it. Figure out how to fix
4187 : // this.
4188 :
4189 : // The name of the object being created, this is changed multiple times as objects are
4190 : // created below
4191 11633 : std::string object_name;
4192 :
4193 : // Create a copy of the supplied parameters to the setting for "_material_data_type" isn't
4194 : // used from a previous tid loop
4195 11633 : InputParameters current_parameters = parameters;
4196 :
4197 : // face material
4198 11633 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4199 : Moose::FACE_MATERIAL_DATA;
4200 11633 : object_name = name + "_face";
4201 : std::shared_ptr<MaterialBase> face_material =
4202 11633 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4203 :
4204 : // neighbor material
4205 23266 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4206 : Moose::NEIGHBOR_MATERIAL_DATA;
4207 11633 : current_parameters.set<bool>("_neighbor") = true;
4208 11633 : object_name = name + "_neighbor";
4209 : std::shared_ptr<MaterialBase> neighbor_material =
4210 11633 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4211 :
4212 : // Store the material objects
4213 11633 : _all_materials.addObjects(material, neighbor_material, face_material, tid);
4214 :
4215 11633 : if (discrete)
4216 73 : _discrete_materials.addObjects(material, neighbor_material, face_material, tid);
4217 : else
4218 23120 : for (auto && warehouse : warehouses)
4219 11560 : warehouse->addObjects(material, neighbor_material, face_material, tid);
4220 :
4221 : // Names of all controllable parameters for this Material object
4222 11633 : const std::string & base = parameters.getBase();
4223 34899 : MooseObjectParameterName name(MooseObjectName(base, material->name()), "*");
4224 : const auto param_names =
4225 11633 : _app.getInputParameterWarehouse().getControllableParameterNames(name);
4226 :
4227 : // Connect parameters of the primary Material object to those on the face and neighbor
4228 : // objects
4229 29560 : for (const auto & p_name : param_names)
4230 : {
4231 35854 : MooseObjectParameterName primary_name(MooseObjectName(base, material->name()),
4232 35854 : p_name.parameter());
4233 35854 : MooseObjectParameterName face_name(MooseObjectName(base, face_material->name()),
4234 35854 : p_name.parameter());
4235 35854 : MooseObjectParameterName neighbor_name(MooseObjectName(base, neighbor_material->name()),
4236 35854 : p_name.parameter());
4237 17927 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4238 : primary_name, face_name, false);
4239 17927 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4240 : primary_name, neighbor_name, false);
4241 17927 : }
4242 11633 : }
4243 14617 : }
4244 13417 : }
4245 :
4246 : void
4247 4681378 : FEProblemBase::prepareMaterials(const std::unordered_set<unsigned int> & consumer_needed_mat_props,
4248 : const SubdomainID blk_id,
4249 : const THREAD_ID tid)
4250 : {
4251 4681378 : std::set<MooseVariableFEBase *> needed_moose_vars;
4252 4681378 : std::unordered_set<unsigned int> needed_mat_props;
4253 :
4254 4681378 : if (_all_materials.hasActiveBlockObjects(blk_id, tid))
4255 : {
4256 590536 : _all_materials.updateVariableDependency(needed_moose_vars, tid);
4257 590536 : _all_materials.updateBlockMatPropDependency(blk_id, needed_mat_props, tid);
4258 : }
4259 :
4260 4681378 : const auto & ids = _mesh.getSubdomainBoundaryIds(blk_id);
4261 21601972 : for (const auto id : ids)
4262 : {
4263 16920594 : _materials.updateBoundaryVariableDependency(id, needed_moose_vars, tid);
4264 16920594 : _materials.updateBoundaryMatPropDependency(id, needed_mat_props, tid);
4265 : }
4266 :
4267 4681378 : const auto & current_active_elemental_moose_variables = getActiveElementalMooseVariables(tid);
4268 4681378 : needed_moose_vars.insert(current_active_elemental_moose_variables.begin(),
4269 : current_active_elemental_moose_variables.end());
4270 :
4271 4681378 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
4272 :
4273 4681378 : setActiveElementalMooseVariables(needed_moose_vars, tid);
4274 4681378 : setActiveMaterialProperties(needed_mat_props, tid);
4275 4681378 : }
4276 :
4277 : void
4278 363618167 : FEProblemBase::reinitMaterials(SubdomainID blk_id, const THREAD_ID tid, bool swap_stateful)
4279 : {
4280 363618167 : if (hasActiveMaterialProperties(tid))
4281 : {
4282 14572523 : auto && elem = _assembly[tid][0]->elem();
4283 14572523 : unsigned int n_points = _assembly[tid][0]->qRule()->n_points();
4284 :
4285 14572523 : auto & material_data = _material_props.getMaterialData(tid);
4286 14572523 : material_data.resize(n_points);
4287 :
4288 : // Only swap if requested
4289 14572523 : if (swap_stateful)
4290 14565981 : material_data.swap(*elem);
4291 :
4292 14572523 : if (_discrete_materials.hasActiveBlockObjects(blk_id, tid))
4293 2698 : material_data.reset(_discrete_materials.getActiveBlockObjects(blk_id, tid));
4294 :
4295 14572520 : if (_materials.hasActiveBlockObjects(blk_id, tid))
4296 14553981 : material_data.reinit(_materials.getActiveBlockObjects(blk_id, tid));
4297 : }
4298 363618112 : }
4299 :
4300 : void
4301 5408724 : FEProblemBase::reinitMaterialsFace(const SubdomainID blk_id,
4302 : const THREAD_ID tid,
4303 : const bool swap_stateful,
4304 : const std::deque<MaterialBase *> * const reinit_mats)
4305 : {
4306 : // we reinit more often than needed here because we dont have a way to check whether
4307 : // we need to compute the face materials on a particular (possibly external) face
4308 5408724 : if (hasActiveMaterialProperties(tid))
4309 : {
4310 1045323 : auto && elem = _assembly[tid][0]->elem();
4311 1045323 : unsigned int side = _assembly[tid][0]->side();
4312 1045323 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4313 :
4314 1045323 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4315 1045323 : bnd_material_data.resize(n_points);
4316 :
4317 1045323 : if (swap_stateful && !bnd_material_data.isSwapped())
4318 1023849 : bnd_material_data.swap(*elem, side);
4319 :
4320 1045323 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4321 0 : bnd_material_data.reset(
4322 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4323 :
4324 1045323 : if (reinit_mats)
4325 21474 : bnd_material_data.reinit(*reinit_mats);
4326 1023849 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4327 1023849 : bnd_material_data.reinit(
4328 1023849 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4329 : }
4330 5408724 : }
4331 :
4332 : void
4333 3900429 : FEProblemBase::reinitMaterialsFaceOnBoundary(const BoundaryID boundary_id,
4334 : const SubdomainID blk_id,
4335 : const THREAD_ID tid,
4336 : const bool swap_stateful,
4337 : const std::deque<MaterialBase *> * const reinit_mats)
4338 : {
4339 4231228 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4340 330799 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4341 330799 : needInternalNeighborSideMaterial(blk_id, tid)))
4342 : {
4343 407314 : const auto * const elem = _assembly[tid][0]->elem();
4344 407314 : unsigned int side = _assembly[tid][0]->side();
4345 407314 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4346 :
4347 407314 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4348 407314 : bnd_material_data.resize(n_points);
4349 :
4350 407314 : if (swap_stateful && !bnd_material_data.isSwapped())
4351 407314 : bnd_material_data.swap(*elem, side);
4352 :
4353 407314 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4354 0 : bnd_material_data.reset(
4355 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4356 :
4357 407314 : if (reinit_mats)
4358 0 : bnd_material_data.reinit(*reinit_mats);
4359 407314 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4360 392922 : bnd_material_data.reinit(
4361 392922 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4362 : }
4363 3900429 : }
4364 :
4365 : void
4366 43901 : FEProblemBase::reinitMaterialsNeighborOnBoundary(
4367 : const BoundaryID boundary_id,
4368 : const SubdomainID blk_id,
4369 : const THREAD_ID tid,
4370 : const bool swap_stateful,
4371 : const std::deque<MaterialBase *> * const reinit_mats)
4372 : {
4373 : // Since objects don't declare whether they need the face or neighbor (side) material properties,
4374 : // we use the same criteria for skipping material property computations as for face material
4375 : // properties This could be a future optimization.
4376 48111 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4377 4210 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4378 4210 : needInternalNeighborSideMaterial(blk_id, tid)))
4379 35956 : reinitMaterialsNeighbor(blk_id, tid, swap_stateful, reinit_mats);
4380 43901 : }
4381 :
4382 : void
4383 4282711 : FEProblemBase::reinitMaterialsNeighbor(const SubdomainID blk_id,
4384 : const THREAD_ID tid,
4385 : const bool swap_stateful,
4386 : const std::deque<MaterialBase *> * const reinit_mats)
4387 : {
4388 4282711 : if (hasActiveMaterialProperties(tid))
4389 : {
4390 : // NOTE: this will not work with h-adaptivity
4391 : // lindsayad: why not?
4392 :
4393 886601 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4394 886601 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(_assembly[tid][0]->elem());
4395 :
4396 : mooseAssert(neighbor, "neighbor should be non-null");
4397 : mooseAssert(blk_id == neighbor->subdomain_id(),
4398 : "The provided blk_id " << blk_id << " and neighbor subdomain ID "
4399 : << neighbor->subdomain_id() << " do not match.");
4400 :
4401 886601 : unsigned int n_points = _assembly[tid][0]->qRuleNeighbor()->n_points();
4402 :
4403 886601 : auto & neighbor_material_data = _neighbor_material_props.getMaterialData(tid);
4404 886601 : neighbor_material_data.resize(n_points);
4405 :
4406 : // Only swap if requested
4407 886601 : if (swap_stateful)
4408 865127 : neighbor_material_data.swap(*neighbor, neighbor_side);
4409 :
4410 886601 : if (_discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4411 0 : neighbor_material_data.reset(
4412 0 : _discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4413 :
4414 886601 : if (reinit_mats)
4415 21474 : neighbor_material_data.reinit(*reinit_mats);
4416 865127 : else if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4417 864864 : neighbor_material_data.reinit(
4418 864864 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4419 : }
4420 4282711 : }
4421 :
4422 : void
4423 4464433 : FEProblemBase::reinitMaterialsBoundary(const BoundaryID boundary_id,
4424 : const THREAD_ID tid,
4425 : const bool swap_stateful,
4426 : const std::deque<MaterialBase *> * const reinit_mats)
4427 : {
4428 4464433 : if (hasActiveMaterialProperties(tid) && needBoundaryMaterialOnSide(boundary_id, tid))
4429 : {
4430 285946 : auto && elem = _assembly[tid][0]->elem();
4431 285946 : unsigned int side = _assembly[tid][0]->side();
4432 285946 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4433 :
4434 285946 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4435 285946 : bnd_material_data.resize(n_points);
4436 :
4437 285946 : if (swap_stateful && !bnd_material_data.isSwapped())
4438 238243 : bnd_material_data.swap(*elem, side);
4439 :
4440 285946 : if (_discrete_materials.hasActiveBoundaryObjects(boundary_id, tid))
4441 0 : bnd_material_data.reset(_discrete_materials.getActiveBoundaryObjects(boundary_id, tid));
4442 :
4443 285946 : if (reinit_mats)
4444 21474 : bnd_material_data.reinit(*reinit_mats);
4445 264472 : else if (_materials.hasActiveBoundaryObjects(boundary_id, tid))
4446 23154 : bnd_material_data.reinit(_materials.getActiveBoundaryObjects(boundary_id, tid));
4447 : }
4448 4464433 : }
4449 :
4450 : void
4451 46758 : FEProblemBase::reinitMaterialsInterface(BoundaryID boundary_id,
4452 : const THREAD_ID tid,
4453 : bool swap_stateful)
4454 : {
4455 46758 : if (hasActiveMaterialProperties(tid) && needInterfaceMaterialOnSide(boundary_id, tid))
4456 : {
4457 36992 : const Elem * const & elem = _assembly[tid][0]->elem();
4458 36992 : unsigned int side = _assembly[tid][0]->side();
4459 36992 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4460 :
4461 36992 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4462 36992 : bnd_material_data.resize(n_points);
4463 :
4464 36992 : if (swap_stateful && !bnd_material_data.isSwapped())
4465 35497 : bnd_material_data.swap(*elem, side);
4466 :
4467 36992 : if (_interface_materials.hasActiveBoundaryObjects(boundary_id, tid))
4468 1171 : bnd_material_data.reinit(_interface_materials.getActiveBoundaryObjects(boundary_id, tid));
4469 : }
4470 46758 : }
4471 :
4472 : void
4473 361949138 : FEProblemBase::swapBackMaterials(const THREAD_ID tid)
4474 : {
4475 361949138 : auto && elem = _assembly[tid][0]->elem();
4476 361949138 : _material_props.getMaterialData(tid).swapBack(*elem);
4477 361949138 : }
4478 :
4479 : void
4480 8745138 : FEProblemBase::swapBackMaterialsFace(const THREAD_ID tid)
4481 : {
4482 8745138 : auto && elem = _assembly[tid][0]->elem();
4483 8745138 : unsigned int side = _assembly[tid][0]->side();
4484 8745138 : _bnd_material_props.getMaterialData(tid).swapBack(*elem, side);
4485 8745138 : }
4486 :
4487 : void
4488 3629422 : FEProblemBase::swapBackMaterialsNeighbor(const THREAD_ID tid)
4489 : {
4490 : // NOTE: this will not work with h-adaptivity
4491 3629422 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4492 : unsigned int neighbor_side =
4493 3629422 : neighbor ? neighbor->which_neighbor_am_i(_assembly[tid][0]->elem()) : libMesh::invalid_uint;
4494 :
4495 3629422 : if (!neighbor)
4496 : {
4497 0 : if (haveFV())
4498 : {
4499 : // If neighbor is null, then we're on the neighbor side of a mesh boundary, e.g. we're off
4500 : // the mesh in ghost-land. If we're using the finite volume method, then variable values and
4501 : // consequently material properties have well-defined values in this ghost region outside of
4502 : // the mesh and we really do want to reinit our neighbor materials in this case. Since we're
4503 : // off in ghost land it's safe to do swaps with `MaterialPropertyStorage` using the elem and
4504 : // elem_side keys
4505 0 : neighbor = _assembly[tid][0]->elem();
4506 0 : neighbor_side = _assembly[tid][0]->side();
4507 : mooseAssert(neighbor, "We should have an appropriate value for elem coming from Assembly");
4508 : }
4509 : else
4510 0 : mooseError("neighbor is null in Assembly!");
4511 : }
4512 :
4513 3629422 : _neighbor_material_props.getMaterialData(tid).swapBack(*neighbor, neighbor_side);
4514 3629422 : }
4515 :
4516 : void
4517 984061 : FEProblemBase::logAdd(const std::string & system,
4518 : const std::string & name,
4519 : const std::string & type,
4520 : const InputParameters & params) const
4521 : {
4522 984061 : if (_verbose_setup != "false")
4523 102 : _console << "[DBG] Adding " << system << " '" << name << "' of type " << type << std::endl;
4524 984061 : if (_verbose_setup == "extra")
4525 0 : _console << params << std::endl;
4526 984061 : }
4527 :
4528 : void
4529 139031 : FEProblemBase::addObjectParamsHelper(InputParameters & parameters,
4530 : const std::string & object_name,
4531 : const std::string & var_param_name)
4532 : {
4533 : // Due to objects like SolutionUserObject which manipulate libmesh objects
4534 : // and variables directly at the back end, we need a default option here
4535 : // which is going to be the pointer to the first solver system within this
4536 : // problem
4537 139031 : unsigned int sys_num = 0;
4538 139031 : if (parameters.isParamValid(var_param_name))
4539 : {
4540 58307 : const auto variable_name = parameters.varName(var_param_name, object_name);
4541 58307 : if (this->hasVariable(variable_name) || this->hasScalarVariable(variable_name))
4542 51584 : sys_num = getSystem(variable_name).number();
4543 58307 : }
4544 278062 : if (parameters.isParamValid("solver_sys"))
4545 : {
4546 3965 : const auto var_sys_num = sys_num;
4547 3965 : sys_num = getSystemBase(parameters.get<SolverSystemName>("solver_sys")).number();
4548 3965 : if (sys_num != var_sys_num && parameters.isParamValid(var_param_name))
4549 0 : mooseError("We dont support setting 'variable' to a variable that is not set to the same "
4550 : "system as the 'solver_sys' parameter");
4551 : }
4552 :
4553 140195 : if (_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4554 140195 : parameters.get<bool>("use_displaced_mesh"))
4555 : {
4556 1132 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4557 566 : if (sys_num == _aux->number())
4558 1104 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->systemBaseAuxiliary();
4559 : else
4560 594 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(sys_num);
4561 : }
4562 : else
4563 : {
4564 : // The object requested use_displaced_mesh, but it was overridden
4565 : // due to there being no displacements variables in the [Mesh] block.
4566 : // If that happened, update the value of use_displaced_mesh appropriately.
4567 224383 : if (!_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4568 224383 : parameters.get<bool>("use_displaced_mesh"))
4569 144 : parameters.set<bool>("use_displaced_mesh") = false;
4570 :
4571 276930 : parameters.set<SubProblem *>("_subproblem") = this;
4572 :
4573 138465 : if (sys_num == _aux->number())
4574 40020 : parameters.set<SystemBase *>("_sys") = _aux.get();
4575 : else
4576 375375 : parameters.set<SystemBase *>("_sys") = _solver_systems[sys_num].get();
4577 : }
4578 139031 : }
4579 :
4580 : void
4581 63034 : FEProblemBase::checkUserObjectNameCollision(const std::string & name,
4582 : const std::string & type) const
4583 : {
4584 63034 : if (hasUserObject(name))
4585 9 : mooseError("A ",
4586 9 : getUserObjectBase(name).typeAndName(),
4587 : " already exists. You may not add a ",
4588 : type,
4589 : " by the same name.");
4590 :
4591 : #ifdef MOOSE_KOKKOS_ENABLED
4592 47237 : if (hasKokkosUserObject(name))
4593 0 : mooseError("A ",
4594 0 : getKokkosUserObject<UserObjectBase>(name).typeAndName(),
4595 : " already exists. You may not add a ",
4596 : type,
4597 : " by the same name.");
4598 : #endif
4599 63025 : }
4600 :
4601 : void
4602 50178 : FEProblemBase::addPostprocessor(const std::string & pp_name,
4603 : const std::string & name,
4604 : InputParameters & parameters)
4605 : {
4606 50178 : checkUserObjectNameCollision(name, "Postprocessor");
4607 :
4608 50175 : addUserObject(pp_name, name, parameters);
4609 50146 : }
4610 :
4611 : void
4612 5693 : FEProblemBase::addVectorPostprocessor(const std::string & pp_name,
4613 : const std::string & name,
4614 : InputParameters & parameters)
4615 : {
4616 5693 : checkUserObjectNameCollision(name, "VectorPostprocessor");
4617 :
4618 5690 : addUserObject(pp_name, name, parameters);
4619 5660 : }
4620 :
4621 : void
4622 4743 : FEProblemBase::addReporter(const std::string & type,
4623 : const std::string & name,
4624 : InputParameters & parameters)
4625 : {
4626 4743 : checkUserObjectNameCollision(name, "Reporter");
4627 :
4628 4740 : addUserObject(type, name, parameters);
4629 4707 : }
4630 :
4631 : std::vector<std::shared_ptr<UserObject>>
4632 70848 : FEProblemBase::addUserObject(const std::string & user_object_name,
4633 : const std::string & name,
4634 : InputParameters & parameters)
4635 : {
4636 : parallel_object_only();
4637 :
4638 70848 : std::vector<std::shared_ptr<UserObject>> uos;
4639 :
4640 : // Add the _subproblem and _sys parameters depending on use_displaced_mesh
4641 70848 : addObjectParamsHelper(parameters, name);
4642 :
4643 115213 : for (const auto tid : make_range(libMesh::n_threads()))
4644 : {
4645 : // Create the UserObject
4646 : std::shared_ptr<UserObject> user_object =
4647 74669 : _factory.create<UserObject>(user_object_name, name, parameters, tid);
4648 74394 : logAdd("UserObject", name, user_object_name, parameters);
4649 74394 : uos.push_back(user_object);
4650 :
4651 74394 : if (tid != 0)
4652 3821 : user_object->setPrimaryThreadCopy(uos[0].get());
4653 :
4654 74394 : theWarehouse().add(user_object);
4655 :
4656 : // Attempt to create all the possible UserObject types
4657 74388 : auto euo = std::dynamic_pointer_cast<ElementUserObject>(user_object);
4658 74388 : auto suo = std::dynamic_pointer_cast<SideUserObject>(user_object);
4659 74388 : auto isuo = std::dynamic_pointer_cast<InternalSideUserObject>(user_object);
4660 74388 : auto iuo = std::dynamic_pointer_cast<InterfaceUserObjectBase>(user_object);
4661 74388 : auto nuo = std::dynamic_pointer_cast<NodalUserObject>(user_object);
4662 74388 : auto duo = std::dynamic_pointer_cast<DomainUserObject>(user_object);
4663 74388 : auto guo = std::dynamic_pointer_cast<GeneralUserObject>(user_object);
4664 74388 : auto tguo = std::dynamic_pointer_cast<ThreadedGeneralUserObject>(user_object);
4665 74388 : auto muo = std::dynamic_pointer_cast<MortarUserObject>(user_object);
4666 :
4667 : // Account for displaced mesh use
4668 74388 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4669 : {
4670 : // Whether to re-init or not depends on the attributes of the base classes.
4671 : // For example, InterfaceUOBase has "_current_side_elem" and "_neighbor_elem"
4672 : // so it needs to reinit on displaced neighbors and faces
4673 : // _reinit_displaced_elem -> _current_elem will be reinited
4674 : // _reinit_displaced_face -> _current_elem, lowerD if any and _current_side_elem to be
4675 : // reinited _reinit_displaced_neighbor -> _current_elem, lowerD if any and _current_neighbor
4676 : // to be reinited Note that as soon as you use materials on the displaced mesh, all three get
4677 : // turned on.
4678 615 : if (euo || nuo || duo)
4679 555 : _reinit_displaced_elem = true;
4680 615 : if (suo || duo || isuo || iuo)
4681 24 : _reinit_displaced_face = true;
4682 615 : if (iuo || duo || isuo)
4683 0 : _reinit_displaced_neighbor = true;
4684 : }
4685 :
4686 : // These objects only require one thread
4687 74388 : if ((guo && !tguo) || muo)
4688 30023 : break;
4689 344595 : }
4690 :
4691 : // Add as a Functor if it is one. We usually need to add the user object from thread 0 as the
4692 : // registered functor for all threads because when user objects are thread joined, generally only
4693 : // the primary thread copy ends up with all the data
4694 148451 : for (const auto tid : make_range(libMesh::n_threads()))
4695 : {
4696 77884 : const decltype(uos)::size_type uo_index = uos.front()->needThreadedCopy() ? tid : 0;
4697 77884 : if (const auto functor = dynamic_cast<Moose::FunctorBase<Real> *>(uos[uo_index].get()))
4698 : {
4699 60855 : this->addFunctor(name, *functor, tid);
4700 60855 : if (_displaced_problem)
4701 729 : _displaced_problem->addFunctor(name, *functor, tid);
4702 : }
4703 : }
4704 :
4705 70567 : return uos;
4706 2 : }
4707 :
4708 : void
4709 2047 : FEProblemBase::addFVInterpolationMethod(const std::string & method_type,
4710 : const std::string & name,
4711 : InputParameters & parameters)
4712 : {
4713 : parallel_object_only();
4714 :
4715 2047 : addObjectParamsHelper(parameters, name);
4716 :
4717 4094 : for (const auto tid : make_range(libMesh::n_threads()))
4718 : {
4719 2047 : auto method = _factory.create<FVInterpolationMethod>(method_type, name, parameters, tid);
4720 2047 : logAdd("FVInterpolationMethod", name, method_type, parameters);
4721 2047 : theWarehouse().add(method);
4722 2047 : }
4723 2047 : }
4724 :
4725 : const UserObject &
4726 285632 : FEProblemBase::getUserObjectBase(const std::string & name, const THREAD_ID tid /* = 0 */) const
4727 : {
4728 285632 : std::vector<UserObject *> objs;
4729 285632 : theWarehouse()
4730 571264 : .query()
4731 285632 : .condition<AttribSystem>("UserObject")
4732 285632 : .condition<AttribThread>(tid)
4733 285632 : .condition<AttribName>(name)
4734 285632 : .queryInto(objs);
4735 285632 : if (objs.empty())
4736 : {
4737 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_user_object"),
4738 : "A UserObject getter was called before UserObjects have been constructed. The "
4739 : "requested UserObject '" +
4740 : name + "' may exist in the input file, but UserObjects are not available yet.");
4741 :
4742 0 : mooseError("Unable to find user object with name '" + name + "'");
4743 : }
4744 : mooseAssert(objs.size() == 1, "Should only find one UO");
4745 571264 : return *(objs[0]);
4746 285632 : }
4747 :
4748 : const Positions &
4749 1142 : FEProblemBase::getPositionsObject(const std::string & name) const
4750 : {
4751 1142 : std::vector<Positions *> objs;
4752 1142 : theWarehouse()
4753 2284 : .query()
4754 1142 : .condition<AttribSystem>("UserObject")
4755 1142 : .condition<AttribName>(name)
4756 1142 : .queryInto(objs);
4757 1142 : if (objs.empty())
4758 0 : mooseError("Unable to find Positions object with name '" + name + "'");
4759 : mooseAssert(objs.size() == 1, "Should only find one Positions");
4760 2284 : return *(objs[0]);
4761 1142 : }
4762 :
4763 : bool
4764 92096 : FEProblemBase::hasUserObject(const std::string & name) const
4765 : {
4766 92096 : std::vector<UserObject *> objs;
4767 92096 : theWarehouse()
4768 92096 : .query()
4769 92096 : .condition<AttribSystem>("UserObject")
4770 184192 : .condition<AttribThread>(0)
4771 92096 : .condition<AttribName>(name)
4772 92096 : .queryInto(objs);
4773 184192 : return !objs.empty();
4774 92096 : }
4775 :
4776 : const FVInterpolationMethod &
4777 586 : FEProblemBase::getFVInterpolationMethod(const InterpolationMethodName & name,
4778 : const THREAD_ID tid) const
4779 : {
4780 586 : std::vector<FVInterpolationMethod *> methods;
4781 586 : theWarehouse()
4782 1172 : .query()
4783 586 : .condition<AttribSystem>("FVInterpolationMethod")
4784 586 : .condition<AttribThread>(tid)
4785 586 : .condition<AttribName>(name)
4786 586 : .queryInto(methods);
4787 :
4788 586 : if (methods.empty())
4789 : {
4790 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_interpolation_method"),
4791 : "An FVInterpolationMethod getter was called before FVInterpolationMethods have "
4792 : "been constructed. If you are attempting to access this object in the constructor "
4793 : "of another object then make sure that the FVInterpolationMethod is constructed "
4794 : "before the object using it.");
4795 :
4796 0 : mooseError("Unable to find FVInterpolationMethod with name '", name, "'");
4797 : }
4798 :
4799 : mooseAssert(methods.size() == 1, "Expected a single FVInterpolationMethod per thread");
4800 1172 : return *(methods[0]);
4801 586 : }
4802 :
4803 : const FVFaceInterpolationMethod &
4804 15 : FEProblemBase::getFVFaceInterpolationMethod(const InterpolationMethodName & name,
4805 : const THREAD_ID tid) const
4806 : {
4807 15 : const auto & method = getFVInterpolationMethod(name, tid);
4808 15 : const auto * face_method = dynamic_cast<const FVFaceInterpolationMethod *>(&method);
4809 :
4810 15 : if (!face_method)
4811 0 : mooseError("FVInterpolationMethod '",
4812 : name,
4813 : "' (",
4814 0 : method.type(),
4815 : ") is not a scalar face interpolation method.");
4816 :
4817 15 : return *face_method;
4818 : }
4819 :
4820 : const FVAdvectedInterpolationMethod &
4821 571 : FEProblemBase::getFVAdvectedInterpolationMethod(const InterpolationMethodName & name,
4822 : const THREAD_ID tid) const
4823 : {
4824 571 : const auto & method = getFVInterpolationMethod(name, tid);
4825 571 : const auto * advected_method = dynamic_cast<const FVAdvectedInterpolationMethod *>(&method);
4826 :
4827 571 : if (!advected_method)
4828 0 : mooseError("FVInterpolationMethod '",
4829 : name,
4830 : "' (",
4831 0 : method.type(),
4832 : ") is not an advected interpolation method.");
4833 :
4834 571 : return *advected_method;
4835 : }
4836 :
4837 : bool
4838 0 : FEProblemBase::hasFVInterpolationMethod(const InterpolationMethodName & name) const
4839 : {
4840 0 : std::vector<FVInterpolationMethod *> methods;
4841 0 : theWarehouse()
4842 0 : .query()
4843 0 : .condition<AttribSystem>("FVInterpolationMethod")
4844 0 : .condition<AttribThread>(0)
4845 0 : .condition<AttribName>(name)
4846 0 : .queryInto(methods);
4847 0 : return !methods.empty();
4848 0 : }
4849 :
4850 : bool
4851 308 : FEProblemBase::hasPostprocessorValueByName(const PostprocessorName & name) const
4852 : {
4853 308 : return _reporter_data.hasReporterValue<PostprocessorValue>(PostprocessorReporterName(name));
4854 : }
4855 :
4856 : const Postprocessor &
4857 2 : FEProblemBase::getPostprocessorObjectByName(const PostprocessorName & object_name,
4858 : const THREAD_ID tid) const
4859 : {
4860 2 : std::vector<Postprocessor *> objs;
4861 2 : theWarehouse()
4862 2 : .query()
4863 4 : .condition<AttribInterfaces>(Interfaces::Postprocessor)
4864 2 : .condition<AttribThread>(tid)
4865 2 : .condition<AttribName>(object_name)
4866 2 : .queryInto(objs);
4867 :
4868 2 : if (objs.empty())
4869 0 : mooseError("Unable to find Postprocessor with name '", object_name, "'");
4870 : mooseAssert(objs.size() == 1,
4871 : "We shouldn't find more than one postprocessor object for a given name");
4872 4 : return *(objs[0]);
4873 2 : }
4874 :
4875 : const PostprocessorValue &
4876 849406 : FEProblemBase::getPostprocessorValueByName(const PostprocessorName & name,
4877 : std::size_t t_index) const
4878 : {
4879 1698812 : return _reporter_data.getReporterValue<PostprocessorValue>(PostprocessorReporterName(name),
4880 1698812 : t_index);
4881 : }
4882 :
4883 : void
4884 578524 : FEProblemBase::setPostprocessorValueByName(const PostprocessorName & name,
4885 : const PostprocessorValue & value,
4886 : std::size_t t_index)
4887 : {
4888 578524 : _reporter_data.setReporterValue<PostprocessorValue>(
4889 1157048 : PostprocessorReporterName(name), value, t_index);
4890 578524 : }
4891 :
4892 : bool
4893 52 : FEProblemBase::hasPostprocessor(const std::string & name) const
4894 : {
4895 52 : mooseDeprecated("FEProblemBase::hasPostprocssor is being removed; use "
4896 : "hasPostprocessorValueByName instead.");
4897 52 : return hasPostprocessorValueByName(name);
4898 : }
4899 :
4900 : const VectorPostprocessorValue &
4901 50 : FEProblemBase::getVectorPostprocessorValueByName(const std::string & object_name,
4902 : const std::string & vector_name,
4903 : std::size_t t_index) const
4904 : {
4905 50 : return _reporter_data.getReporterValue<VectorPostprocessorValue>(
4906 100 : VectorPostprocessorReporterName(object_name, vector_name), t_index);
4907 : }
4908 :
4909 : void
4910 18 : FEProblemBase::setVectorPostprocessorValueByName(const std::string & object_name,
4911 : const std::string & vector_name,
4912 : const VectorPostprocessorValue & value,
4913 : std::size_t t_index)
4914 : {
4915 18 : _reporter_data.setReporterValue<VectorPostprocessorValue>(
4916 36 : VectorPostprocessorReporterName(object_name, vector_name), value, t_index);
4917 18 : }
4918 :
4919 : const VectorPostprocessor &
4920 9882 : FEProblemBase::getVectorPostprocessorObjectByName(const std::string & object_name,
4921 : const THREAD_ID tid) const
4922 : {
4923 9882 : std::vector<VectorPostprocessor *> objs;
4924 9882 : theWarehouse()
4925 9882 : .query()
4926 19764 : .condition<AttribInterfaces>(Interfaces::VectorPostprocessor)
4927 9882 : .condition<AttribThread>(tid)
4928 9882 : .condition<AttribName>(object_name)
4929 9882 : .queryInto(objs);
4930 :
4931 9882 : if (objs.empty())
4932 : {
4933 : mooseAssert(
4934 : getMooseApp().actionWarehouse().isTaskComplete("add_vector_postprocessor"),
4935 : "A VectorPostprocessor getter was called before VectorPostprocessors have been "
4936 : "constructed. The requested VectorPostprocessor '" +
4937 : object_name +
4938 : "' may exist in the input file, but VectorPostprocessors are not available yet.");
4939 :
4940 0 : mooseError("Unable to find VectorPostprocessor with name '", object_name, "'");
4941 : }
4942 : mooseAssert(objs.size() == 1,
4943 : "We shouldn't find more than one vector postprocessor object for a given name");
4944 19764 : return *(objs[0]);
4945 9882 : }
4946 :
4947 : void
4948 70 : FEProblemBase::parentOutputPositionChanged()
4949 : {
4950 2030 : for (const auto & it : _multi_apps)
4951 : {
4952 1960 : const auto & objects = it.second.getActiveObjects();
4953 1979 : for (const auto & obj : objects)
4954 19 : obj->parentOutputPositionChanged();
4955 : }
4956 70 : }
4957 :
4958 : void
4959 0 : FEProblemBase::computeIndicatorsAndMarkers()
4960 : {
4961 0 : computeIndicators();
4962 0 : computeMarkers();
4963 0 : }
4964 :
4965 : void
4966 225671 : FEProblemBase::computeIndicators()
4967 : {
4968 : // Initialize indicator aux variable fields
4969 225671 : if (_indicators.hasActiveObjects() || _internal_side_indicators.hasActiveObjects())
4970 : {
4971 11875 : TIME_SECTION("computeIndicators", 1, "Computing Indicators");
4972 :
4973 : // Internal side indicators may lead to creating a much larger sparsity pattern than dictated by
4974 : // the actual finite element scheme (e.g. CFEM)
4975 2375 : const auto old_do_derivatives = ADReal::do_derivatives;
4976 2375 : ADReal::do_derivatives = false;
4977 :
4978 2375 : std::vector<std::string> fields;
4979 :
4980 : // Indicator Fields
4981 2375 : const auto & indicators = _indicators.getActiveObjects();
4982 2521 : for (const auto & indicator : indicators)
4983 146 : fields.push_back(indicator->name());
4984 :
4985 : // InternalSideIndicator Fields
4986 2375 : const auto & internal_indicators = _internal_side_indicators.getActiveObjects();
4987 4706 : for (const auto & internal_indicator : internal_indicators)
4988 2331 : fields.push_back(internal_indicator->name());
4989 :
4990 2375 : _aux->zeroVariables(fields);
4991 :
4992 : // compute Indicators
4993 2375 : ComputeIndicatorThread cit(*this);
4994 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cit);
4995 2375 : _aux->solution().close();
4996 2375 : _aux->update();
4997 :
4998 2375 : ComputeIndicatorThread finalize_cit(*this, true);
4999 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), finalize_cit);
5000 2375 : _aux->solution().close();
5001 2375 : _aux->update();
5002 :
5003 2375 : ADReal::do_derivatives = old_do_derivatives;
5004 2375 : }
5005 225671 : }
5006 :
5007 : void
5008 225693 : FEProblemBase::computeMarkers()
5009 : {
5010 225693 : if (_markers.hasActiveObjects())
5011 : {
5012 32760 : TIME_SECTION("computeMarkers", 1, "Computing Markers");
5013 :
5014 6552 : std::vector<std::string> fields;
5015 :
5016 : // Marker Fields
5017 6552 : const auto & markers = _markers.getActiveObjects();
5018 14083 : for (const auto & marker : markers)
5019 7531 : fields.push_back(marker->name());
5020 :
5021 6552 : _aux->zeroVariables(fields);
5022 :
5023 6552 : _adaptivity.updateErrorVectors();
5024 :
5025 13736 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5026 : {
5027 7187 : const auto & markers = _markers.getActiveObjects(tid);
5028 15445 : for (const auto & marker : markers)
5029 8261 : marker->markerSetup();
5030 : }
5031 :
5032 6549 : ComputeMarkerThread cmt(*this);
5033 6549 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cmt);
5034 :
5035 6549 : _aux->solution().close();
5036 6549 : _aux->update();
5037 6549 : }
5038 225690 : }
5039 :
5040 : const ExecFlagType &
5041 6241476 : FEProblemBase::getCurrentExecuteOnFlag() const
5042 : {
5043 6241476 : return _current_execute_on_flag;
5044 : }
5045 :
5046 : void
5047 7028176 : FEProblemBase::setCurrentExecuteOnFlag(const ExecFlagType & flag)
5048 : {
5049 7028176 : _current_execute_on_flag = flag;
5050 7028176 : }
5051 :
5052 : void
5053 72 : FEProblemBase::executeAllObjects(const ExecFlagType & /*exec_type*/)
5054 : {
5055 72 : }
5056 :
5057 : void
5058 1801261 : FEProblemBase::customSetup(const ExecFlagType & exec_type)
5059 : {
5060 1801261 : SubProblem::customSetup(exec_type);
5061 :
5062 1801261 : if (_line_search)
5063 0 : _line_search->customSetup(exec_type);
5064 :
5065 1801261 : unsigned int n_threads = libMesh::n_threads();
5066 3780543 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5067 : {
5068 1979282 : _all_materials.customSetup(exec_type, tid);
5069 1979282 : _functions.customSetup(exec_type, tid);
5070 : }
5071 :
5072 : #ifdef MOOSE_KOKKOS_ENABLED
5073 1316524 : _kokkos_functions.customSetup(exec_type);
5074 : #endif
5075 :
5076 1801261 : _aux->customSetup(exec_type);
5077 3606041 : for (auto & nl : _nl)
5078 1804780 : nl->customSetup(exec_type);
5079 :
5080 1801261 : if (_displaced_problem)
5081 145157 : _displaced_problem->customSetup(exec_type);
5082 :
5083 3780543 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5084 : {
5085 1979282 : _internal_side_indicators.customSetup(exec_type, tid);
5086 1979282 : _indicators.customSetup(exec_type, tid);
5087 1979282 : _markers.customSetup(exec_type, tid);
5088 : }
5089 :
5090 1801261 : std::vector<UserObject *> userobjs;
5091 1801261 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
5092 4454721 : for (auto obj : userobjs)
5093 2653460 : obj->customSetup(exec_type);
5094 :
5095 : #ifdef MOOSE_KOKKOS_ENABLED
5096 : {
5097 1316524 : std::vector<UserObjectBase *> userobjs;
5098 1316524 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
5099 1330052 : for (auto obj : userobjs)
5100 13528 : obj->customSetup(exec_type);
5101 1316524 : }
5102 : #endif
5103 :
5104 1801261 : _app.getOutputWarehouse().customSetup(exec_type);
5105 1801261 : }
5106 :
5107 : void
5108 2176192 : FEProblemBase::execute(const ExecFlagType & exec_type)
5109 : {
5110 : // Set the current flag
5111 2176192 : setCurrentExecuteOnFlag(exec_type);
5112 :
5113 2176192 : if (exec_type != EXEC_INITIAL)
5114 2119783 : executeControls(exec_type);
5115 :
5116 : // intentially call this after executing controls because the setups may rely on the controls
5117 : // FIXME: we skip the following flags because they have dedicated setup functions in
5118 : // SetupInterface and it may not be appropriate to call them here.
5119 3979906 : if (!(exec_type == EXEC_INITIAL || exec_type == EXEC_TIMESTEP_BEGIN ||
5120 1803732 : exec_type == EXEC_SUBDOMAIN || exec_type == EXEC_NONLINEAR || exec_type == EXEC_LINEAR))
5121 1801261 : customSetup(exec_type);
5122 :
5123 2176174 : executeSamplers(exec_type);
5124 :
5125 : // Pre-aux UserObjects
5126 2176150 : computeUserObjects(exec_type, Moose::PRE_AUX);
5127 :
5128 : // Systems (includes system time derivative and aux kernel calculations)
5129 2176150 : computeSystems(exec_type);
5130 : // With the auxiliary system solution computed, sync the displaced problem auxiliary solution
5131 : // before computation of post-aux user objects. The undisplaced auxiliary system current local
5132 : // solution is updated (via System::update) within the AuxiliarySystem class's variable
5133 : // computation methods (e.g. computeElementalVarsHelper, computeNodalVarsHelper), so it is safe to
5134 : // use it here
5135 2176117 : if (_displaced_problem)
5136 179606 : _displaced_problem->syncAuxSolution(*getAuxiliarySystem().currentSolution());
5137 :
5138 : // Post-aux UserObjects
5139 2176117 : computeUserObjects(exec_type, Moose::POST_AUX);
5140 :
5141 : // Return the current flag to None
5142 2176028 : setCurrentExecuteOnFlag(EXEC_NONE);
5143 :
5144 2176028 : if (_uo_aux_state_check && !_checking_uo_aux_state)
5145 : {
5146 : // we will only check aux variables and postprocessors
5147 : // checking more reporter data can be added in the future if needed
5148 559 : std::unique_ptr<NumericVector<Number>> x = _aux->currentSolution()->clone();
5149 559 : DenseVector<Real> pp_values = getReporterData().getAllRealReporterValues();
5150 :
5151 : // call THIS execute one more time for checking the possible states
5152 559 : _checking_uo_aux_state = true;
5153 559 : FEProblemBase::execute(exec_type);
5154 559 : _checking_uo_aux_state = false;
5155 :
5156 559 : const Real check_tol = 1e-8;
5157 :
5158 559 : const Real xnorm = x->l2_norm();
5159 559 : *x -= *_aux->currentSolution();
5160 559 : if (x->l2_norm() > check_tol * xnorm)
5161 : {
5162 3 : const auto & sys = _aux->system();
5163 3 : const unsigned int n_vars = sys.n_vars();
5164 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5165 15 : for (const auto i : make_range(n_vars))
5166 : {
5167 12 : const Real vnorm = sys.calculate_norm(*x, i, DISCRETE_L2);
5168 12 : ordered_map.emplace(vnorm, sys.variable_name(i));
5169 : }
5170 :
5171 3 : std::ostringstream oss;
5172 15 : for (const auto & [error_norm, var_name] : ordered_map)
5173 12 : oss << " {" << var_name << ", " << error_norm << "},\n";
5174 :
5175 3 : mooseError("Aux kernels, user objects appear to have states for aux variables on ",
5176 : exec_type,
5177 : ".\nVariable error norms in descending order:\n",
5178 3 : oss.str());
5179 0 : }
5180 :
5181 556 : const DenseVector<Real> new_pp_values = getReporterData().getAllRealReporterValues();
5182 556 : if (pp_values.size() != new_pp_values.size())
5183 0 : mooseError("Second execution for uo/aux state check should not change the number of "
5184 : "real reporter values");
5185 :
5186 556 : const Real ppnorm = pp_values.l2_norm();
5187 556 : pp_values -= new_pp_values;
5188 556 : if (pp_values.l2_norm() > check_tol * ppnorm)
5189 : {
5190 3 : const auto pp_names = getReporterData().getAllRealReporterFullNames();
5191 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5192 12 : for (const auto i : index_range(pp_names))
5193 9 : ordered_map.emplace(std::abs(pp_values(i)), pp_names[i]);
5194 :
5195 3 : std::ostringstream oss;
5196 12 : for (const auto & [error_norm, pp_name] : ordered_map)
5197 9 : oss << " {" << pp_name << ", " << error_norm << "},\n";
5198 :
5199 3 : mooseError("Aux kernels, user objects appear to have states for real reporter values on ",
5200 : exec_type,
5201 : ".\nErrors of real reporter values in descending order:\n",
5202 3 : oss.str());
5203 0 : }
5204 553 : }
5205 2176022 : }
5206 :
5207 : // Finalize, threadJoin, and update PP values of Elemental/Nodal/Side/InternalSideUserObjects
5208 : void
5209 1346426 : FEProblemBase::joinAndFinalize(TheWarehouse::Query query, bool isgen)
5210 : {
5211 1346426 : std::vector<UserObject *> objs;
5212 1346426 : query.queryInto(objs);
5213 1346426 : if (!isgen)
5214 : {
5215 : // join all threaded user objects (i.e. not regular general user objects) to the primary
5216 : // thread
5217 1425991 : for (auto obj : objs)
5218 372019 : if (obj->primaryThreadCopy())
5219 30795 : obj->primaryThreadCopy()->threadJoin(*obj);
5220 : }
5221 :
5222 1346426 : query.condition<AttribThread>(0).queryInto(objs);
5223 :
5224 : // finalize objects and retrieve/store any postprocessor values
5225 1919525 : for (auto obj : objs)
5226 : {
5227 573179 : if (isgen && dynamic_cast<ThreadedGeneralUserObject *>(obj))
5228 133 : continue;
5229 573046 : if (isgen)
5230 : {
5231 : // general user objects are not run in their own threaded loop object - so run them here
5232 231822 : if (shouldPrintExecution(0))
5233 724 : _console << "[DBG] Initializing, executing & finalizing general UO '" << obj->name()
5234 724 : << "' on " << _current_execute_on_flag.name() << std::endl;
5235 231822 : obj->initialize();
5236 231822 : obj->execute();
5237 : }
5238 :
5239 572996 : obj->finalize();
5240 :
5241 : // These have to be stored piecemeal (with every call to this function) because general
5242 : // postprocessors (which run last after other userobjects have been completed) might depend on
5243 : // them being stored. This wouldn't be a problem if all userobjects satisfied the dependency
5244 : // resolver interface and could be sorted appropriately with the general userobjects, but they
5245 : // don't.
5246 572981 : auto pp = dynamic_cast<const Postprocessor *>(obj);
5247 572981 : if (pp)
5248 : {
5249 500138 : _reporter_data.finalize(obj->name());
5250 500138 : setPostprocessorValueByName(obj->name(), pp->getValue());
5251 : }
5252 :
5253 572972 : auto vpp = dynamic_cast<VectorPostprocessor *>(obj);
5254 572972 : if (vpp)
5255 13523 : _reporter_data.finalize(obj->name());
5256 :
5257 : // Update Reporter data
5258 572972 : auto reporter = dynamic_cast<Reporter *>(obj);
5259 572972 : if (reporter)
5260 5333 : _reporter_data.finalize(obj->name());
5261 : }
5262 1346346 : }
5263 :
5264 : TheWarehouse::Query
5265 20025907 : FEProblemBase::getUOQuery(const std::string & system,
5266 : const ExecFlagType & type,
5267 : const Moose::AuxGroup & group) const
5268 : {
5269 : TheWarehouse::Query query =
5270 20025907 : theWarehouse().query().condition<AttribSystem>(system).condition<AttribExecOns>(type);
5271 :
5272 20025907 : if (group == Moose::PRE_IC)
5273 98263 : query.condition<AttribPreIC>(true);
5274 19927644 : else if (group == Moose::PRE_AUX)
5275 9963443 : query.condition<AttribPreAux>(type);
5276 9964201 : else if (group == Moose::POST_AUX)
5277 9964133 : query.condition<AttribPostAux>(type);
5278 :
5279 20025907 : return query;
5280 0 : }
5281 :
5282 : void
5283 20025907 : FEProblemBase::getUOExecutionGroups(TheWarehouse::Query & query,
5284 : std::set<int> & execution_groups) const
5285 : {
5286 20025907 : std::vector<UserObjectBase *> uos;
5287 20025907 : query.queryIntoUnsorted(uos);
5288 20632322 : for (const auto & uo : uos)
5289 1819245 : execution_groups.insert(uo->getParam<int>("execution_order_group"));
5290 20025907 : }
5291 :
5292 : void
5293 56897 : FEProblemBase::computeUserObjectByName(const ExecFlagType & type,
5294 : const Moose::AuxGroup & group,
5295 : const std::string & name)
5296 : {
5297 56897 : const auto old_exec_flag = _current_execute_on_flag;
5298 56897 : _current_execute_on_flag = type;
5299 :
5300 56897 : std::set<int> execution_groups;
5301 :
5302 : #ifdef MOOSE_KOKKOS_ENABLED
5303 : TheWarehouse::Query kokkos_query =
5304 41026 : getUOQuery("KokkosUserObject", type, group).condition<AttribName>(name);
5305 41026 : getUOExecutionGroups(kokkos_query, execution_groups);
5306 : #endif
5307 :
5308 56897 : TheWarehouse::Query query = getUOQuery("UserObject", type, group).condition<AttribName>(name);
5309 56897 : getUOExecutionGroups(query, execution_groups);
5310 :
5311 82848 : for (const auto execution_group : execution_groups)
5312 : {
5313 : #ifdef MOOSE_KOKKOS_ENABLED
5314 18721 : computeKokkosUserObjectsInternal(
5315 18721 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5316 : #endif
5317 :
5318 25951 : computeUserObjectsInternal(type,
5319 25951 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5320 : }
5321 :
5322 56897 : _current_execute_on_flag = old_exec_flag;
5323 56897 : }
5324 :
5325 : void
5326 11524326 : FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGroup & group)
5327 : {
5328 11524326 : std::set<int> execution_groups;
5329 :
5330 : #ifdef MOOSE_KOKKOS_ENABLED
5331 8403658 : TheWarehouse::Query kokkos_query = getUOQuery("KokkosUserObject", type, group);
5332 8403658 : getUOExecutionGroups(kokkos_query, execution_groups);
5333 : #endif
5334 :
5335 11524326 : TheWarehouse::Query query = getUOQuery("UserObject", type, group);
5336 11524326 : getUOExecutionGroups(query, execution_groups);
5337 :
5338 11792465 : for (const auto execution_group : execution_groups)
5339 : {
5340 : #ifdef MOOSE_KOKKOS_ENABLED
5341 195369 : computeKokkosUserObjectsInternal(
5342 195369 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5343 : #endif
5344 :
5345 268228 : computeUserObjectsInternal(type,
5346 268228 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5347 : }
5348 11524237 : }
5349 :
5350 : void
5351 294179 : FEProblemBase::computeUserObjectsInternal(const ExecFlagType & type, TheWarehouse::Query & query)
5352 : {
5353 : try
5354 : {
5355 1470895 : TIME_SECTION("computeUserObjects", 1, "Computing User Objects");
5356 :
5357 294179 : std::vector<GeneralUserObject *> genobjs;
5358 294179 : query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject).queryInto(genobjs);
5359 :
5360 294179 : std::vector<UserObject *> userobjs;
5361 294179 : query.clone()
5362 588358 : .condition<AttribInterfaces>(Interfaces::ElementUserObject | Interfaces::SideUserObject |
5363 : Interfaces::InternalSideUserObject |
5364 588358 : Interfaces::InterfaceUserObject | Interfaces::DomainUserObject)
5365 294179 : .queryInto(userobjs);
5366 :
5367 294179 : std::vector<UserObject *> tgobjs;
5368 294179 : query.clone()
5369 588358 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5370 294179 : .queryInto(tgobjs);
5371 :
5372 294179 : std::vector<UserObject *> nodal;
5373 294179 : query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).queryInto(nodal);
5374 :
5375 294179 : std::vector<MortarUserObject *> mortar;
5376 294179 : query.clone().condition<AttribInterfaces>(Interfaces::MortarUserObject).queryInto(mortar);
5377 :
5378 294179 : if (userobjs.empty() && genobjs.empty() && tgobjs.empty() && nodal.empty() && mortar.empty())
5379 1710 : return;
5380 :
5381 : // Start the timer here since we have at least one active user object
5382 292469 : std::string compute_uo_tag = "computeUserObjects(" + Moose::stringify(type) + ")";
5383 :
5384 : // Perform Residual/Jacobian setups
5385 292469 : if (type == EXEC_LINEAR)
5386 : {
5387 125255 : for (auto obj : userobjs)
5388 71384 : obj->residualSetup();
5389 58329 : for (auto obj : nodal)
5390 4458 : obj->residualSetup();
5391 53871 : for (auto obj : mortar)
5392 0 : obj->residualSetup();
5393 53880 : for (auto obj : tgobjs)
5394 9 : obj->residualSetup();
5395 65554 : for (auto obj : genobjs)
5396 11683 : obj->residualSetup();
5397 : }
5398 238598 : else if (type == EXEC_NONLINEAR)
5399 : {
5400 14080 : for (auto obj : userobjs)
5401 4420 : obj->jacobianSetup();
5402 10052 : for (auto obj : nodal)
5403 392 : obj->jacobianSetup();
5404 9660 : for (auto obj : mortar)
5405 0 : obj->jacobianSetup();
5406 9663 : for (auto obj : tgobjs)
5407 3 : obj->jacobianSetup();
5408 26470 : for (auto obj : genobjs)
5409 16810 : obj->jacobianSetup();
5410 : }
5411 :
5412 643777 : for (auto obj : userobjs)
5413 351308 : obj->initialize();
5414 :
5415 : // Execute Side/InternalSide/Interface/Elemental/DomainUserObjects
5416 292469 : if (!userobjs.empty())
5417 : {
5418 : // non-nodal user objects have to be run separately before the nodal user objects run
5419 : // because some nodal user objects (NodalNormal related) depend on elemental user objects
5420 : // :-(
5421 207504 : ComputeUserObjectsThread cppt(*this, query);
5422 207504 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cppt);
5423 :
5424 : // There is one instance in rattlesnake where an elemental user object's finalize depends
5425 : // on a side user object having been finalized first :-(
5426 207495 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::SideUserObject));
5427 207492 : joinAndFinalize(
5428 414984 : query.clone().condition<AttribInterfaces>(Interfaces::InternalSideUserObject));
5429 207492 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::InterfaceUserObject));
5430 207492 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::ElementUserObject));
5431 207489 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::DomainUserObject));
5432 207489 : }
5433 :
5434 : // if any elemental user object may have written to variables we need to close the aux solution
5435 643670 : for (const auto & uo : userobjs)
5436 351249 : if (auto euo = dynamic_cast<const ElementUserObject *>(uo);
5437 351249 : euo && euo->hasWritableCoupledVariables())
5438 : {
5439 33 : _aux->solution().close();
5440 33 : _aux->system().update();
5441 33 : break;
5442 : }
5443 :
5444 : // Execute NodalUserObjects
5445 : // BISON has an axial reloc elemental user object that has a finalize func that depends on a
5446 : // nodal user object's prev value. So we can't initialize this until after elemental objects
5447 : // have been finalized :-(
5448 312760 : for (auto obj : nodal)
5449 20306 : obj->initialize();
5450 292454 : if (query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).count() > 0)
5451 : {
5452 16379 : ComputeNodalUserObjectsThread cnppt(*this, query);
5453 16379 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), cnppt);
5454 16379 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject));
5455 16379 : }
5456 :
5457 : // if any nodal user object may have written to variables we need to close the aux solution
5458 312724 : for (const auto & uo : nodal)
5459 20292 : if (auto nuo = dynamic_cast<const NodalUserObject *>(uo);
5460 20292 : nuo && nuo->hasWritableCoupledVariables())
5461 : {
5462 22 : _aux->solution().close();
5463 22 : _aux->system().update();
5464 22 : break;
5465 : }
5466 :
5467 : // Execute MortarUserObjects
5468 : {
5469 292476 : for (auto obj : mortar)
5470 22 : obj->initialize();
5471 292454 : if (!mortar.empty())
5472 : {
5473 33 : auto create_and_run_mortar_functors = [this, type, &mortar](const bool displaced)
5474 : {
5475 : // go over mortar interfaces and construct functors
5476 33 : const auto & mortar_interfaces = getMortarInterfaces(displaced);
5477 55 : for (const auto & [primary_secondary_boundary_pair, interface_config] : mortar_interfaces)
5478 : {
5479 : auto mortar_uos_to_execute =
5480 22 : getMortarUserObjects(primary_secondary_boundary_pair.first,
5481 22 : primary_secondary_boundary_pair.second,
5482 : displaced,
5483 22 : mortar);
5484 :
5485 : auto * const subproblem = displaced
5486 22 : ? static_cast<SubProblem *>(_displaced_problem.get())
5487 22 : : static_cast<SubProblem *>(this);
5488 : MortarUserObjectThread muot(mortar_uos_to_execute,
5489 22 : *interface_config.amg,
5490 : *subproblem,
5491 : *this,
5492 : displaced,
5493 22 : subproblem->assembly(0, 0));
5494 :
5495 22 : muot();
5496 22 : }
5497 55 : };
5498 :
5499 22 : create_and_run_mortar_functors(false);
5500 22 : if (_displaced_problem)
5501 11 : create_and_run_mortar_functors(true);
5502 22 : }
5503 292476 : for (auto obj : mortar)
5504 22 : obj->finalize();
5505 : }
5506 :
5507 : // Execute threaded general user objects
5508 292895 : for (auto obj : tgobjs)
5509 441 : obj->initialize();
5510 292454 : std::vector<GeneralUserObject *> tguos_zero;
5511 292454 : query.clone()
5512 292454 : .condition<AttribThread>(0)
5513 584908 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5514 292454 : .queryInto(tguos_zero);
5515 292587 : for (auto obj : tguos_zero)
5516 : {
5517 133 : std::vector<GeneralUserObject *> tguos;
5518 133 : auto q = query.clone()
5519 133 : .condition<AttribName>(obj->name())
5520 133 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject);
5521 133 : q.queryInto(tguos);
5522 :
5523 133 : ComputeThreadedGeneralUserObjectsThread ctguot(*this);
5524 :
5525 : // Force one thread per ThreadedGeneralUserObject via grainsize
5526 266 : Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(),
5527 133 : tguos.end(),
5528 : /*grainsize=*/1),
5529 : ctguot);
5530 133 : joinAndFinalize(q);
5531 133 : }
5532 :
5533 : // Execute general user objects
5534 292454 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject), true);
5535 302640 : }
5536 0 : catch (...)
5537 : {
5538 0 : handleException("computeUserObjectsInternal");
5539 0 : }
5540 : }
5541 :
5542 : void
5543 5737772 : FEProblemBase::executeControls(const ExecFlagType & exec_type)
5544 : {
5545 5737772 : if (_control_warehouse[exec_type].hasActiveObjects())
5546 : {
5547 40205 : TIME_SECTION("executeControls", 1, "Executing Controls");
5548 :
5549 8041 : DependencyResolver<std::shared_ptr<Control>> resolver;
5550 :
5551 8041 : auto controls_wh = _control_warehouse[exec_type];
5552 : // Add all of the dependencies into the resolver and sort them
5553 20003 : for (const auto & it : controls_wh.getActiveObjects())
5554 : {
5555 : // Make sure an item with no dependencies comes out too!
5556 11965 : resolver.addItem(it);
5557 :
5558 11965 : std::vector<std::string> & dependent_controls = it->getDependencies();
5559 14857 : for (const auto & depend_name : dependent_controls)
5560 : {
5561 2895 : if (controls_wh.hasActiveObject(depend_name))
5562 : {
5563 2892 : auto dep_control = controls_wh.getActiveObject(depend_name);
5564 2892 : resolver.addEdge(dep_control, it);
5565 2892 : }
5566 : else
5567 3 : mooseError("The Control \"",
5568 : depend_name,
5569 : "\" was not created, did you make a "
5570 : "spelling mistake or forget to include it "
5571 : "in your input file?");
5572 : }
5573 : }
5574 :
5575 8038 : const auto & ordered_controls = resolver.getSortedValues();
5576 :
5577 8038 : if (!ordered_controls.empty())
5578 : {
5579 : // already called by initialSetup when exec_type == EXEC_INITIAL
5580 8038 : if (exec_type != EXEC_INITIAL)
5581 7130 : _control_warehouse.setup(exec_type);
5582 :
5583 : // Run the controls in the proper order
5584 19958 : for (const auto & control : ordered_controls)
5585 11962 : control->execute();
5586 : }
5587 7996 : }
5588 5737727 : }
5589 :
5590 : void
5591 2176174 : FEProblemBase::executeSamplers(const ExecFlagType & exec_type)
5592 : {
5593 : // TODO: This should be done in a threaded loop, but this should be super quick so for now
5594 : // do a serial loop.
5595 4567414 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5596 : {
5597 2391264 : std::vector<Sampler *> objects;
5598 2391264 : theWarehouse()
5599 4782528 : .query()
5600 2391264 : .condition<AttribSystem>("Sampler")
5601 2391264 : .condition<AttribThread>(tid)
5602 2391264 : .condition<AttribExecOns>(exec_type)
5603 2391264 : .queryInto(objects);
5604 :
5605 2391264 : if (!objects.empty())
5606 : {
5607 1475 : TIME_SECTION("executeSamplers", 1, "Executing Samplers");
5608 295 : FEProblemBase::objectSetupHelper<Sampler>(objects, exec_type);
5609 295 : FEProblemBase::objectExecuteHelper<Sampler>(objects);
5610 271 : }
5611 2391240 : }
5612 2176150 : }
5613 :
5614 : void
5615 312899 : FEProblemBase::updateActiveObjects()
5616 : {
5617 1564495 : TIME_SECTION("updateActiveObjects", 5, "Updating Active Objects");
5618 :
5619 656750 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5620 : {
5621 690105 : for (auto & nl : _nl)
5622 346254 : nl->updateActive(tid);
5623 343851 : _aux->updateActive(tid);
5624 343851 : _indicators.updateActive(tid);
5625 343851 : _internal_side_indicators.updateActive(tid);
5626 343851 : _markers.updateActive(tid);
5627 343851 : _all_materials.updateActive(tid);
5628 343851 : _materials.updateActive(tid);
5629 343851 : _discrete_materials.updateActive(tid);
5630 : }
5631 :
5632 312899 : _control_warehouse.updateActive();
5633 312899 : _multi_apps.updateActive();
5634 312899 : _transient_multi_apps.updateActive();
5635 312899 : _transfers.updateActive();
5636 312899 : _to_multi_app_transfers.updateActive();
5637 312899 : _from_multi_app_transfers.updateActive();
5638 312899 : _between_multi_app_transfers.updateActive();
5639 :
5640 : #ifdef MOOSE_KOKKOS_ENABLED
5641 228221 : _kokkos_materials.updateActive();
5642 : #endif
5643 312899 : }
5644 :
5645 : void
5646 0 : FEProblemBase::reportMooseObjectDependency(MooseObject * /*a*/, MooseObject * /*b*/)
5647 : {
5648 : //<< "Object " << a->name() << " -> " << b->name() << std::endl;
5649 0 : }
5650 :
5651 : void
5652 68258 : FEProblemBase::reinitBecauseOfGhostingOrNewGeomObjects(const bool mortar_changed)
5653 : {
5654 341290 : TIME_SECTION("reinitBecauseOfGhostingOrNewGeomObjects",
5655 : 3,
5656 : "Reinitializing Because of Geometric Search Objects");
5657 :
5658 : // Need to see if _any_ processor has ghosted elems or geometry objects.
5659 68258 : bool needs_reinit = !_ghosted_elems.empty();
5660 136147 : needs_reinit = needs_reinit || !_geometric_search_data._nearest_node_locators.empty() ||
5661 67889 : (_mortar_data->hasObjects() && mortar_changed);
5662 68258 : needs_reinit =
5663 137709 : needs_reinit || (_displaced_problem &&
5664 4423 : (!_displaced_problem->geomSearchData()._nearest_node_locators.empty() ||
5665 68930 : (_mortar_data->hasDisplacedObjects() && mortar_changed)));
5666 68258 : _communicator.max(needs_reinit);
5667 :
5668 68258 : if (needs_reinit)
5669 : {
5670 : // Call reinit to get the ghosted vectors correct now that some geometric search has been done
5671 2211 : es().reinit();
5672 :
5673 2211 : if (_displaced_mesh)
5674 1607 : _displaced_problem->es().reinit();
5675 : }
5676 68258 : }
5677 :
5678 : void
5679 177 : FEProblemBase::addDamper(const std::string & damper_name,
5680 : const std::string & name,
5681 : InputParameters & parameters)
5682 : {
5683 : parallel_object_only();
5684 :
5685 : const auto nl_sys_num =
5686 177 : parameters.isParamValid("variable")
5687 588 : ? determineSolverSystem(parameters.varName("variable", name), true).second
5688 174 : : (unsigned int)0;
5689 :
5690 174 : if (!isSolverSystemNonlinear(nl_sys_num))
5691 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
5692 : "supported at the moment!");
5693 :
5694 348 : parameters.set<SubProblem *>("_subproblem") = this;
5695 348 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
5696 :
5697 174 : _has_dampers = true;
5698 174 : logAdd("Damper", name, damper_name, parameters);
5699 174 : _nl[nl_sys_num]->addDamper(damper_name, name, parameters);
5700 174 : }
5701 :
5702 : void
5703 162 : FEProblemBase::setupDampers()
5704 : {
5705 324 : for (auto & nl : _nl)
5706 162 : nl->setupDampers();
5707 162 : }
5708 :
5709 : void
5710 659 : FEProblemBase::addIndicator(const std::string & indicator_name,
5711 : const std::string & name,
5712 : InputParameters & parameters)
5713 : {
5714 : parallel_object_only();
5715 :
5716 659 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5717 : {
5718 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5719 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5720 0 : _reinit_displaced_elem = true;
5721 : }
5722 : else
5723 : {
5724 659 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5725 : {
5726 : // We allow Indicators to request that they use_displaced_mesh,
5727 : // but then be overridden when no displacements variables are
5728 : // provided in the Mesh block. If that happened, update the value
5729 : // of use_displaced_mesh appropriately for this Indicator.
5730 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5731 0 : parameters.set<bool>("use_displaced_mesh") = false;
5732 : }
5733 :
5734 1318 : parameters.set<SubProblem *>("_subproblem") = this;
5735 1977 : parameters.set<SystemBase *>("_sys") = _aux.get();
5736 : }
5737 :
5738 1382 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5739 : {
5740 : std::shared_ptr<Indicator> indicator =
5741 723 : _factory.create<Indicator>(indicator_name, name, parameters, tid);
5742 723 : logAdd("Indicator", name, indicator_name, parameters);
5743 : std::shared_ptr<InternalSideIndicatorBase> isi =
5744 723 : std::dynamic_pointer_cast<InternalSideIndicatorBase>(indicator);
5745 723 : if (isi)
5746 622 : _internal_side_indicators.addObject(isi, tid);
5747 : else
5748 101 : _indicators.addObject(indicator, tid);
5749 723 : }
5750 659 : }
5751 :
5752 : void
5753 1984 : FEProblemBase::addMarker(const std::string & marker_name,
5754 : const std::string & name,
5755 : InputParameters & parameters)
5756 : {
5757 : parallel_object_only();
5758 :
5759 1984 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5760 : {
5761 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5762 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5763 0 : _reinit_displaced_elem = true;
5764 : }
5765 : else
5766 : {
5767 1984 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5768 : {
5769 : // We allow Markers to request that they use_displaced_mesh,
5770 : // but then be overridden when no displacements variables are
5771 : // provided in the Mesh block. If that happened, update the value
5772 : // of use_displaced_mesh appropriately for this Marker.
5773 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5774 0 : parameters.set<bool>("use_displaced_mesh") = false;
5775 : }
5776 :
5777 3968 : parameters.set<SubProblem *>("_subproblem") = this;
5778 5952 : parameters.set<SystemBase *>("_sys") = _aux.get();
5779 : }
5780 :
5781 4145 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5782 : {
5783 2164 : std::shared_ptr<Marker> marker = _factory.create<Marker>(marker_name, name, parameters, tid);
5784 2161 : logAdd("Marker", name, marker_name, parameters);
5785 2161 : _markers.addObject(marker, tid);
5786 2161 : }
5787 1981 : }
5788 :
5789 : void
5790 8179 : FEProblemBase::addMultiApp(const std::string & multi_app_name,
5791 : const std::string & name,
5792 : InputParameters & parameters)
5793 : {
5794 : parallel_object_only();
5795 :
5796 16358 : parameters.set<MPI_Comm>("_mpi_comm") = _communicator.get();
5797 :
5798 8179 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5799 : {
5800 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5801 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5802 0 : _reinit_displaced_elem = true;
5803 : }
5804 : else
5805 : {
5806 8179 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5807 : {
5808 : // We allow MultiApps to request that they use_displaced_mesh,
5809 : // but then be overridden when no displacements variables are
5810 : // provided in the Mesh block. If that happened, update the value
5811 : // of use_displaced_mesh appropriately for this MultiApp.
5812 42 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5813 84 : parameters.set<bool>("use_displaced_mesh") = false;
5814 : }
5815 :
5816 16358 : parameters.set<SubProblem *>("_subproblem") = this;
5817 24537 : parameters.set<SystemBase *>("_sys") = _aux.get();
5818 : }
5819 :
5820 8179 : std::shared_ptr<MultiApp> multi_app = _factory.create<MultiApp>(multi_app_name, name, parameters);
5821 8167 : logAdd("MultiApp", name, multi_app_name, parameters);
5822 8167 : multi_app->setupPositions();
5823 :
5824 8080 : _multi_apps.addObject(multi_app);
5825 :
5826 : // Store TransientMultiApp objects in another container, this is needed for calling computeDT
5827 : std::shared_ptr<TransientMultiApp> trans_multi_app =
5828 8080 : std::dynamic_pointer_cast<TransientMultiApp>(multi_app);
5829 8080 : if (trans_multi_app)
5830 5395 : _transient_multi_apps.addObject(trans_multi_app);
5831 8080 : }
5832 :
5833 : bool
5834 112600 : FEProblemBase::hasMultiApps(ExecFlagType type) const
5835 : {
5836 112600 : return _multi_apps[type].hasActiveObjects();
5837 : }
5838 :
5839 : bool
5840 26593 : FEProblemBase::hasMultiApp(const std::string & multi_app_name) const
5841 : {
5842 26593 : return _multi_apps.hasActiveObject(multi_app_name);
5843 : }
5844 :
5845 : std::shared_ptr<MultiApp>
5846 26593 : FEProblemBase::getMultiApp(const std::string & multi_app_name) const
5847 : {
5848 26593 : if (!hasMultiApp(multi_app_name))
5849 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_multi_app"),
5850 : "A MultiApp getter was called before MultiApps have been constructed. "
5851 : "If you are attempting to access this object in the constructor of another object "
5852 : "then make sure that the MultiApp is constructed before the object using it.");
5853 :
5854 26593 : return _multi_apps.getObject(multi_app_name);
5855 : }
5856 :
5857 : void
5858 14487621 : FEProblemBase::execMultiAppTransfers(ExecFlagType type, Transfer::DIRECTION direction)
5859 : {
5860 14487621 : bool to_multiapp = direction == MultiAppTransfer::TO_MULTIAPP;
5861 14487621 : bool from_multiapp = direction == MultiAppTransfer::FROM_MULTIAPP;
5862 14487621 : std::string string_direction;
5863 14487621 : if (to_multiapp)
5864 4829261 : string_direction = " To ";
5865 9658360 : else if (from_multiapp)
5866 4829135 : string_direction = " From ";
5867 : else
5868 4829225 : string_direction = " Between ";
5869 :
5870 14487783 : const MooseObjectWarehouse<Transfer> & wh = to_multiapp ? _to_multi_app_transfers[type]
5871 24145891 : : from_multiapp ? _from_multi_app_transfers[type]
5872 33804413 : : _between_multi_app_transfers[type];
5873 :
5874 14487621 : if (wh.hasActiveObjects())
5875 : {
5876 546095 : TIME_SECTION("execMultiAppTransfers", 1, "Executing Transfers");
5877 :
5878 109219 : const auto & transfers = wh.getActiveObjects();
5879 :
5880 109219 : if (_verbose_multiapps)
5881 : {
5882 1120 : _console << COLOR_CYAN << "\nTransfers on " << Moose::stringify(type) << string_direction
5883 1120 : << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
5884 :
5885 : VariadicTable<std::string, std::string, std::string, std::string> table(
5886 2240 : {"Name", "Type", "From", "To"});
5887 :
5888 : // Build Table of Transfer Info
5889 3060 : for (const auto & transfer : transfers)
5890 : {
5891 1940 : auto multiapp_transfer = dynamic_cast<MultiAppTransfer *>(transfer.get());
5892 :
5893 1940 : table.addRow(multiapp_transfer->name(),
5894 1940 : multiapp_transfer->type(),
5895 3880 : multiapp_transfer->getFromName(),
5896 3880 : multiapp_transfer->getToName());
5897 : }
5898 :
5899 : // Print it
5900 1120 : table.print(_console);
5901 1120 : }
5902 :
5903 223479 : for (const auto & transfer : transfers)
5904 : {
5905 114357 : transfer->setCurrentDirection(direction);
5906 114357 : transfer->execute();
5907 : }
5908 :
5909 109122 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5910 :
5911 109122 : if (_verbose_multiapps)
5912 2228 : _console << COLOR_CYAN << "Transfers on " << Moose::stringify(type) << " Are Finished\n"
5913 1114 : << COLOR_DEFAULT << std::endl;
5914 109122 : }
5915 14378402 : else if (_multi_apps[type].getActiveObjects().size())
5916 : {
5917 110610 : if (_verbose_multiapps)
5918 7076 : _console << COLOR_CYAN << "\nNo Transfers on " << Moose::stringify(type) << string_direction
5919 3538 : << "MultiApps\n"
5920 3538 : << COLOR_DEFAULT << std::endl;
5921 : }
5922 14487524 : }
5923 :
5924 : std::vector<std::shared_ptr<Transfer>>
5925 12 : FEProblemBase::getTransfers(ExecFlagType type, Transfer::DIRECTION direction) const
5926 : {
5927 12 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5928 12 : return _to_multi_app_transfers[type].getActiveObjects();
5929 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5930 0 : return _from_multi_app_transfers[type].getActiveObjects();
5931 : else
5932 0 : return _between_multi_app_transfers[type].getActiveObjects();
5933 : }
5934 :
5935 : std::vector<std::shared_ptr<Transfer>>
5936 0 : FEProblemBase::getTransfers(Transfer::DIRECTION direction) const
5937 : {
5938 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5939 0 : return _to_multi_app_transfers.getActiveObjects();
5940 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5941 0 : return _from_multi_app_transfers.getActiveObjects();
5942 : else
5943 0 : return _between_multi_app_transfers.getActiveObjects();
5944 : }
5945 :
5946 : const ExecuteMooseObjectWarehouse<Transfer> &
5947 0 : FEProblemBase::getMultiAppTransferWarehouse(Transfer::DIRECTION direction) const
5948 : {
5949 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5950 0 : return _to_multi_app_transfers;
5951 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5952 0 : return _from_multi_app_transfers;
5953 : else
5954 0 : return _between_multi_app_transfers;
5955 : }
5956 :
5957 : bool
5958 4829264 : FEProblemBase::execMultiApps(ExecFlagType type, bool auto_advance)
5959 : {
5960 : // Active MultiApps
5961 : const std::vector<MooseSharedPointer<MultiApp>> & multi_apps =
5962 4829264 : _multi_apps[type].getActiveObjects();
5963 :
5964 : // Do anything that needs to be done to Apps before transfers
5965 4894498 : for (const auto & multi_app : multi_apps)
5966 65237 : multi_app->preTransfer(_dt, _time);
5967 :
5968 : // Execute Transfers _to_ MultiApps
5969 4829261 : execMultiAppTransfers(type, MultiAppTransfer::TO_MULTIAPP);
5970 :
5971 : // Execute Transfers _between_ Multiapps
5972 4829225 : execMultiAppTransfers(type, MultiAppTransfer::BETWEEN_MULTIAPP);
5973 :
5974 : // Execute MultiApps
5975 4829225 : if (multi_apps.size())
5976 : {
5977 319885 : TIME_SECTION("execMultiApps", 1, "Executing MultiApps", false);
5978 :
5979 63977 : if (_verbose_multiapps)
5980 2988 : _console << COLOR_CYAN << "\nExecuting MultiApps on " << Moose::stringify(type)
5981 1494 : << COLOR_DEFAULT << std::endl;
5982 :
5983 63977 : bool success = true;
5984 :
5985 129086 : for (const auto & multi_app : multi_apps)
5986 : {
5987 65192 : success = multi_app->solveStep(_dt, _time, auto_advance);
5988 : // no need to finish executing the subapps if one fails
5989 65183 : if (!success)
5990 74 : break;
5991 : }
5992 :
5993 63968 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5994 :
5995 63968 : _communicator.min(success);
5996 :
5997 63968 : if (!success)
5998 81 : return false;
5999 :
6000 63887 : if (_verbose_multiapps)
6001 2988 : _console << COLOR_CYAN << "Finished Executing MultiApps on " << Moose::stringify(type) << "\n"
6002 1494 : << COLOR_DEFAULT << std::endl;
6003 63968 : }
6004 :
6005 : // Execute Transfers _from_ MultiApps
6006 4829135 : execMultiAppTransfers(type, MultiAppTransfer::FROM_MULTIAPP);
6007 :
6008 : // If we made it here then everything passed
6009 4829074 : return true;
6010 : }
6011 :
6012 : void
6013 49853 : FEProblemBase::finalizeMultiApps()
6014 : {
6015 49853 : const auto & multi_apps = _multi_apps.getActiveObjects();
6016 :
6017 56917 : for (const auto & multi_app : multi_apps)
6018 7064 : multi_app->finalize();
6019 49853 : }
6020 :
6021 : void
6022 51401 : FEProblemBase::postExecute()
6023 : {
6024 51401 : const auto & multi_apps = _multi_apps.getActiveObjects();
6025 :
6026 58852 : for (const auto & multi_app : multi_apps)
6027 7451 : multi_app->postExecute();
6028 51401 : }
6029 :
6030 : void
6031 670376 : FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
6032 : {
6033 670376 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6034 :
6035 670376 : if (multi_apps.size())
6036 27809 : for (const auto & multi_app : multi_apps)
6037 14076 : multi_app->incrementTStep(_time);
6038 670376 : }
6039 :
6040 : void
6041 36370 : FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
6042 : {
6043 36370 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6044 :
6045 36370 : if (multi_apps.size())
6046 : {
6047 8584 : if (_verbose_multiapps)
6048 270 : _console << COLOR_CYAN << "\nAdvancing MultiApps on " << type.name() << COLOR_DEFAULT
6049 270 : << std::endl;
6050 :
6051 17174 : for (const auto & multi_app : multi_apps)
6052 8590 : multi_app->finishStep(recurse_through_multiapp_levels);
6053 :
6054 8584 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6055 :
6056 8584 : if (_verbose_multiapps)
6057 270 : _console << COLOR_CYAN << "Finished Advancing MultiApps on " << type.name() << "\n"
6058 270 : << COLOR_DEFAULT << std::endl;
6059 : }
6060 36370 : }
6061 :
6062 : void
6063 1123869 : FEProblemBase::backupMultiApps(ExecFlagType type)
6064 : {
6065 1123869 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6066 :
6067 1123869 : if (multi_apps.size())
6068 : {
6069 105985 : TIME_SECTION("backupMultiApps", 5, "Backing Up MultiApp");
6070 :
6071 21197 : if (_verbose_multiapps)
6072 798 : _console << COLOR_CYAN << "\nBacking Up MultiApps on " << type.name() << COLOR_DEFAULT
6073 798 : << std::endl;
6074 :
6075 43543 : for (const auto & multi_app : multi_apps)
6076 22346 : multi_app->backup();
6077 :
6078 21197 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6079 :
6080 21197 : if (_verbose_multiapps)
6081 798 : _console << COLOR_CYAN << "Finished Backing Up MultiApps on " << type.name() << "\n"
6082 798 : << COLOR_DEFAULT << std::endl;
6083 21197 : }
6084 1123869 : }
6085 :
6086 : void
6087 104680 : FEProblemBase::restoreMultiApps(ExecFlagType type, bool force)
6088 : {
6089 104680 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6090 :
6091 104680 : if (multi_apps.size())
6092 : {
6093 43028 : if (_verbose_multiapps)
6094 : {
6095 528 : if (force)
6096 0 : _console << COLOR_CYAN << "\nRestoring Multiapps on " << type.name()
6097 0 : << " because of solve failure!" << COLOR_DEFAULT << std::endl;
6098 : else
6099 528 : _console << COLOR_CYAN << "\nRestoring MultiApps on " << type.name() << COLOR_DEFAULT
6100 528 : << std::endl;
6101 : }
6102 :
6103 86098 : for (const auto & multi_app : multi_apps)
6104 43073 : multi_app->restore(force);
6105 :
6106 43025 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6107 :
6108 43025 : if (_verbose_multiapps)
6109 528 : _console << COLOR_CYAN << "Finished Restoring MultiApps on " << type.name() << "\n"
6110 528 : << COLOR_DEFAULT << std::endl;
6111 : }
6112 104677 : }
6113 :
6114 : Real
6115 695340 : FEProblemBase::computeMultiAppsDT(ExecFlagType type)
6116 : {
6117 695340 : const auto & multi_apps = _transient_multi_apps[type].getActiveObjects();
6118 :
6119 695340 : Real smallest_dt = std::numeric_limits<Real>::max();
6120 :
6121 713135 : for (const auto & multi_app : multi_apps)
6122 17795 : smallest_dt = std::min(smallest_dt, multi_app->computeDT());
6123 :
6124 695340 : return smallest_dt;
6125 : }
6126 :
6127 : void
6128 4774734 : FEProblemBase::execTransfers(ExecFlagType type)
6129 : {
6130 4774734 : if (_transfers[type].hasActiveObjects())
6131 : {
6132 0 : TIME_SECTION("execTransfers", 3, "Executing Transfers");
6133 :
6134 0 : const auto & transfers = _transfers[type].getActiveObjects();
6135 :
6136 0 : for (const auto & transfer : transfers)
6137 0 : transfer->execute();
6138 0 : }
6139 4774734 : }
6140 :
6141 : void
6142 13217 : FEProblemBase::addTransfer(const std::string & transfer_name,
6143 : const std::string & name,
6144 : InputParameters & parameters)
6145 : {
6146 : parallel_object_only();
6147 :
6148 13217 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
6149 : {
6150 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
6151 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
6152 0 : _reinit_displaced_elem = true;
6153 : }
6154 : else
6155 : {
6156 13217 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
6157 : {
6158 : // We allow Transfers to request that they use_displaced_mesh,
6159 : // but then be overridden when no displacements variables are
6160 : // provided in the Mesh block. If that happened, update the value
6161 : // of use_displaced_mesh appropriately for this Transfer.
6162 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
6163 0 : parameters.set<bool>("use_displaced_mesh") = false;
6164 : }
6165 :
6166 26434 : parameters.set<SubProblem *>("_subproblem") = this;
6167 39651 : parameters.set<SystemBase *>("_sys") = _aux.get();
6168 : }
6169 :
6170 : // Handle the "SAME_AS_MULTIAPP" execute option. The get method is used to test for the
6171 : // flag so the set by user flag is not reset, calling set with the true flag causes the set
6172 : // by user status to be reset, which should only be done if the EXEC_SAME_AS_MULTIAPP is
6173 : // being applied to the object.
6174 13217 : if (parameters.get<ExecFlagEnum>("execute_on").isValueSet(EXEC_SAME_AS_MULTIAPP))
6175 : {
6176 11078 : ExecFlagEnum & exec_enum = parameters.set<ExecFlagEnum>("execute_on", true);
6177 11078 : std::shared_ptr<MultiApp> multiapp;
6178 22156 : if (parameters.isParamValid("multi_app"))
6179 50 : multiapp = getMultiApp(parameters.get<MultiAppName>("multi_app"));
6180 : // This catches the sibling transfer case, where we want to be executing only as often as the
6181 : // receiving application. A transfer 'to' a multiapp is executed before that multiapp
6182 22056 : else if (parameters.isParamValid("to_multi_app"))
6183 5743 : multiapp = getMultiApp(parameters.get<MultiAppName>("to_multi_app"));
6184 10570 : else if (parameters.isParamValid("from_multi_app"))
6185 5279 : multiapp = getMultiApp(parameters.get<MultiAppName>("from_multi_app"));
6186 : // else do nothing because the user has provided invalid input. They should get a nice error
6187 : // about this during transfer construction. This necessitates checking for null in this next
6188 : // line, however
6189 11078 : if (multiapp)
6190 33216 : exec_enum = multiapp->getParam<ExecFlagEnum>("execute_on");
6191 11078 : }
6192 :
6193 : // Create the Transfer objects
6194 13217 : std::shared_ptr<Transfer> transfer = _factory.create<Transfer>(transfer_name, name, parameters);
6195 13160 : logAdd("Transfer", name, transfer_name, parameters);
6196 :
6197 : // Add MultiAppTransfer object
6198 : std::shared_ptr<MultiAppTransfer> multi_app_transfer =
6199 13160 : std::dynamic_pointer_cast<MultiAppTransfer>(transfer);
6200 13160 : if (multi_app_transfer)
6201 : {
6202 13160 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::TO_MULTIAPP))
6203 5274 : _to_multi_app_transfers.addObject(multi_app_transfer);
6204 13160 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::FROM_MULTIAPP))
6205 6416 : _from_multi_app_transfers.addObject(multi_app_transfer);
6206 13160 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::BETWEEN_MULTIAPP))
6207 1470 : _between_multi_app_transfers.addObject(multi_app_transfer);
6208 : }
6209 : else
6210 0 : _transfers.addObject(transfer);
6211 13160 : }
6212 :
6213 : bool
6214 1437607 : FEProblemBase::hasVariable(const std::string & var_name) const
6215 : {
6216 2346059 : for (auto & sys : _solver_systems)
6217 1443508 : if (sys->hasVariable(var_name))
6218 535056 : return true;
6219 902551 : if (_aux->hasVariable(var_name))
6220 832291 : return true;
6221 :
6222 70260 : return false;
6223 : }
6224 :
6225 : bool
6226 65 : FEProblemBase::hasSolverVariable(const std::string & var_name) const
6227 : {
6228 78 : for (auto & sys : _solver_systems)
6229 65 : if (sys->hasVariable(var_name))
6230 52 : return true;
6231 :
6232 13 : return false;
6233 : }
6234 :
6235 : const MooseVariableFieldBase &
6236 4313245 : FEProblemBase::getVariable(const THREAD_ID tid,
6237 : const std::string & var_name,
6238 : Moose::VarKindType expected_var_type,
6239 : Moose::VarFieldType expected_var_field_type) const
6240 : {
6241 12939729 : return getVariableHelper(
6242 4313245 : tid, var_name, expected_var_type, expected_var_field_type, _solver_systems, *_aux);
6243 : }
6244 :
6245 : MooseVariable &
6246 7721 : FEProblemBase::getStandardVariable(const THREAD_ID tid, const std::string & var_name)
6247 : {
6248 11623 : for (auto & sys : _solver_systems)
6249 7721 : if (sys->hasVariable(var_name))
6250 3819 : return sys->getFieldVariable<Real>(tid, var_name);
6251 3902 : if (_aux->hasVariable(var_name))
6252 3899 : return _aux->getFieldVariable<Real>(tid, var_name);
6253 :
6254 3 : mooseError("Unknown variable " + var_name);
6255 : }
6256 :
6257 : MooseVariableFieldBase &
6258 238 : FEProblemBase::getActualFieldVariable(const THREAD_ID tid, const std::string & var_name)
6259 : {
6260 283 : for (auto & sys : _solver_systems)
6261 238 : if (sys->hasVariable(var_name))
6262 193 : return sys->getActualFieldVariable<Real>(tid, var_name);
6263 45 : if (_aux->hasVariable(var_name))
6264 45 : return _aux->getActualFieldVariable<Real>(tid, var_name);
6265 :
6266 0 : mooseError("Unknown variable " + var_name);
6267 : }
6268 :
6269 : VectorMooseVariable &
6270 0 : FEProblemBase::getVectorVariable(const THREAD_ID tid, const std::string & var_name)
6271 : {
6272 0 : for (auto & sys : _solver_systems)
6273 0 : if (sys->hasVariable(var_name))
6274 0 : return sys->getFieldVariable<RealVectorValue>(tid, var_name);
6275 0 : if (_aux->hasVariable(var_name))
6276 0 : return _aux->getFieldVariable<RealVectorValue>(tid, var_name);
6277 :
6278 0 : mooseError("Unknown variable " + var_name);
6279 : }
6280 :
6281 : ArrayMooseVariable &
6282 351 : FEProblemBase::getArrayVariable(const THREAD_ID tid, const std::string & var_name)
6283 : {
6284 611 : for (auto & sys : _solver_systems)
6285 351 : if (sys->hasVariable(var_name))
6286 91 : return sys->getFieldVariable<RealEigenVector>(tid, var_name);
6287 260 : if (_aux->hasVariable(var_name))
6288 260 : return _aux->getFieldVariable<RealEigenVector>(tid, var_name);
6289 :
6290 0 : mooseError("Unknown variable " + var_name);
6291 : }
6292 :
6293 : bool
6294 181869 : FEProblemBase::hasScalarVariable(const std::string & var_name) const
6295 : {
6296 343002 : for (auto & sys : _solver_systems)
6297 182894 : if (sys->hasScalarVariable(var_name))
6298 21761 : return true;
6299 160108 : if (_aux->hasScalarVariable(var_name))
6300 10613 : return true;
6301 :
6302 149495 : return false;
6303 : }
6304 :
6305 : MooseVariableScalar &
6306 44346 : FEProblemBase::getScalarVariable(const THREAD_ID tid, const std::string & var_name)
6307 : {
6308 56695 : for (auto & sys : _solver_systems)
6309 44350 : if (sys->hasScalarVariable(var_name))
6310 32001 : return sys->getScalarVariable(tid, var_name);
6311 12345 : if (_aux->hasScalarVariable(var_name))
6312 12345 : return _aux->getScalarVariable(tid, var_name);
6313 :
6314 0 : mooseError("Unknown variable " + var_name);
6315 : }
6316 :
6317 : System &
6318 59051 : FEProblemBase::getSystem(const std::string & var_name)
6319 : {
6320 59051 : const auto [var_in_sys, sys_num] = determineSolverSystem(var_name);
6321 59051 : if (var_in_sys)
6322 40247 : return _solver_systems[sys_num]->system();
6323 18804 : else if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
6324 18804 : return _aux->system();
6325 : else
6326 0 : mooseError("Unable to find a system containing the variable " + var_name);
6327 : }
6328 :
6329 : const RestartableEquationSystems &
6330 0 : FEProblemBase::getRestartableEquationSystems() const
6331 : {
6332 0 : return _req.get();
6333 : }
6334 :
6335 : void
6336 465502 : FEProblemBase::setActiveFEVariableCoupleableMatrixTags(std::set<TagID> & mtags, const THREAD_ID tid)
6337 : {
6338 465502 : SubProblem::setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6339 :
6340 465502 : if (_displaced_problem)
6341 98260 : _displaced_problem->setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6342 465502 : }
6343 :
6344 : void
6345 6244621 : FEProblemBase::setActiveFEVariableCoupleableVectorTags(std::set<TagID> & vtags, const THREAD_ID tid)
6346 : {
6347 6244621 : SubProblem::setActiveFEVariableCoupleableVectorTags(vtags, tid);
6348 :
6349 6244621 : if (_displaced_problem)
6350 340078 : _displaced_problem->setActiveFEVariableCoupleableVectorTags(vtags, tid);
6351 6244621 : }
6352 :
6353 : void
6354 47856 : FEProblemBase::setActiveScalarVariableCoupleableMatrixTags(std::set<TagID> & mtags,
6355 : const THREAD_ID tid)
6356 : {
6357 47856 : SubProblem::setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6358 :
6359 47856 : if (_displaced_problem)
6360 0 : _displaced_problem->setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6361 47856 : }
6362 :
6363 : void
6364 47856 : FEProblemBase::setActiveScalarVariableCoupleableVectorTags(std::set<TagID> & vtags,
6365 : const THREAD_ID tid)
6366 : {
6367 47856 : SubProblem::setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6368 :
6369 47856 : if (_displaced_problem)
6370 0 : _displaced_problem->setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6371 47856 : }
6372 :
6373 : void
6374 9526786 : FEProblemBase::setActiveElementalMooseVariables(const std::set<MooseVariableFEBase *> & moose_vars,
6375 : const THREAD_ID tid)
6376 : {
6377 9526786 : SubProblem::setActiveElementalMooseVariables(moose_vars, tid);
6378 :
6379 9526786 : if (_displaced_problem)
6380 479391 : _displaced_problem->setActiveElementalMooseVariables(moose_vars, tid);
6381 9526786 : }
6382 :
6383 : void
6384 3838574 : FEProblemBase::clearActiveElementalMooseVariables(const THREAD_ID tid)
6385 : {
6386 3838574 : SubProblem::clearActiveElementalMooseVariables(tid);
6387 :
6388 3838574 : if (_displaced_problem)
6389 170465 : _displaced_problem->clearActiveElementalMooseVariables(tid);
6390 3838574 : }
6391 :
6392 : void
6393 207661 : FEProblemBase::clearActiveFEVariableCoupleableMatrixTags(const THREAD_ID tid)
6394 : {
6395 207661 : SubProblem::clearActiveFEVariableCoupleableMatrixTags(tid);
6396 :
6397 207661 : if (_displaced_problem)
6398 41849 : _displaced_problem->clearActiveFEVariableCoupleableMatrixTags(tid);
6399 207661 : }
6400 :
6401 : void
6402 207661 : FEProblemBase::clearActiveFEVariableCoupleableVectorTags(const THREAD_ID tid)
6403 : {
6404 207661 : SubProblem::clearActiveFEVariableCoupleableVectorTags(tid);
6405 :
6406 207661 : if (_displaced_problem)
6407 41849 : _displaced_problem->clearActiveFEVariableCoupleableVectorTags(tid);
6408 207661 : }
6409 :
6410 : void
6411 47856 : FEProblemBase::clearActiveScalarVariableCoupleableMatrixTags(const THREAD_ID tid)
6412 : {
6413 47856 : SubProblem::clearActiveScalarVariableCoupleableMatrixTags(tid);
6414 :
6415 47856 : if (_displaced_problem)
6416 0 : _displaced_problem->clearActiveScalarVariableCoupleableMatrixTags(tid);
6417 47856 : }
6418 :
6419 : void
6420 47856 : FEProblemBase::clearActiveScalarVariableCoupleableVectorTags(const THREAD_ID tid)
6421 : {
6422 47856 : SubProblem::clearActiveScalarVariableCoupleableVectorTags(tid);
6423 :
6424 47856 : if (_displaced_problem)
6425 0 : _displaced_problem->clearActiveScalarVariableCoupleableVectorTags(tid);
6426 47856 : }
6427 :
6428 : void
6429 4861760 : FEProblemBase::setActiveMaterialProperties(const std::unordered_set<unsigned int> & mat_prop_ids,
6430 : const THREAD_ID tid)
6431 : {
6432 : // mark active properties in every material
6433 5982515 : for (auto & mat : _all_materials.getObjects(tid))
6434 1120755 : mat->setActiveProperties(mat_prop_ids);
6435 5741477 : for (auto & mat : _all_materials[Moose::FACE_MATERIAL_DATA].getObjects(tid))
6436 879717 : mat->setActiveProperties(mat_prop_ids);
6437 5741477 : for (auto & mat : _all_materials[Moose::NEIGHBOR_MATERIAL_DATA].getObjects(tid))
6438 879717 : mat->setActiveProperties(mat_prop_ids);
6439 :
6440 4861760 : _has_active_material_properties[tid] = !mat_prop_ids.empty();
6441 4861760 : }
6442 :
6443 : bool
6444 383450727 : FEProblemBase::hasActiveMaterialProperties(const THREAD_ID tid) const
6445 : {
6446 383450727 : return _has_active_material_properties[tid];
6447 : }
6448 :
6449 : void
6450 3981238 : FEProblemBase::clearActiveMaterialProperties(const THREAD_ID tid)
6451 : {
6452 3981238 : _has_active_material_properties[tid] = 0;
6453 3981238 : }
6454 :
6455 : void
6456 60665 : FEProblemBase::addAnyRedistributers()
6457 : {
6458 : #ifdef LIBMESH_ENABLE_AMR
6459 62952 : if ((_adaptivity.isOn() || _num_grid_steps) &&
6460 2287 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
6461 2224 : _neighbor_material_props.hasStatefulProperties()))
6462 : {
6463 : // Even on a serialized Mesh, we don't keep our material
6464 : // properties serialized, so we'll rely on the callback to
6465 : // redistribute() to redistribute properties at the same time
6466 : // libMesh is redistributing elements.
6467 63 : auto add_redistributer = [this](MooseMesh & mesh,
6468 : const std::string & redistributer_name,
6469 : const bool use_displaced_mesh)
6470 : {
6471 63 : InputParameters redistribute_params = RedistributeProperties::validParams();
6472 63 : redistribute_params.set<MooseApp *>(MooseBase::app_param) = &_app;
6473 126 : redistribute_params.set<std::string>("for_whom") = this->name();
6474 189 : redistribute_params.set<MooseMesh *>("mesh") = &mesh;
6475 63 : redistribute_params.set<Moose::RelationshipManagerType>("rm_type") =
6476 : Moose::RelationshipManagerType::GEOMETRIC;
6477 126 : redistribute_params.set<bool>("use_displaced_mesh") = use_displaced_mesh;
6478 63 : redistribute_params.setHitNode(*parameters().getHitNode(), {});
6479 :
6480 : std::shared_ptr<RedistributeProperties> redistributer =
6481 63 : _factory.create<RedistributeProperties>(
6482 126 : "RedistributeProperties", redistributer_name, redistribute_params);
6483 :
6484 63 : if (_material_props.hasStatefulProperties())
6485 63 : redistributer->addMaterialPropertyStorage(_material_props);
6486 :
6487 63 : if (_bnd_material_props.hasStatefulProperties())
6488 63 : redistributer->addMaterialPropertyStorage(_bnd_material_props);
6489 :
6490 63 : if (_neighbor_material_props.hasStatefulProperties())
6491 63 : redistributer->addMaterialPropertyStorage(_neighbor_material_props);
6492 :
6493 63 : mesh.getMesh().add_ghosting_functor(redistributer);
6494 126 : };
6495 :
6496 63 : add_redistributer(_mesh, "mesh_property_redistributer", false);
6497 63 : if (_displaced_problem)
6498 0 : add_redistributer(_displaced_problem->mesh(), "displaced_mesh_property_redistributer", true);
6499 : }
6500 : #endif // LIBMESH_ENABLE_AMR
6501 60665 : }
6502 :
6503 : void
6504 62604 : FEProblemBase::updateMaxQps()
6505 : {
6506 : // Find the maximum number of quadrature points
6507 : {
6508 62604 : MaxQpsThread mqt(*this);
6509 62604 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mqt);
6510 62604 : _max_qps = mqt.max();
6511 :
6512 : // If we have more shape functions or more quadrature points on
6513 : // another processor, then we may need to handle those elements
6514 : // ourselves later after repartitioning.
6515 62604 : _communicator.max(_max_qps);
6516 : }
6517 :
6518 62604 : unsigned int max_qpts = getMaxQps();
6519 62604 : if (max_qpts > Moose::constMaxQpsPerElem)
6520 0 : mooseError("Max quadrature points per element assumptions made in some code (e.g. Coupleable ",
6521 : "and MaterialPropertyInterface classes) have been violated.\n",
6522 : "Complain to Moose developers to have constMaxQpsPerElem increased from ",
6523 : Moose::constMaxQpsPerElem,
6524 : " to ",
6525 : max_qpts);
6526 131854 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6527 : {
6528 : // the highest available order in libMesh is 43
6529 69250 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
6530 69250 : _zero[tid].resize(max_qpts, 0);
6531 69250 : _ad_zero[tid].resize(max_qpts, 0);
6532 69250 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
6533 69250 : _ad_grad_zero[tid].resize(max_qpts, ADRealGradient(0));
6534 69250 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
6535 69250 : _ad_second_zero[tid].resize(max_qpts, ADRealTensorValue(0));
6536 69250 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
6537 69250 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
6538 : }
6539 62604 : }
6540 :
6541 : void
6542 78 : FEProblemBase::bumpVolumeQRuleOrder(Order order, SubdomainID block)
6543 : {
6544 168 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6545 180 : for (const auto i : index_range(_nl))
6546 90 : _assembly[tid][i]->bumpVolumeQRuleOrder(order, block);
6547 :
6548 78 : if (_displaced_problem)
6549 0 : _displaced_problem->bumpVolumeQRuleOrder(order, block);
6550 :
6551 78 : updateMaxQps();
6552 78 : }
6553 :
6554 : void
6555 13 : FEProblemBase::bumpAllQRuleOrder(Order order, SubdomainID block)
6556 : {
6557 28 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6558 30 : for (const auto i : index_range(_nl))
6559 15 : _assembly[tid][i]->bumpAllQRuleOrder(order, block);
6560 :
6561 13 : if (_displaced_problem)
6562 0 : _displaced_problem->bumpAllQRuleOrder(order, block);
6563 :
6564 13 : updateMaxQps();
6565 13 : }
6566 :
6567 : void
6568 62513 : FEProblemBase::createQRules(QuadratureType type,
6569 : Order order,
6570 : Order volume_order,
6571 : Order face_order,
6572 : SubdomainID block,
6573 : const bool allow_negative_qweights)
6574 : {
6575 62513 : if (order == INVALID_ORDER)
6576 : {
6577 : // automatically determine the integration order
6578 61896 : order = _solver_systems[0]->getMinQuadratureOrder();
6579 62224 : for (const auto i : make_range(std::size_t(1), _solver_systems.size()))
6580 328 : if (order < _solver_systems[i]->getMinQuadratureOrder())
6581 0 : order = _solver_systems[i]->getMinQuadratureOrder();
6582 61896 : if (order < _aux->getMinQuadratureOrder())
6583 6081 : order = _aux->getMinQuadratureOrder();
6584 : }
6585 :
6586 62513 : if (volume_order == INVALID_ORDER)
6587 62325 : volume_order = order;
6588 :
6589 62513 : if (face_order == INVALID_ORDER)
6590 62325 : face_order = order;
6591 :
6592 131658 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6593 138679 : for (const auto i : index_range(_solver_systems))
6594 69534 : _assembly[tid][i]->createQRules(
6595 : type, order, volume_order, face_order, block, allow_negative_qweights);
6596 :
6597 62513 : if (_displaced_problem)
6598 2022 : _displaced_problem->createQRules(
6599 : type, order, volume_order, face_order, block, allow_negative_qweights);
6600 :
6601 62513 : updateMaxQps();
6602 62513 : }
6603 :
6604 : void
6605 19943 : FEProblemBase::setCoupling(Moose::CouplingType type)
6606 : {
6607 19943 : if (_trust_user_coupling_matrix)
6608 : {
6609 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6610 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6611 : "haven't been provided a coupling matrix!");
6612 :
6613 : // We've been told to trust the user coupling matrix, so we're going to leave things alone
6614 3 : return;
6615 : }
6616 :
6617 19940 : _coupling = type;
6618 : }
6619 :
6620 : void
6621 0 : FEProblemBase::setCouplingMatrix(CouplingMatrix * cm, const unsigned int i)
6622 : {
6623 : // TODO: Deprecate method
6624 0 : setCoupling(Moose::COUPLING_CUSTOM);
6625 0 : _cm[i].reset(cm);
6626 0 : }
6627 :
6628 : void
6629 13424 : FEProblemBase::setCouplingMatrix(std::unique_ptr<CouplingMatrix> cm, const unsigned int i)
6630 : {
6631 13424 : setCoupling(Moose::COUPLING_CUSTOM);
6632 13424 : _cm[i] = std::move(cm);
6633 13424 : }
6634 :
6635 : void
6636 3 : FEProblemBase::trustUserCouplingMatrix()
6637 : {
6638 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6639 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6640 : "haven't been provided a coupling matrix!");
6641 :
6642 3 : _trust_user_coupling_matrix = true;
6643 3 : }
6644 :
6645 : void
6646 63 : FEProblemBase::setNonlocalCouplingMatrix()
6647 : {
6648 315 : TIME_SECTION("setNonlocalCouplingMatrix", 5, "Setting Nonlocal Coupling Matrix");
6649 :
6650 63 : if (_nl.size() > 1)
6651 0 : mooseError("Nonlocal kernels are weirdly stored on the FEProblem so we don't currently support "
6652 : "multiple nonlinear systems with nonlocal kernels.");
6653 :
6654 126 : for (const auto nl_sys_num : index_range(_nl))
6655 : {
6656 63 : auto & nl = _nl[nl_sys_num];
6657 63 : auto & nonlocal_cm = _nonlocal_cm[nl_sys_num];
6658 63 : unsigned int n_vars = nl->nVariables();
6659 63 : nonlocal_cm.resize(n_vars);
6660 63 : const auto & vars = nl->getVariables(0);
6661 63 : const auto & nonlocal_kernel = _nonlocal_kernels.getObjects();
6662 63 : const auto & nonlocal_integrated_bc = _nonlocal_integrated_bcs.getObjects();
6663 189 : for (const auto & ivar : vars)
6664 : {
6665 196 : for (const auto & kernel : nonlocal_kernel)
6666 : {
6667 140 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6668 70 : if (i == kernel->variable().number())
6669 105 : for (const auto & jvar : vars)
6670 : {
6671 70 : const auto it = _var_dof_map.find(jvar->name());
6672 70 : if (it != _var_dof_map.end())
6673 : {
6674 62 : unsigned int j = jvar->number();
6675 62 : nonlocal_cm(i, j) = 1;
6676 : }
6677 : }
6678 : }
6679 182 : for (const auto & integrated_bc : nonlocal_integrated_bc)
6680 : {
6681 112 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6682 56 : if (i == integrated_bc->variable().number())
6683 84 : for (const auto & jvar : vars)
6684 : {
6685 56 : const auto it = _var_dof_map.find(jvar->name());
6686 56 : if (it != _var_dof_map.end())
6687 : {
6688 28 : unsigned int j = jvar->number();
6689 28 : nonlocal_cm(i, j) = 1;
6690 : }
6691 : }
6692 : }
6693 : }
6694 : }
6695 63 : }
6696 :
6697 : bool
6698 624 : FEProblemBase::areCoupled(const unsigned int ivar,
6699 : const unsigned int jvar,
6700 : const unsigned int nl_sys) const
6701 : {
6702 624 : return (*_cm[nl_sys])(ivar, jvar);
6703 : }
6704 :
6705 : std::vector<std::pair<MooseVariableFieldBase *, MooseVariableFieldBase *>> &
6706 15075988 : FEProblemBase::couplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6707 : {
6708 15075988 : return _assembly[tid][nl_sys]->couplingEntries();
6709 : }
6710 :
6711 : std::vector<std::pair<MooseVariableFieldBase *, MooseVariableFieldBase *>> &
6712 4162 : FEProblemBase::nonlocalCouplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6713 : {
6714 4162 : return _assembly[tid][nl_sys]->nonlocalCouplingEntries();
6715 : }
6716 :
6717 : void
6718 61159 : FEProblemBase::init()
6719 : {
6720 61159 : if (_initialized)
6721 0 : return;
6722 :
6723 305795 : TIME_SECTION("init", 2, "Initializing");
6724 :
6725 : // call executioner's preProblemInit so that it can do some setups before problem init
6726 61159 : _app.getExecutioner()->preProblemInit();
6727 :
6728 : // If we have AD and we are doing global AD indexing, then we should by default set the matrix
6729 : // coupling to full. If the user has told us to trust their coupling matrix, then this call will
6730 : // not do anything
6731 61159 : if (haveADObjects() && Moose::globalADIndexing())
6732 6426 : setCoupling(Moose::COUPLING_FULL);
6733 :
6734 121412 : for (const auto i : index_range(_nl))
6735 : {
6736 60253 : auto & nl = _nl[i];
6737 60253 : auto & cm = _cm[i];
6738 :
6739 60253 : unsigned int n_vars = nl->nVariables();
6740 : {
6741 301265 : TIME_SECTION("fillCouplingMatrix", 3, "Filling Coupling Matrix");
6742 :
6743 60253 : switch (_coupling)
6744 : {
6745 45083 : case Moose::COUPLING_DIAG:
6746 45083 : cm = std::make_unique<CouplingMatrix>(n_vars);
6747 81971 : for (unsigned int i = 0; i < n_vars; i++)
6748 36888 : (*cm)(i, i) = 1;
6749 45083 : break;
6750 :
6751 : // for full jacobian
6752 6579 : case Moose::COUPLING_FULL:
6753 6579 : cm = std::make_unique<CouplingMatrix>(n_vars);
6754 16838 : for (unsigned int i = 0; i < n_vars; i++)
6755 29692 : for (unsigned int j = 0; j < n_vars; j++)
6756 19433 : (*cm)(i, j) = 1;
6757 6579 : break;
6758 :
6759 8591 : case Moose::COUPLING_CUSTOM:
6760 : // do nothing, _cm was already set through couplingMatrix() call
6761 8591 : break;
6762 : }
6763 60253 : }
6764 :
6765 60253 : nl->dofMap()._dof_coupling = cm.get();
6766 :
6767 : // If there are no variables, make sure to pass a nullptr coupling
6768 : // matrix, to avoid warnings about non-nullptr yet empty
6769 : // CouplingMatrices.
6770 60253 : if (n_vars == 0)
6771 14035 : nl->dofMap()._dof_coupling = nullptr;
6772 :
6773 60253 : nl->dofMap().attach_extra_sparsity_function(&extraSparsity, nl.get());
6774 60253 : nl->dofMap().attach_extra_send_list_function(&extraSendList, nl.get());
6775 60253 : _aux->dofMap().attach_extra_send_list_function(&extraSendList, _aux.get());
6776 :
6777 60253 : if (!_skip_nl_system_check && _solve && n_vars == 0)
6778 0 : mooseError("No variables specified in nonlinear system '", nl->name(), "'.");
6779 : }
6780 :
6781 61159 : ghostGhostedBoundaries(); // We do this again right here in case new boundaries have been added
6782 :
6783 : // We may have added element/nodes to the mesh in ghostGhostedBoundaries so we need to update
6784 : // all of our mesh information. We need to make sure that mesh information is up-to-date before
6785 : // EquationSystems::init because that will call through to updateGeomSearch (for sparsity
6786 : // augmentation) and if we haven't added back boundary node information before that latter call,
6787 : // then we're screwed. We'll get things like "Unable to find closest node!"
6788 61159 : _mesh.meshChanged();
6789 61159 : if (_displaced_problem)
6790 2022 : _displaced_mesh->meshChanged();
6791 :
6792 61159 : if (_mesh.doingPRefinement())
6793 : {
6794 240 : preparePRefinement();
6795 240 : if (_displaced_problem)
6796 0 : _displaced_problem->preparePRefinement();
6797 : }
6798 :
6799 : // do not assemble system matrix for JFNK solve
6800 121412 : for (auto & nl : _nl)
6801 60253 : if (solverParams(nl->number())._type == Moose::ST_JFNK)
6802 120 : nl->turnOffJacobian();
6803 :
6804 122640 : for (auto & sys : _solver_systems)
6805 61481 : sys->preInit();
6806 61159 : _aux->preInit();
6807 :
6808 : // Build the mortar segment meshes, if they haven't been already, for a couple reasons:
6809 : // 1) Get the ghosting correct for both static and dynamic meshes
6810 : // 2) Make sure the mortar mesh is built for mortar constraints that live on the static mesh
6811 : //
6812 : // It is worth-while to note that mortar meshes that live on a dynamic mesh will be built
6813 : // during residual and Jacobian evaluation because when displacements are solution variables
6814 : // the mortar mesh will move and change during the course of a non-linear solve. We DO NOT
6815 : // redo ghosting during non-linear solve, so for purpose 1) the below call has to be made
6816 61159 : if (!_mortar_data->initialized())
6817 45558 : updateMortarMesh();
6818 :
6819 : {
6820 305795 : TIME_SECTION("EquationSystems::Init", 2, "Initializing Equation Systems");
6821 61159 : es().init();
6822 61159 : }
6823 :
6824 122640 : for (auto & sys : _solver_systems)
6825 61481 : sys->postInit();
6826 61159 : _aux->postInit();
6827 :
6828 : // Now that the equation system and the dof distribution is done, we can generate the
6829 : // finite volume-related parts if needed.
6830 61159 : if (haveFV())
6831 4343 : _mesh.setupFiniteVolumeMeshData();
6832 :
6833 122640 : for (auto & sys : _solver_systems)
6834 61481 : sys->update();
6835 61159 : _aux->update();
6836 :
6837 128513 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
6838 133825 : for (const auto i : index_range(_nl))
6839 : {
6840 : mooseAssert(
6841 : _cm[i],
6842 : "Coupling matrix not set for system "
6843 : << i
6844 : << ". This should only happen if a preconditioner was not setup for this system");
6845 66471 : _assembly[tid][i]->init(_cm[i].get());
6846 : }
6847 :
6848 61159 : if (_displaced_problem)
6849 2022 : _displaced_problem->init();
6850 :
6851 : #ifdef MOOSE_KOKKOS_ENABLED
6852 46284 : if (_has_kokkos_objects)
6853 2446 : initKokkos();
6854 : #endif
6855 :
6856 61159 : _initialized = true;
6857 61159 : }
6858 :
6859 : unsigned int
6860 12134 : FEProblemBase::nlSysNum(const NonlinearSystemName & nl_sys_name) const
6861 : {
6862 12134 : std::istringstream ss(nl_sys_name);
6863 : unsigned int nl_sys_num;
6864 12134 : if (!(ss >> nl_sys_num) || !ss.eof())
6865 12134 : nl_sys_num = libmesh_map_find(_nl_sys_name_to_num, nl_sys_name);
6866 :
6867 12134 : return nl_sys_num;
6868 12134 : }
6869 :
6870 : unsigned int
6871 77292 : FEProblemBase::linearSysNum(const LinearSystemName & linear_sys_name) const
6872 : {
6873 77292 : std::istringstream ss(linear_sys_name);
6874 : unsigned int linear_sys_num;
6875 77292 : if (!(ss >> linear_sys_num) || !ss.eof())
6876 77292 : linear_sys_num = libmesh_map_find(_linear_sys_name_to_num, linear_sys_name);
6877 :
6878 77292 : return linear_sys_num;
6879 77292 : }
6880 :
6881 : unsigned int
6882 126839 : FEProblemBase::solverSysNum(const SolverSystemName & solver_sys_name) const
6883 : {
6884 126839 : std::istringstream ss(solver_sys_name);
6885 : unsigned int solver_sys_num;
6886 126839 : if (!(ss >> solver_sys_num) || !ss.eof())
6887 : {
6888 126839 : const auto & search = _solver_sys_name_to_num.find(solver_sys_name);
6889 126839 : if (search == _solver_sys_name_to_num.end())
6890 0 : mooseError("The solver system number was requested for system '" + solver_sys_name,
6891 : "' but this system does not exist in the Problem. Systems can be added to the "
6892 : "problem using the 'nl_sys_names'/'linear_sys_names' parameter.\nSystems in the "
6893 0 : "Problem: " +
6894 0 : Moose::stringify(_solver_sys_names));
6895 126839 : solver_sys_num = search->second;
6896 : }
6897 :
6898 126839 : return solver_sys_num;
6899 126839 : }
6900 :
6901 : unsigned int
6902 1644 : FEProblemBase::systemNumForVariable(const VariableName & variable_name) const
6903 : {
6904 1746 : for (const auto & solver_sys : _solver_systems)
6905 1644 : if (solver_sys->hasVariable(variable_name))
6906 1542 : return solver_sys->number();
6907 : mooseAssert(_aux, "Should have an auxiliary system");
6908 102 : if (_aux->hasVariable(variable_name))
6909 102 : return _aux->number();
6910 :
6911 0 : mooseError("Variable '",
6912 : variable_name,
6913 : "' was not found in any solver (nonlinear/linear) or auxiliary system");
6914 : }
6915 :
6916 : void
6917 322792 : FEProblemBase::solve(const unsigned int nl_sys_num)
6918 : {
6919 1613960 : TIME_SECTION("solve", 1, "Solving", false);
6920 :
6921 322792 : setCurrentNonlinearSystem(nl_sys_num);
6922 :
6923 : // This prevents stale dof indices from lingering around and possibly leading to invalid reads
6924 : // and writes. Dof indices may be made stale through operations like mesh adaptivity
6925 322792 : clearAllDofIndices();
6926 322792 : if (_displaced_problem)
6927 32837 : _displaced_problem->clearAllDofIndices();
6928 :
6929 : // Setup the output system for printing linear/nonlinear iteration information and some solver
6930 : // settings, including setting matrix prefixes. This must occur before petscSetOptions
6931 322792 : initPetscOutputAndSomeSolverSettings();
6932 :
6933 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6934 : Moose::PetscSupport::petscSetOptions(
6935 : _petsc_options, _solver_params); // Make sure the PETSc options are setup for this app
6936 : #else
6937 : // Now this database will be the default
6938 : // Each app should have only one database
6939 322792 : if (!_app.isUltimateMaster())
6940 85527 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6941 : // We did not add PETSc options to database yet
6942 322792 : if (!_is_petsc_options_inserted)
6943 : {
6944 : // Insert options for all systems all at once
6945 39421 : Moose::PetscSupport::petscSetOptions(_petsc_options, _solver_params, this);
6946 39418 : _is_petsc_options_inserted = true;
6947 : }
6948 : #endif
6949 :
6950 : // set up DM which is required if use a field split preconditioner
6951 : // We need to setup DM every "solve()" because libMesh destroy SNES after solve()
6952 : // Do not worry, DM setup is very cheap
6953 322789 : _current_nl_sys->setupDM();
6954 :
6955 322789 : possiblyRebuildGeomSearchPatches();
6956 :
6957 : // reset flag so that residual evaluation does not get skipped
6958 : // and the next non-linear iteration does not automatically fail with
6959 : // "DIVERGED_NANORINF", when we throw an exception and stop solve
6960 322789 : _fail_next_system_convergence_check = false;
6961 :
6962 322789 : if (_solve)
6963 : {
6964 289086 : _current_nl_sys->solve();
6965 289031 : _current_nl_sys->update();
6966 : }
6967 :
6968 : // sync solutions in displaced problem
6969 322734 : if (_displaced_problem)
6970 32834 : _displaced_problem->syncSolutions();
6971 :
6972 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6973 322734 : if (!_app.isUltimateMaster())
6974 85527 : LibmeshPetscCall(PetscOptionsPop());
6975 : #endif
6976 322734 : }
6977 :
6978 : void
6979 201 : FEProblemBase::setException(const std::string & message)
6980 : {
6981 201 : _has_exception = true;
6982 201 : _exception_message = message;
6983 201 : }
6984 :
6985 : void
6986 19937463 : FEProblemBase::checkExceptionAndStopSolve(bool print_message)
6987 : {
6988 19937463 : if (_skip_exception_check)
6989 216 : return;
6990 :
6991 59811741 : TIME_SECTION("checkExceptionAndStopSolve", 5);
6992 :
6993 : // See if any processor had an exception. If it did, get back the
6994 : // processor that the exception occurred on.
6995 : unsigned int processor_id;
6996 :
6997 19937247 : _communicator.maxloc(_has_exception, processor_id);
6998 :
6999 19937247 : if (_has_exception)
7000 : {
7001 304 : _communicator.broadcast(_exception_message, processor_id);
7002 :
7003 436 : if (_current_execute_on_flag == EXEC_LINEAR || _current_execute_on_flag == EXEC_NONLINEAR ||
7004 132 : _current_execute_on_flag == EXEC_POSTCHECK)
7005 : {
7006 : // Print the message
7007 304 : if (_communicator.rank() == 0 && print_message)
7008 : {
7009 186 : _console << "\n" << _exception_message << "\n";
7010 186 : if (isTransient())
7011 : _console
7012 : << "To recover, the solution will fail and then be re-attempted with a reduced time "
7013 174 : "step.\n"
7014 174 : << std::endl;
7015 : }
7016 :
7017 : // Stop the solve -- this entails setting
7018 : // SNESSetFunctionDomainError() or directly inserting NaNs in the
7019 : // residual vector to let PETSc >= 3.6 return DIVERGED_NANORINF.
7020 304 : if (_current_nl_sys)
7021 304 : _current_nl_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7022 :
7023 304 : if (_current_linear_sys)
7024 0 : _current_linear_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7025 :
7026 : // and close Aux system (we MUST do this here; see #11525)
7027 304 : _aux->solution().close();
7028 :
7029 : // We've handled this exception, so we no longer have one.
7030 304 : _has_exception = false;
7031 :
7032 : // Force the next non-linear convergence check to fail (and all further residual evaluation
7033 : // to be skipped).
7034 304 : _fail_next_system_convergence_check = true;
7035 :
7036 : // Repropagate the exception, so it can be caught at a higher level, typically
7037 : // this is NonlinearSystem::computeResidual().
7038 304 : throw MooseException(_exception_message);
7039 : }
7040 : else
7041 0 : mooseError("The following parallel-communicated exception was detected during " +
7042 0 : Moose::stringify(_current_execute_on_flag) + " evaluation:\n" +
7043 0 : _exception_message +
7044 : "\nBecause this did not occur during residual evaluation, there"
7045 : " is no way to handle this, so the solution is aborting.\n");
7046 : }
7047 19937247 : }
7048 :
7049 : void
7050 3538411 : FEProblemBase::resetState()
7051 : {
7052 : // Our default state is to allow computing derivatives
7053 3538411 : ADReal::do_derivatives = true;
7054 3538411 : _current_execute_on_flag = EXEC_NONE;
7055 :
7056 : // Clear the VectorTags and MatrixTags
7057 3538411 : clearCurrentResidualVectorTags();
7058 3538411 : clearCurrentJacobianMatrixTags();
7059 :
7060 3538411 : _safe_access_tagged_vectors = true;
7061 3538411 : _safe_access_tagged_matrices = true;
7062 :
7063 3538411 : setCurrentlyComputingResidual(false);
7064 3538411 : setCurrentlyComputingJacobian(false);
7065 3538411 : setCurrentlyComputingResidualAndJacobian(false);
7066 3538411 : if (_displaced_problem)
7067 : {
7068 145293 : _displaced_problem->setCurrentlyComputingResidual(false);
7069 145293 : _displaced_problem->setCurrentlyComputingJacobian(false);
7070 145293 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(false);
7071 : }
7072 3538411 : }
7073 :
7074 : void
7075 25718 : FEProblemBase::solveLinearSystem(const unsigned int linear_sys_num,
7076 : const Moose::PetscSupport::PetscOptions * po)
7077 : {
7078 128590 : TIME_SECTION("solve", 1, "Solving", false);
7079 :
7080 25718 : setCurrentLinearSystem(linear_sys_num);
7081 :
7082 25718 : const Moose::PetscSupport::PetscOptions & options = po ? *po : _petsc_options;
7083 25718 : auto & solver_params = _solver_params[numNonlinearSystems() + linear_sys_num];
7084 :
7085 : // Set custom convergence criteria
7086 25718 : Moose::PetscSupport::petscSetDefaults(*this);
7087 :
7088 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
7089 : LibmeshPetscCall(Moose::PetscSupport::petscSetOptions(
7090 : options, solver_params)); // Make sure the PETSc options are setup for this app
7091 : #else
7092 : // Now this database will be the default
7093 : // Each app should have only one database
7094 25718 : if (!_app.isUltimateMaster())
7095 195 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
7096 :
7097 : // We did not add PETSc options to database yet
7098 25718 : if (!_is_petsc_options_inserted)
7099 : {
7100 1119 : Moose::PetscSupport::petscSetOptions(options, solver_params, this);
7101 1119 : _is_petsc_options_inserted = true;
7102 : }
7103 : #endif
7104 :
7105 25718 : if (_solve)
7106 25708 : _current_linear_sys->solve();
7107 :
7108 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
7109 25718 : if (!_app.isUltimateMaster())
7110 195 : LibmeshPetscCall(PetscOptionsPop());
7111 : #endif
7112 25718 : }
7113 :
7114 : bool
7115 335924 : FEProblemBase::solverSystemConverged(const unsigned int sys_num)
7116 : {
7117 335924 : if (_solve)
7118 335902 : return _solver_systems[sys_num]->converged();
7119 : else
7120 22 : return true;
7121 : }
7122 :
7123 : unsigned int
7124 4901 : FEProblemBase::nNonlinearIterations(const unsigned int nl_sys_num) const
7125 : {
7126 4901 : return _nl[nl_sys_num]->nNonlinearIterations();
7127 : }
7128 :
7129 : unsigned int
7130 4051 : FEProblemBase::nLinearIterations(const unsigned int nl_sys_num) const
7131 : {
7132 4051 : return _nl[nl_sys_num]->nLinearIterations();
7133 : }
7134 :
7135 : Real
7136 242 : FEProblemBase::finalNonlinearResidual(const unsigned int nl_sys_num) const
7137 : {
7138 242 : return _nl[nl_sys_num]->finalNonlinearResidual();
7139 : }
7140 :
7141 : bool
7142 761344 : FEProblemBase::computingPreSMOResidual(const unsigned int nl_sys_num) const
7143 : {
7144 761344 : return _nl[nl_sys_num]->computingPreSMOResidual();
7145 : }
7146 :
7147 : void
7148 55906 : FEProblemBase::copySolutionsBackwards()
7149 : {
7150 279530 : TIME_SECTION("copySolutionsBackwards", 3, "Copying Solutions Backward");
7151 :
7152 112113 : for (auto & sys : _solver_systems)
7153 56207 : sys->copySolutionsBackwards();
7154 55906 : _aux->copySolutionsBackwards();
7155 55906 : }
7156 :
7157 : void
7158 152 : FEProblemBase::skipNextForwardSolutionCopyToOld()
7159 : {
7160 324 : for (auto & sys : _solver_systems)
7161 172 : sys->skipNextSolutionToOldCopy();
7162 152 : _aux->skipNextSolutionToOldCopy();
7163 152 : }
7164 :
7165 : void
7166 234304 : FEProblemBase::advanceState()
7167 : {
7168 1171520 : TIME_SECTION("advanceState", 5, "Advancing State");
7169 :
7170 472015 : for (auto & sys : _solver_systems)
7171 237711 : sys->copyOldSolutions();
7172 234304 : _aux->copyOldSolutions();
7173 :
7174 234304 : if (_displaced_problem)
7175 : {
7176 61858 : for (const auto i : index_range(_solver_systems))
7177 30929 : _displaced_problem->solverSys(i).copyOldSolutions();
7178 30929 : _displaced_problem->auxSys().copyOldSolutions();
7179 : }
7180 :
7181 234304 : _reporter_data.copyValuesBack();
7182 :
7183 234304 : getMooseApp().getChainControlDataSystem().copyValuesBack();
7184 :
7185 234304 : if (_material_props.hasStatefulProperties())
7186 1760 : _material_props.shift();
7187 :
7188 234304 : if (_bnd_material_props.hasStatefulProperties())
7189 1612 : _bnd_material_props.shift();
7190 :
7191 234304 : if (_neighbor_material_props.hasStatefulProperties())
7192 1496 : _neighbor_material_props.shift();
7193 :
7194 : #ifdef MOOSE_KOKKOS_ENABLED
7195 172110 : if (_kokkos_material_props.hasStatefulProperties())
7196 566 : _kokkos_material_props.shift();
7197 :
7198 172110 : if (_kokkos_bnd_material_props.hasStatefulProperties())
7199 641 : _kokkos_bnd_material_props.shift();
7200 :
7201 172110 : if (_kokkos_neighbor_material_props.hasStatefulProperties())
7202 566 : _kokkos_neighbor_material_props.shift();
7203 : #endif
7204 234304 : }
7205 :
7206 : void
7207 3345 : FEProblemBase::restoreSolutions()
7208 : {
7209 16725 : TIME_SECTION("restoreSolutions", 5, "Restoring Solutions");
7210 :
7211 3345 : if (!_not_zeroed_tagged_vectors.empty())
7212 0 : paramError("not_zeroed_tag_vectors",
7213 : "There is currently no way to restore not-zeroed vectors.");
7214 :
7215 6692 : for (auto & sys : _solver_systems)
7216 : {
7217 3347 : if (_verbose_restore)
7218 18 : _console << "Restoring solutions on system " << sys->name() << "..." << std::endl;
7219 3347 : sys->restoreSolutions();
7220 : }
7221 :
7222 3345 : if (_verbose_restore)
7223 18 : _console << "Restoring solutions on Auxiliary system..." << std::endl;
7224 3345 : _aux->restoreSolutions();
7225 :
7226 3345 : if (_verbose_restore)
7227 18 : _console << "Restoring postprocessor, vector-postprocessor, and reporter data..." << std::endl;
7228 3345 : _reporter_data.restoreState(_verbose_restore);
7229 :
7230 3345 : if (_displaced_problem)
7231 132 : _displaced_problem->updateMesh();
7232 3345 : }
7233 :
7234 : void
7235 100 : FEProblemBase::saveOldSolutions()
7236 : {
7237 500 : TIME_SECTION("saveOldSolutions", 5, "Saving Old Solutions");
7238 :
7239 200 : for (auto & sys : _solver_systems)
7240 100 : sys->saveOldSolutions();
7241 100 : _aux->saveOldSolutions();
7242 100 : }
7243 :
7244 : void
7245 100 : FEProblemBase::restoreOldSolutions()
7246 : {
7247 500 : TIME_SECTION("restoreOldSolutions", 5, "Restoring Old Solutions");
7248 :
7249 200 : for (auto & sys : _solver_systems)
7250 100 : sys->restoreOldSolutions();
7251 100 : _aux->restoreOldSolutions();
7252 100 : }
7253 :
7254 : void
7255 1243846 : FEProblemBase::outputStep(ExecFlagType type)
7256 : {
7257 6219230 : TIME_SECTION("outputStep", 1, "Outputting");
7258 :
7259 1243846 : setCurrentExecuteOnFlag(type);
7260 :
7261 2502197 : for (auto & sys : _solver_systems)
7262 1258351 : sys->update();
7263 1243846 : _aux->update();
7264 :
7265 1243846 : if (_displaced_problem)
7266 126253 : _displaced_problem->syncSolutions();
7267 1243846 : _app.getOutputWarehouse().outputStep(type);
7268 :
7269 1243837 : setCurrentExecuteOnFlag(EXEC_NONE);
7270 1243837 : }
7271 :
7272 : void
7273 83891 : FEProblemBase::allowOutput(bool state)
7274 : {
7275 83891 : _app.getOutputWarehouse().allowOutput(state);
7276 83891 : }
7277 :
7278 : void
7279 23 : FEProblemBase::forceOutput()
7280 : {
7281 23 : _app.getOutputWarehouse().forceOutput();
7282 23 : }
7283 :
7284 : void
7285 341817 : FEProblemBase::initPetscOutputAndSomeSolverSettings()
7286 : {
7287 341817 : _app.getOutputWarehouse().solveSetup();
7288 341817 : Moose::PetscSupport::petscSetDefaults(*this);
7289 341817 : }
7290 :
7291 : void
7292 240188 : FEProblemBase::onTimestepBegin()
7293 : {
7294 720564 : TIME_SECTION("onTimestepBegin", 2);
7295 :
7296 483444 : for (auto & nl : _nl)
7297 243256 : nl->onTimestepBegin();
7298 240188 : }
7299 :
7300 : void
7301 311031 : FEProblemBase::onTimestepEnd()
7302 : {
7303 311031 : }
7304 :
7305 : Real
7306 6854511 : FEProblemBase::getTimeFromStateArg(const Moose::StateArg & state) const
7307 : {
7308 6854511 : if (state.iteration_type != Moose::SolutionIterationType::Time)
7309 : // If we are any iteration type other than time (e.g. nonlinear), then temporally we are still
7310 : // in the present time
7311 0 : return time();
7312 :
7313 6854511 : switch (state.state)
7314 : {
7315 6854511 : case 0:
7316 6854511 : return time();
7317 :
7318 0 : case 1:
7319 0 : return timeOld();
7320 :
7321 0 : default:
7322 0 : mooseError("Unhandled state ", state.state, " in FEProblemBase::getTimeFromStateArg");
7323 : }
7324 : }
7325 :
7326 : void
7327 30544 : FEProblemBase::addTimeIntegrator(const std::string & type,
7328 : const std::string & name,
7329 : InputParameters & parameters)
7330 : {
7331 : parallel_object_only();
7332 :
7333 91632 : parameters.set<SubProblem *>("_subproblem") = this;
7334 30544 : logAdd("TimeIntegrator", name, type, parameters);
7335 30544 : _aux->addTimeIntegrator(type, name + ":aux", parameters);
7336 61142 : for (auto & sys : _solver_systems)
7337 30598 : sys->addTimeIntegrator(type, name + ":" + sys->name(), parameters);
7338 30544 : _has_time_integrator = true;
7339 :
7340 : // add vectors to store u_dot, u_dotdot, udot_old, u_dotdot_old and
7341 : // solution vectors older than 2 time steps, if requested by the time
7342 : // integrator
7343 30544 : _aux->addDotVectors();
7344 61111 : for (auto & nl : _nl)
7345 : {
7346 30567 : nl->addDotVectors();
7347 :
7348 30567 : auto tag_udot = nl->getTimeIntegrators()[0]->uDotFactorTag();
7349 30567 : if (!nl->hasVector(tag_udot))
7350 30531 : nl->associateVectorToTag(*nl->solutionUDot(), tag_udot);
7351 30567 : auto tag_udotdot = nl->getTimeIntegrators()[0]->uDotDotFactorTag();
7352 30567 : if (!nl->hasVector(tag_udotdot) && uDotDotRequested())
7353 150 : nl->associateVectorToTag(*nl->solutionUDotDot(), tag_udotdot);
7354 : }
7355 :
7356 30544 : if (_displaced_problem)
7357 : // Time integrator does not exist when displaced problem is created.
7358 1728 : _displaced_problem->addTimeIntegrator();
7359 30544 : }
7360 :
7361 : void
7362 22 : FEProblemBase::addPredictor(const std::string & type,
7363 : const std::string & name,
7364 : InputParameters & parameters)
7365 : {
7366 : parallel_object_only();
7367 :
7368 22 : if (!numNonlinearSystems() && numLinearSystems())
7369 0 : mooseError("Vector bounds cannot be used with LinearSystems!");
7370 :
7371 44 : parameters.set<SubProblem *>("_subproblem") = this;
7372 22 : std::shared_ptr<Predictor> predictor = _factory.create<Predictor>(type, name, parameters);
7373 22 : logAdd("Predictor", name, type, parameters);
7374 :
7375 44 : for (auto & nl : _nl)
7376 22 : nl->setPredictor(predictor);
7377 22 : }
7378 :
7379 : Real
7380 62919 : FEProblemBase::computeResidualL2Norm(NonlinearSystemBase & sys)
7381 : {
7382 62919 : _current_nl_sys = &sys;
7383 62919 : computeResidual(*sys.currentSolution(), sys.RHS(), sys.number());
7384 62919 : return sys.RHS().l2_norm();
7385 : }
7386 :
7387 : Real
7388 84 : FEProblemBase::computeResidualL2Norm(LinearSystem & sys)
7389 : {
7390 84 : _current_linear_sys = &sys;
7391 :
7392 : // We assemble the current system to check the current residual
7393 84 : computeLinearSystemSys(sys.linearImplicitSystem(),
7394 84 : *sys.linearImplicitSystem().matrix,
7395 84 : *sys.linearImplicitSystem().rhs,
7396 : /*compute fresh gradients*/ true);
7397 :
7398 : // Unfortunate, but we have to allocate a new vector for the residual
7399 84 : auto residual = sys.linearImplicitSystem().rhs->clone();
7400 84 : residual->scale(-1.0);
7401 84 : residual->add_vector(*sys.currentSolution(), *sys.linearImplicitSystem().matrix);
7402 168 : return residual->l2_norm();
7403 84 : }
7404 :
7405 : Real
7406 62825 : FEProblemBase::computeResidualL2Norm()
7407 : {
7408 314125 : TIME_SECTION("computeResidualL2Norm", 2, "Computing L2 Norm of Residual");
7409 :
7410 : // We use sum the squared norms of the individual systems and then take the square root of it
7411 62825 : Real l2_norm = 0.0;
7412 125744 : for (auto sys : _nl)
7413 : {
7414 62919 : const auto norm = computeResidualL2Norm(*sys);
7415 62919 : l2_norm += norm * norm;
7416 62919 : }
7417 :
7418 62909 : for (auto sys : _linear_systems)
7419 : {
7420 84 : const auto norm = computeResidualL2Norm(*sys);
7421 84 : l2_norm += norm * norm;
7422 84 : }
7423 :
7424 125650 : return std::sqrt(l2_norm);
7425 62825 : }
7426 :
7427 : void
7428 2948220 : FEProblemBase::computeResidualSys(NonlinearImplicitSystem & sys,
7429 : const NumericVector<Number> & soln,
7430 : NumericVector<Number> & residual)
7431 : {
7432 : parallel_object_only();
7433 :
7434 8844660 : TIME_SECTION("computeResidualSys", 5);
7435 : // Reset before residual setup, calculation & execution
7436 2948220 : _app.solutionInvalidity().resetIterationOccurences();
7437 :
7438 2948220 : computeResidual(soln, residual, sys.number());
7439 2948190 : }
7440 :
7441 : void
7442 0 : FEProblemBase::computeResidual(NonlinearImplicitSystem & sys,
7443 : const NumericVector<Number> & soln,
7444 : NumericVector<Number> & residual)
7445 : {
7446 0 : mooseDeprecated("Please use computeResidualSys");
7447 :
7448 0 : computeResidualSys(sys, soln, residual);
7449 0 : }
7450 :
7451 : void
7452 3007958 : FEProblemBase::computeResidual(const NumericVector<Number> & soln,
7453 : NumericVector<Number> & residual,
7454 : const unsigned int nl_sys_num)
7455 : {
7456 3007958 : setCurrentNonlinearSystem(nl_sys_num);
7457 :
7458 : // We associate the residual tag with the given residual vector to make sure we
7459 : // don't filter it out below
7460 3007958 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7461 3007958 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7462 :
7463 : mooseAssert(_fe_vector_tags.empty(), "This should be empty indicating a clean starting state");
7464 : // We filter out tags which do not have associated vectors in the current nonlinear
7465 : // system. This is essential to be able to use system-dependent residual tags.
7466 3007958 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7467 :
7468 3007958 : computeResidualInternal(soln, residual, _fe_vector_tags);
7469 3007928 : _fe_vector_tags.clear();
7470 3007928 : }
7471 :
7472 : void
7473 9899 : FEProblemBase::computeResidualAndJacobian(const NumericVector<Number> & soln,
7474 : NumericVector<Number> & residual,
7475 : SparseMatrix<Number> & jacobian)
7476 : {
7477 : try
7478 : {
7479 : try
7480 : {
7481 : // vector tags
7482 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7483 9899 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7484 :
7485 : mooseAssert(_fe_vector_tags.empty(),
7486 : "This should be empty indicating a clean starting state");
7487 : // We filter out tags which do not have associated vectors in the current nonlinear
7488 : // system. This is essential to be able to use system-dependent residual tags.
7489 9899 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7490 :
7491 9899 : setCurrentResidualVectorTags(_fe_vector_tags);
7492 :
7493 : // matrix tags
7494 : {
7495 9899 : _fe_matrix_tags.clear();
7496 :
7497 9899 : auto & tags = getMatrixTags();
7498 29697 : for (auto & tag : tags)
7499 19798 : _fe_matrix_tags.insert(tag.second);
7500 : }
7501 :
7502 9899 : _current_nl_sys->setSolution(soln);
7503 :
7504 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7505 9899 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7506 :
7507 29697 : for (const auto tag : _fe_matrix_tags)
7508 19798 : if (_current_nl_sys->hasMatrix(tag))
7509 : {
7510 9899 : auto & matrix = _current_nl_sys->getMatrix(tag);
7511 9899 : matrix.zero();
7512 9899 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7513 : // PETSc algorithms require diagonal allocations regardless of whether there is non-zero
7514 : // diagonal dependence. With global AD indexing we only add non-zero
7515 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7516 123018 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7517 120004 : matrix.add(index, index, 0);
7518 : }
7519 :
7520 9899 : _aux->zeroVariablesForResidual();
7521 :
7522 9899 : unsigned int n_threads = libMesh::n_threads();
7523 :
7524 9899 : _current_execute_on_flag = EXEC_LINEAR;
7525 :
7526 : // Random interface objects
7527 9899 : for (const auto & it : _random_data_objects)
7528 0 : it.second->updateSeeds(EXEC_LINEAR);
7529 :
7530 9899 : setCurrentlyComputingResidual(true);
7531 9899 : setCurrentlyComputingJacobian(true);
7532 9899 : setCurrentlyComputingResidualAndJacobian(true);
7533 9899 : if (_displaced_problem)
7534 : {
7535 1057 : _displaced_problem->setCurrentlyComputingResidual(true);
7536 1057 : _displaced_problem->setCurrentlyComputingJacobian(true);
7537 1057 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(true);
7538 : }
7539 :
7540 9899 : execTransfers(EXEC_LINEAR);
7541 :
7542 9899 : execMultiApps(EXEC_LINEAR);
7543 :
7544 21234 : for (unsigned int tid = 0; tid < n_threads; tid++)
7545 11335 : reinitScalars(tid);
7546 :
7547 9899 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7548 :
7549 9899 : _aux->residualSetup();
7550 :
7551 9899 : if (_displaced_problem)
7552 : {
7553 1057 : computeSystems(EXEC_PRE_DISPLACE);
7554 1057 : _displaced_problem->updateMesh();
7555 1057 : if (_mortar_data->hasDisplacedObjects())
7556 1057 : updateMortarMesh();
7557 : }
7558 :
7559 21234 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7560 : {
7561 11335 : _all_materials.residualSetup(tid);
7562 11335 : _functions.residualSetup(tid);
7563 : }
7564 :
7565 : #ifdef MOOSE_KOKKOS_ENABLED
7566 8839 : _kokkos_functions.residualSetup();
7567 : #endif
7568 :
7569 9899 : computeSystems(EXEC_LINEAR);
7570 :
7571 9899 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7572 :
7573 9899 : executeControls(EXEC_LINEAR);
7574 :
7575 9899 : _app.getOutputWarehouse().residualSetup();
7576 :
7577 9899 : _safe_access_tagged_vectors = false;
7578 9899 : _safe_access_tagged_matrices = false;
7579 :
7580 9899 : _current_nl_sys->computeResidualAndJacobianTags(_fe_vector_tags, _fe_matrix_tags);
7581 :
7582 9899 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7583 9899 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7584 : }
7585 0 : catch (...)
7586 : {
7587 0 : handleException("computeResidualAndJacobian");
7588 0 : }
7589 : }
7590 0 : catch (const MooseException &)
7591 : {
7592 : // The buck stops here, we have already handled the exception by
7593 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7594 : // "diverged" reason during the next solve.
7595 0 : }
7596 0 : catch (...)
7597 : {
7598 0 : mooseError("Unexpected exception type");
7599 0 : }
7600 :
7601 9899 : resetState();
7602 9899 : _fe_vector_tags.clear();
7603 9899 : _fe_matrix_tags.clear();
7604 9899 : }
7605 :
7606 : void
7607 0 : FEProblemBase::computeResidualTag(const NumericVector<Number> & soln,
7608 : NumericVector<Number> & residual,
7609 : TagID tag)
7610 : {
7611 : try
7612 : {
7613 0 : _current_nl_sys->setSolution(soln);
7614 :
7615 0 : _current_nl_sys->associateVectorToTag(residual, tag);
7616 :
7617 0 : computeResidualTags({tag});
7618 :
7619 0 : _current_nl_sys->disassociateVectorFromTag(residual, tag);
7620 : }
7621 0 : catch (MooseException & e)
7622 : {
7623 : // If a MooseException propagates all the way to here, it means
7624 : // that it was thrown from a MOOSE system where we do not
7625 : // (currently) properly support the throwing of exceptions, and
7626 : // therefore we have no choice but to error out. It may be
7627 : // *possible* to handle exceptions from other systems, but in the
7628 : // meantime, we don't want to silently swallow any unhandled
7629 : // exceptions here.
7630 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7631 : "contact the MOOSE team for assistance.");
7632 0 : }
7633 0 : }
7634 :
7635 : void
7636 3011545 : FEProblemBase::computeResidualInternal(const NumericVector<Number> & soln,
7637 : NumericVector<Number> & residual,
7638 : const std::set<TagID> & tags)
7639 : {
7640 : parallel_object_only();
7641 :
7642 9034635 : TIME_SECTION("computeResidualInternal", 1);
7643 :
7644 : try
7645 : {
7646 3011545 : _current_nl_sys->setSolution(soln);
7647 :
7648 3011545 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7649 :
7650 3011545 : computeResidualTags(tags);
7651 :
7652 3011515 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7653 : }
7654 0 : catch (MooseException & e)
7655 : {
7656 : // If a MooseException propagates all the way to here, it means
7657 : // that it was thrown from a MOOSE system where we do not
7658 : // (currently) properly support the throwing of exceptions, and
7659 : // therefore we have no choice but to error out. It may be
7660 : // *possible* to handle exceptions from other systems, but in the
7661 : // meantime, we don't want to silently swallow any unhandled
7662 : // exceptions here.
7663 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7664 : "contact the MOOSE team for assistance.");
7665 0 : }
7666 3011515 : }
7667 :
7668 : void
7669 0 : FEProblemBase::computeResidualType(const NumericVector<Number> & soln,
7670 : NumericVector<Number> & residual,
7671 : TagID tag)
7672 : {
7673 0 : TIME_SECTION("computeResidualType", 5);
7674 :
7675 : try
7676 : {
7677 0 : _current_nl_sys->setSolution(soln);
7678 :
7679 0 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7680 :
7681 0 : computeResidualTags({tag, _current_nl_sys->residualVectorTag()});
7682 :
7683 0 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7684 : }
7685 0 : catch (MooseException & e)
7686 : {
7687 : // If a MooseException propagates all the way to here, it means
7688 : // that it was thrown from a MOOSE system where we do not
7689 : // (currently) properly support the throwing of exceptions, and
7690 : // therefore we have no choice but to error out. It may be
7691 : // *possible* to handle exceptions from other systems, but in the
7692 : // meantime, we don't want to silently swallow any unhandled
7693 : // exceptions here.
7694 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7695 : "contact the MOOSE team for assistance.");
7696 0 : }
7697 0 : }
7698 :
7699 : void
7700 3 : FEProblemBase::handleException(const std::string & calling_method)
7701 : {
7702 : auto create_exception_message =
7703 3 : [&calling_method](const std::string & exception_type, const auto & exception)
7704 : {
7705 : return std::string("A " + exception_type + " was raised during FEProblemBase::" +
7706 6 : calling_method + "\n" + std::string(exception.what()));
7707 3 : };
7708 :
7709 : try
7710 : {
7711 3 : throw;
7712 : }
7713 3 : catch (const MooseException & e)
7714 : {
7715 0 : setException(create_exception_message("MooseException", e));
7716 0 : }
7717 0 : catch (const MetaPhysicL::LogicError & e)
7718 : {
7719 0 : moose::translateMetaPhysicLError(e);
7720 0 : }
7721 3 : catch (const libMesh::PetscSolverException & e)
7722 : {
7723 : // One PETSc solver exception that we cannot currently recover from are new nonzero errors. In
7724 : // particular I have observed the following scenario in a parallel test:
7725 : // - Both processes throw because of a new nonzero during MOOSE's computeJacobianTags
7726 : // - We potentially handle the exceptions nicely here
7727 : // - When the matrix is closed in libMesh's libmesh_petsc_snes_solver, there is a new nonzero
7728 : // throw which we do not catch here in MOOSE and the simulation terminates. This only appears
7729 : // in parallel (and not all the time; a test I was examining threw with distributed mesh, but
7730 : // not with replicated). In serial there are no new throws from libmesh_petsc_snes_solver.
7731 : // So for uniformity of behavior across serial/parallel, we will choose to abort here and always
7732 : // produce a non-zero exit code
7733 6 : mooseError(create_exception_message("libMesh::PetscSolverException", e));
7734 0 : }
7735 0 : catch (const std::exception & e)
7736 : {
7737 : // This might be libMesh detecting a degenerate Jacobian or matrix
7738 0 : if (strstr(e.what(), "Jacobian") || strstr(e.what(), "singular") ||
7739 0 : strstr(e.what(), "det != 0"))
7740 0 : setException(create_exception_message("libMesh DegenerateMap", e));
7741 : else
7742 : {
7743 0 : const auto message = create_exception_message("std::exception", e);
7744 0 : if (_regard_general_exceptions_as_errors)
7745 0 : mooseError(message);
7746 : else
7747 0 : setException(message);
7748 0 : }
7749 0 : }
7750 :
7751 0 : checkExceptionAndStopSolve();
7752 0 : }
7753 :
7754 : void
7755 3046948 : FEProblemBase::computeResidualTags(const std::set<TagID> & tags)
7756 : {
7757 : parallel_object_only();
7758 :
7759 : try
7760 : {
7761 : try
7762 : {
7763 15234740 : TIME_SECTION("computeResidualTags", 5, "Computing Residual");
7764 :
7765 3046948 : ADReal::do_derivatives = false;
7766 :
7767 3046948 : setCurrentResidualVectorTags(tags);
7768 :
7769 3046948 : _aux->zeroVariablesForResidual();
7770 :
7771 3046948 : unsigned int n_threads = libMesh::n_threads();
7772 :
7773 3046948 : _current_execute_on_flag = EXEC_LINEAR;
7774 :
7775 : // Random interface objects
7776 3059830 : for (const auto & it : _random_data_objects)
7777 12882 : it.second->updateSeeds(EXEC_LINEAR);
7778 :
7779 3046948 : execTransfers(EXEC_LINEAR);
7780 :
7781 3046948 : execMultiApps(EXEC_LINEAR);
7782 :
7783 6401970 : for (unsigned int tid = 0; tid < n_threads; tid++)
7784 3355022 : reinitScalars(tid);
7785 :
7786 3046948 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7787 :
7788 3046948 : _aux->residualSetup();
7789 :
7790 3046948 : if (_displaced_problem)
7791 : {
7792 123092 : computeSystems(EXEC_PRE_DISPLACE);
7793 123092 : _displaced_problem->updateMesh();
7794 123092 : if (_mortar_data->hasDisplacedObjects())
7795 2211 : updateMortarMesh();
7796 : }
7797 :
7798 6401970 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7799 : {
7800 3355022 : _all_materials.residualSetup(tid);
7801 3355022 : _functions.residualSetup(tid);
7802 : }
7803 :
7804 : #ifdef MOOSE_KOKKOS_ENABLED
7805 2216329 : _kokkos_functions.residualSetup();
7806 : #endif
7807 :
7808 3046948 : computeSystems(EXEC_LINEAR);
7809 :
7810 3046948 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7811 :
7812 3046948 : executeControls(EXEC_LINEAR);
7813 :
7814 3046948 : _app.getOutputWarehouse().residualSetup();
7815 :
7816 3046948 : _safe_access_tagged_vectors = false;
7817 3046948 : _current_nl_sys->computeResidualTags(tags);
7818 3046918 : }
7819 0 : catch (...)
7820 : {
7821 0 : handleException("computeResidualTags");
7822 0 : }
7823 : }
7824 0 : catch (const MooseException &)
7825 : {
7826 : // The buck stops here, we have already handled the exception by
7827 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7828 : // "diverged" reason during the next solve.
7829 0 : }
7830 0 : catch (...)
7831 : {
7832 0 : mooseError("Unexpected exception type");
7833 0 : }
7834 :
7835 3046918 : resetState();
7836 3046918 : }
7837 :
7838 : void
7839 473203 : FEProblemBase::computeJacobianSys(NonlinearImplicitSystem & sys,
7840 : const NumericVector<Number> & soln,
7841 : SparseMatrix<Number> & jacobian)
7842 : {
7843 : // Reset before Jacobian setup, calculation & execution
7844 473203 : _app.solutionInvalidity().resetIterationOccurences();
7845 473203 : computeJacobian(soln, jacobian, sys.number());
7846 473190 : }
7847 :
7848 : void
7849 4467 : FEProblemBase::computeJacobianTag(const NumericVector<Number> & soln,
7850 : SparseMatrix<Number> & jacobian,
7851 : TagID tag)
7852 : {
7853 4467 : _current_nl_sys->setSolution(soln);
7854 :
7855 4467 : _current_nl_sys->associateMatrixToTag(jacobian, tag);
7856 :
7857 8934 : computeJacobianTags({tag});
7858 :
7859 4467 : _current_nl_sys->disassociateMatrixFromTag(jacobian, tag);
7860 4467 : }
7861 :
7862 : void
7863 472359 : FEProblemBase::computeJacobian(const NumericVector<Number> & soln,
7864 : SparseMatrix<Number> & jacobian,
7865 : const unsigned int nl_sys_num)
7866 : {
7867 472359 : setCurrentNonlinearSystem(nl_sys_num);
7868 :
7869 472359 : _fe_matrix_tags.clear();
7870 :
7871 472359 : auto & tags = getMatrixTags();
7872 1417339 : for (auto & tag : tags)
7873 944980 : _fe_matrix_tags.insert(tag.second);
7874 :
7875 472359 : computeJacobianInternal(soln, jacobian, _fe_matrix_tags);
7876 472346 : }
7877 :
7878 : void
7879 472359 : FEProblemBase::computeJacobianInternal(const NumericVector<Number> & soln,
7880 : SparseMatrix<Number> & jacobian,
7881 : const std::set<TagID> & tags)
7882 : {
7883 1417077 : TIME_SECTION("computeJacobianInternal", 1);
7884 :
7885 472359 : _current_nl_sys->setSolution(soln);
7886 :
7887 472359 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7888 :
7889 472359 : computeJacobianTags(tags);
7890 :
7891 472346 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7892 472346 : }
7893 :
7894 : void
7895 481607 : FEProblemBase::computeJacobianTags(const std::set<TagID> & tags)
7896 : {
7897 : try
7898 : {
7899 : try
7900 : {
7901 481607 : if (!_has_jacobian || !_const_jacobian)
7902 : {
7903 2375760 : TIME_SECTION("computeJacobianTags", 5, "Computing Jacobian");
7904 :
7905 1418126 : for (auto tag : tags)
7906 942974 : if (_current_nl_sys->hasMatrix(tag))
7907 : {
7908 477070 : auto & matrix = _current_nl_sys->getMatrix(tag);
7909 477070 : if (_restore_original_nonzero_pattern)
7910 7033 : matrix.restore_original_nonzero_pattern();
7911 : else
7912 470037 : matrix.zero();
7913 477070 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7914 : // PETSc algorithms require diagonal allocations regardless of whether there is
7915 : // non-zero diagonal dependence. With global AD indexing we only add non-zero
7916 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7917 4868311 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7918 4824447 : matrix.add(index, index, 0);
7919 : }
7920 :
7921 475152 : _aux->zeroVariablesForJacobian();
7922 :
7923 475152 : unsigned int n_threads = libMesh::n_threads();
7924 :
7925 : // Random interface objects
7926 477174 : for (const auto & it : _random_data_objects)
7927 2022 : it.second->updateSeeds(EXEC_NONLINEAR);
7928 :
7929 475152 : _current_execute_on_flag = EXEC_NONLINEAR;
7930 475152 : _currently_computing_jacobian = true;
7931 475152 : if (_displaced_problem)
7932 21147 : _displaced_problem->setCurrentlyComputingJacobian(true);
7933 :
7934 475152 : execTransfers(EXEC_NONLINEAR);
7935 475152 : execMultiApps(EXEC_NONLINEAR);
7936 :
7937 1000455 : for (unsigned int tid = 0; tid < n_threads; tid++)
7938 525303 : reinitScalars(tid);
7939 :
7940 475152 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7941 :
7942 475152 : _aux->jacobianSetup();
7943 :
7944 475152 : if (_displaced_problem)
7945 : {
7946 21147 : computeSystems(EXEC_PRE_DISPLACE);
7947 21147 : _displaced_problem->updateMesh();
7948 : }
7949 :
7950 1000448 : for (unsigned int tid = 0; tid < n_threads; tid++)
7951 : {
7952 525299 : _all_materials.jacobianSetup(tid);
7953 525299 : _functions.jacobianSetup(tid);
7954 : }
7955 :
7956 : #ifdef MOOSE_KOKKOS_ENABLED
7957 346499 : _kokkos_functions.jacobianSetup();
7958 : #endif
7959 :
7960 475149 : computeSystems(EXEC_NONLINEAR);
7961 :
7962 475149 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7963 :
7964 475149 : executeControls(EXEC_NONLINEAR);
7965 :
7966 475149 : _app.getOutputWarehouse().jacobianSetup();
7967 :
7968 475149 : _safe_access_tagged_matrices = false;
7969 :
7970 475149 : _current_nl_sys->computeJacobianTags(tags);
7971 :
7972 : // For explicit Euler calculations for example we often compute the Jacobian one time and
7973 : // then re-use it over and over. If we're performing automatic scaling, we don't want to
7974 : // use that kernel, diagonal-block only Jacobian for our actual matrix when performing
7975 : // solves!
7976 475139 : if (!_current_nl_sys->computingScalingJacobian())
7977 474574 : _has_jacobian = true;
7978 475142 : }
7979 : }
7980 3 : catch (...)
7981 : {
7982 3 : handleException("computeJacobianTags");
7983 0 : }
7984 : }
7985 0 : catch (const MooseException &)
7986 : {
7987 : // The buck stops here, we have already handled the exception by
7988 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7989 : // "diverged" reason during the next solve.
7990 0 : }
7991 0 : catch (...)
7992 : {
7993 0 : mooseError("Unexpected exception type");
7994 0 : }
7995 :
7996 481594 : resetState();
7997 481594 : }
7998 :
7999 : void
8000 263 : FEProblemBase::computeJacobianBlocks(std::vector<JacobianBlock *> & blocks,
8001 : const unsigned int nl_sys_num)
8002 : {
8003 789 : TIME_SECTION("computeTransientImplicitJacobian", 2);
8004 263 : setCurrentNonlinearSystem(nl_sys_num);
8005 :
8006 263 : if (_displaced_problem)
8007 : {
8008 0 : computeSystems(EXEC_PRE_DISPLACE);
8009 0 : _displaced_problem->updateMesh();
8010 : }
8011 :
8012 263 : computeSystems(EXEC_NONLINEAR);
8013 :
8014 263 : _currently_computing_jacobian = true;
8015 263 : _current_nl_sys->computeJacobianBlocks(blocks);
8016 263 : _currently_computing_jacobian = false;
8017 263 : }
8018 :
8019 : void
8020 0 : FEProblemBase::computeJacobianBlock(SparseMatrix<Number> & jacobian,
8021 : libMesh::System & precond_system,
8022 : unsigned int ivar,
8023 : unsigned int jvar)
8024 : {
8025 0 : JacobianBlock jac_block(precond_system, jacobian, ivar, jvar);
8026 0 : std::vector<JacobianBlock *> blocks = {&jac_block};
8027 : mooseAssert(_current_nl_sys, "This should be non-null");
8028 0 : computeJacobianBlocks(blocks, _current_nl_sys->number());
8029 0 : }
8030 :
8031 : void
8032 714 : FEProblemBase::computeBounds(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8033 : NumericVector<Number> & lower,
8034 : NumericVector<Number> & upper)
8035 : {
8036 : try
8037 : {
8038 : try
8039 : {
8040 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8041 : "I expect these system numbers to be the same");
8042 :
8043 3570 : if (!_current_nl_sys->hasVector("lower_bound") || !_current_nl_sys->hasVector("upper_bound"))
8044 0 : return;
8045 :
8046 3570 : TIME_SECTION("computeBounds", 1, "Computing Bounds");
8047 :
8048 1428 : NumericVector<Number> & _lower = _current_nl_sys->getVector("lower_bound");
8049 1428 : NumericVector<Number> & _upper = _current_nl_sys->getVector("upper_bound");
8050 714 : _lower.swap(lower);
8051 714 : _upper.swap(upper);
8052 1492 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
8053 778 : _all_materials.residualSetup(tid);
8054 :
8055 714 : _aux->residualSetup();
8056 714 : computeSystems(EXEC_LINEAR);
8057 714 : _lower.swap(lower);
8058 714 : _upper.swap(upper);
8059 714 : }
8060 0 : catch (...)
8061 : {
8062 0 : handleException("computeBounds");
8063 0 : }
8064 : }
8065 0 : catch (MooseException & e)
8066 : {
8067 0 : mooseError("Irrecoverable exception: " + std::string(e.what()));
8068 0 : }
8069 0 : catch (...)
8070 : {
8071 0 : mooseError("Unexpected exception type");
8072 0 : }
8073 : }
8074 :
8075 : void
8076 25792 : FEProblemBase::computeLinearSystemSys(LinearImplicitSystem & sys,
8077 : SparseMatrix<Number> & system_matrix,
8078 : NumericVector<Number> & rhs,
8079 : const bool compute_gradients)
8080 : {
8081 77376 : TIME_SECTION("computeLinearSystemSys", 5);
8082 :
8083 25792 : setCurrentLinearSystem(linearSysNum(sys.name()));
8084 :
8085 25792 : _current_linear_sys->associateVectorToTag(rhs, _current_linear_sys->rightHandSideVectorTag());
8086 25792 : _current_linear_sys->associateMatrixToTag(system_matrix, _current_linear_sys->systemMatrixTag());
8087 :
8088 : // We are using the residual tag system for right hand sides so we fetch everything
8089 25792 : const auto & vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
8090 :
8091 : // We filter out tags which do not have associated vectors in the current
8092 : // system. This is essential to be able to use system-dependent vector tags.
8093 25792 : selectVectorTagsFromSystem(*_current_linear_sys, vector_tags, _linear_vector_tags);
8094 25792 : selectMatrixTagsFromSystem(*_current_linear_sys, getMatrixTags(), _linear_matrix_tags);
8095 :
8096 25792 : computeLinearSystemTags(*(_current_linear_sys->currentSolution()),
8097 25792 : _linear_vector_tags,
8098 25792 : _linear_matrix_tags,
8099 : compute_gradients);
8100 :
8101 25792 : _current_linear_sys->disassociateMatrixFromTag(system_matrix,
8102 25792 : _current_linear_sys->systemMatrixTag());
8103 25792 : _current_linear_sys->disassociateVectorFromTag(rhs,
8104 25792 : _current_linear_sys->rightHandSideVectorTag());
8105 : // We reset the tags to the default containers for further operations
8106 25792 : _current_linear_sys->associateVectorToTag(_current_linear_sys->getRightHandSideVector(),
8107 25792 : _current_linear_sys->rightHandSideVectorTag());
8108 25792 : _current_linear_sys->associateMatrixToTag(_current_linear_sys->getSystemMatrix(),
8109 25792 : _current_linear_sys->systemMatrixTag());
8110 25792 : }
8111 :
8112 : void
8113 25792 : FEProblemBase::computeLinearSystemTags(const NumericVector<Number> & soln,
8114 : const std::set<TagID> & vector_tags,
8115 : const std::set<TagID> & matrix_tags,
8116 : const bool compute_gradients)
8117 : {
8118 128960 : TIME_SECTION("computeLinearSystemTags", 5, "Computing Linear System");
8119 :
8120 25792 : _current_linear_sys->setSolution(soln);
8121 :
8122 51604 : for (auto tag : matrix_tags)
8123 : {
8124 25812 : auto & matrix = _current_linear_sys->getMatrix(tag);
8125 25812 : matrix.zero();
8126 : }
8127 :
8128 25792 : unsigned int n_threads = libMesh::n_threads();
8129 :
8130 25792 : _current_execute_on_flag = EXEC_NONLINEAR;
8131 :
8132 : // Random interface objects
8133 25792 : for (const auto & it : _random_data_objects)
8134 0 : it.second->updateSeeds(EXEC_NONLINEAR);
8135 :
8136 25792 : execTransfers(EXEC_NONLINEAR);
8137 25792 : execMultiApps(EXEC_NONLINEAR);
8138 :
8139 25792 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
8140 :
8141 25792 : _aux->jacobianSetup();
8142 :
8143 51584 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
8144 : {
8145 25792 : _functions.jacobianSetup(tid);
8146 : }
8147 :
8148 : #ifdef MOOSE_KOKKOS_ENABLED
8149 17842 : _kokkos_functions.jacobianSetup();
8150 : #endif
8151 :
8152 : try
8153 : {
8154 25792 : computeSystems(EXEC_NONLINEAR);
8155 : }
8156 0 : catch (MooseException & e)
8157 : {
8158 0 : _console << "\nA MooseException was raised during Auxiliary variable computation.\n"
8159 0 : << "The next solve will fail, the timestep will be reduced, and we will try again.\n"
8160 0 : << std::endl;
8161 :
8162 : // We know the next solve is going to fail, so there's no point in
8163 : // computing anything else after this. Plus, using incompletely
8164 : // computed AuxVariables in subsequent calculations could lead to
8165 : // other errors or unhandled exceptions being thrown.
8166 0 : return;
8167 0 : }
8168 :
8169 25792 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
8170 25792 : executeControls(EXEC_NONLINEAR);
8171 :
8172 25792 : _app.getOutputWarehouse().jacobianSetup();
8173 :
8174 25792 : _current_linear_sys->computeLinearSystemTags(vector_tags, matrix_tags, compute_gradients);
8175 :
8176 : // Reset execution flag as after this point we are no longer on LINEAR
8177 25792 : _current_execute_on_flag = EXEC_NONE;
8178 :
8179 : // These are the relevant parts of resetState()
8180 25792 : _safe_access_tagged_vectors = true;
8181 25792 : _safe_access_tagged_matrices = true;
8182 25792 : }
8183 :
8184 : void
8185 296215 : FEProblemBase::computeNearNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8186 : std::vector<NumericVector<Number> *> & sp)
8187 : {
8188 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8189 : "I expect these system numbers to be the same");
8190 :
8191 296215 : sp.clear();
8192 888645 : for (unsigned int i = 0; i < subspaceDim("NearNullSpace"); ++i)
8193 : {
8194 0 : std::stringstream postfix;
8195 0 : postfix << "_" << i;
8196 0 : std::string modename = "NearNullSpace" + postfix.str();
8197 0 : sp.push_back(&_current_nl_sys->getVector(modename));
8198 0 : }
8199 296215 : }
8200 :
8201 : void
8202 296215 : FEProblemBase::computeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8203 : std::vector<NumericVector<Number> *> & sp)
8204 : {
8205 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8206 : "I expect these system numbers to be the same");
8207 296215 : sp.clear();
8208 888711 : for (unsigned int i = 0; i < subspaceDim("NullSpace"); ++i)
8209 : {
8210 22 : std::stringstream postfix;
8211 22 : postfix << "_" << i;
8212 22 : sp.push_back(&_current_nl_sys->getVector("NullSpace" + postfix.str()));
8213 22 : }
8214 296215 : }
8215 :
8216 : void
8217 296215 : FEProblemBase::computeTransposeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8218 : std::vector<NumericVector<Number> *> & sp)
8219 : {
8220 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8221 : "I expect these system numbers to be the same");
8222 296215 : sp.clear();
8223 888678 : for (unsigned int i = 0; i < subspaceDim("TransposeNullSpace"); ++i)
8224 : {
8225 11 : std::stringstream postfix;
8226 11 : postfix << "_" << i;
8227 11 : sp.push_back(&_current_nl_sys->getVector("TransposeNullSpace" + postfix.str()));
8228 11 : }
8229 296215 : }
8230 :
8231 : void
8232 2128 : FEProblemBase::computePostCheck(NonlinearImplicitSystem & sys,
8233 : const NumericVector<Number> & old_soln,
8234 : NumericVector<Number> & search_direction,
8235 : NumericVector<Number> & new_soln,
8236 : bool & changed_search_direction,
8237 : bool & changed_new_soln)
8238 : {
8239 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8240 : "I expect these system numbers to be the same");
8241 :
8242 : // This function replaces the old PetscSupport::dampedCheck() function.
8243 : //
8244 : // 1.) Recreate code in PetscSupport::dampedCheck() for constructing
8245 : // ghosted "soln" and "update" vectors.
8246 : // 2.) Call FEProblemBase::computeDamping() with these ghost vectors.
8247 : // 3.) Recreate the code in PetscSupport::dampedCheck() to actually update
8248 : // the solution vector based on the damping, and set the "changed" flags
8249 : // appropriately.
8250 :
8251 10640 : TIME_SECTION("computePostCheck", 2, "Computing Post Check");
8252 :
8253 2128 : _current_execute_on_flag = EXEC_POSTCHECK;
8254 :
8255 : // MOOSE's FEProblemBase doesn't update the solution during the
8256 : // postcheck, but FEProblemBase-derived classes might.
8257 2128 : if (_has_dampers || shouldUpdateSolution())
8258 : {
8259 : // We need ghosted versions of new_soln and search_direction (the
8260 : // ones we get from libmesh/PETSc are PARALLEL vectors. To make
8261 : // our lives simpler, we use the same ghosting pattern as the
8262 : // system's current_local_solution to create new ghosted vectors.
8263 :
8264 : // Construct zeroed-out clones with the same ghosted dofs as the
8265 : // System's current_local_solution.
8266 : std::unique_ptr<NumericVector<Number>> ghosted_solution =
8267 1625 : sys.current_local_solution->zero_clone(),
8268 : ghosted_search_direction =
8269 1625 : sys.current_local_solution->zero_clone();
8270 :
8271 : // Copy values from input vectors into clones with ghosted values.
8272 1625 : *ghosted_solution = new_soln;
8273 1625 : *ghosted_search_direction = search_direction;
8274 :
8275 1625 : if (_has_dampers)
8276 : {
8277 : // Compute the damping coefficient using the ghosted vectors
8278 1625 : Real damping = computeDamping(*ghosted_solution, *ghosted_search_direction);
8279 :
8280 : // If some non-trivial damping was computed, update the new_soln
8281 : // vector accordingly.
8282 1625 : if (damping < 1.0)
8283 : {
8284 1223 : new_soln = old_soln;
8285 1223 : new_soln.add(-damping, search_direction);
8286 1223 : changed_new_soln = true;
8287 : }
8288 : }
8289 :
8290 1625 : if (shouldUpdateSolution())
8291 : {
8292 : // Update the ghosted copy of the new solution, if necessary.
8293 0 : if (changed_new_soln)
8294 0 : *ghosted_solution = new_soln;
8295 :
8296 0 : bool updated_solution = updateSolution(new_soln, *ghosted_solution);
8297 0 : if (updated_solution)
8298 0 : changed_new_soln = true;
8299 : }
8300 1625 : }
8301 :
8302 2128 : if (vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
8303 : {
8304 503 : _current_nl_sys->setPreviousNewtonSolution(old_soln);
8305 503 : _aux->copyCurrentIntoPreviousNL();
8306 : }
8307 :
8308 : // MOOSE doesn't change the search_direction
8309 2128 : changed_search_direction = false;
8310 :
8311 2128 : _current_execute_on_flag = EXEC_NONE;
8312 2128 : }
8313 :
8314 : Real
8315 1625 : FEProblemBase::computeDamping(const NumericVector<Number> & soln,
8316 : const NumericVector<Number> & update)
8317 : {
8318 : // Default to no damping
8319 1625 : Real damping = 1.0;
8320 :
8321 1625 : if (_has_dampers)
8322 : {
8323 8125 : TIME_SECTION("computeDamping", 1, "Computing Damping");
8324 :
8325 : // Save pointer to the current solution
8326 1625 : const NumericVector<Number> * _saved_current_solution = _current_nl_sys->currentSolution();
8327 :
8328 1625 : _current_nl_sys->setSolution(soln);
8329 : // For now, do not re-compute auxiliary variables. Doing so allows a wild solution increment
8330 : // to get to the material models, which may not be able to cope with drastically different
8331 : // values. Once more complete dependency checking is in place, auxiliary variables (and
8332 : // material properties) will be computed as needed by dampers.
8333 : // _aux.compute();
8334 1625 : damping = _current_nl_sys->computeDamping(soln, update);
8335 :
8336 : // restore saved solution
8337 1625 : _current_nl_sys->setSolution(*_saved_current_solution);
8338 1625 : }
8339 :
8340 1625 : return damping;
8341 : }
8342 :
8343 : bool
8344 290811 : FEProblemBase::shouldUpdateSolution()
8345 : {
8346 290811 : return false;
8347 : }
8348 :
8349 : bool
8350 0 : FEProblemBase::updateSolution(NumericVector<Number> & /*vec_solution*/,
8351 : NumericVector<Number> & /*ghosted_solution*/)
8352 : {
8353 0 : return false;
8354 : }
8355 :
8356 : void
8357 203 : FEProblemBase::predictorCleanup(NumericVector<Number> & /*ghosted_solution*/)
8358 : {
8359 203 : }
8360 :
8361 : void
8362 2022 : FEProblemBase::addDisplacedProblem(std::shared_ptr<DisplacedProblem> displaced_problem)
8363 : {
8364 : parallel_object_only();
8365 :
8366 2022 : _displaced_mesh = &displaced_problem->mesh();
8367 2022 : _displaced_problem = displaced_problem;
8368 2022 : }
8369 :
8370 : void
8371 122210 : FEProblemBase::updateGeomSearch(GeometricSearchData::GeometricSearchType type)
8372 : {
8373 611050 : TIME_SECTION("updateGeometricSearch", 3, "Updating Geometric Search");
8374 :
8375 122210 : _geometric_search_data.update(type);
8376 :
8377 122210 : if (_displaced_problem)
8378 4209 : _displaced_problem->updateGeomSearch(type);
8379 122210 : }
8380 :
8381 : void
8382 64547 : FEProblemBase::updateMortarMesh()
8383 : {
8384 322735 : TIME_SECTION("updateMortarMesh", 5, "Updating Mortar Mesh");
8385 :
8386 64547 : FloatingPointExceptionGuard fpe_guard(_app);
8387 :
8388 64547 : _mortar_data->update();
8389 64541 : }
8390 :
8391 : void
8392 1481 : FEProblemBase::createMortarInterface(
8393 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8394 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8395 : bool on_displaced,
8396 : bool periodic,
8397 : const bool debug,
8398 : const bool correct_edge_dropping,
8399 : const Real minimum_projection_angle,
8400 : const Mortar3DSubpatchPlane mortar_3d_subpatch_plane,
8401 : const MooseEnum & triangulation,
8402 : const bool triangulate_triangles,
8403 : const Mortar3DQuadraturePointMapping mortar_3d_qp_mapping)
8404 : {
8405 1481 : _has_mortar = true;
8406 :
8407 1481 : if (on_displaced)
8408 254 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8409 : primary_secondary_subdomain_pair,
8410 127 : *_displaced_problem,
8411 : on_displaced,
8412 : periodic,
8413 : debug,
8414 : correct_edge_dropping,
8415 : minimum_projection_angle,
8416 : mortar_3d_subpatch_plane,
8417 : triangulation,
8418 : triangulate_triangles,
8419 127 : mortar_3d_qp_mapping);
8420 : else
8421 1354 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8422 : primary_secondary_subdomain_pair,
8423 : *this,
8424 : on_displaced,
8425 : periodic,
8426 : debug,
8427 : correct_edge_dropping,
8428 : minimum_projection_angle,
8429 : mortar_3d_subpatch_plane,
8430 : triangulation,
8431 : triangulate_triangles,
8432 1348 : mortar_3d_qp_mapping);
8433 : }
8434 :
8435 : const AutomaticMortarGeneration &
8436 0 : FEProblemBase::getMortarInterface(
8437 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8438 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8439 : bool on_displaced) const
8440 : {
8441 0 : return _mortar_data->getMortarInterface(
8442 0 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8443 : }
8444 :
8445 : AutomaticMortarGeneration &
8446 256642 : FEProblemBase::getMortarInterface(
8447 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8448 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8449 : bool on_displaced)
8450 : {
8451 256642 : return _mortar_data->getMortarInterface(
8452 256642 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8453 : }
8454 :
8455 : void
8456 322789 : FEProblemBase::possiblyRebuildGeomSearchPatches()
8457 : {
8458 322789 : if (_displaced_problem) // Only need to do this if things are moving...
8459 : {
8460 164185 : TIME_SECTION("possiblyRebuildGeomSearchPatches", 5, "Rebuilding Geometric Search Patches");
8461 :
8462 32837 : switch (_mesh.getPatchUpdateStrategy())
8463 : {
8464 31742 : case Moose::Never:
8465 31742 : break;
8466 365 : case Moose::Iteration:
8467 : // Update the list of ghosted elements at the start of the time step
8468 365 : _geometric_search_data.updateGhostedElems();
8469 365 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8470 :
8471 365 : _displaced_problem->geomSearchData().updateGhostedElems();
8472 365 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8473 :
8474 : // The commands below ensure that the sparsity of the Jacobian matrix is
8475 : // augmented at the start of the time step using neighbor nodes from the end
8476 : // of the previous time step.
8477 :
8478 365 : reinitBecauseOfGhostingOrNewGeomObjects();
8479 :
8480 : // This is needed to reinitialize PETSc output
8481 365 : initPetscOutputAndSomeSolverSettings();
8482 :
8483 365 : break;
8484 :
8485 331 : case Moose::Auto:
8486 : {
8487 331 : Real max = _displaced_problem->geomSearchData().maxPatchPercentage();
8488 331 : _communicator.max(max);
8489 :
8490 : // If we haven't moved very far through the patch
8491 331 : if (max < 0.4)
8492 298 : break;
8493 : }
8494 : libmesh_fallthrough();
8495 :
8496 : // Let this fall through if things do need to be updated...
8497 : case Moose::Always:
8498 : // Flush output here to see the message before the reinitialization, which could take a
8499 : // while
8500 432 : _console << "\n\nUpdating geometric search patches\n" << std::endl;
8501 :
8502 432 : _geometric_search_data.clearNearestNodeLocators();
8503 432 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8504 :
8505 432 : _displaced_problem->geomSearchData().clearNearestNodeLocators();
8506 432 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8507 :
8508 432 : reinitBecauseOfGhostingOrNewGeomObjects();
8509 :
8510 : // This is needed to reinitialize PETSc output
8511 432 : initPetscOutputAndSomeSolverSettings();
8512 : }
8513 32837 : }
8514 322789 : }
8515 :
8516 : #ifdef LIBMESH_ENABLE_AMR
8517 : void
8518 56406 : FEProblemBase::initialAdaptMesh()
8519 : {
8520 56406 : unsigned int n = adaptivity().getInitialSteps();
8521 56406 : _cycles_completed = 0;
8522 56406 : if (n)
8523 : {
8524 649 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8525 3 : mooseError("HFEM does not support mesh adaptivity currently.");
8526 :
8527 3230 : TIME_SECTION("initialAdaptMesh", 2, "Performing Initial Adaptivity");
8528 :
8529 1546 : for (unsigned int i = 0; i < n; i++)
8530 : {
8531 1075 : computeIndicators();
8532 1075 : computeMarkers();
8533 :
8534 1075 : if (_adaptivity.initialAdaptMesh())
8535 : {
8536 900 : meshChanged(
8537 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8538 :
8539 : // reproject the initial condition
8540 900 : projectSolution();
8541 :
8542 900 : _cycles_completed++;
8543 : }
8544 : else
8545 : {
8546 175 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8547 175 : return;
8548 : }
8549 : }
8550 646 : }
8551 : }
8552 :
8553 : bool
8554 168510 : FEProblemBase::adaptMesh()
8555 : {
8556 : // reset cycle counter
8557 168510 : _cycles_completed = 0;
8558 :
8559 168510 : if (!_adaptivity.isAdaptivityDue())
8560 163933 : return false;
8561 :
8562 22885 : TIME_SECTION("adaptMesh", 3, "Adapting Mesh");
8563 :
8564 4577 : unsigned int cycles_per_step = _adaptivity.getCyclesPerStep();
8565 :
8566 4577 : bool mesh_changed = false;
8567 :
8568 8056 : for (unsigned int i = 0; i < cycles_per_step; ++i)
8569 : {
8570 4729 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8571 0 : mooseError("HFEM does not support mesh adaptivity currently.");
8572 :
8573 : // Markers were already computed once by Executioner
8574 4729 : if (_adaptivity.getRecomputeMarkersFlag() && i > 0)
8575 22 : computeMarkers();
8576 :
8577 : bool mesh_changed_this_step;
8578 4729 : mesh_changed_this_step = _adaptivity.adaptMesh();
8579 :
8580 4729 : if (mesh_changed_this_step)
8581 : {
8582 3479 : mesh_changed = true;
8583 :
8584 3479 : meshChanged(
8585 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8586 3479 : _cycles_completed++;
8587 : }
8588 : else
8589 : {
8590 : // If the mesh didn't change, we still need to update the displaced mesh
8591 : // to undo the undisplacement performed in Adaptivity::adaptMesh
8592 1250 : if (_displaced_problem)
8593 44 : _displaced_problem->updateMesh();
8594 :
8595 1250 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8596 1250 : break;
8597 : }
8598 :
8599 : // Show adaptivity progress
8600 3479 : _console << std::flush;
8601 : }
8602 :
8603 : // We're done with all intermediate changes; now get systems ready
8604 : // for real if necessary.
8605 4577 : if (mesh_changed)
8606 3338 : es().reinit_systems();
8607 :
8608 : // Execute multi-apps that need to run after adaptivity, but before the next timestep.
8609 4577 : execMultiApps(EXEC_POST_ADAPTIVITY);
8610 :
8611 4577 : return mesh_changed;
8612 4577 : }
8613 : #endif // LIBMESH_ENABLE_AMR
8614 :
8615 : void
8616 0 : FEProblemBase::initXFEM(std::shared_ptr<XFEMInterface> xfem)
8617 : {
8618 0 : _xfem = xfem;
8619 0 : _xfem->setMesh(&_mesh);
8620 0 : if (_displaced_mesh)
8621 0 : _xfem->setDisplacedMesh(_displaced_mesh);
8622 :
8623 0 : auto fill_data = [](auto & storage)
8624 : {
8625 0 : std::vector<MaterialData *> data(libMesh::n_threads());
8626 0 : for (const auto tid : make_range(libMesh::n_threads()))
8627 0 : data[tid] = &storage.getMaterialData(tid);
8628 0 : return data;
8629 0 : };
8630 0 : _xfem->setMaterialData(fill_data(_material_props));
8631 0 : _xfem->setBoundaryMaterialData(fill_data(_bnd_material_props));
8632 :
8633 0 : unsigned int n_threads = libMesh::n_threads();
8634 0 : for (unsigned int i = 0; i < n_threads; ++i)
8635 0 : for (const auto nl_sys_num : index_range(_nl))
8636 : {
8637 0 : _assembly[i][nl_sys_num]->setXFEM(_xfem);
8638 0 : if (_displaced_problem)
8639 0 : _displaced_problem->assembly(i, nl_sys_num).setXFEM(_xfem);
8640 : }
8641 0 : }
8642 :
8643 : bool
8644 0 : FEProblemBase::updateMeshXFEM()
8645 : {
8646 0 : TIME_SECTION("updateMeshXFEM", 5, "Updating XFEM");
8647 :
8648 0 : bool updated = false;
8649 0 : if (haveXFEM())
8650 : {
8651 0 : if (_xfem->updateHeal())
8652 : // XFEM exodiff tests rely on a given numbering because they cannot use map = true due to
8653 : // having coincident elements. While conceptually speaking we do not need to contract the
8654 : // mesh, we need its call to renumber_nodes_and_elements in order to preserve these tests
8655 0 : meshChanged(
8656 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8657 :
8658 0 : updated = _xfem->update(_time, _nl, *_aux);
8659 0 : if (updated)
8660 : {
8661 0 : meshChanged(
8662 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8663 0 : _xfem->initSolution(_nl, *_aux);
8664 0 : restoreSolutions();
8665 0 : _console << "\nXFEM update complete: Mesh modified" << std::endl;
8666 : }
8667 : else
8668 0 : _console << "\nXFEM update complete: Mesh not modified" << std::endl;
8669 : }
8670 0 : return updated;
8671 0 : }
8672 :
8673 : void
8674 7044 : FEProblemBase::meshChanged(const bool intermediate_change,
8675 : const bool contract_mesh,
8676 : const bool clean_refinement_flags)
8677 : {
8678 35220 : TIME_SECTION("meshChanged", 3, "Handling Mesh Changes");
8679 :
8680 7044 : _app.markMeshChangedForBackup();
8681 :
8682 13880 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8683 6836 : _neighbor_material_props.hasStatefulProperties())
8684 208 : _mesh.cacheChangedLists(); // Currently only used with adaptivity and stateful material
8685 : // properties
8686 :
8687 : // Clear these out because they corresponded to the old mesh
8688 7044 : _ghosted_elems.clear();
8689 7044 : ghostGhostedBoundaries();
8690 :
8691 : // The mesh changed. We notify the MooseMesh first, because
8692 : // callbacks (e.g. for sparsity calculations) triggered by the
8693 : // EquationSystems reinit may require up-to-date MooseMesh caches.
8694 7044 : _mesh.meshChanged();
8695 :
8696 : // If we're just going to alter the mesh again, all we need to
8697 : // handle here is AMR and projections, not full system reinit
8698 7044 : if (intermediate_change)
8699 3510 : es().reinit_solutions();
8700 : else
8701 3534 : es().reinit();
8702 :
8703 7044 : if (contract_mesh)
8704 : // Once vectors are restricted, we can delete children of coarsened elements
8705 4470 : _mesh.getMesh().contract();
8706 7044 : if (clean_refinement_flags)
8707 : {
8708 : // Finally clear refinement flags so that if someone tries to project vectors again without
8709 : // an intervening mesh refinement to clear flags they won't run into trouble
8710 4470 : MeshRefinement refinement(_mesh.getMesh());
8711 4470 : refinement.clean_refinement_flags();
8712 4470 : }
8713 :
8714 7044 : if (!intermediate_change)
8715 : {
8716 : // Since the mesh has changed, we need to make sure that we update any of our
8717 : // MOOSE-system specific data.
8718 7068 : for (auto & sys : _solver_systems)
8719 3534 : sys->reinit();
8720 3534 : _aux->reinit();
8721 : }
8722 :
8723 : // Updating MooseMesh first breaks other adaptivity code, unless we
8724 : // then *again* update the MooseMesh caches. E.g. the definition of
8725 : // "active" and "local" may have been *changed* by refinement and
8726 : // repartitioning done in EquationSystems::reinit().
8727 7044 : _mesh.meshChanged();
8728 :
8729 : // If we have finite volume variables, we will need to recompute additional elemental/face
8730 : // quantities
8731 7044 : if (haveFV() && _mesh.isFiniteVolumeInfoDirty())
8732 351 : _mesh.setupFiniteVolumeMeshData();
8733 :
8734 : // Let the meshChangedInterface notify the mesh changed event before we update the active
8735 : // semilocal nodes, because the set of ghosted elements may potentially be updated during a mesh
8736 : // changed event.
8737 125709 : for (const auto & mci : _notify_when_mesh_changes)
8738 118665 : mci->meshChanged();
8739 :
8740 : // Since the Mesh changed, update the PointLocator object used by DiracKernels.
8741 7044 : _dirac_kernel_info.updatePointLocator(_mesh);
8742 :
8743 : // Need to redo ghosting
8744 7044 : _geometric_search_data.reinit();
8745 :
8746 7044 : if (_displaced_problem)
8747 : {
8748 569 : _displaced_problem->meshChanged(contract_mesh, clean_refinement_flags);
8749 569 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8750 : }
8751 :
8752 7044 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8753 :
8754 7044 : _evaluable_local_elem_range.reset();
8755 7044 : _nl_evaluable_local_elem_range.reset();
8756 :
8757 : // Just like we reinitialized our geometric search objects, we also need to reinitialize our
8758 : // mortar meshes. Note that this needs to happen after DisplacedProblem::meshChanged because the
8759 : // mortar mesh discretization will depend necessarily on the displaced mesh being re-displaced
8760 7044 : _mortar_data->meshChanged();
8761 :
8762 : // Nonlinear systems hold the mortar mesh functors. The domains of definition of the mortar
8763 : // functors might have changed when the mesh changed.
8764 14078 : for (auto & nl_sys : _nl)
8765 7034 : nl_sys->reinitMortarFunctors();
8766 :
8767 7044 : reinitBecauseOfGhostingOrNewGeomObjects(/*mortar_changed=*/true);
8768 :
8769 : // We need to create new storage for newly active elements, and copy
8770 : // stateful properties from the old elements.
8771 7252 : if (_has_initialized_stateful &&
8772 208 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties()))
8773 : {
8774 208 : if (havePRefinement())
8775 66 : _mesh.buildPRefinementAndCoarseningMaps(_assembly[0][0].get());
8776 :
8777 : // Prolong properties onto newly refined elements' children
8778 : {
8779 : ProjectMaterialProperties pmp(
8780 208 : /* refine = */ true, *this, _material_props, _bnd_material_props, _assembly);
8781 208 : const auto & range = *_mesh.refinedElementRange();
8782 208 : Threads::parallel_reduce(range, pmp);
8783 :
8784 : // Concurrent erasure from the shared hash map is not safe while we are reading from it in
8785 : // ProjectMaterialProperties, so we handle erasure here. Moreover, erasure based on key is
8786 : // not thread safe in and of itself because it is a read-write operation. Note that we do not
8787 : // do the erasure for p-refinement because the coarse level element is the same as our active
8788 : // refined level element
8789 208 : if (!doingPRefinement())
8790 3272 : for (const auto & elem : range)
8791 : {
8792 3130 : _material_props.eraseProperty(elem);
8793 3130 : _bnd_material_props.eraseProperty(elem);
8794 3130 : _neighbor_material_props.eraseProperty(elem);
8795 : }
8796 208 : }
8797 :
8798 : // Restrict properties onto newly coarsened elements
8799 : {
8800 : ProjectMaterialProperties pmp(
8801 208 : /* refine = */ false, *this, _material_props, _bnd_material_props, _assembly);
8802 208 : const auto & range = *_mesh.coarsenedElementRange();
8803 208 : Threads::parallel_reduce(range, pmp);
8804 : // Note that we do not do the erasure for p-refinement because the coarse level element is the
8805 : // same as our active refined level element
8806 208 : if (!doingPRefinement())
8807 1322 : for (const auto & elem : range)
8808 : {
8809 1180 : auto && coarsened_children = _mesh.coarsenedElementChildren(elem);
8810 7732 : for (auto && child : coarsened_children)
8811 : {
8812 6552 : _material_props.eraseProperty(child);
8813 6552 : _bnd_material_props.eraseProperty(child);
8814 6552 : _neighbor_material_props.eraseProperty(child);
8815 : }
8816 : }
8817 208 : }
8818 : }
8819 :
8820 7044 : if (_calculate_jacobian_in_uo)
8821 0 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
8822 :
8823 7044 : _has_jacobian = false; // we have to recompute jacobian when mesh changed
8824 :
8825 : // Now for backwards compatibility with user code that overrode the old no-arg meshChanged we must
8826 : // call it here
8827 7044 : meshChanged();
8828 7044 : }
8829 :
8830 : void
8831 928792 : FEProblemBase::notifyWhenMeshChanges(MeshChangedInterface * mci)
8832 : {
8833 928792 : _notify_when_mesh_changes.push_back(mci);
8834 928792 : }
8835 :
8836 : void
8837 77412 : FEProblemBase::notifyWhenMeshDisplaces(MeshDisplacedInterface * mdi)
8838 : {
8839 77412 : _notify_when_mesh_displaces.push_back(mdi);
8840 77412 : }
8841 :
8842 : void
8843 63048 : FEProblemBase::meshDisplaced()
8844 : {
8845 91725 : for (const auto & mdi : _notify_when_mesh_displaces)
8846 28677 : mdi->meshDisplaced();
8847 63048 : }
8848 :
8849 : void
8850 9805 : FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const bool threaded)
8851 : {
8852 : ComputeMaterialsObjectThread cmt(
8853 9805 : *this, _material_props, _bnd_material_props, _neighbor_material_props, _assembly);
8854 9805 : if (threaded)
8855 9805 : Threads::parallel_reduce(elem_range, cmt);
8856 : else
8857 0 : cmt(elem_range, true);
8858 :
8859 : #ifdef MOOSE_KOKKOS_ENABLED
8860 7383 : if (_has_kokkos_objects)
8861 781 : initKokkosStatefulProps();
8862 : #endif
8863 9805 : }
8864 :
8865 : void
8866 61107 : FEProblemBase::checkProblemIntegrity()
8867 : {
8868 183321 : TIME_SECTION("checkProblemIntegrity", 5);
8869 :
8870 : // Subdomains specified by the "Problem/block" parameter
8871 122214 : const auto & subdomain_names = getParam<std::vector<SubdomainName>>("block");
8872 61107 : auto mesh_subdomains_vec = MooseMeshUtils::getSubdomainIDs(_mesh, subdomain_names);
8873 61107 : std::set<SubdomainID> mesh_subdomains(mesh_subdomains_vec.begin(), mesh_subdomains_vec.end());
8874 :
8875 : // Check kernel coverage of subdomains (blocks) in the mesh
8876 61107 : if (!_skip_nl_system_check && _solve && _kernel_coverage_check != CoverageCheckMode::FALSE &&
8877 41822 : _kernel_coverage_check != CoverageCheckMode::OFF)
8878 : {
8879 41810 : std::set<SubdomainID> blocks;
8880 41810 : if (_kernel_coverage_check == CoverageCheckMode::TRUE ||
8881 231 : _kernel_coverage_check == CoverageCheckMode::ON)
8882 41579 : blocks = mesh_subdomains;
8883 231 : else if (_kernel_coverage_check == CoverageCheckMode::SKIP_LIST)
8884 : {
8885 12 : blocks = mesh_subdomains;
8886 24 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8887 : {
8888 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8889 12 : if (id == Moose::INVALID_BLOCK_ID)
8890 0 : paramError("kernel_coverage_block_list",
8891 : "Subdomain \"",
8892 : subdomain_name,
8893 : "\" not found in mesh.");
8894 12 : blocks.erase(id);
8895 : }
8896 : }
8897 219 : else if (_kernel_coverage_check == CoverageCheckMode::ONLY_LIST)
8898 438 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8899 : {
8900 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8901 219 : if (id == Moose::INVALID_BLOCK_ID)
8902 0 : paramError("kernel_coverage_block_list",
8903 : "Subdomain \"",
8904 : subdomain_name,
8905 : "\" not found in mesh.");
8906 219 : blocks.insert(id);
8907 : }
8908 41810 : if (!blocks.empty())
8909 82689 : for (auto & nl : _nl)
8910 40891 : nl->checkKernelCoverage(blocks);
8911 41798 : }
8912 :
8913 : // Check materials
8914 : {
8915 : #ifdef LIBMESH_ENABLE_AMR
8916 63382 : if ((_adaptivity.isOn() || _num_grid_steps) &&
8917 2287 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8918 2224 : _neighbor_material_props.hasStatefulProperties()))
8919 : {
8920 63 : _console << "Using EXPERIMENTAL Stateful Material Property projection with Adaptivity!\n"
8921 63 : << std::flush;
8922 : }
8923 : #endif
8924 :
8925 61095 : std::set<SubdomainID> local_mesh_subs(mesh_subdomains);
8926 :
8927 61095 : if (_material_coverage_check != CoverageCheckMode::FALSE &&
8928 61011 : _material_coverage_check != CoverageCheckMode::OFF)
8929 : {
8930 : /**
8931 : * If a material is specified for any block in the simulation, then all blocks must
8932 : * have a material specified.
8933 : */
8934 61011 : bool check_material_coverage = false;
8935 61011 : std::set<SubdomainID> ids = _all_materials.getActiveBlocks();
8936 73686 : for (const auto & id : ids)
8937 : {
8938 12675 : local_mesh_subs.erase(id);
8939 12675 : check_material_coverage = true;
8940 : }
8941 :
8942 : // did the user limit the subdomains to be checked?
8943 61011 : if (_material_coverage_check == CoverageCheckMode::SKIP_LIST)
8944 : {
8945 24 : for (const auto & subdomain_name : _material_coverage_blocks)
8946 : {
8947 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8948 12 : if (id == Moose::INVALID_BLOCK_ID)
8949 0 : paramError("material_coverage_block_list",
8950 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8951 12 : local_mesh_subs.erase(id);
8952 : }
8953 : }
8954 60999 : else if (_material_coverage_check == CoverageCheckMode::ONLY_LIST)
8955 : {
8956 219 : std::set<SubdomainID> blocks(local_mesh_subs);
8957 438 : for (const auto & subdomain_name : _material_coverage_blocks)
8958 : {
8959 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8960 219 : if (id == Moose::INVALID_BLOCK_ID)
8961 0 : paramError("material_coverage_block_list",
8962 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8963 219 : blocks.erase(id);
8964 : }
8965 231 : for (const auto id : blocks)
8966 12 : local_mesh_subs.erase(id);
8967 219 : }
8968 :
8969 : // also exclude mortar spaces from the material check
8970 61011 : auto && mortar_subdomain_ids = _mortar_data->getMortarSubdomainIDs();
8971 63139 : for (auto subdomain_id : mortar_subdomain_ids)
8972 2128 : local_mesh_subs.erase(subdomain_id);
8973 :
8974 : // Check Material Coverage
8975 61011 : if (check_material_coverage && !local_mesh_subs.empty())
8976 : {
8977 6 : std::stringstream extra_subdomain_ids;
8978 : /// unsigned int is necessary to print SubdomainIDs in the statement below
8979 6 : std::copy(local_mesh_subs.begin(),
8980 : local_mesh_subs.end(),
8981 12 : std::ostream_iterator<unsigned int>(extra_subdomain_ids, " "));
8982 : /// vector is necessary to get the subdomain names
8983 : std::vector<SubdomainID> local_mesh_subs_vec(local_mesh_subs.begin(),
8984 6 : local_mesh_subs.end());
8985 :
8986 18 : mooseError("The following blocks from your input mesh do not contain an active material: " +
8987 12 : extra_subdomain_ids.str() +
8988 18 : "(names: " + Moose::stringify(_mesh.getSubdomainNames(local_mesh_subs_vec)) +
8989 : ")\nWhen ANY mesh block contains a Material object, "
8990 : "all blocks must contain a Material object.\n");
8991 0 : }
8992 61005 : }
8993 :
8994 : // Check material properties on blocks and boundaries
8995 61089 : checkBlockMatProps();
8996 61057 : checkBoundaryMatProps();
8997 :
8998 : // Check that material properties exist when requested by other properties on a given block
8999 61048 : const auto & materials = _all_materials.getActiveObjects();
9000 75249 : for (const auto & material : materials)
9001 14201 : material->checkStatefulSanity();
9002 :
9003 : // auto mats_to_check = _materials.getActiveBlockObjects();
9004 : // const auto & discrete_materials = _discrete_materials.getActiveBlockObjects();
9005 : // for (const auto & map_it : discrete_materials)
9006 : // for (const auto & container_element : map_it.second)
9007 : // mats_to_check[map_it.first].push_back(container_element);
9008 61048 : if (_material_dependency_check)
9009 61024 : checkDependMaterialsHelper(_all_materials.getActiveBlockObjects());
9010 61035 : }
9011 :
9012 61035 : checkUserObjects();
9013 :
9014 : // Verify that we don't have any Element type/Coordinate Type conflicts
9015 61035 : checkCoordinateSystems();
9016 :
9017 : // Coordinate transforms are only intended for use with MultiApps at this time. If you are not
9018 : // using multiapps but still require these, contact a moose developer
9019 61172 : if (_mesh.coordTransform().hasScalingOrRotationTransformation() && _app.isUltimateMaster() &&
9020 140 : !hasMultiApps())
9021 3 : mooseError("Coordinate transformation parameters, listed below, are only to be used in the "
9022 : "context of application to application field transfers at this time. The mesh is "
9023 : "not modified by these parameters within an application.\n"
9024 : "You should likely use a 'TransformGenerator' in the [Mesh] block to achieve the "
9025 : "desired mesh modification.\n\n",
9026 3 : Moose::stringify(MooseAppCoordTransform::validParams()));
9027 :
9028 : // If using displacements, verify that the order of the displacement
9029 : // variables matches the order of the elements in the displaced
9030 : // mesh.
9031 61029 : checkDisplacementOrders();
9032 :
9033 : // Check for postprocessor names with same name as a scalar variable
9034 61026 : checkDuplicatePostprocessorVariableNames();
9035 61026 : }
9036 :
9037 : void
9038 61029 : FEProblemBase::checkDisplacementOrders()
9039 : {
9040 61029 : if (_displaced_problem)
9041 : {
9042 2022 : bool mesh_has_second_order_elements = false;
9043 4044 : for (const auto & elem : as_range(_displaced_mesh->activeLocalElementsBegin(),
9044 464948 : _displaced_mesh->activeLocalElementsEnd()))
9045 : {
9046 229785 : if (elem->default_order() == SECOND)
9047 : {
9048 344 : mesh_has_second_order_elements = true;
9049 344 : break;
9050 : }
9051 2022 : }
9052 :
9053 : // We checked our local elements, so take the max over all processors.
9054 2022 : _displaced_mesh->comm().max(mesh_has_second_order_elements);
9055 :
9056 : // If the Mesh has second order elements, make sure the
9057 : // displacement variables are second-order.
9058 2022 : if (mesh_has_second_order_elements)
9059 : {
9060 : const std::vector<std::string> & displacement_variables =
9061 344 : _displaced_problem->getDisplacementVarNames();
9062 :
9063 1133 : for (const auto & var_name : displacement_variables)
9064 : {
9065 : MooseVariableFEBase & mv =
9066 792 : _displaced_problem->getVariable(/*tid=*/0,
9067 : var_name,
9068 : Moose::VarKindType::VAR_ANY,
9069 : Moose::VarFieldType::VAR_FIELD_STANDARD);
9070 792 : if (mv.order() != SECOND)
9071 3 : mooseError("Error: mesh has SECOND order elements, so all displacement variables must be "
9072 : "SECOND order.");
9073 : }
9074 : }
9075 : }
9076 61026 : }
9077 :
9078 : void
9079 61035 : FEProblemBase::checkUserObjects()
9080 : {
9081 : // Check user_objects block coverage
9082 61035 : std::set<SubdomainID> mesh_subdomains = _mesh.meshSubdomains();
9083 61035 : std::set<SubdomainID> user_objects_blocks;
9084 :
9085 : // gather names of all user_objects that were defined in the input file
9086 : // and the blocks that they are defined on
9087 61035 : std::set<std::string> names;
9088 :
9089 61035 : std::vector<UserObjectBase *> objects;
9090 61035 : theWarehouse().query().condition<AttribInterfaces>(Interfaces::UserObject).queryInto(objects);
9091 :
9092 136150 : for (const auto & obj : objects)
9093 75115 : names.insert(obj->name());
9094 :
9095 : // See if all referenced blocks are covered
9096 61035 : std::set<SubdomainID> difference;
9097 61035 : std::set_difference(user_objects_blocks.begin(),
9098 : user_objects_blocks.end(),
9099 : mesh_subdomains.begin(),
9100 : mesh_subdomains.end(),
9101 : std::inserter(difference, difference.end()));
9102 :
9103 61035 : if (!difference.empty())
9104 : {
9105 0 : std::ostringstream oss;
9106 0 : oss << "One or more UserObjects is referencing a nonexistent block:\n";
9107 0 : for (const auto & id : difference)
9108 0 : oss << id << "\n";
9109 0 : mooseError(oss.str());
9110 0 : }
9111 61035 : }
9112 :
9113 : void
9114 61024 : FEProblemBase::checkDependMaterialsHelper(
9115 : const std::map<SubdomainID, std::vector<std::shared_ptr<MaterialBase>>> & materials_map)
9116 : {
9117 73641 : for (const auto & it : materials_map)
9118 : {
9119 : /// These two sets are used to make sure that all dependent props on a block are actually supplied
9120 12625 : std::set<std::string> block_depend_props, block_supplied_props;
9121 :
9122 31390 : for (const auto & mat1 : it.second)
9123 : {
9124 18765 : auto & alldeps = mat1->getMatPropDependencies(); // includes requested stateful props
9125 21084 : for (auto & dep : alldeps)
9126 2319 : block_depend_props.insert(_material_prop_registry.getName(dep));
9127 :
9128 : // See if any of the active materials supply this property
9129 57970 : for (const auto & mat2 : it.second)
9130 : {
9131 39205 : const std::set<std::string> & supplied_props = mat2->MaterialBase::getSuppliedItems();
9132 39205 : block_supplied_props.insert(supplied_props.begin(), supplied_props.end());
9133 : }
9134 : }
9135 :
9136 : // Add zero material properties specific to this block and unrestricted
9137 12625 : block_supplied_props.insert(_zero_block_material_props[it.first].begin(),
9138 12625 : _zero_block_material_props[it.first].end());
9139 :
9140 : // Error check to make sure all properties consumed by materials are supplied on this block
9141 12625 : std::set<std::string> difference;
9142 12625 : std::set_difference(block_depend_props.begin(),
9143 : block_depend_props.end(),
9144 : block_supplied_props.begin(),
9145 : block_supplied_props.end(),
9146 : std::inserter(difference, difference.end()));
9147 :
9148 12625 : if (!difference.empty())
9149 : {
9150 8 : std::ostringstream oss;
9151 8 : oss << "One or more Material Properties were not supplied on block ";
9152 8 : const std::string & subdomain_name = _mesh.getSubdomainName(it.first);
9153 8 : if (subdomain_name.length() > 0)
9154 0 : oss << subdomain_name << " (" << it.first << ")";
9155 : else
9156 8 : oss << it.first;
9157 8 : oss << ":\n";
9158 16 : for (const auto & name : difference)
9159 8 : oss << name << "\n";
9160 8 : mooseError(oss.str());
9161 0 : }
9162 12617 : }
9163 :
9164 : // This loop checks that materials are not supplied by multiple Material objects
9165 73628 : for (const auto & it : materials_map)
9166 : {
9167 12617 : const auto & materials = it.second;
9168 12617 : std::set<std::string> inner_supplied, outer_supplied;
9169 :
9170 31362 : for (const auto & outer_mat : materials)
9171 : {
9172 : // Storage for properties for this material (outer) and all other materials (inner)
9173 18750 : outer_supplied = outer_mat->getSuppliedItems();
9174 18750 : inner_supplied.clear();
9175 :
9176 : // Property to material map for error reporting
9177 18750 : std::map<std::string, std::set<std::string>> prop_to_mat;
9178 39632 : for (const auto & name : outer_supplied)
9179 20882 : prop_to_mat[name].insert(outer_mat->name());
9180 :
9181 57931 : for (const auto & inner_mat : materials)
9182 : {
9183 39181 : if (outer_mat == inner_mat)
9184 18750 : continue;
9185 :
9186 : // Check whether these materials are an AD pair
9187 20431 : auto outer_mat_type = outer_mat->type();
9188 20431 : auto inner_mat_type = inner_mat->type();
9189 40862 : removeSubstring(outer_mat_type, "<RESIDUAL>");
9190 40862 : removeSubstring(outer_mat_type, "<JACOBIAN>");
9191 40862 : removeSubstring(inner_mat_type, "<RESIDUAL>");
9192 20431 : removeSubstring(inner_mat_type, "<JACOBIAN>");
9193 20431 : if (outer_mat_type == inner_mat_type && outer_mat_type != outer_mat->type() &&
9194 0 : inner_mat_type != inner_mat->type())
9195 0 : continue;
9196 :
9197 20431 : inner_supplied.insert(inner_mat->getSuppliedItems().begin(),
9198 20431 : inner_mat->getSuppliedItems().end());
9199 :
9200 115501 : for (const auto & inner_supplied_name : inner_supplied)
9201 95070 : prop_to_mat[inner_supplied_name].insert(inner_mat->name());
9202 20431 : }
9203 :
9204 : // Test that a property isn't supplied on multiple blocks
9205 18750 : std::set<std::string> intersection;
9206 18750 : std::set_intersection(outer_supplied.begin(),
9207 : outer_supplied.end(),
9208 : inner_supplied.begin(),
9209 : inner_supplied.end(),
9210 : std::inserter(intersection, intersection.end()));
9211 :
9212 18750 : if (!intersection.empty())
9213 : {
9214 5 : std::ostringstream oss;
9215 5 : oss << "The following material properties are declared on block " << it.first
9216 5 : << " by multiple materials:\n";
9217 10 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << "Material Property"
9218 5 : << "Material Objects\n";
9219 20 : for (const auto & outer_name : intersection)
9220 : {
9221 15 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << outer_name;
9222 45 : for (const auto & inner_name : prop_to_mat[outer_name])
9223 30 : oss << inner_name << " ";
9224 15 : oss << '\n';
9225 : }
9226 :
9227 5 : mooseError(oss.str());
9228 : break;
9229 0 : }
9230 18745 : }
9231 12612 : }
9232 61011 : }
9233 :
9234 : void
9235 61035 : FEProblemBase::checkCoordinateSystems()
9236 : {
9237 61035 : _mesh.checkCoordinateSystems();
9238 61032 : }
9239 :
9240 : void
9241 469 : FEProblemBase::setRestartFile(const std::string & file_name)
9242 : {
9243 469 : if (_app.isRecovering())
9244 : {
9245 23 : mooseInfo("Restart file ", file_name, " is NOT being used since we are performing recovery.");
9246 : }
9247 : else
9248 : {
9249 446 : _app.setRestart(true);
9250 446 : _app.setRestartRecoverFileBase(file_name);
9251 446 : mooseInfo("Using ", file_name, " for restart.");
9252 : }
9253 469 : }
9254 :
9255 : std::vector<VariableName>
9256 359883 : FEProblemBase::getVariableNames()
9257 : {
9258 359883 : std::vector<VariableName> names;
9259 :
9260 724157 : for (auto & sys : _solver_systems)
9261 : {
9262 364274 : const std::vector<VariableName> & var_names = sys->getVariableNames();
9263 364274 : names.insert(names.end(), var_names.begin(), var_names.end());
9264 : }
9265 :
9266 359883 : const std::vector<VariableName> & aux_var_names = _aux->getVariableNames();
9267 359883 : names.insert(names.end(), aux_var_names.begin(), aux_var_names.end());
9268 :
9269 359883 : return names;
9270 0 : }
9271 :
9272 : SolverParams &
9273 1633120 : FEProblemBase::solverParams(const unsigned int solver_sys_num)
9274 : {
9275 : mooseAssert(solver_sys_num < numSolverSystems(),
9276 : "Solver system number '" << solver_sys_num << "' is out of bounds. We have '"
9277 : << numSolverSystems() << "' solver systems");
9278 1633120 : return _solver_params[solver_sys_num];
9279 : }
9280 :
9281 : const SolverParams &
9282 14652 : FEProblemBase::solverParams(const unsigned int solver_sys_num) const
9283 : {
9284 14652 : return const_cast<FEProblemBase *>(this)->solverParams(solver_sys_num);
9285 : }
9286 :
9287 : void
9288 371 : FEProblemBase::registerRandomInterface(RandomInterface & random_interface, const std::string & name)
9289 : {
9290 371 : auto insert_pair = moose_try_emplace(
9291 371 : _random_data_objects, name, std::make_unique<RandomData>(*this, random_interface));
9292 :
9293 371 : auto random_data_ptr = insert_pair.first->second.get();
9294 371 : random_interface.setRandomDataPointer(random_data_ptr);
9295 371 : }
9296 :
9297 : bool
9298 1603469 : FEProblemBase::needBoundaryMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9299 : {
9300 1603469 : if (_bnd_mat_side_cache[tid].find(bnd_id) == _bnd_mat_side_cache[tid].end())
9301 : {
9302 28384 : auto & bnd_mat_side_cache = _bnd_mat_side_cache[tid][bnd_id];
9303 28384 : bnd_mat_side_cache = false;
9304 :
9305 : // Check systems
9306 28384 : if (_aux->needMaterialOnSide(bnd_id))
9307 : {
9308 506 : bnd_mat_side_cache = true;
9309 506 : return true;
9310 : }
9311 53484 : for (auto & nl : _nl)
9312 27690 : if (nl->needBoundaryMaterialOnSide(bnd_id, tid))
9313 : {
9314 2084 : bnd_mat_side_cache = true;
9315 2084 : return true;
9316 : }
9317 :
9318 : // TODO: these objects should be checked for whether they actually consume materials
9319 : // NOTE: InterfaceUO can use use boundary properties too
9320 25794 : if (theWarehouse()
9321 51588 : .query()
9322 25794 : .condition<AttribThread>(tid)
9323 25794 : .condition<AttribInterfaces>(Interfaces::SideUserObject | Interfaces::DomainUserObject |
9324 : Interfaces::InterfaceUserObject)
9325 25794 : .condition<AttribBoundaries>(bnd_id)
9326 25794 : .count() > 0)
9327 : {
9328 560 : bnd_mat_side_cache = true;
9329 560 : return true;
9330 : }
9331 : }
9332 :
9333 1600319 : return _bnd_mat_side_cache[tid][bnd_id];
9334 : }
9335 :
9336 : bool
9337 386533 : FEProblemBase::needInterfaceMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9338 : {
9339 386533 : if (_interface_mat_side_cache[tid].find(bnd_id) == _interface_mat_side_cache[tid].end())
9340 : {
9341 2700 : auto & interface_mat_side_cache = _interface_mat_side_cache[tid][bnd_id];
9342 2700 : interface_mat_side_cache = false;
9343 :
9344 : // Aux-system has not needed interface materials so far
9345 5184 : for (auto & nl : _nl)
9346 2700 : if (nl->needInterfaceMaterialOnSide(bnd_id, tid))
9347 : {
9348 216 : interface_mat_side_cache = true;
9349 216 : return true;
9350 : }
9351 :
9352 : // TODO: these objects should be checked for whether they actually consume materials
9353 2484 : if (theWarehouse()
9354 4968 : .query()
9355 2484 : .condition<AttribThread>(tid)
9356 2484 : .condition<AttribInterfaces>(Interfaces::InterfaceUserObject |
9357 : Interfaces::DomainUserObject)
9358 2484 : .condition<AttribBoundaries>(bnd_id)
9359 2484 : .count() > 0)
9360 : {
9361 79 : interface_mat_side_cache = true;
9362 79 : return true;
9363 : }
9364 2405 : else if (_interface_materials.hasActiveBoundaryObjects(bnd_id, tid))
9365 : {
9366 9 : interface_mat_side_cache = true;
9367 9 : return true;
9368 : }
9369 : }
9370 386229 : return _interface_mat_side_cache[tid][bnd_id];
9371 : }
9372 :
9373 : bool
9374 422805 : FEProblemBase::needInternalNeighborSideMaterial(SubdomainID subdomain_id, const THREAD_ID tid)
9375 : {
9376 422805 : if (_block_mat_side_cache[tid].find(subdomain_id) == _block_mat_side_cache[tid].end())
9377 : {
9378 12258 : _block_mat_side_cache[tid][subdomain_id] = false;
9379 :
9380 24071 : for (auto & nl : _nl)
9381 12223 : if (nl->needInternalNeighborSideMaterial(subdomain_id, tid))
9382 : {
9383 410 : _block_mat_side_cache[tid][subdomain_id] = true;
9384 410 : return true;
9385 : }
9386 :
9387 : // TODO: these objects should be checked for whether they actually consume materials
9388 11848 : if (theWarehouse()
9389 23696 : .query()
9390 11848 : .condition<AttribThread>(tid)
9391 11848 : .condition<AttribInterfaces>(Interfaces::InternalSideUserObject |
9392 : Interfaces::DomainUserObject)
9393 11848 : .condition<AttribSubdomains>(subdomain_id)
9394 11848 : .count() > 0)
9395 : {
9396 33 : _block_mat_side_cache[tid][subdomain_id] = true;
9397 33 : return true;
9398 : }
9399 : }
9400 :
9401 422362 : return _block_mat_side_cache[tid][subdomain_id];
9402 : }
9403 :
9404 : bool
9405 288683 : FEProblemBase::needsPreviousNewtonIteration() const
9406 : {
9407 288683 : return vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG);
9408 : }
9409 :
9410 : void
9411 76 : FEProblemBase::needsPreviousNewtonIteration(bool state)
9412 : {
9413 76 : if (state && !vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
9414 0 : mooseError("Previous nonlinear solution is required but not added through "
9415 : "Problem/previous_nl_solution_required=true");
9416 76 : }
9417 :
9418 : void
9419 52 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(bool needed,
9420 : const unsigned int solver_sys_num)
9421 : {
9422 52 : _previous_multiapp_fp_nl_solution_required[solver_sys_num] = needed;
9423 52 : }
9424 :
9425 : bool
9426 58919 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(
9427 : const unsigned int solver_sys_num) const
9428 : {
9429 58919 : return _previous_multiapp_fp_nl_solution_required[solver_sys_num];
9430 : }
9431 :
9432 : void
9433 13 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary(bool state)
9434 : {
9435 13 : _previous_multiapp_fp_aux_solution_required = state;
9436 13 : }
9437 :
9438 : bool
9439 58720 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary() const
9440 : {
9441 58720 : return _previous_multiapp_fp_aux_solution_required;
9442 : }
9443 :
9444 : bool
9445 7563758 : FEProblemBase::hasJacobian() const
9446 : {
9447 7563758 : return _has_jacobian;
9448 : }
9449 :
9450 : bool
9451 7079230 : FEProblemBase::constJacobian() const
9452 : {
9453 7079230 : return _const_jacobian;
9454 : }
9455 :
9456 : void
9457 290990 : FEProblemBase::addOutput(const std::string & object_type,
9458 : const std::string & object_name,
9459 : InputParameters & parameters)
9460 : {
9461 : parallel_object_only();
9462 :
9463 : // Get a reference to the OutputWarehouse
9464 290990 : OutputWarehouse & output_warehouse = _app.getOutputWarehouse();
9465 :
9466 : // Reject the reserved names for objects not built by MOOSE
9467 290990 : if (!parameters.get<bool>("_built_by_moose") && output_warehouse.isReservedName(object_name))
9468 6 : mooseError("The name '", object_name, "' is a reserved name for output objects");
9469 :
9470 : // Check that an object by the same name does not already exist; this must be done before the
9471 : // object is created to avoid getting misleading errors from the Parser
9472 290984 : if (output_warehouse.hasOutput(object_name))
9473 3 : mooseError("An output object named '", object_name, "' already exists");
9474 :
9475 : // Add a pointer to the FEProblemBase class
9476 581962 : parameters.addPrivateParam<FEProblemBase *>("_fe_problem_base", this);
9477 :
9478 : // --show-input should enable the display of the input file on the screen
9479 705275 : if (object_type == "Console" && _app.getParam<bool>("show_input") &&
9480 291008 : parameters.get<bool>("output_screen"))
9481 54 : parameters.set<ExecFlagEnum>("execute_input_on") = EXEC_INITIAL;
9482 :
9483 : // Apply only user-set parameters from the common [Outputs] block so that
9484 : // each output type's own defaults are not overridden by common defaults.
9485 290981 : const InputParameters * common = output_warehouse.getCommonParameters();
9486 290981 : if (common)
9487 290981 : parameters.applyCommonUserSetParameters(*common);
9488 :
9489 : // Set the correct value for the binary flag for XDA/XDR output
9490 290981 : if (object_type == "XDR")
9491 120 : parameters.set<bool>("_binary") = true;
9492 290921 : else if (object_type == "XDA")
9493 244 : parameters.set<bool>("_binary") = false;
9494 :
9495 : // Adjust the checkpoint suffix if auto recovery was enabled
9496 290981 : if (object_name == "auto_recovery_checkpoint")
9497 0 : parameters.set<std::string>("suffix") = "auto_recovery";
9498 :
9499 : // Create the object and add it to the warehouse
9500 290981 : std::shared_ptr<Output> output = _factory.create<Output>(object_type, object_name, parameters);
9501 290969 : logAdd("Output", object_name, object_type, parameters);
9502 290969 : output_warehouse.addOutput(output);
9503 290969 : }
9504 :
9505 : void
9506 23098 : FEProblemBase::haveADObjects(const bool have_ad_objects)
9507 : {
9508 23098 : _have_ad_objects = have_ad_objects;
9509 23098 : if (_displaced_problem)
9510 227 : _displaced_problem->SubProblem::haveADObjects(have_ad_objects);
9511 23098 : }
9512 :
9513 : const SystemBase &
9514 0 : FEProblemBase::getSystemBase(const unsigned int sys_num) const
9515 : {
9516 0 : if (sys_num < _solver_systems.size())
9517 0 : return *_solver_systems[sys_num];
9518 :
9519 0 : return *_aux;
9520 : }
9521 :
9522 : SystemBase &
9523 3965 : FEProblemBase::getSystemBase(const std::string & sys_name)
9524 : {
9525 3965 : if (std::find(_solver_sys_names.begin(), _solver_sys_names.end(), sys_name) !=
9526 7930 : _solver_sys_names.end())
9527 3965 : return getSystemBase(solverSysNum(sys_name));
9528 0 : else if (sys_name == "aux0")
9529 0 : return *_aux;
9530 : else
9531 0 : mooseError("System '" + sys_name + "' was requested from problem but does not exist.");
9532 : }
9533 :
9534 : SystemBase &
9535 5609 : FEProblemBase::getSystemBase(const unsigned int sys_num)
9536 : {
9537 5609 : if (sys_num < _solver_systems.size())
9538 5507 : return *_solver_systems[sys_num];
9539 :
9540 102 : return *_aux;
9541 : }
9542 :
9543 : const SystemBase &
9544 12824 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num) const
9545 : {
9546 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9547 12824 : return *_nl[sys_num];
9548 : }
9549 :
9550 : SystemBase &
9551 2193236 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num)
9552 : {
9553 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9554 2193236 : return *_nl[sys_num];
9555 : }
9556 :
9557 : const SystemBase &
9558 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num) const
9559 : {
9560 : mooseAssert(sys_num < _linear_systems.size(),
9561 : "System number greater than the number of linear systems");
9562 0 : return *_linear_systems[sys_num];
9563 : }
9564 :
9565 : SystemBase &
9566 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num)
9567 : {
9568 : mooseAssert(sys_num < _linear_systems.size(),
9569 : "System number greater than the number of linear systems");
9570 0 : return *_linear_systems[sys_num];
9571 : }
9572 :
9573 : const SystemBase &
9574 0 : FEProblemBase::systemBaseSolver(const unsigned int sys_num) const
9575 : {
9576 : mooseAssert(sys_num < _solver_systems.size(),
9577 : "System number greater than the number of solver systems");
9578 0 : return *_solver_systems[sys_num];
9579 : }
9580 :
9581 : SystemBase &
9582 6364736 : FEProblemBase::systemBaseSolver(const unsigned int sys_num)
9583 : {
9584 : mooseAssert(sys_num < _solver_systems.size(),
9585 : "System number greater than the number of solver systems");
9586 6364736 : return *_solver_systems[sys_num];
9587 : }
9588 :
9589 : const SystemBase &
9590 417 : FEProblemBase::systemBaseAuxiliary() const
9591 : {
9592 417 : return *_aux;
9593 : }
9594 :
9595 : SystemBase &
9596 10028171 : FEProblemBase::systemBaseAuxiliary()
9597 : {
9598 10028171 : return *_aux;
9599 : }
9600 :
9601 : void
9602 3898381 : FEProblemBase::computingNonlinearResid(bool computing_nonlinear_residual)
9603 : {
9604 : parallel_object_only();
9605 :
9606 3898381 : if (_displaced_problem)
9607 192294 : _displaced_problem->computingNonlinearResid(computing_nonlinear_residual);
9608 3898381 : _computing_nonlinear_residual = computing_nonlinear_residual;
9609 3898381 : }
9610 :
9611 : void
9612 9642323 : FEProblemBase::setCurrentlyComputingResidual(bool currently_computing_residual)
9613 : {
9614 9642323 : if (_displaced_problem)
9615 392534 : _displaced_problem->setCurrentlyComputingResidual(currently_computing_residual);
9616 9642323 : _currently_computing_residual = currently_computing_residual;
9617 9642323 : }
9618 :
9619 : void
9620 50 : FEProblemBase::uniformRefine()
9621 : {
9622 : // ResetDisplacedMeshThread::onNode looks up the reference mesh by ID, so we need to make sure
9623 : // we undisplace before adapting the reference mesh
9624 50 : if (_displaced_problem)
9625 34 : _displaced_problem->undisplaceMesh();
9626 :
9627 50 : Adaptivity::uniformRefine(&_mesh, 1);
9628 50 : if (_displaced_problem)
9629 34 : Adaptivity::uniformRefine(&_displaced_problem->mesh(), 1);
9630 :
9631 50 : meshChanged(
9632 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
9633 50 : }
9634 :
9635 : void
9636 60789 : FEProblemBase::automaticScaling(bool automatic_scaling)
9637 : {
9638 60789 : if (_displaced_problem)
9639 2022 : _displaced_problem->automaticScaling(automatic_scaling);
9640 :
9641 60789 : SubProblem::automaticScaling(automatic_scaling);
9642 60789 : }
9643 :
9644 : void
9645 564004 : FEProblemBase::reinitElemFaceRef(const Elem * elem,
9646 : unsigned int side,
9647 : Real tolerance,
9648 : const std::vector<Point> * const pts,
9649 : const std::vector<Real> * const weights,
9650 : const THREAD_ID tid)
9651 : {
9652 564004 : SubProblem::reinitElemFaceRef(elem, side, tolerance, pts, weights, tid);
9653 :
9654 564004 : if (_displaced_problem)
9655 20096 : _displaced_problem->reinitElemFaceRef(
9656 20096 : _displaced_mesh->elemPtr(elem->id()), side, tolerance, pts, weights, tid);
9657 564004 : }
9658 :
9659 : void
9660 564004 : FEProblemBase::reinitNeighborFaceRef(const Elem * neighbor_elem,
9661 : unsigned int neighbor_side,
9662 : Real tolerance,
9663 : const std::vector<Point> * const pts,
9664 : const std::vector<Real> * const weights,
9665 : const THREAD_ID tid)
9666 : {
9667 564004 : SubProblem::reinitNeighborFaceRef(neighbor_elem, neighbor_side, tolerance, pts, weights, tid);
9668 :
9669 564004 : if (_displaced_problem)
9670 20096 : _displaced_problem->reinitNeighborFaceRef(
9671 20096 : _displaced_mesh->elemPtr(neighbor_elem->id()), neighbor_side, tolerance, pts, weights, tid);
9672 564004 : }
9673 :
9674 : void
9675 3036200 : FEProblemBase::getFVMatsAndDependencies(
9676 : const SubdomainID blk_id,
9677 : std::vector<std::shared_ptr<MaterialBase>> & face_materials,
9678 : std::vector<std::shared_ptr<MaterialBase>> & neighbor_materials,
9679 : std::set<MooseVariableFieldBase *> & variables,
9680 : const THREAD_ID tid)
9681 : {
9682 3036200 : if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9683 : {
9684 : auto & this_face_mats =
9685 3544 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9686 7268 : for (std::shared_ptr<MaterialBase> face_mat : this_face_mats)
9687 3724 : if (face_mat->ghostable())
9688 : {
9689 3724 : face_materials.push_back(face_mat);
9690 3724 : auto & var_deps = face_mat->getMooseVariableDependencies();
9691 4084 : for (auto * var : var_deps)
9692 : {
9693 360 : if (!var->isFV())
9694 0 : mooseError(
9695 : "Ghostable materials should only have finite volume variables coupled into them.");
9696 360 : else if (face_mat->hasStatefulProperties())
9697 0 : mooseError("Finite volume materials do not currently support stateful properties.");
9698 360 : variables.insert(var);
9699 : }
9700 3724 : }
9701 : }
9702 :
9703 3036200 : if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9704 : {
9705 : auto & this_neighbor_mats =
9706 3544 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9707 7268 : for (std::shared_ptr<MaterialBase> neighbor_mat : this_neighbor_mats)
9708 3724 : if (neighbor_mat->ghostable())
9709 : {
9710 3724 : neighbor_materials.push_back(neighbor_mat);
9711 : #ifndef NDEBUG
9712 : auto & var_deps = neighbor_mat->getMooseVariableDependencies();
9713 : for (auto * var : var_deps)
9714 : {
9715 : if (!var->isFV())
9716 : mooseError(
9717 : "Ghostable materials should only have finite volume variables coupled into them.");
9718 : else if (neighbor_mat->hasStatefulProperties())
9719 : mooseError("Finite volume materials do not currently support stateful properties.");
9720 : auto pr = variables.insert(var);
9721 : mooseAssert(!pr.second,
9722 : "We should not have inserted any new variables dependencies from our "
9723 : "neighbor materials that didn't exist for our face materials");
9724 : }
9725 : #endif
9726 3724 : }
9727 : }
9728 3036200 : }
9729 :
9730 : void
9731 31850919 : FEProblemBase::resizeMaterialData(const Moose::MaterialDataType data_type,
9732 : const unsigned int nqp,
9733 : const THREAD_ID tid)
9734 : {
9735 31850919 : getMaterialData(data_type, tid).resize(nqp);
9736 31850919 : }
9737 :
9738 : void
9739 60729 : FEProblemBase::setNonlinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9740 : {
9741 60729 : if (convergence_names.size() != numNonlinearSystems())
9742 0 : paramError("nonlinear_convergence",
9743 : "There must be one convergence object per nonlinear system");
9744 60729 : _nonlinear_convergence_names = convergence_names;
9745 60729 : }
9746 :
9747 : void
9748 62298 : FEProblemBase::setMultiAppFixedPointConvergenceName(const ConvergenceName & convergence_name)
9749 : {
9750 62298 : _multiapp_fixed_point_convergence_name = convergence_name;
9751 62298 : }
9752 :
9753 : void
9754 30478 : FEProblemBase::setSteadyStateConvergenceName(const ConvergenceName & convergence_name)
9755 : {
9756 30478 : _steady_state_convergence_name = convergence_name;
9757 30478 : }
9758 :
9759 : const std::vector<ConvergenceName> &
9760 989314 : FEProblemBase::getNonlinearConvergenceNames() const
9761 : {
9762 989314 : if (_nonlinear_convergence_names)
9763 989314 : return *_nonlinear_convergence_names;
9764 0 : mooseError("The nonlinear system convergence name(s) have not been set.");
9765 : }
9766 :
9767 : bool
9768 26206 : FEProblemBase::hasLinearConvergenceObjects() const
9769 : {
9770 : // If false,this means we have not set one, not that we are querying this too early
9771 : // TODO: once there is a default linear CV object, error on the 'not set' case
9772 26206 : return _linear_convergence_names.has_value();
9773 : }
9774 :
9775 : void
9776 134 : FEProblemBase::setLinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9777 : {
9778 134 : if (convergence_names.size() != numLinearSystems())
9779 0 : paramError("linear_convergence", "There must be one convergence object per linear system");
9780 134 : _linear_convergence_names = convergence_names;
9781 134 : }
9782 :
9783 : const std::vector<ConvergenceName> &
9784 4281 : FEProblemBase::getLinearConvergenceNames() const
9785 : {
9786 4281 : if (_linear_convergence_names)
9787 4281 : return *_linear_convergence_names;
9788 0 : mooseError("The linear convergence name(s) have not been set.");
9789 : }
9790 :
9791 : const ConvergenceName &
9792 251694 : FEProblemBase::getMultiAppFixedPointConvergenceName() const
9793 : {
9794 251694 : if (_multiapp_fixed_point_convergence_name)
9795 251694 : return _multiapp_fixed_point_convergence_name.value();
9796 : else
9797 0 : mooseError("The fixed point convergence name has not been set.");
9798 : }
9799 :
9800 : const ConvergenceName &
9801 103532 : FEProblemBase::getSteadyStateConvergenceName() const
9802 : {
9803 103532 : if (_steady_state_convergence_name)
9804 103532 : return _steady_state_convergence_name.value();
9805 : else
9806 0 : mooseError("The steady convergence name has not been set.");
9807 : }
9808 :
9809 : void
9810 3056943 : FEProblemBase::residualSetup()
9811 : {
9812 3056943 : SubProblem::residualSetup();
9813 : // We need to setup all the nonlinear systems other than our current one which actually called
9814 : // this method (so we have to make sure we don't go in a circle)
9815 6203692 : for (const auto i : make_range(numNonlinearSystems()))
9816 3146749 : if (i != currentNlSysNum())
9817 89806 : _nl[i]->residualSetup();
9818 : // We don't setup the aux sys because that's been done elsewhere
9819 3056943 : if (_displaced_problem)
9820 124149 : _displaced_problem->residualSetup();
9821 3056943 : }
9822 :
9823 : void
9824 475149 : FEProblemBase::jacobianSetup()
9825 : {
9826 475149 : SubProblem::jacobianSetup();
9827 : // We need to setup all the nonlinear systems other than our current one which actually called
9828 : // this method (so we have to make sure we don't go in a circle)
9829 965831 : for (const auto i : make_range(numNonlinearSystems()))
9830 490682 : if (i != currentNlSysNum())
9831 15533 : _nl[i]->jacobianSetup();
9832 : // We don't setup the aux sys because that's been done elsewhere
9833 475149 : if (_displaced_problem)
9834 21144 : _displaced_problem->jacobianSetup();
9835 475149 : }
9836 :
9837 : MooseAppCoordTransform &
9838 96024 : FEProblemBase::coordTransform()
9839 : {
9840 96024 : return mesh().coordTransform();
9841 : }
9842 :
9843 : unsigned int
9844 477938418 : FEProblemBase::currentNlSysNum() const
9845 : {
9846 : // If we don't have nonlinear systems this should be an invalid number
9847 477938418 : unsigned int current_nl_sys_num = libMesh::invalid_uint;
9848 477938418 : if (_nl.size())
9849 477936306 : current_nl_sys_num = currentNonlinearSystem().number();
9850 :
9851 477938418 : return current_nl_sys_num;
9852 : }
9853 :
9854 : unsigned int
9855 0 : FEProblemBase::currentLinearSysNum() const
9856 : {
9857 : // If we don't have linear systems this should be an invalid number
9858 0 : unsigned int current_linear_sys_num = libMesh::invalid_uint;
9859 0 : if (_linear_systems.size())
9860 0 : current_linear_sys_num = currentLinearSystem().number();
9861 :
9862 0 : return current_linear_sys_num;
9863 : }
9864 :
9865 : bool
9866 123317300 : FEProblemBase::shouldPrintExecution(const THREAD_ID tid) const
9867 : {
9868 : // For now, only support printing from thread 0
9869 123317300 : if (tid != 0)
9870 516904 : return false;
9871 :
9872 245363785 : if (_print_execution_on.isValueSet(_current_execute_on_flag) ||
9873 122563389 : _print_execution_on.isValueSet(EXEC_ALWAYS))
9874 337439 : return true;
9875 : else
9876 122462957 : return false;
9877 : }
9878 :
9879 : std::vector<MortarUserObject *>
9880 562830 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9881 : const BoundaryID secondary_boundary_id,
9882 : const bool displaced,
9883 : const std::vector<MortarUserObject *> & mortar_uo_superset)
9884 : {
9885 562830 : std::vector<MortarUserObject *> mortar_uos;
9886 562830 : auto * const subproblem = displaced ? static_cast<SubProblem *>(_displaced_problem.get())
9887 562830 : : static_cast<SubProblem *>(this);
9888 562852 : for (auto * const obj : mortar_uo_superset)
9889 44 : if (obj->onInterface(primary_boundary_id, secondary_boundary_id) &&
9890 22 : (&obj->getSubProblem() == subproblem))
9891 22 : mortar_uos.push_back(obj);
9892 :
9893 562830 : return mortar_uos;
9894 0 : }
9895 :
9896 : std::vector<MortarUserObject *>
9897 562808 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9898 : const BoundaryID secondary_boundary_id,
9899 : const bool displaced)
9900 : {
9901 562808 : std::vector<MortarUserObject *> mortar_uos;
9902 562808 : theWarehouse()
9903 562808 : .query()
9904 1125616 : .condition<AttribInterfaces>(Interfaces::MortarUserObject)
9905 562808 : .queryInto(mortar_uos);
9906 1125616 : return getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced, mortar_uos);
9907 562808 : }
9908 :
9909 : void
9910 562808 : FEProblemBase::reinitMortarUserObjects(const BoundaryID primary_boundary_id,
9911 : const BoundaryID secondary_boundary_id,
9912 : const bool displaced)
9913 : {
9914 : const auto mortar_uos =
9915 562808 : getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced);
9916 562808 : for (auto * const mortar_uo : mortar_uos)
9917 : {
9918 0 : mortar_uo->setNormals();
9919 0 : mortar_uo->reinit();
9920 : }
9921 562808 : }
9922 :
9923 : void
9924 0 : FEProblemBase::setVerboseProblem(bool verbose)
9925 : {
9926 0 : _verbose_setup = verbose ? "true" : "false";
9927 0 : _verbose_multiapps = verbose;
9928 0 : _verbose_restore = verbose;
9929 0 : }
9930 :
9931 : void
9932 111832 : FEProblemBase::setCurrentLowerDElem(const Elem * const lower_d_elem, const THREAD_ID tid)
9933 : {
9934 111832 : SubProblem::setCurrentLowerDElem(lower_d_elem, tid);
9935 111832 : if (_displaced_problem)
9936 27619 : _displaced_problem->setCurrentLowerDElem(
9937 0 : lower_d_elem ? _displaced_mesh->elemPtr(lower_d_elem->id()) : nullptr, tid);
9938 111832 : }
9939 :
9940 : void
9941 121800780 : FEProblemBase::setCurrentBoundaryID(BoundaryID bid, const THREAD_ID tid)
9942 : {
9943 121800780 : SubProblem::setCurrentBoundaryID(bid, tid);
9944 121800780 : if (_displaced_problem)
9945 8992867 : _displaced_problem->setCurrentBoundaryID(bid, tid);
9946 121800780 : }
9947 :
9948 : void
9949 7422908 : FEProblemBase::setCurrentNonlinearSystem(const unsigned int nl_sys_num)
9950 : {
9951 : mooseAssert(nl_sys_num < _nl.size(),
9952 : "System number greater than the number of nonlinear systems");
9953 7422908 : _current_nl_sys = _nl[nl_sys_num].get();
9954 7422908 : _current_solver_sys = _current_nl_sys;
9955 7422908 : }
9956 :
9957 : void
9958 77302 : FEProblemBase::setCurrentLinearSystem(const unsigned int sys_num)
9959 : {
9960 : mooseAssert(sys_num < _linear_systems.size(),
9961 : "System number greater than the number of linear systems");
9962 77302 : _current_linear_sys = _linear_systems[sys_num].get();
9963 77302 : _current_solver_sys = _current_linear_sys;
9964 77302 : }
9965 :
9966 : void
9967 5880535 : FEProblemBase::computeSystems(const ExecFlagType & type)
9968 : {
9969 : // When performing an adjoint solve in the optimization module, the current solver system is the
9970 : // adjoint. However, the adjoint solve requires having accurate time derivative calculations for
9971 : // the forward system. The cleanest way to handle such uses is just to compute the time
9972 : // derivatives for all solver systems instead of trying to guess which ones we need and don't need
9973 11912864 : for (auto & solver_sys : _solver_systems)
9974 6032329 : solver_sys->compute(type);
9975 :
9976 5880535 : _aux->compute(type);
9977 5880502 : }
9978 :
9979 : const ConstElemRange &
9980 3942725 : FEProblemBase::getCurrentAlgebraicElementRange()
9981 : {
9982 3942725 : if (!_current_algebraic_elem_range)
9983 3942725 : return *_mesh.getActiveLocalElementRange();
9984 :
9985 0 : return *_current_algebraic_elem_range;
9986 : }
9987 : const ConstNodeRange &
9988 96793 : FEProblemBase::getCurrentAlgebraicNodeRange()
9989 : {
9990 96793 : if (!_current_algebraic_node_range)
9991 96793 : return *_mesh.getLocalNodeRange();
9992 :
9993 0 : return *_current_algebraic_node_range;
9994 : }
9995 : const ConstBndNodeRange &
9996 3490319 : FEProblemBase::getCurrentAlgebraicBndNodeRange()
9997 : {
9998 3490319 : if (!_current_algebraic_bnd_node_range)
9999 3490319 : return *_mesh.getBoundaryNodeRange();
10000 :
10001 0 : return *_current_algebraic_bnd_node_range;
10002 : }
10003 :
10004 : void
10005 0 : FEProblemBase::setCurrentAlgebraicElementRange(ConstElemRange * range)
10006 : {
10007 0 : if (!range)
10008 : {
10009 0 : _current_algebraic_elem_range = nullptr;
10010 0 : return;
10011 : }
10012 :
10013 0 : _current_algebraic_elem_range = std::make_unique<ConstElemRange>(*range);
10014 : }
10015 : void
10016 0 : FEProblemBase::setCurrentAlgebraicNodeRange(ConstNodeRange * range)
10017 : {
10018 0 : if (!range)
10019 : {
10020 0 : _current_algebraic_node_range = nullptr;
10021 0 : return;
10022 : }
10023 :
10024 0 : _current_algebraic_node_range = std::make_unique<ConstNodeRange>(*range);
10025 : }
10026 : void
10027 0 : FEProblemBase::setCurrentAlgebraicBndNodeRange(ConstBndNodeRange * range)
10028 : {
10029 0 : if (!range)
10030 : {
10031 0 : _current_algebraic_bnd_node_range = nullptr;
10032 0 : return;
10033 : }
10034 :
10035 0 : _current_algebraic_bnd_node_range = std::make_unique<ConstBndNodeRange>(*range);
10036 : }
10037 :
10038 : unsigned short
10039 59473 : FEProblemBase::getCurrentICState()
10040 : {
10041 59473 : return _current_ic_state;
10042 : }
10043 :
10044 : std::string
10045 54130 : FEProblemBase::solverTypeString(const unsigned int solver_sys_num)
10046 : {
10047 54130 : return Moose::stringify(solverParams(solver_sys_num)._type);
10048 : }
10049 :
10050 : SolverParams
10051 1264 : FEProblemBase::makeLinearSolverParams()
10052 : {
10053 1264 : SolverParams solver_params;
10054 1264 : solver_params._type = Moose::SolveType::ST_LINEAR;
10055 1264 : solver_params._line_search = Moose::LineSearchType::LS_NONE;
10056 1264 : return solver_params;
10057 : }
10058 :
10059 : const libMesh::CouplingMatrix &
10060 72576 : FEProblemBase::nonlocalCouplingMatrix(const unsigned i) const
10061 : {
10062 72576 : return _nonlocal_cm[i];
10063 : }
10064 :
10065 : bool
10066 90510839 : FEProblemBase::checkNonlocalCouplingRequirement() const
10067 : {
10068 90510839 : return _requires_nonlocal_coupling;
10069 : }
10070 :
10071 : const std::unordered_map<std::pair<BoundaryID, BoundaryID>, MortarInterfaceConfig> &
10072 119113 : FEProblemBase::getMortarInterfaces(bool on_displaced) const
10073 : {
10074 119113 : return _mortar_data->getMortarInterfaces(on_displaced);
10075 : }
|
