Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #ifdef MOOSE_KOKKOS_ENABLED
11 : #include "KokkosMaterialPropertyStorage.h"
12 : #endif
13 :
14 : #include "FEProblemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialPropertyStorage.h"
17 : #include "MooseEnum.h"
18 : #include "Factory.h"
19 : #include "MooseUtils.h"
20 : #include "DisplacedProblem.h"
21 : #include "SystemBase.h"
22 : #include "MaterialData.h"
23 : #include "ComputeUserObjectsThread.h"
24 : #include "ComputeNodalUserObjectsThread.h"
25 : #include "ComputeThreadedGeneralUserObjectsThread.h"
26 : #include "ComputeMaterialsObjectThread.h"
27 : #include "ProjectMaterialProperties.h"
28 : #include "ComputeIndicatorThread.h"
29 : #include "ComputeMarkerThread.h"
30 : #include "ComputeInitialConditionThread.h"
31 : #include "ComputeFVInitialConditionThread.h"
32 : #include "ComputeBoundaryInitialConditionThread.h"
33 : #include "MaxQpsThread.h"
34 : #include "ActionWarehouse.h"
35 : #include "Conversion.h"
36 : #include "Material.h"
37 : #include "FunctorMaterial.h"
38 : #include "ConstantIC.h"
39 : #include "Parser.h"
40 : #include "ElementH1Error.h"
41 : #include "Function.h"
42 : #include "Convergence.h"
43 : #include "NonlinearSystem.h"
44 : #include "LinearSystem.h"
45 : #include "SolverSystem.h"
46 : #include "Distribution.h"
47 : #include "Sampler.h"
48 : #include "FVAdvectedInterpolationMethod.h"
49 : #include "FVFaceInterpolationMethod.h"
50 : #include "FVInterpolationMethod.h"
51 : #include "PetscSupport.h"
52 : #include "RandomInterface.h"
53 : #include "RandomData.h"
54 : #include "MooseEigenSystem.h"
55 : #include "MooseParsedFunction.h"
56 : #include "MeshChangedInterface.h"
57 : #include "MeshDisplacedInterface.h"
58 : #include "ComputeJacobianBlocksThread.h"
59 : #include "ScalarInitialCondition.h"
60 : #include "FVInitialConditionTempl.h"
61 : #include "ElementPostprocessor.h"
62 : #include "NodalPostprocessor.h"
63 : #include "SidePostprocessor.h"
64 : #include "InternalSidePostprocessor.h"
65 : #include "InterfacePostprocessor.h"
66 : #include "GeneralPostprocessor.h"
67 : #include "ElementVectorPostprocessor.h"
68 : #include "NodalVectorPostprocessor.h"
69 : #include "SideVectorPostprocessor.h"
70 : #include "InternalSideVectorPostprocessor.h"
71 : #include "GeneralVectorPostprocessor.h"
72 : #include "Positions.h"
73 : #include "Indicator.h"
74 : #include "Marker.h"
75 : #include "MultiApp.h"
76 : #include "MultiAppTransfer.h"
77 : #include "TransientMultiApp.h"
78 : #include "ElementUserObject.h"
79 : #include "DomainUserObject.h"
80 : #include "NodalUserObject.h"
81 : #include "SideUserObject.h"
82 : #include "InternalSideUserObject.h"
83 : #include "InterfaceUserObject.h"
84 : #include "GeneralUserObject.h"
85 : #include "ThreadedGeneralUserObject.h"
86 : #include "InternalSideIndicatorBase.h"
87 : #include "Transfer.h"
88 : #include "MultiAppTransfer.h"
89 : #include "MultiMooseEnum.h"
90 : #include "Predictor.h"
91 : #include "Assembly.h"
92 : #include "Control.h"
93 : #include "XFEMInterface.h"
94 : #include "ConsoleUtils.h"
95 : #include "NonlocalKernel.h"
96 : #include "NonlocalIntegratedBC.h"
97 : #include "ShapeElementUserObject.h"
98 : #include "ShapeSideUserObject.h"
99 : #include "MooseVariableFE.h"
100 : #include "MooseVariableScalar.h"
101 : #include "InputParameterWarehouse.h"
102 : #include "TimeIntegrator.h"
103 : #include "LineSearch.h"
104 : #include "FloatingPointExceptionGuard.h"
105 : #include "MaxVarNDofsPerElem.h"
106 : #include "MaxVarNDofsPerNode.h"
107 : #include "FVKernel.h"
108 : #include "LinearFVKernel.h"
109 : #include "FVTimeKernel.h"
110 : #include "MooseVariableFV.h"
111 : #include "MooseLinearVariableFV.h"
112 : #include "FVBoundaryCondition.h"
113 : #include "LinearFVBoundaryCondition.h"
114 : #include "FVInterfaceKernel.h"
115 : #include "Reporter.h"
116 : #include "ADUtils.h"
117 : #include "Executioner.h"
118 : #include "VariadicTable.h"
119 : #include "BoundaryNodeIntegrityCheckThread.h"
120 : #include "BoundaryElemIntegrityCheckThread.h"
121 : #include "NodalBCBase.h"
122 : #include "MortarUserObject.h"
123 : #include "MortarUserObjectThread.h"
124 : #include "RedistributeProperties.h"
125 : #include "Checkpoint.h"
126 : #include "MortarInterfaceWarehouse.h"
127 : #include "AutomaticMortarGeneration.h"
128 :
129 : #include "libmesh/exodusII_io.h"
130 : #include "libmesh/quadrature.h"
131 : #include "libmesh/coupling_matrix.h"
132 : #include "libmesh/nonlinear_solver.h"
133 : #include "libmesh/sparse_matrix.h"
134 : #include "libmesh/string_to_enum.h"
135 : #include "libmesh/fe_interface.h"
136 : #include "libmesh/enum_norm_type.h"
137 : #include "libmesh/petsc_solver_exception.h"
138 :
139 : #include "metaphysicl/dualnumber.h"
140 :
141 : // C++
142 : #include <cstring> // for "Jacobian" exception test
143 :
144 : using namespace libMesh;
145 :
146 : // Anonymous namespace for helper function
147 : namespace
148 : {
149 : /**
150 : * Method for sorting the MooseVariableFEBases based on variable numbers
151 : */
152 : bool
153 61 : sortMooseVariables(const MooseVariableFEBase * a, const MooseVariableFEBase * b)
154 : {
155 61 : return a->number() < b->number();
156 : }
157 : } // namespace
158 :
159 : Threads::spin_mutex get_function_mutex;
160 :
161 : InputParameters
162 192130 : FEProblemBase::validParams()
163 : {
164 192130 : InputParameters params = SubProblem::validParams();
165 768520 : params.addParam<unsigned int>("null_space_dimension", 0, "The dimension of the nullspace");
166 576390 : params.addParam<unsigned int>(
167 384260 : "transpose_null_space_dimension", 0, "The dimension of the transpose nullspace");
168 576390 : params.addParam<unsigned int>(
169 384260 : "near_null_space_dimension", 0, "The dimension of the near nullspace");
170 576390 : params.addParam<bool>("solve",
171 384260 : true,
172 : "Whether or not to actually solve the Nonlinear system. "
173 : "This is handy in the case that all you want to do is "
174 : "execute AuxKernels, Transfers, etc. without actually "
175 : "solving anything");
176 576390 : params.addParam<bool>("use_nonlinear",
177 384260 : true,
178 : "Determines whether to use a Nonlinear vs a "
179 : "Eigenvalue system (Automatically determined based "
180 : "on executioner)");
181 768520 : params.addParam<bool>("error_on_jacobian_nonzero_reallocation",
182 : "This causes PETSc to error if it had to reallocate memory in the Jacobian "
183 : "matrix due to not having enough nonzeros");
184 576390 : params.addParam<bool>("ignore_zeros_in_jacobian",
185 384260 : false,
186 : "Do not explicitly store zero values in "
187 : "the Jacobian matrix if true");
188 576390 : params.addParam<bool>("force_restart",
189 384260 : false,
190 : "EXPERIMENTAL: If true, a sub_app may use a "
191 : "restart file instead of using of using the master "
192 : "backup file");
193 960650 : params.addDeprecatedParam<bool>("skip_additional_restart_data",
194 384260 : false,
195 : "True to skip additional data in equation system for restart.",
196 : "This parameter is no longer used, as we do not load additional "
197 : "vectors by default with restart");
198 576390 : params.addParam<bool>("skip_nl_system_check",
199 384260 : false,
200 : "True to skip the NonlinearSystem check for work to do (e.g. Make sure "
201 : "that there are variables to solve for).");
202 576390 : params.addParam<bool>("allow_initial_conditions_with_restart",
203 384260 : false,
204 : "True to allow the user to specify initial conditions when restarting. "
205 : "Initial conditions can override any restarted field");
206 :
207 384260 : auto coverage_check_description = [](std::string scope, std::string list_param_name)
208 : {
209 768520 : return "Controls, if and how a " + scope +
210 : " subdomain coverage check is performed. "
211 : "With 'TRUE' or 'ON' all subdomains are checked (the default). Setting 'FALSE' or 'OFF' "
212 : "will disable the check for all subdomains. "
213 : "To exclude a predefined set of subdomains 'SKIP_LIST' is to "
214 768520 : "be used, while the subdomains to skip are to be defined in the parameter '" +
215 768520 : list_param_name +
216 : "'. To limit the check to a list of subdomains, 'ONLY_LIST' is to "
217 1152780 : "be used (again, using the parameter '" +
218 768520 : list_param_name + "').";
219 : };
220 :
221 960650 : params.addParam<std::vector<SubdomainName>>(
222 : "block",
223 : {"ANY_BLOCK_ID"},
224 : "List of subdomains for kernel coverage and material coverage checks. Setting this parameter "
225 : "is equivalent to setting 'kernel_coverage_block_list' and 'material_coverage_block_list' as "
226 : "well as using 'ONLY_LIST' as the coverage check mode.");
227 :
228 768520 : MooseEnum kernel_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
229 192130 : params.addParam<MooseEnum>("kernel_coverage_check",
230 : kernel_coverage_check_modes,
231 960650 : coverage_check_description("kernel", "kernel_coverage_block_list"));
232 768520 : params.addParam<std::vector<SubdomainName>>(
233 : "kernel_coverage_block_list",
234 : {},
235 : "List of subdomains for kernel coverage check. The meaning of this list is controlled by the "
236 : "parameter 'kernel_coverage_check' (whether this is the list of subdomains to be checked, "
237 : "not to be checked or not taken into account).");
238 576390 : params.addParam<bool>(
239 : "boundary_restricted_node_integrity_check",
240 384260 : true,
241 : "Set to false to disable checking of boundary restricted nodal object variable dependencies, "
242 : "e.g. are the variable dependencies defined on the selected boundaries?");
243 576390 : params.addParam<bool>("boundary_restricted_elem_integrity_check",
244 384260 : true,
245 : "Set to false to disable checking of boundary restricted elemental object "
246 : "variable dependencies, e.g. are the variable dependencies defined on the "
247 : "selected boundaries?");
248 768520 : MooseEnum material_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
249 192130 : params.addParam<MooseEnum>(
250 : "material_coverage_check",
251 : material_coverage_check_modes,
252 960650 : coverage_check_description("material", "material_coverage_block_list"));
253 768520 : params.addParam<std::vector<SubdomainName>>(
254 : "material_coverage_block_list",
255 : {},
256 : "List of subdomains for material coverage check. The meaning of this list is controlled by "
257 : "the parameter 'material_coverage_check' (whether this is the list of subdomains to be "
258 : "checked, not to be checked or not taken into account).");
259 :
260 576390 : params.addParam<bool>("fv_bcs_integrity_check",
261 384260 : true,
262 : "Set to false to disable checking of overlapping Dirichlet and Flux BCs "
263 : "and/or multiple DirichletBCs per sideset");
264 :
265 576390 : params.addParam<bool>(
266 384260 : "material_dependency_check", true, "Set to false to disable material dependency check");
267 576390 : params.addParam<bool>("parallel_barrier_messaging",
268 384260 : false,
269 : "Displays messaging from parallel "
270 : "barrier notifications when executing "
271 : "or transferring to/from Multiapps "
272 : "(default: false)");
273 :
274 768520 : MooseEnum verbosity("false true extra", "false");
275 768520 : params.addParam<MooseEnum>("verbose_setup",
276 : verbosity,
277 : "Set to 'true' to have the problem report on any object created. Set "
278 : "to 'extra' to also display all parameters.");
279 576390 : params.addParam<bool>("verbose_multiapps",
280 384260 : false,
281 : "Set to True to enable verbose screen printing related to MultiApps");
282 576390 : params.addParam<bool>(
283 : "verbose_restore",
284 384260 : false,
285 : "Set to True to enable verbose screen printing related to solution restoration");
286 :
287 768520 : params.addParam<FileNameNoExtension>("restart_file_base",
288 : "File base name used for restart (e.g. "
289 : "<path>/<filebase> or <path>/LATEST to "
290 : "grab the latest file available)");
291 :
292 768520 : params.addParam<std::vector<std::vector<TagName>>>(
293 : "extra_tag_vectors",
294 : {},
295 : "Extra vectors to add to the system that can be filled by objects which compute residuals "
296 : "and Jacobians (Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
297 : "nonlinear system the extra tag vectors should be added for");
298 :
299 768520 : params.addParam<std::vector<std::vector<TagName>>>(
300 : "not_zeroed_tag_vectors",
301 : {},
302 : "Extra vector tags which the sytem will not zero when other vector tags are zeroed. "
303 : "The outer index is for which nonlinear system the extra tag vectors should be added for");
304 :
305 768520 : params.addParam<std::vector<std::vector<TagName>>>(
306 : "extra_tag_matrices",
307 : {},
308 : "Extra matrices to add to the system that can be filled "
309 : "by objects which compute residuals and Jacobians "
310 : "(Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
311 : "nonlinear system the extra tag vectors should be added for");
312 :
313 768520 : params.addParam<std::vector<TagName>>(
314 : "extra_tag_solutions",
315 : {},
316 : "Extra solution vectors to add to the system that can be used by "
317 : "objects for coupling variable values stored in them.");
318 :
319 576390 : params.addParam<bool>("previous_nl_solution_required",
320 384260 : false,
321 : "True to indicate that this calculation requires a solution vector for "
322 : "storing the previous nonlinear iteration.");
323 :
324 576390 : params.addParam<std::vector<NonlinearSystemName>>(
325 768520 : "nl_sys_names", std::vector<NonlinearSystemName>{"nl0"}, "The nonlinear system names");
326 :
327 768520 : params.addParam<std::vector<LinearSystemName>>("linear_sys_names", {}, "The linear system names");
328 :
329 576390 : params.addParam<bool>("check_uo_aux_state",
330 384260 : false,
331 : "True to turn on a check that no state presents during the evaluation of "
332 : "user objects and aux kernels");
333 :
334 192130 : params.addPrivateParam<MooseMesh *>("mesh");
335 :
336 576390 : params.declareControllable("solve");
337 :
338 576390 : params.addParam<bool>(
339 : "allow_invalid_solution",
340 384260 : false,
341 : "Set to true to allow convergence even though the solution has been marked as 'invalid'");
342 576390 : params.addParam<bool>("show_invalid_solution_console",
343 384260 : true,
344 : "Set to true to show the invalid solution occurrence summary in console");
345 576390 : params.addParam<bool>("immediately_print_invalid_solution",
346 384260 : false,
347 : "Whether or not to report invalid solution warnings at the time the "
348 : "warning is produced instead of after the calculation");
349 :
350 576390 : params.addParam<bool>(
351 : "identify_variable_groups_in_nl",
352 384260 : true,
353 : "Whether to identify variable groups in nonlinear systems. This affects dof ordering");
354 :
355 576390 : params.addParam<bool>(
356 : "regard_general_exceptions_as_errors",
357 384260 : false,
358 : "If we catch an exception during residual/Jacobian evaluaton for which we don't have "
359 : "specific handling, immediately error instead of allowing the time step to be cut");
360 :
361 576390 : params.addParam<bool>("use_hash_table_matrix_assembly",
362 384260 : false,
363 : "Whether to assemble matrices using hash tables instead of preallocating "
364 : "matrix memory. This can be a good option if the sparsity pattern changes "
365 : "throughout the course of the simulation.");
366 768520 : params.addParam<bool>(
367 : "restore_original_nonzero_pattern",
368 : "Whether we should reset matrix memory for every Jacobian evaluation. This option is useful "
369 : "if the sparsity pattern is constantly changing and you are using hash table assembly or if "
370 : "you wish to continually restore the matrix to the originally preallocated sparsity pattern "
371 : "computed by relationship managers.");
372 :
373 768520 : params.addParamNamesToGroup(
374 : "skip_nl_system_check kernel_coverage_check kernel_coverage_block_list "
375 : "boundary_restricted_node_integrity_check "
376 : "boundary_restricted_elem_integrity_check material_coverage_check "
377 : "material_coverage_block_list fv_bcs_integrity_check "
378 : "material_dependency_check check_uo_aux_state error_on_jacobian_nonzero_reallocation",
379 : "Simulation checks");
380 768520 : params.addParamNamesToGroup("use_nonlinear previous_nl_solution_required nl_sys_names "
381 : "ignore_zeros_in_jacobian identify_variable_groups_in_nl "
382 : "use_hash_table_matrix_assembly restore_original_nonzero_pattern",
383 : "Nonlinear system(s)");
384 768520 : params.addParamNamesToGroup(
385 : "restart_file_base force_restart allow_initial_conditions_with_restart", "Restart");
386 768520 : params.addParamNamesToGroup(
387 : "verbose_setup verbose_multiapps verbose_restore parallel_barrier_messaging", "Verbosity");
388 768520 : params.addParamNamesToGroup(
389 : "null_space_dimension transpose_null_space_dimension near_null_space_dimension",
390 : "Null space removal");
391 768520 : params.addParamNamesToGroup(
392 : "extra_tag_vectors extra_tag_matrices extra_tag_solutions not_zeroed_tag_vectors",
393 : "Contribution to tagged field data");
394 576390 : params.addParamNamesToGroup(
395 : "allow_invalid_solution show_invalid_solution_console immediately_print_invalid_solution",
396 : "Solution validity control");
397 :
398 384260 : return params;
399 576390 : }
400 :
401 62453 : FEProblemBase::FEProblemBase(const InputParameters & parameters)
402 : : SubProblem(parameters),
403 : Restartable(this, "FEProblemBase"),
404 249812 : _mesh(*getCheckedPointerParam<MooseMesh *>("mesh")),
405 124906 : _req(declareManagedRestartableDataWithContext<RestartableEquationSystems>(
406 : "equation_systems", nullptr, _mesh)),
407 62453 : _initialized(false),
408 124906 : _solve(getParam<bool>("solve")),
409 62453 : _transient(false),
410 124906 : _time(declareRestartableData<Real>("time")),
411 124906 : _time_old(declareRestartableData<Real>("time_old")),
412 124906 : _t_step(declareRecoverableData<int>("t_step")),
413 124906 : _dt(declareRestartableData<Real>("dt")),
414 124906 : _dt_old(declareRestartableData<Real>("dt_old")),
415 62453 : _need_to_add_default_nonlinear_convergence(false),
416 62453 : _need_to_add_default_multiapp_fixed_point_convergence(false),
417 62453 : _need_to_add_default_steady_state_convergence(false),
418 124906 : _linear_sys_names(getParam<std::vector<LinearSystemName>>("linear_sys_names")),
419 62453 : _num_linear_sys(_linear_sys_names.size()),
420 124906 : _linear_systems(_num_linear_sys, nullptr),
421 62453 : _current_linear_sys(nullptr),
422 124906 : _using_default_nl(!isParamSetByUser("nl_sys_names")),
423 185399 : _nl_sys_names(!_using_default_nl || (_using_default_nl && !_linear_sys_names.size())
424 62453 : ? getParam<std::vector<NonlinearSystemName>>("nl_sys_names")
425 : : std::vector<NonlinearSystemName>()),
426 62453 : _num_nl_sys(_nl_sys_names.size()),
427 124906 : _nl(_num_nl_sys, nullptr),
428 62453 : _current_nl_sys(nullptr),
429 124906 : _solver_systems(_num_nl_sys + _num_linear_sys, nullptr),
430 62453 : _aux(nullptr),
431 62453 : _coupling(Moose::COUPLING_DIAG),
432 : #ifdef MOOSE_KOKKOS_ENABLED
433 47210 : _kokkos_assembly(*this),
434 : #endif
435 62453 : _mesh_divisions(/*threaded=*/true),
436 62453 : _material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
437 62453 : "material_props", &_mesh, _material_prop_registry, *this)),
438 62453 : _bnd_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
439 62453 : "bnd_material_props", &_mesh, _material_prop_registry, *this)),
440 62453 : _neighbor_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
441 62453 : "neighbor_material_props", &_mesh, _material_prop_registry, *this)),
442 : #ifdef MOOSE_KOKKOS_ENABLED
443 47210 : _kokkos_material_props(
444 47210 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
445 47210 : "kokkos_material_props", &_mesh, _material_prop_registry, *this)),
446 47210 : _kokkos_bnd_material_props(
447 47210 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
448 47210 : "kokkos_bnd_material_props", &_mesh, _material_prop_registry, *this)),
449 47210 : _kokkos_neighbor_material_props(
450 47210 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
451 47210 : "kokkos_neighbor_material_props", &_mesh, _material_prop_registry, *this)),
452 : #endif
453 62453 : _reporter_data(_app),
454 62453 : _multi_apps(_app.getExecuteOnEnum()),
455 62453 : _transient_multi_apps(_app.getExecuteOnEnum()),
456 62453 : _transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
457 62453 : _to_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
458 62453 : _from_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
459 62453 : _between_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
460 : #ifdef LIBMESH_ENABLE_AMR
461 62453 : _adaptivity(*this),
462 62453 : _cycles_completed(0),
463 : #endif
464 62453 : _displaced_mesh(nullptr),
465 62453 : _geometric_search_data(*this, _mesh),
466 62453 : _mortar_data(std::make_unique<MortarInterfaceWarehouse>(*this)),
467 62453 : _reinit_displaced_elem(false),
468 62453 : _reinit_displaced_face(false),
469 62453 : _reinit_displaced_neighbor(false),
470 62453 : _input_file_saved(false),
471 62453 : _has_dampers(false),
472 62453 : _has_constraints(false),
473 62453 : _snesmf_reuse_base(true),
474 62453 : _skip_exception_check(false),
475 62453 : _snesmf_reuse_base_set_by_user(false),
476 62453 : _has_initialized_stateful(false),
477 62453 : _const_jacobian(false),
478 62453 : _has_jacobian(false),
479 62453 : _needs_old_newton_iter(false),
480 124906 : _previous_nl_solution_required(getParam<bool>("previous_nl_solution_required")),
481 124906 : _previous_multiapp_fp_nl_solution_required(_num_nl_sys + _num_linear_sys, false),
482 62453 : _previous_multiapp_fp_aux_solution_required(false),
483 62453 : _has_nonlocal_coupling(false),
484 62453 : _calculate_jacobian_in_uo(false),
485 62453 : _kernel_coverage_check(
486 124906 : getParam<MooseEnum>("kernel_coverage_check").getEnum<CoverageCheckMode>()),
487 124906 : _kernel_coverage_blocks(getParam<std::vector<SubdomainName>>("kernel_coverage_block_list")),
488 62453 : _boundary_restricted_node_integrity_check(
489 124906 : getParam<bool>("boundary_restricted_node_integrity_check")),
490 62453 : _boundary_restricted_elem_integrity_check(
491 124906 : getParam<bool>("boundary_restricted_elem_integrity_check")),
492 62453 : _material_coverage_check(
493 124906 : getParam<MooseEnum>("material_coverage_check").getEnum<CoverageCheckMode>()),
494 124906 : _material_coverage_blocks(getParam<std::vector<SubdomainName>>("material_coverage_block_list")),
495 124906 : _fv_bcs_integrity_check(getParam<bool>("fv_bcs_integrity_check")),
496 124906 : _material_dependency_check(getParam<bool>("material_dependency_check")),
497 124906 : _uo_aux_state_check(getParam<bool>("check_uo_aux_state")),
498 : #ifndef NDEBUG
499 : _check_residual_for_nans(false),
500 : #endif
501 62453 : _max_qps(std::numeric_limits<unsigned int>::max()),
502 62453 : _max_scalar_order(INVALID_ORDER),
503 62453 : _has_time_integrator(false),
504 62453 : _has_exception(false),
505 124906 : _parallel_barrier_messaging(getParam<bool>("parallel_barrier_messaging")),
506 124906 : _verbose_setup(getParam<MooseEnum>("verbose_setup")),
507 124906 : _verbose_multiapps(getParam<bool>("verbose_multiapps")),
508 124906 : _verbose_restore(getParam<bool>("verbose_restore")),
509 62453 : _current_execute_on_flag(EXEC_NONE),
510 62453 : _control_warehouse(_app.getExecuteOnEnum(), /*threaded=*/false),
511 62453 : _is_petsc_options_inserted(false),
512 62453 : _line_search(nullptr),
513 62453 : _using_ad_mat_props(false),
514 62453 : _current_ic_state(0),
515 124906 : _use_hash_table_matrix_assembly(getParam<bool>("use_hash_table_matrix_assembly")),
516 62453 : _error_on_jacobian_nonzero_reallocation(
517 124906 : isParamValid("error_on_jacobian_nonzero_reallocation")
518 125433 : ? getParam<bool>("error_on_jacobian_nonzero_reallocation")
519 61926 : : _app.errorOnJacobianNonzeroReallocation()),
520 124906 : _restore_original_nonzero_pattern(isParamValid("restore_original_nonzero_pattern")
521 124906 : ? getParam<bool>("restore_original_nonzero_pattern")
522 62453 : : _use_hash_table_matrix_assembly),
523 124906 : _ignore_zeros_in_jacobian(getParam<bool>("ignore_zeros_in_jacobian")),
524 62453 : _preserve_matrix_sparsity_pattern(true),
525 124906 : _force_restart(getParam<bool>("force_restart")),
526 124906 : _allow_ics_during_restart(getParam<bool>("allow_initial_conditions_with_restart")),
527 124906 : _skip_nl_system_check(getParam<bool>("skip_nl_system_check")),
528 62453 : _fail_next_system_convergence_check(false),
529 124906 : _allow_invalid_solution(getParam<bool>("allow_invalid_solution")),
530 124906 : _show_invalid_solution_console(getParam<bool>("show_invalid_solution_console")),
531 124906 : _immediately_print_invalid_solution(getParam<bool>("immediately_print_invalid_solution")),
532 62453 : _started_initial_setup(false),
533 62453 : _has_internal_edge_residual_objects(false),
534 62453 : _u_dot_requested(false),
535 62453 : _u_dotdot_requested(false),
536 62453 : _u_dot_old_requested(false),
537 62453 : _u_dotdot_old_requested(false),
538 62453 : _has_mortar(false),
539 62453 : _num_grid_steps(0),
540 62453 : _print_execution_on(),
541 124906 : _identify_variable_groups_in_nl(getParam<bool>("identify_variable_groups_in_nl")),
542 62453 : _regard_general_exceptions_as_errors(getParam<bool>("regard_general_exceptions_as_errors")),
543 1186607 : _requires_nonlocal_coupling(false)
544 : {
545 : auto checkCoverageCheckConflict =
546 124906 : [this](const std::string & coverage_check,
547 : const CoverageCheckMode & coverage_check_mode,
548 : const std::vector<SubdomainName> & coverage_blocks) -> void
549 : {
550 124906 : if (coverage_check_mode != CoverageCheckMode::FALSE &&
551 120483 : coverage_check_mode != CoverageCheckMode::OFF)
552 120465 : if (coverage_blocks.size() > 1)
553 0 : if (std::find(coverage_blocks.begin(), coverage_blocks.end(), "ANY_BLOCK_ID") !=
554 0 : coverage_blocks.end())
555 0 : paramError(coverage_check,
556 : "The list of blocks used for ",
557 : coverage_check,
558 : " cannot contain 'ANY_BLOCK_ID' along with other blocks. ");
559 124906 : };
560 :
561 62453 : checkCoverageCheckConflict(
562 62453 : "kernel_coverage_check", _kernel_coverage_check, _kernel_coverage_blocks);
563 62453 : checkCoverageCheckConflict(
564 62453 : "material_coverage_check", _material_coverage_check, _material_coverage_blocks);
565 :
566 : // Initialize static do_derivatives member. We initialize this to true so that all the
567 : // default AD things that we setup early in the simulation actually get their derivative
568 : // vectors initalized. We will toggle this to false when doing residual evaluations
569 62453 : ADReal::do_derivatives = true;
570 :
571 : // Disable refinement/coarsening in EquationSystems::reinit because we already do this ourselves
572 62453 : es().disable_refine_in_reinit();
573 :
574 62453 : _solver_params.reserve(_num_nl_sys + _num_linear_sys);
575 : // Default constructor fine for nonlinear because it will be populated later by framework
576 : // executioner/solve object parameters
577 62453 : _solver_params.resize(_num_nl_sys);
578 124178 : for (const auto i : index_range(_nl_sys_names))
579 : {
580 61725 : const auto & name = _nl_sys_names[i];
581 61725 : _nl_sys_name_to_num[name] = i;
582 61725 : _solver_sys_name_to_num[name] = i;
583 61725 : _solver_sys_names.push_back(name);
584 : }
585 :
586 63485 : for (const auto i : index_range(_linear_sys_names))
587 : {
588 1032 : const auto & name = _linear_sys_names[i];
589 1032 : _linear_sys_name_to_num[name] = i;
590 1032 : _solver_sys_name_to_num[name] = i + _num_nl_sys;
591 1032 : _solver_sys_names.push_back(name);
592 : // Unlike for nonlinear these are basically dummy parameters
593 1032 : _solver_params.push_back(makeLinearSolverParams());
594 : }
595 :
596 62453 : _nonlocal_cm.resize(numSolverSystems());
597 62453 : _cm.resize(numSolverSystems());
598 :
599 62453 : _time = 0.0;
600 62453 : _time_old = 0.0;
601 62453 : _t_step = 0;
602 62453 : _dt = 0;
603 62453 : _dt_old = _dt;
604 :
605 62453 : unsigned int n_threads = libMesh::n_threads();
606 :
607 62453 : _real_zero.resize(n_threads, 0.);
608 62453 : _scalar_zero.resize(n_threads);
609 62453 : _zero.resize(n_threads);
610 62453 : _phi_zero.resize(n_threads);
611 62453 : _ad_zero.resize(n_threads);
612 62453 : _grad_zero.resize(n_threads);
613 62453 : _ad_grad_zero.resize(n_threads);
614 62453 : _grad_phi_zero.resize(n_threads);
615 62453 : _second_zero.resize(n_threads);
616 62453 : _ad_second_zero.resize(n_threads);
617 62453 : _second_phi_zero.resize(n_threads);
618 62453 : _point_zero.resize(n_threads);
619 62453 : _vector_zero.resize(n_threads);
620 62453 : _vector_curl_zero.resize(n_threads);
621 62453 : _uo_jacobian_moose_vars.resize(n_threads);
622 :
623 62453 : _has_active_material_properties.resize(n_threads, 0);
624 :
625 62453 : _block_mat_side_cache.resize(n_threads);
626 62453 : _bnd_mat_side_cache.resize(n_threads);
627 62453 : _interface_mat_side_cache.resize(n_threads);
628 :
629 124906 : es().parameters.set<FEProblemBase *>("_fe_problem_base") = this;
630 :
631 187359 : if (isParamValid("restart_file_base"))
632 : {
633 938 : std::string restart_file_base = getParam<FileNameNoExtension>("restart_file_base");
634 :
635 : // This check reverts to old behavior of providing "restart_file_base=" to mean
636 : // don't restart... BISON currently relies on this. It could probably be removed.
637 : // The new MooseUtils::convertLatestCheckpoint will error out if a checkpoint file
638 : // is not found, which I think makes sense. Which means, without this, if you
639 : // set "restart_file_base=", you'll get a "No checkpoint file found" error
640 469 : if (restart_file_base.size())
641 : {
642 469 : restart_file_base = MooseUtils::convertLatestCheckpoint(restart_file_base);
643 469 : setRestartFile(restart_file_base);
644 : }
645 469 : }
646 :
647 : // // Generally speaking, the mesh is prepared for use, and consequently remote elements are deleted
648 : // // well before our Problem(s) are constructed. Historically, in MooseMesh we have a bunch of
649 : // // needs_prepare type flags that make it so we never call prepare_for_use (and consequently
650 : // // delete_remote_elements) again. So the below line, historically, has had no impact. HOWEVER:
651 : // // I've added some code in SetupMeshCompleteAction for deleting remote elements post
652 : // // EquationSystems::init. If I execute that code without default ghosting, then I get > 40 MOOSE
653 : // // test failures, so we clearly have some simulations that are not yet covered properly by
654 : // // relationship managers. Until that is resolved, I am going to retain default geometric ghosting
655 : // if (!_default_ghosting)
656 : // _mesh.getMesh().remove_ghosting_functor(_mesh.getMesh().default_ghosting());
657 :
658 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
659 : // Main app should hold the default database to handle system petsc options
660 62453 : if (!_app.isUltimateMaster())
661 12244 : LibmeshPetscCall(PetscOptionsCreate(&_petsc_option_data_base));
662 : #endif
663 :
664 62453 : if (!_solve)
665 : {
666 : // If we are not solving, we do not care about seeing unused petsc options
667 50091 : Moose::PetscSupport::setSinglePetscOption("-options_left", "0");
668 : // We don't want petscSetOptions being called in solve and clearing the option that was just set
669 16697 : _is_petsc_options_inserted = true;
670 : }
671 62453 : }
672 :
673 : const MooseMesh &
674 0 : FEProblemBase::mesh(bool use_displaced) const
675 : {
676 0 : if (use_displaced && !_displaced_problem)
677 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
678 : "Regular mesh will be returned");
679 0 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
680 : }
681 :
682 : MooseMesh &
683 359630 : FEProblemBase::mesh(bool use_displaced)
684 : {
685 359630 : if (use_displaced && !_displaced_problem)
686 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
687 : "Regular mesh will be returned");
688 359630 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
689 : }
690 :
691 : void
692 62453 : FEProblemBase::createTagVectors()
693 : {
694 : // add vectors and their tags to system
695 124906 : auto & vectors = getParam<std::vector<std::vector<TagName>>>("extra_tag_vectors");
696 63217 : for (const auto sys_num : index_range(vectors))
697 1836 : for (auto & vector : vectors[sys_num])
698 : {
699 1072 : auto tag = addVectorTag(vector);
700 1072 : _solver_systems[sys_num]->addVector(tag, false, libMesh::GHOSTED);
701 : }
702 :
703 124906 : auto & not_zeroed_vectors = getParam<std::vector<std::vector<TagName>>>("not_zeroed_tag_vectors");
704 62464 : for (const auto sys_num : index_range(not_zeroed_vectors))
705 22 : for (auto & vector : not_zeroed_vectors[sys_num])
706 : {
707 11 : auto tag = addVectorTag(vector);
708 11 : _solver_systems[sys_num]->addVector(tag, false, GHOSTED);
709 11 : addNotZeroedVectorTag(tag);
710 : }
711 62453 : }
712 :
713 : void
714 61557 : FEProblemBase::createTagMatrices(CreateTaggedMatrixKey)
715 : {
716 123114 : auto & matrices = getParam<std::vector<std::vector<TagName>>>("extra_tag_matrices");
717 61803 : for (const auto sys_num : index_range(matrices))
718 692 : for (auto & matrix : matrices[sys_num])
719 : {
720 446 : auto tag = addMatrixTag(matrix);
721 446 : _solver_systems[sys_num]->addMatrix(tag);
722 : }
723 :
724 123394 : for (auto & sys : _solver_systems)
725 61837 : sys->sizeVariableMatrixData();
726 61557 : _aux->sizeVariableMatrixData();
727 61557 : }
728 :
729 : void
730 62453 : FEProblemBase::createTagSolutions()
731 : {
732 187384 : for (auto & vector : getParam<std::vector<TagName>>("extra_tag_solutions"))
733 : {
734 25 : auto tag = addVectorTag(vector, Moose::VECTOR_TAG_SOLUTION);
735 50 : for (auto & sys : _solver_systems)
736 25 : sys->addVector(tag, false, libMesh::GHOSTED);
737 25 : _aux->addVector(tag, false, libMesh::GHOSTED);
738 : }
739 :
740 62453 : if (_previous_nl_solution_required)
741 : {
742 : // We'll populate the zeroth state of the nonlinear iterations with the current solution for
743 : // ease of use in doing things like copying solutions backwards. We're just storing pointers in
744 : // the solution states containers so populating the zeroth state does not cost us the memory of
745 : // a new vector
746 83 : needSolutionState(1, Moose::SolutionIterationType::Nonlinear);
747 : }
748 :
749 62453 : auto tag = addVectorTag(Moose::SOLUTION_TAG, Moose::VECTOR_TAG_SOLUTION);
750 125210 : for (auto & sys : _solver_systems)
751 62757 : sys->associateVectorToTag(*sys->system().current_local_solution.get(), tag);
752 62453 : _aux->associateVectorToTag(*_aux->system().current_local_solution.get(), tag);
753 62453 : }
754 :
755 : void
756 135 : FEProblemBase::needSolutionState(unsigned int state, Moose::SolutionIterationType iteration_type)
757 : {
758 270 : for (auto & sys : _solver_systems)
759 135 : sys->needSolutionState(state, iteration_type);
760 135 : _aux->needSolutionState(state, iteration_type);
761 135 : }
762 :
763 : bool
764 48 : FEProblemBase::hasSolutionState(unsigned int state,
765 : Moose::SolutionIterationType iteration_type) const
766 : {
767 48 : bool has_solution_state = false;
768 107 : for (auto & sys : _solver_systems)
769 59 : has_solution_state |= sys->hasSolutionState(state, iteration_type);
770 48 : has_solution_state |= _aux->hasSolutionState(state, iteration_type);
771 48 : return has_solution_state;
772 : }
773 :
774 : void
775 62453 : FEProblemBase::newAssemblyArray(std::vector<std::shared_ptr<SolverSystem>> & solver_systems)
776 : {
777 62453 : unsigned int n_threads = libMesh::n_threads();
778 :
779 62453 : _assembly.resize(n_threads);
780 131801 : for (const auto i : make_range(n_threads))
781 : {
782 69348 : _assembly[i].resize(solver_systems.size());
783 139028 : for (const auto j : index_range(solver_systems))
784 69680 : _assembly[i][j] = std::make_unique<Assembly>(*solver_systems[j], i);
785 : }
786 62453 : }
787 :
788 : void
789 59531 : FEProblemBase::initNullSpaceVectors(const InputParameters & parameters,
790 : std::vector<std::shared_ptr<NonlinearSystemBase>> & nls)
791 : {
792 297655 : TIME_SECTION("initNullSpaceVectors", 5, "Initializing Null Space Vectors");
793 :
794 59531 : unsigned int dimNullSpace = parameters.get<unsigned int>("null_space_dimension");
795 : unsigned int dimTransposeNullSpace =
796 59531 : parameters.get<unsigned int>("transpose_null_space_dimension");
797 59531 : unsigned int dimNearNullSpace = parameters.get<unsigned int>("near_null_space_dimension");
798 59555 : for (unsigned int i = 0; i < dimNullSpace; ++i)
799 : {
800 24 : std::ostringstream oss;
801 24 : oss << "_" << i;
802 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
803 : // builder might march over all nodes
804 48 : for (auto & nl : nls)
805 24 : nl->addVector("NullSpace" + oss.str(), false, libMesh::GHOSTED);
806 24 : }
807 119062 : _subspace_dim["NullSpace"] = dimNullSpace;
808 59543 : for (unsigned int i = 0; i < dimTransposeNullSpace; ++i)
809 : {
810 12 : std::ostringstream oss;
811 12 : oss << "_" << i;
812 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
813 : // builder might march over all nodes
814 24 : for (auto & nl : nls)
815 12 : nl->addVector("TransposeNullSpace" + oss.str(), false, libMesh::GHOSTED);
816 12 : }
817 119062 : _subspace_dim["TransposeNullSpace"] = dimTransposeNullSpace;
818 59531 : for (unsigned int i = 0; i < dimNearNullSpace; ++i)
819 : {
820 0 : std::ostringstream oss;
821 0 : oss << "_" << i;
822 : // do not project, since this will be recomputed, but make it ghosted, since the near-nullspace
823 : // builder might march over all semilocal nodes
824 0 : for (auto & nl : nls)
825 0 : nl->addVector("NearNullSpace" + oss.str(), false, libMesh::GHOSTED);
826 0 : }
827 119062 : _subspace_dim["NearNullSpace"] = dimNearNullSpace;
828 59531 : }
829 :
830 178080 : FEProblemBase::~FEProblemBase()
831 : {
832 : // Flush the Console stream, the underlying call to Console::mooseConsole
833 : // relies on a call to Output::checkInterval that has references to
834 : // _time, etc. If it is not flushed here memory problems arise if you have
835 : // an unflushed stream and start destructing things.
836 59360 : _console << std::flush;
837 :
838 59360 : unsigned int n_threads = libMesh::n_threads();
839 124315 : for (unsigned int i = 0; i < n_threads; i++)
840 : {
841 64955 : _zero[i].release();
842 64955 : _phi_zero[i].release();
843 64955 : _scalar_zero[i].release();
844 64955 : _grad_zero[i].release();
845 64955 : _grad_phi_zero[i].release();
846 64955 : _second_zero[i].release();
847 64955 : _second_phi_zero[i].release();
848 64955 : _vector_zero[i].release();
849 64955 : _vector_curl_zero[i].release();
850 64955 : _ad_zero[i].release();
851 64955 : _ad_grad_zero[i].release();
852 64955 : _ad_second_zero[i].release();
853 : }
854 :
855 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
856 59360 : if (!_app.isUltimateMaster())
857 : {
858 11480 : auto ierr = PetscOptionsDestroy(&_petsc_option_data_base);
859 : // Don't throw on destruction
860 11480 : CHKERRABORT(this->comm().get(), ierr);
861 : }
862 : #endif
863 59360 : }
864 :
865 : void
866 0 : FEProblemBase::setCoordSystem(const std::vector<SubdomainName> & blocks,
867 : const MultiMooseEnum & coord_sys)
868 : {
869 0 : TIME_SECTION("setCoordSystem", 5, "Setting Coordinate System");
870 0 : _mesh.setCoordSystem(blocks, coord_sys);
871 0 : }
872 :
873 : void
874 0 : FEProblemBase::setAxisymmetricCoordAxis(const MooseEnum & rz_coord_axis)
875 : {
876 0 : _mesh.setAxisymmetricCoordAxis(rz_coord_axis);
877 0 : }
878 :
879 : const ConstElemRange &
880 1383 : FEProblemBase::getEvaluableElementRange()
881 : {
882 1383 : if (!_evaluable_local_elem_range)
883 : {
884 677 : std::vector<const DofMap *> dof_maps(es().n_systems());
885 2031 : for (const auto i : make_range(es().n_systems()))
886 : {
887 1354 : const auto & sys = es().get_system(i);
888 1354 : dof_maps[i] = &sys.get_dof_map();
889 : }
890 : _evaluable_local_elem_range =
891 1354 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
892 2031 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
893 677 : }
894 1383 : return *_evaluable_local_elem_range;
895 : }
896 :
897 : const ConstElemRange &
898 208 : FEProblemBase::getNonlinearEvaluableElementRange()
899 : {
900 208 : if (!_nl_evaluable_local_elem_range)
901 : {
902 208 : std::vector<const DofMap *> dof_maps(_nl.size());
903 416 : for (const auto i : index_range(dof_maps))
904 208 : dof_maps[i] = &_nl[i]->dofMap();
905 : _nl_evaluable_local_elem_range =
906 416 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
907 624 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
908 208 : }
909 :
910 208 : return *_nl_evaluable_local_elem_range;
911 : }
912 :
913 : void
914 60178 : FEProblemBase::initialSetup()
915 : {
916 300890 : TIME_SECTION("initialSetup", 2, "Performing Initial Setup");
917 :
918 60178 : SubProblem::initialSetup();
919 :
920 60178 : if (_app.isRecovering() + _app.isRestarting() + bool(_app.getExReaderForRestart()) > 1)
921 0 : mooseError("Checkpoint recovery and restart and exodus restart are all mutually exclusive.");
922 :
923 60178 : if (_skip_exception_check)
924 9 : mooseWarning("MOOSE may fail to catch an exception when the \"skip_exception_check\" parameter "
925 : "is used. If you receive a terse MPI error during execution, remove this "
926 : "parameter and rerun your simulation");
927 :
928 : // set state flag indicating that we are in or beyond initialSetup.
929 : // This can be used to throw errors in methods that _must_ be called at construction time.
930 60178 : _started_initial_setup = true;
931 60178 : setCurrentExecuteOnFlag(EXEC_INITIAL);
932 :
933 : // Setup the solution states (current, old, etc) in each system based on
934 : // its default and the states requested of each of its variables
935 120636 : for (const auto i : index_range(_solver_systems))
936 : {
937 60458 : _solver_systems[i]->initSolutionState();
938 60458 : if (getDisplacedProblem())
939 2013 : getDisplacedProblem()->solverSys(i).initSolutionState();
940 : }
941 60178 : _aux->initSolutionState();
942 60178 : if (getDisplacedProblem())
943 2013 : getDisplacedProblem()->auxSys().initSolutionState();
944 :
945 : // always execute to get the max number of DoF per element and node needed to initialize phi_zero
946 : // variables
947 60178 : dof_id_type global_max_var_n_dofs_per_elem = 0;
948 120636 : for (const auto i : index_range(_solver_systems))
949 : {
950 60458 : auto & sys = *_solver_systems[i];
951 : dof_id_type max_var_n_dofs_per_elem;
952 : dof_id_type max_var_n_dofs_per_node;
953 : {
954 302290 : TIME_SECTION("computingMaxDofs", 3, "Computing Max Dofs Per Element");
955 :
956 60458 : MaxVarNDofsPerElem mvndpe(*this, sys);
957 60458 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mvndpe);
958 60458 : max_var_n_dofs_per_elem = mvndpe.max();
959 60458 : _communicator.max(max_var_n_dofs_per_elem);
960 :
961 60458 : MaxVarNDofsPerNode mvndpn(*this, sys);
962 60458 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), mvndpn);
963 60458 : max_var_n_dofs_per_node = mvndpn.max();
964 60458 : _communicator.max(max_var_n_dofs_per_node);
965 60458 : global_max_var_n_dofs_per_elem =
966 60458 : std::max(global_max_var_n_dofs_per_elem, max_var_n_dofs_per_elem);
967 60458 : }
968 :
969 : {
970 302290 : TIME_SECTION("assignMaxDofs", 5, "Assigning Maximum Dofs Per Elem");
971 :
972 60458 : sys.assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
973 60458 : auto displaced_problem = getDisplacedProblem();
974 60458 : if (displaced_problem)
975 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
976 :
977 60458 : sys.assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
978 60458 : if (displaced_problem)
979 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
980 60458 : }
981 : }
982 :
983 : {
984 300890 : TIME_SECTION("resizingVarValues", 5, "Resizing Variable Values");
985 :
986 126385 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
987 : {
988 132414 : _phi_zero[tid].resize(global_max_var_n_dofs_per_elem, std::vector<Real>(getMaxQps(), 0.));
989 132414 : _grad_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
990 132414 : std::vector<RealGradient>(getMaxQps(), RealGradient(0.)));
991 132414 : _second_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
992 132414 : std::vector<RealTensor>(getMaxQps(), RealTensor(0.)));
993 : }
994 60178 : }
995 :
996 : // Set up stateful material property redistribution, if we suspect
997 : // it may be necessary later.
998 60178 : addAnyRedistributers();
999 :
1000 60178 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1001 : {
1002 : // Only load all of the vectors if we're recovering
1003 4537 : _req.set().setLoadAllVectors(_app.isRecovering());
1004 :
1005 : // This forces stateful material property loading to be an exact one-to-one match
1006 4537 : if (_app.isRecovering())
1007 : {
1008 16020 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1009 12015 : props->setRecovering();
1010 :
1011 : #ifdef MOOSE_KOKKOS_ENABLED
1012 15880 : for (auto props :
1013 19850 : {&_kokkos_material_props, &_kokkos_bnd_material_props, &_kokkos_neighbor_material_props})
1014 11910 : props->setRecovering();
1015 : #endif
1016 : }
1017 :
1018 22685 : TIME_SECTION("restore", 3, "Restoring from backup");
1019 :
1020 : // We could have a cached backup when this app is a sub-app and has been given a Backup
1021 4537 : if (!_app.hasInitialBackup())
1022 3756 : _app.restore(_app.restartFolderBase(_app.getRestartRecoverFileBase()), _app.isRestarting());
1023 : else
1024 781 : _app.restoreFromInitialBackup(_app.isRestarting());
1025 :
1026 : /**
1027 : * If this is a restart run, the user may want to override the start time, which we already set
1028 : * in the constructor. "_time" however will have been "restored" from the restart file. We need
1029 : * to honor the original request of the developer now that the restore has been completed.
1030 : */
1031 4504 : if (_app.isRestarting())
1032 : {
1033 499 : if (_app.hasStartTime())
1034 173 : _time = _time_old = _app.getStartTime();
1035 : else
1036 326 : _time_old = _time;
1037 : }
1038 4504 : }
1039 : else
1040 : {
1041 55641 : libMesh::ExodusII_IO * reader = _app.getExReaderForRestart();
1042 :
1043 55641 : if (reader)
1044 : {
1045 1890 : TIME_SECTION("copyingFromExodus", 3, "Copying Variables From Exodus");
1046 :
1047 764 : for (auto & sys : _solver_systems)
1048 389 : sys->copyVars(*reader);
1049 375 : _aux->copyVars(*reader);
1050 375 : }
1051 : else
1052 : {
1053 55263 : if (_solver_systems[0]->hasVarCopy() || _aux->hasVarCopy())
1054 0 : mooseError("Need Exodus reader to restart variables but the reader is not available\n"
1055 : "Use either FileMesh with an Exodus mesh file or FileMeshGenerator with an "
1056 : "Exodus mesh file and with use_for_exodus_restart equal to true");
1057 : }
1058 : }
1059 :
1060 : // Perform output related setups
1061 60142 : _app.getOutputWarehouse().initialSetup();
1062 :
1063 : // Flush all output to _console that occur during construction and initialization of objects
1064 60096 : _app.getOutputWarehouse().mooseConsole();
1065 :
1066 : // Build Refinement and Coarsening maps for stateful material projections if necessary
1067 62365 : if ((_adaptivity.isOn() || _num_grid_steps) &&
1068 2269 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1069 2206 : _neighbor_material_props.hasStatefulProperties()))
1070 : {
1071 63 : if (_has_internal_edge_residual_objects)
1072 6 : mooseError("Stateful neighbor material properties do not work with mesh adaptivity");
1073 :
1074 57 : _mesh.buildRefinementAndCoarseningMaps(_assembly[0][0].get());
1075 : }
1076 :
1077 60090 : if (!_app.isRecovering())
1078 : {
1079 : /**
1080 : * If we are not recovering but we are doing restart (_app.getExodusFileRestart() == true) with
1081 : * additional uniform refinements. We have to delay the refinement until this point
1082 : * in time so that the equation systems are initialized and projections can be performed.
1083 : */
1084 56085 : if (_mesh.uniformRefineLevel() > 0 && _app.getExodusFileRestart())
1085 : {
1086 10 : if (!_app.isUltimateMaster())
1087 0 : mooseError(
1088 : "Doing extra refinements when restarting is NOT supported for sub-apps of a MultiApp");
1089 :
1090 10 : adaptivity().uniformRefineWithProjection();
1091 : }
1092 : }
1093 :
1094 60090 : unsigned int n_threads = libMesh::n_threads();
1095 :
1096 : // Convergence initial setup
1097 : {
1098 300450 : TIME_SECTION("convergenceInitialSetup", 5, "Initializing Convergence objects");
1099 :
1100 126158 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1101 66083 : _convergences.initialSetup(tid);
1102 60075 : }
1103 :
1104 : // UserObject initialSetup
1105 60075 : std::set<std::string> depend_objects_ic = _ics.getDependObjects();
1106 60075 : std::set<std::string> depend_objects_aux = _aux->getDependObjects();
1107 :
1108 60075 : std::map<int, std::vector<UserObjectBase *>> group_userobjs;
1109 :
1110 : // This replaces all prior updateDependObjects calls on the old user object warehouses.
1111 60075 : TheWarehouse::Query uo_query = theWarehouse().query().condition<AttribSystem>("UserObject");
1112 60075 : std::vector<UserObjectBase *> userobjs;
1113 60075 : uo_query.queryInto(userobjs);
1114 60075 : groupUserObjects(
1115 60075 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1116 :
1117 132520 : for (auto obj : userobjs)
1118 217335 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1119 :
1120 : #ifdef MOOSE_KOKKOS_ENABLED
1121 : {
1122 : TheWarehouse::Query uo_query =
1123 45460 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject");
1124 45460 : std::vector<UserObjectBase *> userobjs;
1125 45460 : uo_query.queryInto(userobjs);
1126 45460 : groupUserObjects(
1127 45460 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1128 :
1129 46685 : for (auto obj : userobjs)
1130 3675 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1131 45460 : }
1132 : #endif
1133 :
1134 90619 : for (auto & [group, objs] : group_userobjs)
1135 104161 : for (auto obj : objs)
1136 73617 : obj->initialSetup();
1137 :
1138 : // check if jacobian calculation is done in userobject
1139 125957 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1140 65962 : checkUserObjectJacobianRequirement(tid);
1141 :
1142 : // Check whether nonlocal coupling is required or not
1143 59995 : checkNonlocalCoupling();
1144 59995 : if (_requires_nonlocal_coupling)
1145 63 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
1146 :
1147 : {
1148 299975 : TIME_SECTION("initializingFunctions", 5, "Initializing Functions");
1149 :
1150 : // Call the initialSetup methods for functions
1151 125925 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1152 : {
1153 65954 : reinitScalars(tid); // initialize scalars so they are properly sized for use as input into
1154 : // ParsedFunctions
1155 65954 : _functions.initialSetup(tid);
1156 : }
1157 :
1158 : #ifdef MOOSE_KOKKOS_ENABLED
1159 45391 : _kokkos_functions.initialSetup();
1160 : #endif
1161 59971 : }
1162 :
1163 : {
1164 299855 : TIME_SECTION("initializingRandomObjects", 5, "Initializing Random Objects");
1165 :
1166 : // Random interface objects
1167 60295 : for (const auto & it : _random_data_objects)
1168 324 : it.second->updateSeeds(EXEC_INITIAL);
1169 59971 : }
1170 :
1171 59971 : if (!_app.isRecovering())
1172 : {
1173 55966 : computeUserObjects(EXEC_INITIAL, Moose::PRE_IC);
1174 :
1175 : {
1176 279830 : TIME_SECTION("ICinitialSetup", 5, "Setting Up Initial Conditions");
1177 :
1178 117838 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1179 61878 : _ics.initialSetup(tid);
1180 :
1181 55960 : _scalar_ics.initialSetup();
1182 55960 : }
1183 :
1184 55960 : projectSolution();
1185 : }
1186 :
1187 : // Materials
1188 59959 : if (_all_materials.hasActiveObjects(0))
1189 : {
1190 42085 : TIME_SECTION("materialInitialSetup", 3, "Setting Up Materials");
1191 :
1192 17565 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1193 : {
1194 : // Sort the Material objects, these will be actually computed by MOOSE in reinit methods.
1195 9166 : _materials.sort(tid);
1196 9163 : _interface_materials.sort(tid);
1197 :
1198 : // Call initialSetup on all material objects
1199 9163 : _all_materials.initialSetup(tid);
1200 :
1201 : // Discrete materials may insert additional dependencies on materials during the initial
1202 : // setup. Therefore we resolve the dependencies once more, now with the additional
1203 : // dependencies due to discrete materials.
1204 9148 : if (_discrete_materials.hasActiveObjects())
1205 : {
1206 56 : _materials.sort(tid);
1207 56 : _interface_materials.sort(tid);
1208 : }
1209 : }
1210 :
1211 : #ifdef MOOSE_KOKKOS_ENABLED
1212 6317 : _kokkos_materials.sort(0, true);
1213 : #endif
1214 :
1215 : {
1216 41985 : TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");
1217 :
1218 8397 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1219 :
1220 16119 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1221 7722 : _neighbor_material_props.hasStatefulProperties())
1222 675 : _has_initialized_stateful = true;
1223 : #ifdef MOOSE_KOKKOS_ENABLED
1224 6315 : if (_kokkos_material_props.hasStatefulProperties() ||
1225 12413 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1226 6098 : _kokkos_neighbor_material_props.hasStatefulProperties())
1227 217 : _has_initialized_stateful = true;
1228 : #endif
1229 8397 : }
1230 8397 : }
1231 :
1232 : // setRestartInPlace() is set because the property maps have now been setup and we can
1233 : // dataLoad() them directly in place
1234 : // setRecovering() is set because from now on we require a one-to-one mapping of
1235 : // stateful properties because we shouldn't be declaring any more
1236 239756 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1237 : {
1238 179817 : props->setRestartInPlace();
1239 179817 : props->setRecovering();
1240 : }
1241 :
1242 125826 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1243 : {
1244 65887 : _internal_side_indicators.initialSetup(tid);
1245 65887 : _indicators.initialSetup(tid);
1246 65887 : _markers.sort(tid);
1247 65887 : _markers.initialSetup(tid);
1248 : }
1249 :
1250 : #ifdef LIBMESH_ENABLE_AMR
1251 :
1252 59939 : if (!_app.isRecovering() && !_app.restoredInitialBackupMesh())
1253 : {
1254 55932 : unsigned int n = adaptivity().getInitialSteps();
1255 55932 : if (n && !_app.isUltimateMaster() && _app.isRestarting())
1256 0 : mooseError("Cannot perform initial adaptivity during restart on sub-apps of a MultiApp!");
1257 :
1258 55932 : initialAdaptMesh();
1259 : }
1260 :
1261 : #endif // LIBMESH_ENABLE_AMR
1262 :
1263 59936 : if (!_app.isRecovering() && !_app.isRestarting())
1264 : {
1265 : // During initial setup the solution is copied to the older solution states (old, older, etc)
1266 55432 : copySolutionsBackwards();
1267 :
1268 : // Check if there are old state initial conditions
1269 55432 : auto ics = _ics.getActiveObjects();
1270 55432 : auto fv_ics = _fv_ics.getActiveObjects();
1271 55432 : auto scalar_ics = _scalar_ics.getActiveObjects();
1272 55432 : unsigned short ic_state_max = 0;
1273 :
1274 166296 : auto findMax = [&ic_state_max](const auto & obj_list)
1275 : {
1276 197017 : for (auto ic : obj_list.getActiveObjects())
1277 30721 : ic_state_max = std::max(ic_state_max, ic->getState());
1278 221728 : };
1279 55432 : findMax(_ics);
1280 55432 : findMax(_fv_ics);
1281 55432 : findMax(_scalar_ics);
1282 :
1283 : // if there are old state ICs, compute them and write to old states accordingly
1284 55432 : if (ic_state_max > 0)
1285 : {
1286 : // state 0 copy (we'll overwrite current state when evaluating ICs and need to restore it once
1287 : // we're done with the old/older state ICs)
1288 0 : std::vector<std::unique_ptr<NumericVector<Real>>> state0_sys_buffers(_solver_systems.size());
1289 0 : std::unique_ptr<NumericVector<Real>> state0_aux_buffer;
1290 :
1291 : // save state 0
1292 0 : for (const auto i : index_range(_solver_systems))
1293 0 : state0_sys_buffers[i] = _solver_systems[i]->solutionState(0).clone();
1294 :
1295 0 : state0_aux_buffer = _aux->solutionState(0).clone();
1296 :
1297 : // compute old state ICs
1298 0 : for (_current_ic_state = 1; _current_ic_state <= ic_state_max; _current_ic_state++)
1299 : {
1300 0 : projectSolution();
1301 :
1302 0 : for (auto & sys : _solver_systems)
1303 0 : sys->solutionState(_current_ic_state) = sys->solutionState(0);
1304 :
1305 0 : _aux->solutionState(_current_ic_state) = _aux->solutionState(0);
1306 : }
1307 0 : _current_ic_state = 0;
1308 :
1309 : // recover state 0
1310 0 : for (const auto i : index_range(_solver_systems))
1311 : {
1312 0 : _solver_systems[i]->solutionState(0) = *state0_sys_buffers[i];
1313 0 : _solver_systems[i]->solutionState(0).close();
1314 0 : _solver_systems[i]->update();
1315 : }
1316 0 : _aux->solutionState(0) = *state0_aux_buffer;
1317 0 : _aux->solutionState(0).close();
1318 0 : _aux->update();
1319 0 : }
1320 55432 : }
1321 :
1322 59936 : if (!_app.isRecovering())
1323 : {
1324 55931 : if (haveXFEM())
1325 0 : updateMeshXFEM();
1326 : }
1327 :
1328 : // Call initialSetup on the solver systems
1329 120152 : for (auto & sys : _solver_systems)
1330 60216 : sys->initialSetup();
1331 :
1332 : // Auxilary variable initialSetup calls
1333 59936 : _aux->initialSetup();
1334 :
1335 59930 : if (_displaced_problem)
1336 : // initialSetup for displaced systems
1337 2013 : _displaced_problem->initialSetup();
1338 :
1339 120140 : for (auto & sys : _solver_systems)
1340 60210 : sys->setSolution(*(sys->system().current_local_solution.get()));
1341 :
1342 : // Update the nearest node searches (has to be called after the problem is all set up)
1343 : // We do this here because this sets up the Element's DoFs to ghost
1344 59930 : updateGeomSearch(GeometricSearchData::NEAREST_NODE);
1345 :
1346 59930 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
1347 59930 : if (_displaced_mesh)
1348 2013 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
1349 :
1350 : // We need to move the mesh in order to build a map between mortar secondary and primary
1351 : // interfaces. This map will then be used by the AgumentSparsityOnInterface ghosting functor to
1352 : // know which dofs we need ghosted when we call EquationSystems::reinit
1353 59930 : if (_displaced_problem && _mortar_data->hasDisplacedObjects())
1354 : {
1355 114 : _displaced_problem->updateMesh();
1356 : // if displacements were applied to the mesh, the mortar mesh should be updated too
1357 114 : updateMortarMesh();
1358 : }
1359 :
1360 : // Possibly reinit one more time to get ghosting correct
1361 59930 : reinitBecauseOfGhostingOrNewGeomObjects();
1362 :
1363 59930 : if (_displaced_mesh)
1364 2013 : _displaced_problem->updateMesh();
1365 :
1366 59930 : updateGeomSearch(); // Call all of the rest of the geometric searches
1367 :
1368 120137 : for (auto & sys : _solver_systems)
1369 : {
1370 60210 : const auto & tis = sys->getTimeIntegrators();
1371 :
1372 : {
1373 301050 : TIME_SECTION("timeIntegratorInitialSetup", 5, "Initializing Time Integrator");
1374 89980 : for (auto & ti : tis)
1375 29773 : ti->initialSetup();
1376 60207 : }
1377 : }
1378 :
1379 : // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ main-app restartable data
1380 : // has been restored
1381 :
1382 : // Call initialSetup on the MultiApps
1383 59927 : if (_multi_apps.hasObjects())
1384 : {
1385 35305 : TIME_SECTION("initialSetupMultiApps", 2, "Initializing MultiApps", false);
1386 7061 : _multi_apps.initialSetup();
1387 7055 : }
1388 :
1389 : // Call initialSetup on the transfers
1390 : {
1391 299605 : TIME_SECTION("initialSetupTransfers", 2, "Initializing Transfers");
1392 :
1393 59921 : _transfers.initialSetup();
1394 :
1395 : // Call initialSetup on the MultiAppTransfers to be executed on TO_MULTIAPP
1396 59921 : const auto & to_multi_app_objects = _to_multi_app_transfers.getActiveObjects();
1397 65057 : for (const auto & transfer : to_multi_app_objects)
1398 : {
1399 5184 : transfer->setCurrentDirection(Transfer::DIRECTION::TO_MULTIAPP);
1400 5184 : transfer->initialSetup();
1401 : }
1402 :
1403 : // Call initialSetup on the MultiAppTransfers to be executed on FROM_MULTIAPP
1404 59873 : const auto & from_multi_app_objects = _from_multi_app_transfers.getActiveObjects();
1405 66109 : for (const auto & transfer : from_multi_app_objects)
1406 : {
1407 6278 : transfer->setCurrentDirection(Transfer::DIRECTION::FROM_MULTIAPP);
1408 6278 : transfer->initialSetup();
1409 : }
1410 :
1411 : // Call initialSetup on the MultiAppTransfers to be executed on BETWEEN_MULTIAPP
1412 59831 : const auto & between_multi_app_objects = _between_multi_app_transfers.getActiveObjects();
1413 61301 : for (const auto & transfer : between_multi_app_objects)
1414 : {
1415 1470 : transfer->setCurrentDirection(Transfer::DIRECTION::BETWEEN_MULTIAPP);
1416 1470 : transfer->initialSetup();
1417 : }
1418 59831 : }
1419 :
1420 59831 : if (_boundary_restricted_node_integrity_check)
1421 : {
1422 179457 : TIME_SECTION("BoundaryRestrictedNodeIntegrityCheck", 5);
1423 :
1424 : // check that variables are defined along boundaries of boundary restricted nodal objects
1425 59819 : const auto & bnd_nodes = getCurrentAlgebraicBndNodeRange();
1426 59819 : BoundaryNodeIntegrityCheckThread bnict(*this, uo_query);
1427 59819 : Threads::parallel_reduce(bnd_nodes, bnict);
1428 :
1429 : // Nodal bcs aren't threaded
1430 118861 : for (auto & nl : _nl)
1431 : {
1432 59060 : const auto & nodal_bcs = nl->getNodalBCWarehouse();
1433 59060 : if (!nodal_bcs.hasBoundaryObjects())
1434 27640 : continue;
1435 :
1436 2470552 : for (const auto & bnode : bnd_nodes)
1437 : {
1438 2439141 : const auto boundary_id = bnode->_bnd_id;
1439 2439141 : const Node * const node = bnode->_node;
1440 :
1441 2439141 : if (node->processor_id() != this->processor_id())
1442 555220 : continue;
1443 :
1444 1883921 : const auto & bnd_name = _mesh.getBoundaryName(boundary_id);
1445 :
1446 : // Avoid assertion in getBoundaryObjects that we have boundary objects for this boundary ID
1447 1883921 : if (!nodal_bcs.hasBoundaryObjects(boundary_id))
1448 965879 : continue;
1449 :
1450 918042 : const auto & bnd_objects = nodal_bcs.getBoundaryObjects(boundary_id);
1451 1927590 : for (const auto & bnd_object : bnd_objects)
1452 : {
1453 1009557 : const auto & bnd_variable = bnd_object->variable();
1454 : // Skip if this object uses geometric search because coupled variables may be defined on
1455 : // paired boundaries instead of the boundary this node is on. Also skip if this boundary
1456 : // condition isn't applicable to the current node, e.g. if the node doesn't have any
1457 : // degrees of freedom for the boundary condition's variable
1458 1009557 : if (!bnd_object->requiresGeometricSearch() &&
1459 2019114 : bnd_object->checkVariableBoundaryIntegrity() &&
1460 1009557 : node->n_dofs(nl->number(), bnd_variable.number()))
1461 : {
1462 : std::set<MooseVariableFieldBase *> vars_to_omit = {
1463 : &static_cast<MooseVariableFieldBase &>(
1464 1927646 : const_cast<MooseVariableBase &>(bnd_variable))};
1465 :
1466 1927646 : boundaryIntegrityCheckError(
1467 1927637 : *bnd_object, bnd_object->checkAllVariables(*node, vars_to_omit), bnd_name);
1468 963814 : }
1469 : }
1470 : }
1471 : }
1472 59801 : }
1473 :
1474 59813 : if (_boundary_restricted_elem_integrity_check)
1475 : {
1476 179349 : TIME_SECTION("BoundaryRestrictedElemIntegrityCheck", 5);
1477 :
1478 : // check that variables are defined along boundaries of boundary restricted elemental objects
1479 59783 : ConstBndElemRange & bnd_elems = *mesh().getBoundaryElementRange();
1480 59783 : BoundaryElemIntegrityCheckThread beict(*this, uo_query);
1481 59783 : Threads::parallel_reduce(bnd_elems, beict);
1482 59771 : }
1483 :
1484 59801 : if (!_app.isRecovering())
1485 : {
1486 55796 : execTransfers(EXEC_INITIAL);
1487 :
1488 55796 : bool converged = execMultiApps(EXEC_INITIAL);
1489 55790 : if (!converged)
1490 3 : mooseError("failed to converge initial MultiApp");
1491 :
1492 : // We'll backup the Multiapp here
1493 55787 : backupMultiApps(EXEC_INITIAL);
1494 :
1495 117428 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1496 61641 : reinitScalars(tid);
1497 :
1498 55787 : execute(EXEC_INITIAL);
1499 :
1500 : // The FEProblemBase::execute method doesn't call all the systems on EXEC_INITIAL, but it does
1501 : // set/unset the current flag. Therefore, this resets the current flag to EXEC_INITIAL so that
1502 : // subsequent calls (e.g., executeControls) have the proper flag.
1503 55709 : setCurrentExecuteOnFlag(EXEC_INITIAL);
1504 : }
1505 :
1506 : // Here we will initialize the stateful properties once more since they may have been updated
1507 : // during initialSetup by calls to computeProperties.
1508 : //
1509 : // It's really bad that we don't allow this during restart. It means that we can't add new
1510 : // stateful materials
1511 : // during restart. This is only happening because this _has_ to be below initial userobject
1512 : // execution.
1513 : // Otherwise this could be done up above... _before_ restoring restartable data... which would
1514 : // allow you to have
1515 : // this happen during restart. I honestly have no idea why this has to happen after initial user
1516 : // object computation.
1517 : // THAT is something we should fix... so I've opened this ticket: #5804
1518 59714 : if (!_app.isRecovering() && !_app.isRestarting())
1519 : {
1520 109848 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1521 54638 : _neighbor_material_props.hasStatefulProperties())
1522 : {
1523 2860 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1524 :
1525 572 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1526 572 : }
1527 : #ifdef MOOSE_KOKKOS_ENABLED
1528 40879 : if (_kokkos_material_props.hasStatefulProperties() ||
1529 81593 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1530 40714 : _kokkos_neighbor_material_props.hasStatefulProperties())
1531 : {
1532 825 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1533 :
1534 165 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1535 165 : }
1536 : #endif
1537 : }
1538 :
1539 : // Control Logic
1540 59714 : _control_warehouse.initialSetup();
1541 59714 : executeControls(EXEC_INITIAL);
1542 :
1543 : // Scalar variables need to reinited for the initial conditions to be available for output
1544 125239 : for (unsigned int tid = 0; tid < n_threads; tid++)
1545 65552 : reinitScalars(tid);
1546 :
1547 59687 : if (_displaced_mesh)
1548 2013 : _displaced_problem->syncSolutions();
1549 :
1550 : // Writes all calls to _console from initialSetup() methods
1551 59687 : _app.getOutputWarehouse().mooseConsole();
1552 :
1553 59687 : if (_requires_nonlocal_coupling)
1554 : {
1555 63 : setNonlocalCouplingMatrix();
1556 126 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1557 126 : for (auto & assembly : _assembly[tid])
1558 63 : assembly->initNonlocalCoupling();
1559 : }
1560 :
1561 : {
1562 298435 : TIME_SECTION("lineSearchInitialSetup", 5, "Initializing Line Search");
1563 :
1564 59687 : if (_line_search)
1565 0 : _line_search->initialSetup();
1566 59687 : }
1567 :
1568 : // Perform Reporter get/declare check
1569 59687 : _reporter_data.check();
1570 :
1571 : // We do this late to allow objects to get late restartable data
1572 59687 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1573 4504 : _app.finalizeRestore();
1574 :
1575 59687 : setCurrentExecuteOnFlag(EXEC_NONE);
1576 59687 : }
1577 :
1578 : void
1579 60539 : FEProblemBase::checkDuplicatePostprocessorVariableNames()
1580 : {
1581 111934 : for (const auto & pp : _reporter_data.getPostprocessorNames())
1582 51395 : if (hasScalarVariable(pp))
1583 0 : mooseError("Postprocessor \"" + pp +
1584 60539 : "\" has the same name as a scalar variable in the system.");
1585 60539 : }
1586 :
1587 : void
1588 269309 : FEProblemBase::timestepSetup()
1589 : {
1590 269309 : SubProblem::timestepSetup();
1591 :
1592 269309 : if (_t_step > 1 && _num_grid_steps)
1593 : {
1594 31 : libMesh::MeshRefinement mesh_refinement(_mesh);
1595 31 : std::unique_ptr<libMesh::MeshRefinement> displaced_mesh_refinement(nullptr);
1596 31 : if (_displaced_mesh)
1597 23 : displaced_mesh_refinement = std::make_unique<libMesh::MeshRefinement>(*_displaced_mesh);
1598 :
1599 62 : for (MooseIndex(_num_grid_steps) i = 0; i < _num_grid_steps; ++i)
1600 : {
1601 31 : if (_displaced_problem)
1602 : // If the DisplacedProblem is active, undisplace the DisplacedMesh in preparation for
1603 : // refinement. We can't safely refine the DisplacedMesh directly, since the Hilbert keys
1604 : // computed on the inconsistenly-displaced Mesh are different on different processors,
1605 : // leading to inconsistent Hilbert keys. We must do this before the undisplaced Mesh is
1606 : // coarsensed, so that the element and node numbering is still consistent. We also have to
1607 : // make sure this is done during every step of coarsening otherwise different partitions
1608 : // will be generated for the reference and displaced meshes (even for replicated)
1609 23 : _displaced_problem->undisplaceMesh();
1610 :
1611 31 : mesh_refinement.uniformly_coarsen();
1612 31 : if (_displaced_mesh)
1613 23 : displaced_mesh_refinement->uniformly_coarsen();
1614 :
1615 : // Mark this as an intermediate change because we do not yet want to reinit_systems. E.g. we
1616 : // need things to happen in the following order for the undisplaced problem:
1617 : // u1) EquationSystems::reinit_solutions. This will restrict the solution vectors and then
1618 : // contract the mesh
1619 : // u2) MooseMesh::meshChanged. This will update the node/side lists and other
1620 : // things which needs to happen after the contraction
1621 : // u3) GeometricSearchData::reinit. Once the node/side lists are updated we can perform our
1622 : // geometric searches which will aid in determining sparsity patterns
1623 : //
1624 : // We do these things for the displaced problem (if it exists)
1625 : // d1) EquationSystems::reinit. Restrict the displaced problem vector copies and then contract
1626 : // the mesh. It's safe to do a full reinit with the displaced because there are no
1627 : // matrices that sparsity pattern calculations will be conducted for
1628 : // d2) MooseMesh::meshChanged. This will update the node/side lists and other
1629 : // things which needs to happen after the contraction
1630 : // d3) UpdateDisplacedMeshThread::operator(). Re-displace the mesh using the *displaced*
1631 : // solution vector copy because we don't know the state of the reference solution vector.
1632 : // It's safe to use the displaced copy because we are outside of a non-linear solve,
1633 : // and there is no concern about differences between solution and current_local_solution
1634 : // d4) GeometricSearchData::reinit. With the node/side lists updated and the mesh
1635 : // re-displaced, we can perform our geometric searches, which will aid in determining the
1636 : // sparsity pattern of the matrix held by the libMesh::ImplicitSystem held by the
1637 : // NonlinearSystem held by this
1638 31 : meshChanged(
1639 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
1640 : }
1641 :
1642 : // u4) Now that all the geometric searches have been done (both undisplaced and displaced),
1643 : // we're ready to update the sparsity pattern
1644 31 : es().reinit_systems();
1645 31 : }
1646 :
1647 269309 : _control_warehouse.timestepSetup();
1648 269309 : if (_line_search)
1649 0 : _line_search->timestepSetup();
1650 :
1651 : // Random interface objects
1652 270689 : for (const auto & it : _random_data_objects)
1653 1380 : it.second->updateSeeds(EXEC_TIMESTEP_BEGIN);
1654 :
1655 269309 : unsigned int n_threads = libMesh::n_threads();
1656 565322 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1657 : {
1658 296013 : _all_materials.timestepSetup(tid);
1659 296013 : _functions.timestepSetup(tid);
1660 : }
1661 :
1662 : #ifdef MOOSE_KOKKOS_ENABLED
1663 197461 : _kokkos_functions.timestepSetup();
1664 : #endif
1665 :
1666 269309 : _aux->timestepSetup();
1667 542102 : for (auto & sys : _solver_systems)
1668 272793 : sys->timestepSetup();
1669 :
1670 269309 : if (_displaced_problem)
1671 : // timestepSetup for displaced systems
1672 30674 : _displaced_problem->timestepSetup();
1673 :
1674 565322 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1675 : {
1676 296013 : _internal_side_indicators.timestepSetup(tid);
1677 296013 : _indicators.timestepSetup(tid);
1678 296013 : _markers.timestepSetup(tid);
1679 : }
1680 :
1681 269309 : std::vector<UserObject *> userobjs;
1682 269309 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
1683 603752 : for (auto obj : userobjs)
1684 334443 : obj->timestepSetup();
1685 :
1686 : #ifdef MOOSE_KOKKOS_ENABLED
1687 : {
1688 197461 : std::vector<UserObjectBase *> userobjs;
1689 197461 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
1690 199640 : for (auto obj : userobjs)
1691 2179 : obj->timestepSetup();
1692 197461 : }
1693 : #endif
1694 :
1695 : // Timestep setup of output objects
1696 269309 : _app.getOutputWarehouse().timestepSetup();
1697 :
1698 269309 : if (_requires_nonlocal_coupling)
1699 97 : if (_nonlocal_kernels.hasActiveObjects() || _nonlocal_integrated_bcs.hasActiveObjects())
1700 97 : _has_nonlocal_coupling = true;
1701 269309 : }
1702 :
1703 : unsigned int
1704 757159 : FEProblemBase::getMaxQps() const
1705 : {
1706 757159 : if (_max_qps == std::numeric_limits<unsigned int>::max())
1707 0 : mooseError("Max QPS uninitialized");
1708 757159 : return _max_qps;
1709 : }
1710 :
1711 : Order
1712 52 : FEProblemBase::getMaxScalarOrder() const
1713 : {
1714 52 : return _max_scalar_order;
1715 : }
1716 :
1717 : void
1718 59995 : FEProblemBase::checkNonlocalCoupling()
1719 : {
1720 299975 : TIME_SECTION("checkNonlocalCoupling", 5, "Checking Nonlocal Coupling");
1721 :
1722 125957 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
1723 131194 : for (auto & nl : _nl)
1724 : {
1725 65232 : const auto & all_kernels = nl->getKernelWarehouse();
1726 65232 : const auto & kernels = all_kernels.getObjects(tid);
1727 149820 : for (const auto & kernel : kernels)
1728 : {
1729 : std::shared_ptr<NonlocalKernel> nonlocal_kernel =
1730 84588 : std::dynamic_pointer_cast<NonlocalKernel>(kernel);
1731 84588 : if (nonlocal_kernel)
1732 : {
1733 35 : if (_calculate_jacobian_in_uo)
1734 35 : _requires_nonlocal_coupling = true;
1735 35 : _nonlocal_kernels.addObject(kernel, tid);
1736 : }
1737 84588 : }
1738 : const MooseObjectWarehouse<IntegratedBCBase> & all_integrated_bcs =
1739 65232 : nl->getIntegratedBCWarehouse();
1740 65232 : const auto & integrated_bcs = all_integrated_bcs.getObjects(tid);
1741 74143 : for (const auto & integrated_bc : integrated_bcs)
1742 : {
1743 : std::shared_ptr<NonlocalIntegratedBC> nonlocal_integrated_bc =
1744 8911 : std::dynamic_pointer_cast<NonlocalIntegratedBC>(integrated_bc);
1745 8911 : if (nonlocal_integrated_bc)
1746 : {
1747 28 : if (_calculate_jacobian_in_uo)
1748 28 : _requires_nonlocal_coupling = true;
1749 28 : _nonlocal_integrated_bcs.addObject(integrated_bc, tid);
1750 : }
1751 8911 : }
1752 : }
1753 59995 : }
1754 :
1755 : void
1756 65962 : FEProblemBase::checkUserObjectJacobianRequirement(THREAD_ID tid)
1757 : {
1758 65962 : std::set<const MooseVariableFEBase *> uo_jacobian_moose_vars;
1759 : {
1760 65962 : std::vector<ShapeElementUserObject *> objs;
1761 65962 : theWarehouse()
1762 65962 : .query()
1763 131924 : .condition<AttribInterfaces>(Interfaces::ShapeElementUserObject)
1764 65962 : .condition<AttribThread>(tid)
1765 65962 : .queryInto(objs);
1766 :
1767 66006 : for (const auto & uo : objs)
1768 : {
1769 44 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1770 44 : const auto & mv_deps = uo->jacobianMooseVariables();
1771 44 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1772 : }
1773 65962 : }
1774 : {
1775 65962 : std::vector<ShapeSideUserObject *> objs;
1776 65962 : theWarehouse()
1777 65962 : .query()
1778 131924 : .condition<AttribInterfaces>(Interfaces::ShapeSideUserObject)
1779 65962 : .condition<AttribThread>(tid)
1780 65962 : .queryInto(objs);
1781 66018 : for (const auto & uo : objs)
1782 : {
1783 56 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1784 56 : const auto & mv_deps = uo->jacobianMooseVariables();
1785 56 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1786 : }
1787 65962 : }
1788 :
1789 65962 : _uo_jacobian_moose_vars[tid].assign(uo_jacobian_moose_vars.begin(), uo_jacobian_moose_vars.end());
1790 131924 : std::sort(
1791 131924 : _uo_jacobian_moose_vars[tid].begin(), _uo_jacobian_moose_vars[tid].end(), sortMooseVariables);
1792 65962 : }
1793 :
1794 : void
1795 63 : FEProblemBase::setVariableAllDoFMap(const std::vector<const MooseVariableFEBase *> & moose_vars)
1796 : {
1797 153 : for (unsigned int i = 0; i < moose_vars.size(); ++i)
1798 : {
1799 90 : VariableName var_name = moose_vars[i]->name();
1800 90 : auto & sys = _solver_systems[moose_vars[i]->sys().number()];
1801 90 : sys->setVariableGlobalDoFs(var_name);
1802 90 : _var_dof_map[var_name] = sys->getVariableGlobalDoFs();
1803 90 : }
1804 63 : }
1805 :
1806 : void
1807 374960844 : FEProblemBase::prepare(const Elem * elem, const THREAD_ID tid)
1808 : {
1809 752634437 : for (const auto i : index_range(_solver_systems))
1810 : {
1811 377673593 : _assembly[tid][i]->reinit(elem);
1812 377673593 : _solver_systems[i]->prepare(tid);
1813 :
1814 377673593 : if (i < _num_nl_sys)
1815 : {
1816 : // This method is called outside of residual/Jacobian callbacks during initial condition
1817 : // evaluation
1818 377105545 : if ((!_has_jacobian || !_const_jacobian) && currentlyComputingJacobian())
1819 47388802 : _assembly[tid][i]->prepareJacobianBlock();
1820 377105545 : _assembly[tid][i]->prepareResidual();
1821 377105545 : if (_has_nonlocal_coupling && currentlyComputingJacobian())
1822 8824 : _assembly[tid][i]->prepareNonlocal();
1823 : }
1824 : }
1825 374960844 : _aux->prepare(tid);
1826 :
1827 386605339 : if (_displaced_problem &&
1828 : // _reinit_displaced_neighbor applies to interface type objects which will do computations
1829 : // based on both elem and neighbor. Consequently, despite what you might think by its name, we
1830 : // must make sure we prepare the displaced elem
1831 386605339 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1832 : {
1833 7561302 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), tid);
1834 7561278 : if (_has_nonlocal_coupling)
1835 0 : _displaced_problem->prepareNonlocal(tid);
1836 : }
1837 374960820 : }
1838 :
1839 : void
1840 25659 : FEProblemBase::prepareFace(const Elem * elem, const THREAD_ID tid)
1841 : {
1842 51318 : for (auto & nl : _nl)
1843 25659 : nl->prepareFace(tid, true);
1844 25659 : _aux->prepareFace(tid, false);
1845 :
1846 25659 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1847 0 : _displaced_problem->prepareFace(_displaced_mesh->elemPtr(elem->id()), tid);
1848 25659 : }
1849 :
1850 : void
1851 0 : FEProblemBase::prepare(const Elem * elem,
1852 : unsigned int ivar,
1853 : unsigned int jvar,
1854 : const std::vector<dof_id_type> & dof_indices,
1855 : const THREAD_ID tid)
1856 : {
1857 0 : for (const auto i : index_range(_nl))
1858 : {
1859 0 : _assembly[tid][i]->reinit(elem);
1860 0 : _nl[i]->prepare(tid);
1861 : }
1862 :
1863 0 : _aux->prepare(tid);
1864 0 : const auto current_nl_sys_num = _current_nl_sys->number();
1865 0 : _assembly[tid][current_nl_sys_num]->prepareBlock(ivar, jvar, dof_indices);
1866 0 : if (_has_nonlocal_coupling)
1867 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1868 : {
1869 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1870 0 : _assembly[tid][current_nl_sys_num]->prepareBlockNonlocal(
1871 : ivar, jvar, dof_indices, jv.allDofIndices());
1872 : }
1873 :
1874 0 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1875 : {
1876 0 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), ivar, jvar, dof_indices, tid);
1877 0 : if (_has_nonlocal_coupling)
1878 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1879 : {
1880 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1881 0 : _displaced_problem->prepareBlockNonlocal(ivar, jvar, dof_indices, jv.allDofIndices(), tid);
1882 : }
1883 : }
1884 0 : }
1885 :
1886 : void
1887 406293947 : FEProblemBase::setCurrentSubdomainID(const Elem * elem, const THREAD_ID tid)
1888 : {
1889 406293947 : SubdomainID did = elem->subdomain_id();
1890 816310577 : for (const auto i : index_range(_solver_systems))
1891 : {
1892 410016630 : _assembly[tid][i]->setCurrentSubdomainID(did);
1893 422478166 : if (_displaced_problem &&
1894 422478166 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1895 7814055 : _displaced_problem->assembly(tid, i).setCurrentSubdomainID(did);
1896 : }
1897 406293947 : }
1898 :
1899 : void
1900 1457428245 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, unsigned int side, const THREAD_ID tid)
1901 : {
1902 1457428245 : SubdomainID did = elem->neighbor_ptr(side)->subdomain_id();
1903 2919960961 : for (const auto i : index_range(_nl))
1904 : {
1905 1462532716 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1906 1506109174 : if (_displaced_problem &&
1907 1506109174 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1908 25839139 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1909 : }
1910 1457428245 : }
1911 :
1912 : void
1913 15257576 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, const THREAD_ID tid)
1914 : {
1915 15257576 : SubdomainID did = elem->subdomain_id();
1916 31466464 : for (const auto i : index_range(_nl))
1917 : {
1918 16208888 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1919 16260184 : if (_displaced_problem &&
1920 16260184 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1921 51296 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1922 : }
1923 15257576 : }
1924 :
1925 : void
1926 136662 : FEProblemBase::prepareAssembly(const THREAD_ID tid)
1927 : {
1928 136662 : _assembly[tid][_current_nl_sys->number()]->prepare();
1929 136662 : if (_has_nonlocal_coupling)
1930 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
1931 :
1932 136662 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1933 : {
1934 51296 : _displaced_problem->prepareAssembly(tid);
1935 51296 : if (_has_nonlocal_coupling)
1936 0 : _displaced_problem->prepareNonlocal(tid);
1937 : }
1938 136662 : }
1939 :
1940 : void
1941 266613 : FEProblemBase::addResidual(const THREAD_ID tid)
1942 : {
1943 533226 : _assembly[tid][_current_nl_sys->number()]->addResidual(Assembly::GlobalDataKey{},
1944 266613 : currentResidualVectorTags());
1945 :
1946 266613 : if (_displaced_problem)
1947 4776 : _displaced_problem->addResidual(tid);
1948 266613 : }
1949 :
1950 : void
1951 1985081 : FEProblemBase::addResidualNeighbor(const THREAD_ID tid)
1952 : {
1953 3970162 : _assembly[tid][_current_nl_sys->number()]->addResidualNeighbor(Assembly::GlobalDataKey{},
1954 1985081 : currentResidualVectorTags());
1955 :
1956 1985081 : if (_displaced_problem)
1957 61744 : _displaced_problem->addResidualNeighbor(tid);
1958 1985081 : }
1959 :
1960 : void
1961 1967412 : FEProblemBase::addResidualLower(const THREAD_ID tid)
1962 : {
1963 3934824 : _assembly[tid][_current_nl_sys->number()]->addResidualLower(Assembly::GlobalDataKey{},
1964 1967412 : currentResidualVectorTags());
1965 :
1966 1967412 : if (_displaced_problem)
1967 61956 : _displaced_problem->addResidualLower(tid);
1968 1967412 : }
1969 :
1970 : void
1971 48426 : FEProblemBase::addResidualScalar(const THREAD_ID tid /* = 0*/)
1972 : {
1973 96852 : _assembly[tid][_current_nl_sys->number()]->addResidualScalar(Assembly::GlobalDataKey{},
1974 48426 : currentResidualVectorTags());
1975 48426 : }
1976 :
1977 : void
1978 288471664 : FEProblemBase::cacheResidual(const THREAD_ID tid)
1979 : {
1980 288471664 : SubProblem::cacheResidual(tid);
1981 288471664 : if (_displaced_problem)
1982 7590898 : _displaced_problem->cacheResidual(tid);
1983 288471664 : }
1984 :
1985 : void
1986 50077 : FEProblemBase::cacheResidualNeighbor(const THREAD_ID tid)
1987 : {
1988 50077 : SubProblem::cacheResidualNeighbor(tid);
1989 50077 : if (_displaced_problem)
1990 32 : _displaced_problem->cacheResidualNeighbor(tid);
1991 50077 : }
1992 :
1993 : void
1994 17008051 : FEProblemBase::addCachedResidual(const THREAD_ID tid)
1995 : {
1996 17008051 : SubProblem::addCachedResidual(tid);
1997 17008051 : if (_displaced_problem)
1998 495407 : _displaced_problem->addCachedResidual(tid);
1999 17008051 : }
2000 :
2001 : void
2002 11332 : FEProblemBase::addCachedResidualDirectly(NumericVector<Number> & residual, const THREAD_ID tid)
2003 : {
2004 11332 : if (_current_nl_sys->hasVector(_current_nl_sys->timeVectorTag()))
2005 30252 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2006 30252 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->timeVectorTag()));
2007 :
2008 11332 : if (_current_nl_sys->hasVector(_current_nl_sys->nonTimeVectorTag()))
2009 33996 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2010 33996 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->nonTimeVectorTag()));
2011 :
2012 11332 : std::vector<VectorTag> extra_residual_vector_tags;
2013 11332 : extra_residual_vector_tags.reserve(currentResidualVectorTags().size());
2014 11332 : const auto time_tag = _current_nl_sys->timeVectorTag();
2015 11332 : const auto non_time_tag = _current_nl_sys->nonTimeVectorTag();
2016 44152 : for (const auto & vector_tag : currentResidualVectorTags())
2017 32820 : if (vector_tag._id != time_tag && vector_tag._id != non_time_tag)
2018 11404 : extra_residual_vector_tags.push_back(vector_tag);
2019 :
2020 : // Flush extra vector tag caches (e.g. from extra_vector_tags on NodalConstraints)
2021 : // to their respective system vectors after the standard TIME/NONTIME caches above.
2022 : // Without this, NodalConstraint contributions to extra vector tags are silently
2023 : // discarded by the blanket clearCachedResiduals.
2024 11332 : _assembly[tid][_current_nl_sys->number()]->addCachedResiduals(Assembly::GlobalDataKey{},
2025 : extra_residual_vector_tags);
2026 :
2027 : // We do this because by adding the cached residual directly, we cannot ensure that all of the
2028 : // cached residuals are emptied after only the two add calls above
2029 11332 : _assembly[tid][_current_nl_sys->number()]->clearCachedResiduals(Assembly::GlobalDataKey{});
2030 :
2031 11332 : if (_displaced_problem)
2032 35 : _displaced_problem->addCachedResidualDirectly(residual, tid);
2033 11332 : }
2034 :
2035 : void
2036 0 : FEProblemBase::setResidual(NumericVector<Number> & residual, const THREAD_ID tid)
2037 : {
2038 0 : _assembly[tid][_current_nl_sys->number()]->setResidual(
2039 : residual,
2040 0 : Assembly::GlobalDataKey{},
2041 0 : getVectorTag(_nl[_current_nl_sys->number()]->residualVectorTag()));
2042 0 : if (_displaced_problem)
2043 0 : _displaced_problem->setResidual(residual, tid);
2044 0 : }
2045 :
2046 : void
2047 0 : FEProblemBase::setResidualNeighbor(NumericVector<Number> & residual, const THREAD_ID tid)
2048 : {
2049 0 : _assembly[tid][_current_nl_sys->number()]->setResidualNeighbor(
2050 0 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->residualVectorTag()));
2051 0 : if (_displaced_problem)
2052 0 : _displaced_problem->setResidualNeighbor(residual, tid);
2053 0 : }
2054 :
2055 : void
2056 37415 : FEProblemBase::addJacobian(const THREAD_ID tid)
2057 : {
2058 37415 : _assembly[tid][_current_nl_sys->number()]->addJacobian(Assembly::GlobalDataKey{});
2059 37415 : if (_has_nonlocal_coupling)
2060 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianNonlocal(Assembly::GlobalDataKey{});
2061 37415 : if (_displaced_problem)
2062 : {
2063 200 : _displaced_problem->addJacobian(tid);
2064 200 : if (_has_nonlocal_coupling)
2065 0 : _displaced_problem->addJacobianNonlocal(tid);
2066 : }
2067 37415 : }
2068 :
2069 : void
2070 7852 : FEProblemBase::addJacobianNeighbor(const THREAD_ID tid)
2071 : {
2072 7852 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighbor(Assembly::GlobalDataKey{});
2073 7852 : if (_displaced_problem)
2074 44 : _displaced_problem->addJacobianNeighbor(tid);
2075 7852 : }
2076 :
2077 : void
2078 109320 : FEProblemBase::addJacobianNeighborLowerD(const THREAD_ID tid)
2079 : {
2080 109320 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborLowerD(Assembly::GlobalDataKey{});
2081 109320 : if (_displaced_problem)
2082 3072 : _displaced_problem->addJacobianNeighborLowerD(tid);
2083 109320 : }
2084 :
2085 : void
2086 4696 : FEProblemBase::addJacobianLowerD(const THREAD_ID tid)
2087 : {
2088 4696 : _assembly[tid][_current_nl_sys->number()]->addJacobianLowerD(Assembly::GlobalDataKey{});
2089 4696 : if (_displaced_problem)
2090 192 : _displaced_problem->addJacobianLowerD(tid);
2091 4696 : }
2092 :
2093 : void
2094 11603 : FEProblemBase::addJacobianScalar(const THREAD_ID tid /* = 0*/)
2095 : {
2096 11603 : _assembly[tid][_current_nl_sys->number()]->addJacobianScalar(Assembly::GlobalDataKey{});
2097 11603 : }
2098 :
2099 : void
2100 30002 : FEProblemBase::addJacobianOffDiagScalar(unsigned int ivar, const THREAD_ID tid /* = 0*/)
2101 : {
2102 60004 : _assembly[tid][_current_nl_sys->number()]->addJacobianOffDiagScalar(ivar,
2103 30002 : Assembly::GlobalDataKey{});
2104 30002 : }
2105 :
2106 : void
2107 46790896 : FEProblemBase::cacheJacobian(const THREAD_ID tid)
2108 : {
2109 46790896 : SubProblem::cacheJacobian(tid);
2110 46790896 : if (_displaced_problem)
2111 1544138 : _displaced_problem->cacheJacobian(tid);
2112 46790896 : }
2113 :
2114 : void
2115 9409 : FEProblemBase::cacheJacobianNeighbor(const THREAD_ID tid)
2116 : {
2117 9409 : SubProblem::cacheJacobianNeighbor(tid);
2118 9409 : if (_displaced_problem)
2119 0 : _displaced_problem->cacheJacobianNeighbor(tid);
2120 9409 : }
2121 :
2122 : void
2123 2892807 : FEProblemBase::addCachedJacobian(const THREAD_ID tid)
2124 : {
2125 2892807 : SubProblem::addCachedJacobian(tid);
2126 2892804 : if (_displaced_problem)
2127 96451 : _displaced_problem->addCachedJacobian(tid);
2128 2892804 : }
2129 :
2130 : void
2131 70832 : FEProblemBase::addJacobianBlockTags(SparseMatrix<Number> & jacobian,
2132 : unsigned int ivar,
2133 : unsigned int jvar,
2134 : const DofMap & dof_map,
2135 : std::vector<dof_id_type> & dof_indices,
2136 : const std::set<TagID> & tags,
2137 : const THREAD_ID tid)
2138 : {
2139 141664 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockTags(
2140 70832 : jacobian, ivar, jvar, dof_map, dof_indices, Assembly::GlobalDataKey{}, tags);
2141 :
2142 70832 : if (_has_nonlocal_coupling)
2143 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2144 : {
2145 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2146 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockNonlocalTags(
2147 : jacobian,
2148 : ivar,
2149 : jvar,
2150 : dof_map,
2151 : dof_indices,
2152 : jv.allDofIndices(),
2153 0 : Assembly::GlobalDataKey{},
2154 : tags);
2155 : }
2156 :
2157 70832 : if (_displaced_problem)
2158 : {
2159 0 : _displaced_problem->addJacobianBlockTags(jacobian, ivar, jvar, dof_map, dof_indices, tags, tid);
2160 0 : if (_has_nonlocal_coupling)
2161 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2162 : {
2163 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2164 0 : _displaced_problem->addJacobianBlockNonlocal(
2165 : jacobian, ivar, jvar, dof_map, dof_indices, jv.allDofIndices(), tags, tid);
2166 : }
2167 : }
2168 70832 : }
2169 :
2170 : void
2171 768 : FEProblemBase::addJacobianNeighbor(SparseMatrix<Number> & jacobian,
2172 : unsigned int ivar,
2173 : unsigned int jvar,
2174 : const DofMap & dof_map,
2175 : std::vector<dof_id_type> & dof_indices,
2176 : std::vector<dof_id_type> & neighbor_dof_indices,
2177 : const std::set<TagID> & tags,
2178 : const THREAD_ID tid)
2179 : {
2180 1536 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborTags(jacobian,
2181 : ivar,
2182 : jvar,
2183 : dof_map,
2184 : dof_indices,
2185 : neighbor_dof_indices,
2186 768 : Assembly::GlobalDataKey{},
2187 : tags);
2188 768 : if (_displaced_problem)
2189 0 : _displaced_problem->addJacobianNeighbor(
2190 : jacobian, ivar, jvar, dof_map, dof_indices, neighbor_dof_indices, tags, tid);
2191 768 : }
2192 :
2193 : void
2194 125946060 : FEProblemBase::prepareShapes(unsigned int var, const THREAD_ID tid)
2195 : {
2196 125946060 : _assembly[tid][_current_nl_sys->number()]->copyShapes(var);
2197 125946060 : }
2198 :
2199 : void
2200 596432 : FEProblemBase::prepareFaceShapes(unsigned int var, const THREAD_ID tid)
2201 : {
2202 596432 : _assembly[tid][_current_nl_sys->number()]->copyFaceShapes(var);
2203 596432 : }
2204 :
2205 : void
2206 185398 : FEProblemBase::prepareNeighborShapes(unsigned int var, const THREAD_ID tid)
2207 : {
2208 185398 : _assembly[tid][_current_nl_sys->number()]->copyNeighborShapes(var);
2209 185398 : }
2210 :
2211 : void
2212 874909 : FEProblemBase::addGhostedElem(dof_id_type elem_id)
2213 : {
2214 874909 : if (_mesh.elemPtr(elem_id)->processor_id() != processor_id())
2215 205736 : _ghosted_elems.insert(elem_id);
2216 874909 : }
2217 :
2218 : void
2219 29442 : FEProblemBase::addGhostedBoundary(BoundaryID boundary_id)
2220 : {
2221 29442 : _mesh.addGhostedBoundary(boundary_id);
2222 29442 : if (_displaced_problem)
2223 26702 : _displaced_mesh->addGhostedBoundary(boundary_id);
2224 29442 : }
2225 :
2226 : void
2227 67698 : FEProblemBase::ghostGhostedBoundaries()
2228 : {
2229 338490 : TIME_SECTION("ghostGhostedBoundaries", 3, "Ghosting Ghosted Boundaries");
2230 :
2231 67698 : _mesh.ghostGhostedBoundaries();
2232 :
2233 67698 : if (_displaced_problem)
2234 2591 : _displaced_mesh->ghostGhostedBoundaries();
2235 67698 : }
2236 :
2237 : void
2238 0 : FEProblemBase::sizeZeroes(unsigned int /*size*/, const THREAD_ID /*tid*/)
2239 : {
2240 0 : mooseDoOnce(mooseWarning(
2241 : "This function is deprecated and no longer performs any function. Please do not call it."));
2242 0 : }
2243 :
2244 : bool
2245 304028 : FEProblemBase::reinitDirac(const Elem * elem, const THREAD_ID tid)
2246 : {
2247 304028 : std::vector<Point> & points = _dirac_kernel_info.getPoints()[elem].first;
2248 :
2249 304028 : unsigned int n_points = points.size();
2250 :
2251 304028 : if (n_points)
2252 : {
2253 299052 : if (n_points > _max_qps)
2254 : {
2255 0 : _max_qps = n_points;
2256 :
2257 : /**
2258 : * The maximum number of qps can rise if several Dirac points are added to a single element.
2259 : * In that case we need to resize the zeros to compensate.
2260 : */
2261 0 : unsigned int max_qpts = getMaxQps();
2262 0 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
2263 : {
2264 : // the highest available order in libMesh is 43
2265 0 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
2266 0 : _zero[tid].resize(max_qpts, 0);
2267 0 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
2268 0 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
2269 0 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
2270 0 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
2271 : }
2272 : }
2273 :
2274 598104 : for (const auto i : index_range(_nl))
2275 : {
2276 299052 : _assembly[tid][i]->reinitAtPhysical(elem, points);
2277 299052 : _nl[i]->prepare(tid);
2278 : }
2279 299052 : _aux->prepare(tid);
2280 :
2281 299052 : reinitElem(elem, tid);
2282 : }
2283 :
2284 304028 : _assembly[tid][_current_nl_sys->number()]->prepare();
2285 304028 : if (_has_nonlocal_coupling)
2286 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
2287 :
2288 304028 : bool have_points = n_points > 0;
2289 304028 : if (_displaced_problem && (_reinit_displaced_elem))
2290 : {
2291 4976 : have_points |= _displaced_problem->reinitDirac(_displaced_mesh->elemPtr(elem->id()), tid);
2292 4976 : if (_has_nonlocal_coupling)
2293 0 : _displaced_problem->prepareNonlocal(tid);
2294 : }
2295 :
2296 304028 : return have_points;
2297 : }
2298 :
2299 : void
2300 375207036 : FEProblemBase::reinitElem(const Elem * elem, const THREAD_ID tid)
2301 : {
2302 753126821 : for (auto & sys : _solver_systems)
2303 377919785 : sys->reinitElem(elem, tid);
2304 375207036 : _aux->reinitElem(elem, tid);
2305 :
2306 375207036 : if (_displaced_problem && _reinit_displaced_elem)
2307 4423675 : _displaced_problem->reinitElem(_displaced_mesh->elemPtr(elem->id()), tid);
2308 375207036 : }
2309 :
2310 : void
2311 58516 : FEProblemBase::reinitElemPhys(const Elem * const elem,
2312 : const std::vector<Point> & phys_points_in_elem,
2313 : const THREAD_ID tid)
2314 : {
2315 : mooseAssert(_mesh.queryElemPtr(elem->id()) == elem,
2316 : "Are you calling this method with a displaced mesh element?");
2317 :
2318 117032 : for (const auto i : index_range(_solver_systems))
2319 : {
2320 58516 : _assembly[tid][i]->reinitAtPhysical(elem, phys_points_in_elem);
2321 58516 : _solver_systems[i]->prepare(tid);
2322 58516 : _assembly[tid][i]->prepare();
2323 58516 : if (_has_nonlocal_coupling)
2324 0 : _assembly[tid][i]->prepareNonlocal();
2325 : }
2326 58516 : _aux->prepare(tid);
2327 :
2328 58516 : reinitElem(elem, tid);
2329 58516 : }
2330 :
2331 : void
2332 0 : FEProblemBase::reinitElemFace(const Elem * const elem,
2333 : const unsigned int side,
2334 : const BoundaryID,
2335 : const THREAD_ID tid)
2336 : {
2337 0 : mooseDeprecated(
2338 : "reinitElemFace with a BoundaryID argument is deprecated because the boundary id was never "
2339 : "used. Please call reinitElemFace without the BoundaryID argument instead");
2340 :
2341 0 : reinitElemFace(elem, side, tid);
2342 0 : }
2343 :
2344 : void
2345 5188323 : FEProblemBase::reinitElemFace(const Elem * const elem, const unsigned int side, const THREAD_ID tid)
2346 : {
2347 10376994 : for (const auto i : index_range(_solver_systems))
2348 : {
2349 5188671 : _assembly[tid][i]->reinit(elem, side);
2350 5188671 : _solver_systems[i]->reinitElemFace(elem, side, tid);
2351 : }
2352 5188323 : _aux->reinitElemFace(elem, side, tid);
2353 :
2354 5188323 : if (_displaced_problem && _reinit_displaced_face)
2355 82248 : _displaced_problem->reinitElemFace(_displaced_mesh->elemPtr(elem->id()), side, tid);
2356 5188323 : }
2357 :
2358 : void
2359 489413 : FEProblemBase::reinitLowerDElem(const Elem * lower_d_elem,
2360 : const THREAD_ID tid,
2361 : const std::vector<Point> * const pts,
2362 : const std::vector<Real> * const weights)
2363 : {
2364 489413 : SubProblem::reinitLowerDElem(lower_d_elem, tid, pts, weights);
2365 :
2366 489413 : if (_displaced_problem && _displaced_mesh)
2367 960 : _displaced_problem->reinitLowerDElem(
2368 960 : _displaced_mesh->elemPtr(lower_d_elem->id()), tid, pts, weights);
2369 489413 : }
2370 :
2371 : void
2372 25953949 : FEProblemBase::reinitNode(const Node * node, const THREAD_ID tid)
2373 : {
2374 25953949 : if (_displaced_problem && _reinit_displaced_elem)
2375 1001711 : _displaced_problem->reinitNode(&_displaced_mesh->nodeRef(node->id()), tid);
2376 :
2377 51910814 : for (const auto i : index_range(_nl))
2378 : {
2379 25956865 : _assembly[tid][i]->reinit(node);
2380 25956865 : _nl[i]->reinitNode(node, tid);
2381 : }
2382 25953949 : _aux->reinitNode(node, tid);
2383 25953949 : }
2384 :
2385 : void
2386 65205553 : FEProblemBase::reinitNodeFace(const Node * node, BoundaryID bnd_id, const THREAD_ID tid)
2387 : {
2388 65205553 : if (_displaced_problem && _reinit_displaced_face)
2389 3388880 : _displaced_problem->reinitNodeFace(&_displaced_mesh->nodeRef(node->id()), bnd_id, tid);
2390 :
2391 131205232 : for (const auto i : index_range(_nl))
2392 : {
2393 65999679 : _assembly[tid][i]->reinit(node);
2394 65999679 : _nl[i]->reinitNodeFace(node, bnd_id, tid);
2395 : }
2396 65205553 : _aux->reinitNodeFace(node, bnd_id, tid);
2397 65205553 : }
2398 :
2399 : void
2400 5097 : FEProblemBase::reinitNodes(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2401 : {
2402 5097 : if (_displaced_problem && _reinit_displaced_elem)
2403 0 : _displaced_problem->reinitNodes(nodes, tid);
2404 :
2405 10194 : for (auto & nl : _nl)
2406 5097 : nl->reinitNodes(nodes, tid);
2407 5097 : _aux->reinitNodes(nodes, tid);
2408 5097 : }
2409 :
2410 : void
2411 1003 : FEProblemBase::reinitNodesNeighbor(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2412 : {
2413 1003 : if (_displaced_problem && _reinit_displaced_elem)
2414 0 : _displaced_problem->reinitNodesNeighbor(nodes, tid);
2415 :
2416 2006 : for (auto & nl : _nl)
2417 1003 : nl->reinitNodesNeighbor(nodes, tid);
2418 1003 : _aux->reinitNodesNeighbor(nodes, tid);
2419 1003 : }
2420 :
2421 : void
2422 8439441 : FEProblemBase::reinitScalars(const THREAD_ID tid, bool reinit_for_derivative_reordering /*=false*/)
2423 : {
2424 42197205 : TIME_SECTION("reinitScalars", 3, "Reinitializing Scalar Variables");
2425 :
2426 8439441 : if (_displaced_problem && _reinit_displaced_elem)
2427 113549 : _displaced_problem->reinitScalars(tid, reinit_for_derivative_reordering);
2428 :
2429 17115268 : for (auto & nl : _nl)
2430 8675827 : nl->reinitScalars(tid, reinit_for_derivative_reordering);
2431 8439441 : _aux->reinitScalars(tid, reinit_for_derivative_reordering);
2432 :
2433 : // This is called outside of residual/Jacobian call-backs
2434 17120592 : for (auto & assembly : _assembly[tid])
2435 8681151 : assembly->prepareScalar();
2436 8439441 : }
2437 :
2438 : void
2439 185551 : FEProblemBase::reinitOffDiagScalars(const THREAD_ID tid)
2440 : {
2441 185551 : _assembly[tid][_current_nl_sys->number()]->prepareOffDiagScalar();
2442 185551 : if (_displaced_problem)
2443 60 : _displaced_problem->reinitOffDiagScalars(tid);
2444 185551 : }
2445 :
2446 : void
2447 3643016 : FEProblemBase::reinitNeighbor(const Elem * elem, unsigned int side, const THREAD_ID tid)
2448 : {
2449 3643016 : setNeighborSubdomainID(elem, side, tid);
2450 :
2451 3643016 : const Elem * neighbor = elem->neighbor_ptr(side);
2452 3643016 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(elem);
2453 :
2454 7286059 : for (const auto i : index_range(_nl))
2455 : {
2456 3643043 : _assembly[tid][i]->reinitElemAndNeighbor(elem, side, neighbor, neighbor_side);
2457 3643043 : _nl[i]->prepareNeighbor(tid);
2458 : // Called during stateful material property evaluation outside of solve
2459 3643043 : _assembly[tid][i]->prepareNeighbor();
2460 : }
2461 3643016 : _aux->prepareNeighbor(tid);
2462 :
2463 7286059 : for (auto & nl : _nl)
2464 : {
2465 3643043 : nl->reinitElemFace(elem, side, tid);
2466 3643043 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2467 : }
2468 3643016 : _aux->reinitElemFace(elem, side, tid);
2469 3643016 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2470 :
2471 3643016 : if (_displaced_problem && _reinit_displaced_neighbor)
2472 : {
2473 : // There are cases like for cohesive zone modeling without significant sliding where we cannot
2474 : // use FEInterface::inverse_map in Assembly::reinitElemAndNeighbor in the displaced problem
2475 : // because the physical points coming from the element don't actually lie on the neighbor.
2476 : // Moreover, what's the point of doing another physical point inversion in other cases? We only
2477 : // care about the reference points which we can just take from the undisplaced computation
2478 64800 : const auto & displaced_ref_pts = _assembly[tid][0]->qRuleNeighbor()->get_points();
2479 :
2480 64800 : _displaced_problem->reinitNeighbor(
2481 64800 : _displaced_mesh->elemPtr(elem->id()), side, tid, &displaced_ref_pts);
2482 : }
2483 3643016 : }
2484 :
2485 : void
2486 2069965 : FEProblemBase::reinitElemNeighborAndLowerD(const Elem * elem,
2487 : unsigned int side,
2488 : const THREAD_ID tid)
2489 : {
2490 2069965 : reinitNeighbor(elem, side, tid);
2491 :
2492 2069965 : const Elem * lower_d_elem = _mesh.getLowerDElem(elem, side);
2493 2069965 : if (lower_d_elem && _mesh.interiorLowerDBlocks().count(lower_d_elem->subdomain_id()) > 0)
2494 10332 : reinitLowerDElem(lower_d_elem, tid);
2495 : else
2496 : {
2497 : // with mesh refinement, lower-dimensional element might be defined on neighbor side
2498 2059633 : auto & neighbor = _assembly[tid][0]->neighbor();
2499 2059633 : auto & neighbor_side = _assembly[tid][0]->neighborSide();
2500 2059633 : const Elem * lower_d_elem_neighbor = _mesh.getLowerDElem(neighbor, neighbor_side);
2501 2059633 : if (lower_d_elem_neighbor &&
2502 2059633 : _mesh.interiorLowerDBlocks().count(lower_d_elem_neighbor->subdomain_id()) > 0)
2503 : {
2504 0 : auto qps = _assembly[tid][0]->qPointsFaceNeighbor().stdVector();
2505 0 : std::vector<Point> reference_points;
2506 0 : FEMap::inverse_map(
2507 0 : lower_d_elem_neighbor->dim(), lower_d_elem_neighbor, qps, reference_points);
2508 0 : reinitLowerDElem(lower_d_elem_neighbor, tid, &reference_points);
2509 0 : }
2510 : }
2511 :
2512 2069965 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
2513 64740 : _displaced_problem->reinitElemNeighborAndLowerD(
2514 64740 : _displaced_mesh->elemPtr(elem->id()), side, tid);
2515 2069965 : }
2516 :
2517 : void
2518 97230 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2519 : unsigned int neighbor_side,
2520 : const std::vector<Point> & physical_points,
2521 : const THREAD_ID tid)
2522 : {
2523 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2524 : "Are you calling this method with a displaced mesh element?");
2525 :
2526 194460 : for (const auto i : index_range(_nl))
2527 : {
2528 : // Reinits shape the functions at the physical points
2529 97230 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, neighbor_side, physical_points);
2530 :
2531 : // Sets the neighbor dof indices
2532 97230 : _nl[i]->prepareNeighbor(tid);
2533 : }
2534 97230 : _aux->prepareNeighbor(tid);
2535 :
2536 : // Resizes Re and Ke
2537 97230 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2538 :
2539 : // Compute the values of each variable at the points
2540 194460 : for (auto & nl : _nl)
2541 97230 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2542 97230 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2543 97230 : }
2544 :
2545 : void
2546 19880 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2547 : const std::vector<Point> & physical_points,
2548 : const THREAD_ID tid)
2549 : {
2550 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2551 : "Are you calling this method with a displaced mesh element?");
2552 :
2553 39760 : for (const auto i : index_range(_nl))
2554 : {
2555 : // Reinits shape the functions at the physical points
2556 19880 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, physical_points);
2557 :
2558 : // Sets the neighbor dof indices
2559 19880 : _nl[i]->prepareNeighbor(tid);
2560 : }
2561 19880 : _aux->prepareNeighbor(tid);
2562 :
2563 : // Resizes Re and Ke
2564 19880 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2565 :
2566 : // Compute the values of each variable at the points
2567 39760 : for (auto & nl : _nl)
2568 19880 : nl->reinitNeighbor(neighbor, tid);
2569 19880 : _aux->reinitNeighbor(neighbor, tid);
2570 19880 : }
2571 :
2572 : void
2573 35518 : FEProblemBase::getDiracElements(std::set<const Elem *> & elems)
2574 : {
2575 : // First add in the undisplaced elements
2576 35518 : elems = _dirac_kernel_info.getElements();
2577 :
2578 35518 : if (_displaced_problem)
2579 : {
2580 2359 : std::set<const Elem *> displaced_elements;
2581 2359 : _displaced_problem->getDiracElements(displaced_elements);
2582 :
2583 : { // Use the ids from the displaced elements to get the undisplaced elements
2584 : // and add them to the list
2585 7335 : for (const auto & elem : displaced_elements)
2586 4976 : elems.insert(_mesh.elemPtr(elem->id()));
2587 : }
2588 2359 : }
2589 35518 : }
2590 :
2591 : void
2592 3520923 : FEProblemBase::clearDiracInfo()
2593 : {
2594 3520923 : _dirac_kernel_info.clearPoints();
2595 :
2596 3520923 : if (_displaced_problem)
2597 144138 : _displaced_problem->clearDiracInfo();
2598 3520923 : }
2599 :
2600 : void
2601 5096411 : FEProblemBase::subdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2602 : {
2603 5096411 : _all_materials.subdomainSetup(subdomain, tid);
2604 : // Call the subdomain methods of the output system, these are not threaded so only call it once
2605 5096411 : if (tid == 0)
2606 5083659 : _app.getOutputWarehouse().subdomainSetup();
2607 :
2608 10326649 : for (auto & nl : _nl)
2609 5230238 : nl->subdomainSetup(subdomain, tid);
2610 :
2611 : // FIXME: call displaced_problem->subdomainSetup() ?
2612 : // When adding possibility with materials being evaluated on displaced mesh
2613 5096411 : }
2614 :
2615 : void
2616 16555076 : FEProblemBase::neighborSubdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2617 : {
2618 16555076 : _all_materials.neighborSubdomainSetup(subdomain, tid);
2619 16555076 : }
2620 :
2621 : void
2622 49454 : FEProblemBase::addFunction(const std::string & type,
2623 : const std::string & name,
2624 : InputParameters & parameters)
2625 : {
2626 : parallel_object_only();
2627 :
2628 98908 : parameters.set<SubProblem *>("_subproblem") = this;
2629 :
2630 103238 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2631 : {
2632 53890 : std::shared_ptr<Function> func = _factory.create<Function>(type, name, parameters, tid);
2633 53784 : logAdd("Function", name, type, parameters);
2634 53784 : _functions.addObject(func, tid);
2635 :
2636 53784 : if (auto * const functor = dynamic_cast<Moose::FunctorBase<Real> *>(func.get()))
2637 : {
2638 53784 : this->addFunctor(name, *functor, tid);
2639 53784 : if (_displaced_problem)
2640 1780 : _displaced_problem->addFunctor(name, *functor, tid);
2641 : }
2642 : else
2643 0 : mooseError("Unrecognized function functor type");
2644 53784 : }
2645 49348 : }
2646 :
2647 : void
2648 152792 : FEProblemBase::addConvergence(const std::string & type,
2649 : const std::string & name,
2650 : InputParameters & parameters)
2651 : {
2652 : parallel_object_only();
2653 :
2654 322009 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2655 : {
2656 169244 : std::shared_ptr<Convergence> conv = _factory.create<Convergence>(type, name, parameters, tid);
2657 169217 : _convergences.addObject(conv, tid);
2658 169217 : }
2659 152765 : }
2660 :
2661 : void
2662 59746 : FEProblemBase::addDefaultNonlinearConvergence(const InputParameters & params_to_apply)
2663 : {
2664 59746 : const std::string class_name = "DefaultNonlinearConvergence";
2665 59746 : InputParameters params = _factory.getValidParams(class_name);
2666 59746 : params.applyParameters(params_to_apply);
2667 59746 : params.applyParameters(parameters());
2668 59746 : params.set<bool>("added_as_default") = true;
2669 118725 : for (const auto & conv_name : getNonlinearConvergenceNames())
2670 58979 : addConvergence(class_name, conv_name, params);
2671 59746 : }
2672 :
2673 : void
2674 61706 : FEProblemBase::addDefaultMultiAppFixedPointConvergence(const InputParameters & params_to_apply)
2675 : {
2676 61706 : const std::string class_name = "DefaultMultiAppFixedPointConvergence";
2677 61706 : InputParameters params = _factory.getValidParams(class_name);
2678 61706 : params.applyParameters(params_to_apply);
2679 61706 : params.applyParameters(parameters());
2680 61706 : params.set<bool>("added_as_default") = true;
2681 61706 : addConvergence(class_name, getMultiAppFixedPointConvergenceName(), params);
2682 61697 : }
2683 :
2684 : void
2685 30314 : FEProblemBase::addDefaultSteadyStateConvergence(const InputParameters & params_to_apply)
2686 : {
2687 30314 : const std::string class_name = "DefaultSteadyStateConvergence";
2688 30314 : InputParameters params = _factory.getValidParams(class_name);
2689 30314 : params.applyParameters(params_to_apply);
2690 30314 : params.applyParameters(parameters());
2691 30314 : params.set<bool>("added_as_default") = true;
2692 30314 : addConvergence(class_name, getSteadyStateConvergenceName(), params);
2693 30314 : }
2694 :
2695 : bool
2696 86830 : FEProblemBase::hasFunction(const std::string & name, const THREAD_ID tid)
2697 : {
2698 86830 : return _functions.hasActiveObject(name, tid);
2699 : }
2700 :
2701 : Function &
2702 63085 : FEProblemBase::getFunction(const std::string & name, const THREAD_ID tid)
2703 : {
2704 : // This thread lock is necessary since this method will create functions
2705 : // for all threads if one is missing.
2706 63085 : Threads::spin_mutex::scoped_lock lock(get_function_mutex);
2707 :
2708 63085 : if (!hasFunction(name, tid))
2709 : {
2710 : // If we didn't find a function, it might be a default function, attempt to construct one now
2711 20435 : std::istringstream ss(name);
2712 : Real real_value;
2713 :
2714 : // First see if it's just a constant. If it is, build a ConstantFunction
2715 20435 : if (ss >> real_value && ss.eof())
2716 : {
2717 12758 : InputParameters params = _factory.getValidParams("ConstantFunction");
2718 12758 : params.set<Real>("value") = real_value;
2719 19137 : addFunction("ConstantFunction", ss.str(), params);
2720 6379 : }
2721 : else
2722 : {
2723 14056 : FunctionParserBase<Real> fp;
2724 14056 : std::string vars = "x,y,z,t,NaN,pi,e";
2725 14056 : if (fp.Parse(name, vars) == -1) // -1 for success
2726 : {
2727 : // It parsed ok, so build a MooseParsedFunction
2728 42153 : InputParameters params = _factory.getValidParams("ParsedFunction");
2729 14051 : params.set<std::string>("expression") = name;
2730 28102 : addFunction("ParsedFunction", name, params);
2731 14051 : }
2732 14056 : }
2733 :
2734 : // Try once more
2735 20435 : if (!hasFunction(name, tid))
2736 : {
2737 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_function"),
2738 : "getFunction() was called before Functions have been constructed. The requested "
2739 : "Function '" +
2740 : name + "' may exist in the input file, but Functions are not available yet.");
2741 :
2742 7 : mooseError("Unable to find function " + name);
2743 : }
2744 20432 : }
2745 :
2746 63080 : auto * const ret = dynamic_cast<Function *>(_functions.getActiveObject(name, tid).get());
2747 63080 : if (!ret)
2748 0 : mooseError("No function named ", name, " of appropriate type");
2749 :
2750 63080 : return *ret;
2751 63082 : }
2752 :
2753 : bool
2754 211658 : FEProblemBase::hasConvergence(const std::string & name, const THREAD_ID tid) const
2755 : {
2756 211658 : return _convergences.hasActiveObject(name, tid);
2757 : }
2758 :
2759 : Convergence &
2760 1133847 : FEProblemBase::getConvergence(const std::string & name, const THREAD_ID tid) const
2761 : {
2762 1133847 : auto * const ret = dynamic_cast<Convergence *>(_convergences.getActiveObject(name, tid).get());
2763 1133847 : if (!ret)
2764 0 : mooseError("The Convergence object '", name, "' does not exist.");
2765 :
2766 1133847 : return *ret;
2767 : }
2768 :
2769 : const std::vector<std::shared_ptr<Convergence>> &
2770 193914 : FEProblemBase::getConvergenceObjects(const THREAD_ID tid) const
2771 : {
2772 193914 : return _convergences.getActiveObjects(tid);
2773 : }
2774 :
2775 : void
2776 687 : FEProblemBase::addMeshDivision(const std::string & type,
2777 : const std::string & name,
2778 : InputParameters & parameters)
2779 : {
2780 : parallel_object_only();
2781 1374 : parameters.set<FEProblemBase *>("_fe_problem_base") = this;
2782 1374 : parameters.set<SubProblem *>("_subproblem") = this;
2783 1462 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2784 : {
2785 784 : std::shared_ptr<MeshDivision> func = _factory.create<MeshDivision>(type, name, parameters, tid);
2786 775 : _mesh_divisions.addObject(func, tid);
2787 775 : }
2788 678 : }
2789 :
2790 : MeshDivision &
2791 1567 : FEProblemBase::getMeshDivision(const std::string & name, const THREAD_ID tid) const
2792 : {
2793 1567 : auto * const ret = dynamic_cast<MeshDivision *>(_mesh_divisions.getActiveObject(name, tid).get());
2794 1567 : if (!ret)
2795 0 : mooseError("No MeshDivision object named ", name, " of appropriate type");
2796 1567 : return *ret;
2797 : }
2798 :
2799 : void
2800 0 : FEProblemBase::lineSearch()
2801 : {
2802 0 : _line_search->lineSearch();
2803 0 : }
2804 :
2805 : NonlinearSystem &
2806 0 : FEProblemBase::getNonlinearSystem(const unsigned int sys_num)
2807 : {
2808 0 : mooseDeprecated("FEProblemBase::getNonlinearSystem() is deprecated, please use "
2809 : "FEProblemBase::getNonlinearSystemBase() \n");
2810 :
2811 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
2812 0 : auto nl_sys = std::dynamic_pointer_cast<NonlinearSystem>(_nl[sys_num]);
2813 :
2814 0 : if (!nl_sys)
2815 0 : mooseError("This is not a NonlinearSystem");
2816 :
2817 0 : return *nl_sys;
2818 0 : }
2819 :
2820 : void
2821 2 : FEProblemBase::addDistribution(const std::string & type,
2822 : const std::string & name,
2823 : InputParameters & parameters)
2824 : {
2825 4 : parameters.set<std::string>("type") = type;
2826 2 : addObject<Distribution>(type, name, parameters, /* threaded = */ false);
2827 2 : }
2828 :
2829 : bool
2830 4 : FEProblemBase::hasDistribution(const std::string & name) const
2831 : {
2832 4 : std::vector<Distribution *> objs;
2833 4 : theWarehouse()
2834 8 : .query()
2835 4 : .condition<AttribSystem>("Distribution")
2836 4 : .condition<AttribName>(name)
2837 4 : .queryInto(objs);
2838 8 : return !objs.empty();
2839 4 : }
2840 :
2841 : Distribution &
2842 4 : FEProblemBase::getDistribution(const std::string & name)
2843 : {
2844 4 : std::vector<Distribution *> objs;
2845 4 : theWarehouse()
2846 8 : .query()
2847 4 : .condition<AttribSystem>("Distribution")
2848 4 : .condition<AttribName>(name)
2849 4 : .queryInto(objs);
2850 4 : if (objs.empty())
2851 : {
2852 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_distribution"),
2853 : "A Distribution getter was called before Distributions have been constructed. "
2854 : "If you are attempting to access this object in the constructor of another object "
2855 : "then make sure that the Distribution is constructed before the object using it.");
2856 0 : mooseError("Unable to find Distribution with name '" + name + "'");
2857 : }
2858 8 : return *(objs[0]);
2859 4 : }
2860 :
2861 : void
2862 275 : FEProblemBase::addSampler(const std::string & type,
2863 : const std::string & name,
2864 : InputParameters & parameters)
2865 : {
2866 275 : const auto samplers = addObject<Sampler>(type, name, parameters);
2867 569 : for (auto & sampler : samplers)
2868 303 : sampler->init();
2869 266 : }
2870 :
2871 : Sampler &
2872 266 : FEProblemBase::getSampler(const std::string & name, const THREAD_ID tid)
2873 : {
2874 266 : std::vector<Sampler *> objs;
2875 266 : theWarehouse()
2876 532 : .query()
2877 266 : .condition<AttribSystem>("Sampler")
2878 266 : .condition<AttribThread>(tid)
2879 266 : .condition<AttribName>(name)
2880 266 : .queryInto(objs);
2881 266 : if (objs.empty())
2882 : {
2883 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_sampler"),
2884 : "A Sampler getter was called before Samplers have been constructed. "
2885 : "If you are attempting to access this object in the constructor of another object "
2886 : "then make sure that the Sampler is constructed before the object using it.");
2887 :
2888 0 : mooseError(
2889 0 : "Unable to find Sampler with name '" + name +
2890 : "', if you are attempting to access this object in the constructor of another object then "
2891 : "make sure that the Sampler is constructed before the object using it.");
2892 : }
2893 532 : return *(objs[0]);
2894 266 : }
2895 :
2896 : bool
2897 154176 : FEProblemBase::duplicateVariableCheck(const std::string & var_name,
2898 : const FEType & type,
2899 : bool is_aux,
2900 : const std::set<SubdomainID> * const active_subdomains)
2901 : {
2902 154176 : std::set<SubdomainID> subdomainIDs;
2903 154176 : if (active_subdomains->size() == 0)
2904 : {
2905 145897 : const auto subdomains = _mesh.meshSubdomains();
2906 145897 : subdomainIDs.insert(subdomains.begin(), subdomains.end());
2907 145897 : }
2908 : else
2909 8279 : subdomainIDs.insert(active_subdomains->begin(), active_subdomains->end());
2910 :
2911 309131 : for (auto & sys : _solver_systems)
2912 : {
2913 154967 : SystemBase * curr_sys_ptr = sys.get();
2914 154967 : SystemBase * other_sys_ptr = _aux.get();
2915 154967 : std::string error_prefix = "";
2916 154967 : if (is_aux)
2917 : {
2918 94165 : curr_sys_ptr = _aux.get();
2919 94165 : other_sys_ptr = sys.get();
2920 94165 : error_prefix = "aux";
2921 : }
2922 :
2923 154967 : if (other_sys_ptr->hasVariable(var_name))
2924 3 : mooseError("Cannot have an auxiliary variable and a solver variable with the same name: ",
2925 : var_name);
2926 :
2927 154964 : if (curr_sys_ptr->hasVariable(var_name))
2928 : {
2929 : const Variable & var =
2930 9 : curr_sys_ptr->system().variable(curr_sys_ptr->system().variable_number(var_name));
2931 :
2932 : // variable type
2933 9 : if (var.type() != type)
2934 : {
2935 12 : const auto stringifyType = [](FEType t)
2936 12 : { return Moose::stringify(t.family) + " of order " + Moose::stringify(t.order); };
2937 :
2938 6 : mooseError("Mismatching types are specified for ",
2939 : error_prefix,
2940 : "variable with name '",
2941 : var_name,
2942 : "': '",
2943 6 : stringifyType(var.type()),
2944 : "' and '",
2945 6 : stringifyType(type),
2946 : "'");
2947 : }
2948 :
2949 : // block-restriction
2950 3 : if (!(active_subdomains->size() == 0 && var.active_subdomains().size() == 0))
2951 : {
2952 3 : const auto varActiveSubdomains = var.active_subdomains();
2953 3 : std::set<SubdomainID> varSubdomainIDs;
2954 3 : if (varActiveSubdomains.size() == 0)
2955 : {
2956 0 : const auto subdomains = _mesh.meshSubdomains();
2957 0 : varSubdomainIDs.insert(subdomains.begin(), subdomains.end());
2958 0 : }
2959 : else
2960 3 : varSubdomainIDs.insert(varActiveSubdomains.begin(), varActiveSubdomains.end());
2961 :
2962 : // Is subdomainIDs a subset of varSubdomainIDs? With this we allow the case that the newly
2963 : // requested block restriction is only a subset of the existing one.
2964 3 : const auto isSubset = std::includes(varSubdomainIDs.begin(),
2965 : varSubdomainIDs.end(),
2966 : subdomainIDs.begin(),
2967 : subdomainIDs.end());
2968 :
2969 3 : if (!isSubset)
2970 : {
2971 : // helper function: make a string from a set of subdomain ids
2972 6 : const auto stringifySubdomains = [this](std::set<SubdomainID> subdomainIDs)
2973 : {
2974 6 : std::stringstream s;
2975 15 : for (auto const i : subdomainIDs)
2976 : {
2977 : // do we need to insert a comma?
2978 9 : if (s.tellp() != 0)
2979 3 : s << ", ";
2980 :
2981 : // insert subdomain name and id -or- only the id (if no name is given)
2982 9 : const auto subdomainName = _mesh.getSubdomainName(i);
2983 9 : if (subdomainName.empty())
2984 9 : s << i;
2985 : else
2986 0 : s << subdomainName << " (" << i << ")";
2987 9 : }
2988 12 : return s.str();
2989 6 : };
2990 :
2991 6 : const std::string msg = "Mismatching block-restrictions are specified for " +
2992 6 : error_prefix + "variable with name '" + var_name + "': {" +
2993 12 : stringifySubdomains(varSubdomainIDs) + "} and {" +
2994 9 : stringifySubdomains(subdomainIDs) + "}";
2995 :
2996 3 : mooseError(msg);
2997 0 : }
2998 0 : }
2999 :
3000 0 : return true;
3001 : }
3002 154955 : }
3003 :
3004 154164 : return false;
3005 154164 : }
3006 :
3007 : void
3008 60143 : FEProblemBase::addVariable(const std::string & var_type,
3009 : const std::string & var_name,
3010 : InputParameters & params)
3011 : {
3012 : parallel_object_only();
3013 :
3014 60143 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3015 60143 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3016 60143 : const auto fe_type = FEType(order, family);
3017 :
3018 : const auto active_subdomains_vector =
3019 60143 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3020 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3021 60143 : active_subdomains_vector.end());
3022 :
3023 60143 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ false, &active_subdomains))
3024 0 : return;
3025 :
3026 180420 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3027 60140 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_SOLVER;
3028 60140 : SolverSystemName sys_name = params.get<SolverSystemName>("solver_sys");
3029 :
3030 60140 : const auto solver_system_number = solverSysNum(sys_name);
3031 60140 : logAdd("Variable", var_name, var_type, params);
3032 60140 : _solver_systems[solver_system_number]->addVariable(var_type, var_name, params);
3033 60128 : if (_displaced_problem)
3034 : // MooseObjects need to be unique so change the name here
3035 3453 : _displaced_problem->addVariable(var_type, var_name, params, solver_system_number);
3036 :
3037 60128 : _solver_var_to_sys_num[var_name] = solver_system_number;
3038 :
3039 60128 : markFamilyPRefinement(params);
3040 60128 : if (_displaced_problem)
3041 3453 : _displaced_problem->markFamilyPRefinement(params);
3042 60128 : }
3043 :
3044 : std::pair<bool, unsigned int>
3045 4556000 : FEProblemBase::determineSolverSystem(const std::string & var_name,
3046 : const bool error_if_not_found) const
3047 : {
3048 4556000 : auto map_it = _solver_var_to_sys_num.find(var_name);
3049 4556000 : const bool var_in_sys = map_it != _solver_var_to_sys_num.end();
3050 4556000 : if (var_in_sys)
3051 : mooseAssert(_solver_systems[map_it->second]->hasVariable(var_name) ||
3052 : _solver_systems[map_it->second]->hasScalarVariable(var_name),
3053 : "If the variable is in our FEProblem solver system map, then it must be in the "
3054 : "solver system we expect");
3055 3102333 : else if (error_if_not_found)
3056 : {
3057 32 : if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
3058 21 : mooseError("No solver variable named ",
3059 : var_name,
3060 : " found. Did you specify an auxiliary variable when you meant to specify a "
3061 : "solver variable?");
3062 : else
3063 11 : mooseError("Unknown variable '",
3064 : var_name,
3065 : "'. It does not exist in the solver system(s) or auxiliary system");
3066 : }
3067 :
3068 9111936 : return std::make_pair(var_in_sys, var_in_sys ? map_it->second : libMesh::invalid_uint);
3069 : }
3070 :
3071 : void
3072 160433 : FEProblemBase::setResidualObjectParamsAndLog(const std::string & ro_name,
3073 : const std::string & name,
3074 : InputParameters & parameters,
3075 : const unsigned int nl_sys_num,
3076 : const std::string & base_name,
3077 : bool & reinit_displaced)
3078 : {
3079 160433 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3080 : {
3081 1920 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3082 1920 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3083 960 : reinit_displaced = true;
3084 : }
3085 : else
3086 : {
3087 159473 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3088 : {
3089 : // We allow Kernels to request that they use_displaced_mesh,
3090 : // but then be overridden when no displacements variables are
3091 : // provided in the Mesh block. If that happened, update the value
3092 : // of use_displaced_mesh appropriately for this Kernel.
3093 105 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3094 210 : parameters.set<bool>("use_displaced_mesh") = false;
3095 : }
3096 :
3097 318946 : parameters.set<SubProblem *>("_subproblem") = this;
3098 478419 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3099 : }
3100 :
3101 160433 : logAdd(base_name, name, ro_name, parameters);
3102 160433 : }
3103 :
3104 : void
3105 65139 : FEProblemBase::setAuxKernelParamsAndLog(const std::string & ak_name,
3106 : const std::string & name,
3107 : InputParameters & parameters,
3108 : const std::string & base_name)
3109 : {
3110 65139 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3111 : {
3112 22872 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3113 22872 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3114 22872 : parameters.set<SystemBase *>("_nl_sys") = &_displaced_problem->solverSys(0);
3115 11436 : if (!parameters.get<std::vector<BoundaryName>>("boundary").empty())
3116 10990 : _reinit_displaced_face = true;
3117 : else
3118 446 : _reinit_displaced_elem = true;
3119 : }
3120 : else
3121 : {
3122 53703 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3123 : {
3124 : // We allow AuxKernels to request that they use_displaced_mesh,
3125 : // but then be overridden when no displacements variables are
3126 : // provided in the Mesh block. If that happened, update the value
3127 : // of use_displaced_mesh appropriately for this AuxKernel.
3128 795 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3129 1590 : parameters.set<bool>("use_displaced_mesh") = false;
3130 : }
3131 :
3132 107406 : parameters.set<SubProblem *>("_subproblem") = this;
3133 107406 : parameters.set<SystemBase *>("_sys") = _aux.get();
3134 161109 : parameters.set<SystemBase *>("_nl_sys") = _solver_systems[0].get();
3135 : }
3136 :
3137 65139 : logAdd(base_name, name, ak_name, parameters);
3138 65139 : }
3139 :
3140 : void
3141 77251 : FEProblemBase::addKernel(const std::string & kernel_name,
3142 : const std::string & name,
3143 : InputParameters & parameters)
3144 : {
3145 : parallel_object_only();
3146 154502 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3147 77239 : if (!isSolverSystemNonlinear(nl_sys_num))
3148 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
3149 : "supported at the moment!");
3150 77239 : setResidualObjectParamsAndLog(
3151 77239 : kernel_name, name, parameters, nl_sys_num, "Kernel", _reinit_displaced_elem);
3152 :
3153 77239 : _nl[nl_sys_num]->addKernel(kernel_name, name, parameters);
3154 77086 : }
3155 :
3156 : void
3157 431 : FEProblemBase::addHDGKernel(const std::string & kernel_name,
3158 : const std::string & name,
3159 : InputParameters & parameters)
3160 : {
3161 : parallel_object_only();
3162 862 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3163 431 : if (!isSolverSystemNonlinear(nl_sys_num))
3164 0 : mooseError("You are trying to add a HDGKernel to a linear variable/system, which is not "
3165 : "supported at the moment!");
3166 431 : setResidualObjectParamsAndLog(
3167 431 : kernel_name, name, parameters, nl_sys_num, "HDGKernel", _reinit_displaced_elem);
3168 :
3169 431 : _nl[nl_sys_num]->addHDGKernel(kernel_name, name, parameters);
3170 431 : }
3171 :
3172 : void
3173 602 : FEProblemBase::addNodalKernel(const std::string & kernel_name,
3174 : const std::string & name,
3175 : InputParameters & parameters)
3176 : {
3177 : parallel_object_only();
3178 :
3179 1204 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3180 599 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3181 : {
3182 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3183 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3184 0 : _reinit_displaced_elem = true;
3185 : }
3186 : else
3187 : {
3188 599 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3189 : {
3190 : // We allow NodalKernels to request that they use_displaced_mesh,
3191 : // but then be overridden when no displacements variables are
3192 : // provided in the Mesh block. If that happened, update the value
3193 : // of use_displaced_mesh appropriately for this NodalKernel.
3194 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3195 0 : parameters.set<bool>("use_displaced_mesh") = false;
3196 : }
3197 :
3198 1198 : parameters.set<SubProblem *>("_subproblem") = this;
3199 1797 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3200 : }
3201 599 : logAdd("NodalKernel", name, kernel_name, parameters);
3202 599 : _nl[nl_sys_num]->addNodalKernel(kernel_name, name, parameters);
3203 599 : }
3204 :
3205 : void
3206 1319 : FEProblemBase::addScalarKernel(const std::string & kernel_name,
3207 : const std::string & name,
3208 : InputParameters & parameters)
3209 : {
3210 : parallel_object_only();
3211 :
3212 2638 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3213 1319 : if (!isSolverSystemNonlinear(nl_sys_num))
3214 0 : mooseError("You are trying to add a ScalarKernel to a linear variable/system, which is not "
3215 : "supported at the moment!");
3216 :
3217 1319 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3218 : {
3219 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3220 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3221 : }
3222 : else
3223 : {
3224 1319 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3225 : {
3226 : // We allow ScalarKernels to request that they use_displaced_mesh,
3227 : // but then be overridden when no displacements variables are
3228 : // provided in the Mesh block. If that happened, update the value
3229 : // of use_displaced_mesh appropriately for this ScalarKernel.
3230 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3231 0 : parameters.set<bool>("use_displaced_mesh") = false;
3232 : }
3233 :
3234 2638 : parameters.set<SubProblem *>("_subproblem") = this;
3235 3957 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3236 : }
3237 :
3238 1319 : logAdd("ScalarKernel", name, kernel_name, parameters);
3239 1319 : _nl[nl_sys_num]->addScalarKernel(kernel_name, name, parameters);
3240 1313 : }
3241 :
3242 : void
3243 75263 : FEProblemBase::addBoundaryCondition(const std::string & bc_name,
3244 : const std::string & name,
3245 : InputParameters & parameters)
3246 : {
3247 : parallel_object_only();
3248 :
3249 150530 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3250 75255 : if (!isSolverSystemNonlinear(nl_sys_num))
3251 0 : mooseError(
3252 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
3253 : "supported at the moment!");
3254 :
3255 75255 : setResidualObjectParamsAndLog(
3256 75255 : bc_name, name, parameters, nl_sys_num, "BoundaryCondition", _reinit_displaced_face);
3257 75255 : _nl[nl_sys_num]->addBoundaryCondition(bc_name, name, parameters);
3258 75211 : }
3259 :
3260 : void
3261 1713 : FEProblemBase::addConstraint(const std::string & c_name,
3262 : const std::string & name,
3263 : InputParameters & parameters)
3264 : {
3265 : parallel_object_only();
3266 :
3267 1713 : _has_constraints = true;
3268 :
3269 1713 : auto determine_var_param_name = [¶meters, this]()
3270 : {
3271 5139 : if (parameters.isParamValid("variable"))
3272 1152 : return "variable";
3273 : else
3274 : {
3275 : // must be a mortar constraint
3276 1122 : const bool has_secondary_var = parameters.isParamValid("secondary_variable");
3277 1122 : const bool has_primary_var = parameters.isParamValid("primary_variable");
3278 561 : if (!has_secondary_var && !has_primary_var)
3279 0 : mooseError(
3280 : "Either a 'secondary_variable' or 'primary_variable' parameter must be supplied for '",
3281 0 : parameters.getObjectName(),
3282 : "'");
3283 561 : return has_secondary_var ? "secondary_variable" : "primary_variable";
3284 : }
3285 1713 : };
3286 :
3287 : const auto nl_sys_num =
3288 3426 : determineSolverSystem(parameters.varName(determine_var_param_name(), name), true).second;
3289 1710 : if (!isSolverSystemNonlinear(nl_sys_num))
3290 0 : mooseError("You are trying to add a Constraint to a linear variable/system, which is not "
3291 : "supported at the moment!");
3292 :
3293 1710 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3294 : {
3295 266 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3296 266 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3297 133 : _reinit_displaced_face = true;
3298 : }
3299 : else
3300 : {
3301 : // It might _want_ to use a displaced mesh... but we're not so set it to false
3302 1577 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3303 3154 : parameters.set<bool>("use_displaced_mesh") = false;
3304 :
3305 3154 : parameters.set<SubProblem *>("_subproblem") = this;
3306 4731 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3307 : }
3308 :
3309 1710 : logAdd("Constraint", name, c_name, parameters);
3310 1710 : _nl[nl_sys_num]->addConstraint(c_name, name, parameters);
3311 1695 : }
3312 :
3313 : void
3314 94033 : FEProblemBase::addAuxVariable(const std::string & var_type,
3315 : const std::string & var_name,
3316 : InputParameters & params)
3317 : {
3318 : parallel_object_only();
3319 :
3320 94033 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3321 94033 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3322 94033 : const auto fe_type = FEType(order, family);
3323 :
3324 : const auto active_subdomains_vector =
3325 94033 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3326 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3327 94033 : active_subdomains_vector.end());
3328 :
3329 94033 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ true, &active_subdomains))
3330 0 : return;
3331 :
3332 282072 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3333 188048 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3334 :
3335 94024 : logAdd("AuxVariable", var_name, var_type, params);
3336 94024 : _aux->addVariable(var_type, var_name, params);
3337 94024 : if (_displaced_problem)
3338 : // MooseObjects need to be unique so change the name here
3339 10166 : _displaced_problem->addAuxVariable(var_type, var_name, params);
3340 :
3341 94024 : markFamilyPRefinement(params);
3342 94024 : if (_displaced_problem)
3343 10166 : _displaced_problem->markFamilyPRefinement(params);
3344 94024 : }
3345 :
3346 : void
3347 2628 : FEProblemBase::addElementalFieldVariable(const std::string & var_type,
3348 : const std::string & var_name,
3349 : InputParameters & params)
3350 : {
3351 2628 : addAuxVariable(var_type, var_name, params);
3352 2628 : }
3353 :
3354 : void
3355 0 : FEProblemBase::addAuxVariable(const std::string & var_name,
3356 : const FEType & type,
3357 : const std::set<SubdomainID> * const active_subdomains)
3358 : {
3359 : parallel_object_only();
3360 :
3361 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3362 :
3363 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3364 0 : return;
3365 :
3366 0 : std::string var_type;
3367 0 : if (type == FEType(0, MONOMIAL))
3368 0 : var_type = "MooseVariableConstMonomial";
3369 0 : else if (type.family == SCALAR)
3370 0 : var_type = "MooseVariableScalar";
3371 0 : else if (FEInterface::field_type(type) == TYPE_VECTOR)
3372 0 : var_type = "VectorMooseVariable";
3373 : else
3374 0 : var_type = "MooseVariable";
3375 :
3376 0 : InputParameters params = _factory.getValidParams(var_type);
3377 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3378 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3379 0 : params.set<MooseEnum>("order") = type.order.get_order();
3380 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3381 :
3382 0 : if (active_subdomains)
3383 0 : for (const SubdomainID & id : *active_subdomains)
3384 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3385 :
3386 0 : logAdd("AuxVariable", var_name, var_type, params);
3387 0 : _aux->addVariable(var_type, var_name, params);
3388 0 : if (_displaced_problem)
3389 0 : _displaced_problem->addAuxVariable("MooseVariable", var_name, params);
3390 :
3391 0 : markFamilyPRefinement(params);
3392 0 : if (_displaced_problem)
3393 0 : _displaced_problem->markFamilyPRefinement(params);
3394 0 : }
3395 :
3396 : void
3397 0 : FEProblemBase::addAuxArrayVariable(const std::string & var_name,
3398 : const FEType & type,
3399 : unsigned int components,
3400 : const std::set<SubdomainID> * const active_subdomains)
3401 : {
3402 : parallel_object_only();
3403 :
3404 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3405 :
3406 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3407 0 : return;
3408 :
3409 0 : InputParameters params = _factory.getValidParams("ArrayMooseVariable");
3410 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3411 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3412 0 : params.set<MooseEnum>("order") = type.order.get_order();
3413 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3414 0 : params.set<unsigned int>("components") = components;
3415 :
3416 0 : if (active_subdomains)
3417 0 : for (const SubdomainID & id : *active_subdomains)
3418 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3419 :
3420 0 : logAdd("Variable", var_name, "ArrayMooseVariable", params);
3421 0 : _aux->addVariable("ArrayMooseVariable", var_name, params);
3422 0 : if (_displaced_problem)
3423 0 : _displaced_problem->addAuxVariable("ArrayMooseVariable", var_name, params);
3424 :
3425 0 : markFamilyPRefinement(params);
3426 0 : if (_displaced_problem)
3427 0 : _displaced_problem->markFamilyPRefinement(params);
3428 0 : }
3429 :
3430 : void
3431 0 : FEProblemBase::addAuxScalarVariable(const std::string & var_name,
3432 : Order order,
3433 : Real /*scale_factor*/,
3434 : const std::set<SubdomainID> * const active_subdomains)
3435 : {
3436 : parallel_object_only();
3437 :
3438 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3439 :
3440 0 : if (order > _max_scalar_order)
3441 0 : _max_scalar_order = order;
3442 :
3443 0 : FEType type(order, SCALAR);
3444 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3445 0 : return;
3446 :
3447 0 : InputParameters params = _factory.getValidParams("MooseVariableScalar");
3448 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3449 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3450 :
3451 0 : params.set<MooseEnum>("order") = type.order.get_order();
3452 0 : params.set<MooseEnum>("family") = "SCALAR";
3453 0 : params.set<std::vector<Real>>("scaling") = std::vector<Real>{1};
3454 0 : if (active_subdomains)
3455 0 : for (const SubdomainID & id : *active_subdomains)
3456 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3457 :
3458 0 : logAdd("ScalarVariable", var_name, "MooseVariableScalar", params);
3459 0 : _aux->addVariable("MooseVariableScalar", var_name, params);
3460 0 : if (_displaced_problem)
3461 0 : _displaced_problem->addAuxVariable("MooseVariableScalar", var_name, params);
3462 0 : }
3463 :
3464 : void
3465 64518 : FEProblemBase::addAuxKernel(const std::string & kernel_name,
3466 : const std::string & name,
3467 : InputParameters & parameters)
3468 : {
3469 : parallel_object_only();
3470 :
3471 64518 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "AuxKernel");
3472 :
3473 64518 : _aux->addKernel(kernel_name, name, parameters);
3474 64404 : }
3475 :
3476 : void
3477 475 : FEProblemBase::addAuxScalarKernel(const std::string & kernel_name,
3478 : const std::string & name,
3479 : InputParameters & parameters)
3480 : {
3481 : parallel_object_only();
3482 :
3483 475 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3484 : {
3485 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3486 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3487 : }
3488 : else
3489 : {
3490 475 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3491 : {
3492 : // We allow AuxScalarKernels to request that they use_displaced_mesh,
3493 : // but then be overridden when no displacements variables are
3494 : // provided in the Mesh block. If that happened, update the value
3495 : // of use_displaced_mesh appropriately for this AuxScalarKernel.
3496 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3497 0 : parameters.set<bool>("use_displaced_mesh") = false;
3498 : }
3499 :
3500 950 : parameters.set<SubProblem *>("_subproblem") = this;
3501 1425 : parameters.set<SystemBase *>("_sys") = _aux.get();
3502 : }
3503 :
3504 475 : logAdd("AuxScalarKernel", name, kernel_name, parameters);
3505 475 : _aux->addScalarKernel(kernel_name, name, parameters);
3506 472 : }
3507 :
3508 : void
3509 871 : FEProblemBase::addDiracKernel(const std::string & kernel_name,
3510 : const std::string & name,
3511 : InputParameters & parameters)
3512 : {
3513 : parallel_object_only();
3514 :
3515 1742 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3516 868 : if (!isSolverSystemNonlinear(nl_sys_num))
3517 0 : mooseError("You are trying to add a DiracKernel to a linear variable/system, which is not "
3518 : "supported at the moment!");
3519 :
3520 868 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3521 : {
3522 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3523 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3524 12 : _reinit_displaced_elem = true;
3525 : }
3526 : else
3527 : {
3528 856 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3529 : {
3530 : // We allow DiracKernels to request that they use_displaced_mesh,
3531 : // but then be overridden when no displacements variables are
3532 : // provided in the Mesh block. If that happened, update the value
3533 : // of use_displaced_mesh appropriately for this DiracKernel.
3534 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3535 0 : parameters.set<bool>("use_displaced_mesh") = false;
3536 : }
3537 :
3538 1712 : parameters.set<SubProblem *>("_subproblem") = this;
3539 2568 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3540 : }
3541 :
3542 868 : logAdd("DiracKernel", name, kernel_name, parameters);
3543 868 : _nl[nl_sys_num]->addDiracKernel(kernel_name, name, parameters);
3544 862 : }
3545 :
3546 : // DGKernels ////
3547 :
3548 : void
3549 1298 : FEProblemBase::addDGKernel(const std::string & dg_kernel_name,
3550 : const std::string & name,
3551 : InputParameters & parameters)
3552 : {
3553 : parallel_object_only();
3554 :
3555 2596 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3556 1295 : if (!isSolverSystemNonlinear(nl_sys_num))
3557 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
3558 : "supported at the moment!");
3559 :
3560 1295 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3561 : {
3562 48 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3563 48 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3564 24 : _reinit_displaced_neighbor = true;
3565 : }
3566 : else
3567 : {
3568 1271 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3569 : {
3570 : // We allow DGKernels to request that they use_displaced_mesh,
3571 : // but then be overridden when no displacements variables are
3572 : // provided in the Mesh block. If that happened, update the value
3573 : // of use_displaced_mesh appropriately for this DGKernel.
3574 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3575 0 : parameters.set<bool>("use_displaced_mesh") = false;
3576 : }
3577 :
3578 2542 : parameters.set<SubProblem *>("_subproblem") = this;
3579 3813 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3580 : }
3581 :
3582 1295 : logAdd("DGKernel", name, dg_kernel_name, parameters);
3583 1295 : _nl[nl_sys_num]->addDGKernel(dg_kernel_name, name, parameters);
3584 :
3585 1295 : _has_internal_edge_residual_objects = true;
3586 1295 : }
3587 :
3588 : void
3589 6609 : FEProblemBase::addFVKernel(const std::string & fv_kernel_name,
3590 : const std::string & name,
3591 : InputParameters & parameters)
3592 : {
3593 6609 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3594 : // FVElementalKernels are computed in the historically finite element threaded loops. They rely
3595 : // on Assembly data like _current_elem. When we call reinit on the FEProblemBase we will only
3596 : // reinit the DisplacedProblem and its associated Assembly objects if we mark this boolean as
3597 : // true
3598 0 : _reinit_displaced_elem = true;
3599 6609 : addObject<FVKernel>(fv_kernel_name, name, parameters);
3600 6603 : }
3601 :
3602 : void
3603 5855 : FEProblemBase::addFVBC(const std::string & fv_bc_name,
3604 : const std::string & name,
3605 : InputParameters & parameters)
3606 : {
3607 5855 : addObject<FVBoundaryCondition>(fv_bc_name, name, parameters);
3608 5855 : }
3609 :
3610 : void
3611 238 : FEProblemBase::addFVInterfaceKernel(const std::string & fv_ik_name,
3612 : const std::string & name,
3613 : InputParameters & parameters)
3614 : {
3615 : /// We assume that variable1 and variable2 can live on different systems, in this case
3616 : /// the user needs to create two interface kernels with flipped variables and parameters
3617 238 : addObject<FVInterfaceKernel>(
3618 : fv_ik_name, name, parameters, /*threaded=*/true, /*variable_param_name=*/"variable1");
3619 229 : }
3620 :
3621 : void
3622 2157 : FEProblemBase::addLinearFVKernel(const std::string & kernel_name,
3623 : const std::string & name,
3624 : InputParameters & parameters)
3625 : {
3626 2157 : addObject<LinearFVKernel>(kernel_name, name, parameters);
3627 2157 : }
3628 :
3629 : void
3630 2064 : FEProblemBase::addLinearFVBC(const std::string & bc_name,
3631 : const std::string & name,
3632 : InputParameters & parameters)
3633 : {
3634 2064 : addObject<LinearFVBoundaryCondition>(bc_name, name, parameters);
3635 2064 : }
3636 :
3637 : // InterfaceKernels ////
3638 :
3639 : void
3640 800 : FEProblemBase::addInterfaceKernel(const std::string & interface_kernel_name,
3641 : const std::string & name,
3642 : InputParameters & parameters)
3643 : {
3644 : parallel_object_only();
3645 :
3646 1600 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3647 797 : if (!isSolverSystemNonlinear(nl_sys_num))
3648 0 : mooseError("You are trying to add a InterfaceKernel to a linear variable/system, which is not "
3649 : "supported at the moment!");
3650 :
3651 797 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3652 : {
3653 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3654 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3655 12 : _reinit_displaced_neighbor = true;
3656 : }
3657 : else
3658 : {
3659 785 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3660 : {
3661 : // We allow InterfaceKernels to request that they use_displaced_mesh,
3662 : // but then be overridden when no displacements variables are
3663 : // provided in the Mesh block. If that happened, update the value
3664 : // of use_displaced_mesh appropriately for this InterfaceKernel.
3665 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3666 0 : parameters.set<bool>("use_displaced_mesh") = false;
3667 : }
3668 :
3669 1570 : parameters.set<SubProblem *>("_subproblem") = this;
3670 2355 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3671 : }
3672 :
3673 797 : logAdd("InterfaceKernel", name, interface_kernel_name, parameters);
3674 797 : _nl[nl_sys_num]->addInterfaceKernel(interface_kernel_name, name, parameters);
3675 :
3676 797 : _has_internal_edge_residual_objects = true;
3677 797 : }
3678 :
3679 : void
3680 33826 : FEProblemBase::checkICRestartError(const std::string & ic_name,
3681 : const std::string & name,
3682 : const VariableName & var_name)
3683 : {
3684 33826 : if (!_allow_ics_during_restart)
3685 : {
3686 33747 : std::string restart_method = "";
3687 33747 : if (_app.isRestarting())
3688 : restart_method =
3689 0 : "a checkpoint restart, by IC object '" + ic_name + "' for variable '" + name + "'";
3690 33747 : else if (_app.getExReaderForRestart())
3691 : {
3692 3 : std::vector<std::string> restarted_vars = _app.getExReaderForRestart()->get_elem_var_names();
3693 3 : const auto nodal_vars = _app.getExReaderForRestart()->get_nodal_var_names();
3694 3 : const auto global_vars = _app.getExReaderForRestart()->get_global_var_names();
3695 3 : restarted_vars.insert(restarted_vars.end(), nodal_vars.begin(), nodal_vars.end());
3696 3 : restarted_vars.insert(restarted_vars.end(), global_vars.begin(), global_vars.end());
3697 :
3698 3 : if (std::find(restarted_vars.begin(), restarted_vars.end(), var_name) != restarted_vars.end())
3699 6 : restart_method = "an Exodus restart, by IC object '" + ic_name + "' for variable '" + name +
3700 3 : "' that is also being restarted";
3701 3 : }
3702 33747 : if (!restart_method.empty())
3703 3 : mooseError(
3704 : "Initial conditions have been specified during ",
3705 : restart_method,
3706 : ".\nThis is only allowed if you specify 'allow_initial_conditions_with_restart' to "
3707 : "the [Problem], as initial conditions can override restarted fields");
3708 33744 : }
3709 33823 : }
3710 :
3711 : void
3712 31560 : FEProblemBase::addInitialCondition(const std::string & ic_name,
3713 : const std::string & name,
3714 : InputParameters & parameters)
3715 : {
3716 : parallel_object_only();
3717 :
3718 : // before we start to mess with the initial condition, we need to check parameters for errors.
3719 31560 : parameters.checkParams(name);
3720 31557 : const std::string & var_name = parameters.get<VariableName>("variable");
3721 :
3722 : // Forbid initial conditions on a restarted problem, as they would override the restart
3723 31557 : checkICRestartError(ic_name, name, var_name);
3724 :
3725 63108 : parameters.set<SubProblem *>("_subproblem") = this;
3726 :
3727 : // field IC
3728 31554 : if (hasVariable(var_name))
3729 : {
3730 63446 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3731 : {
3732 33452 : MooseVariableFEBase & var = getVariable(
3733 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3734 66904 : parameters.set<SystemBase *>("_sys") = &var.sys();
3735 33452 : std::shared_ptr<InitialConditionBase> ic;
3736 33452 : if (dynamic_cast<MooseVariable *>(&var))
3737 31331 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3738 2121 : else if (dynamic_cast<VectorMooseVariable *>(&var))
3739 250 : ic = _factory.create<VectorInitialCondition>(ic_name, name, parameters, tid);
3740 1871 : else if (dynamic_cast<ArrayMooseVariable *>(&var))
3741 1207 : ic = _factory.create<ArrayInitialCondition>(ic_name, name, parameters, tid);
3742 664 : else if (dynamic_cast<MooseVariableFVReal *>(&var))
3743 664 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3744 0 : else if (dynamic_cast<MooseLinearVariableFVReal *>(&var))
3745 0 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3746 : else
3747 0 : mooseError("Your FE variable in initial condition ",
3748 : name,
3749 : " must be either of scalar or vector type");
3750 33431 : logAdd("IC", name, ic_name, parameters);
3751 33431 : _ics.addObject(ic, tid);
3752 33416 : }
3753 : }
3754 :
3755 : // scalar IC
3756 1524 : else if (hasScalarVariable(var_name))
3757 : {
3758 1524 : MooseVariableScalar & var = getScalarVariable(0, var_name);
3759 3048 : parameters.set<SystemBase *>("_sys") = &var.sys();
3760 : std::shared_ptr<ScalarInitialCondition> ic =
3761 1524 : _factory.create<ScalarInitialCondition>(ic_name, name, parameters);
3762 1524 : logAdd("ScalarIC", name, ic_name, parameters);
3763 1524 : _scalar_ics.addObject(ic);
3764 1524 : }
3765 :
3766 : else
3767 0 : mooseError(
3768 : "Variable '", var_name, "' requested in initial condition '", name, "' does not exist.");
3769 31518 : }
3770 :
3771 : void
3772 2269 : FEProblemBase::addFVInitialCondition(const std::string & ic_name,
3773 : const std::string & name,
3774 : InputParameters & parameters)
3775 : {
3776 : parallel_object_only();
3777 :
3778 : // before we start to mess with the initial condition, we need to check parameters for errors.
3779 2269 : parameters.checkParams(name);
3780 2269 : const std::string & var_name = parameters.get<VariableName>("variable");
3781 :
3782 : // Forbid initial conditions on a restarted problem, as they would override the restart
3783 2269 : checkICRestartError(ic_name, name, var_name);
3784 :
3785 4538 : parameters.set<SubProblem *>("_subproblem") = this;
3786 :
3787 : // field IC
3788 2269 : if (hasVariable(var_name))
3789 : {
3790 4620 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3791 : {
3792 2351 : auto & var = getVariable(
3793 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3794 4702 : parameters.set<SystemBase *>("_sys") = &var.sys();
3795 2351 : std::shared_ptr<FVInitialConditionBase> ic;
3796 2351 : if (var.isFV())
3797 2351 : ic = _factory.create<FVInitialCondition>(ic_name, name, parameters, tid);
3798 : else
3799 0 : mooseError(
3800 : "Your variable for an FVInitialCondition needs to be an a finite volume variable!");
3801 2351 : _fv_ics.addObject(ic, tid);
3802 2351 : }
3803 : }
3804 : else
3805 0 : mooseError("Variable '",
3806 : var_name,
3807 : "' requested in finite volume initial condition '",
3808 : name,
3809 : "' does not exist.");
3810 2269 : }
3811 :
3812 : void
3813 56842 : FEProblemBase::projectSolution()
3814 : {
3815 284210 : TIME_SECTION("projectSolution", 2, "Projecting Initial Solutions")
3816 :
3817 56842 : FloatingPointExceptionGuard fpe_guard(_app);
3818 :
3819 56842 : ComputeInitialConditionThread cic(*this);
3820 56842 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cic);
3821 :
3822 56836 : if (haveFV())
3823 : {
3824 : using ElemInfoRange = StoredRange<MooseMesh::const_elem_info_iterator, const ElemInfo *>;
3825 3932 : ElemInfoRange elem_info_range(_mesh.ownedElemInfoBegin(), _mesh.ownedElemInfoEnd());
3826 :
3827 3932 : ComputeFVInitialConditionThread cfvic(*this);
3828 3932 : Threads::parallel_reduce(elem_info_range, cfvic);
3829 3932 : }
3830 :
3831 : // Need to close the solution vector here so that boundary ICs take precendence
3832 112933 : for (auto & nl : _nl)
3833 56097 : nl->solution().close();
3834 56836 : _aux->solution().close();
3835 :
3836 : // now run boundary-restricted initial conditions
3837 56836 : ComputeBoundaryInitialConditionThread cbic(*this);
3838 56836 : Threads::parallel_reduce(getCurrentAlgebraicBndNodeRange(), cbic);
3839 :
3840 112933 : for (auto & nl : _nl)
3841 56097 : nl->solution().close();
3842 56836 : _aux->solution().close();
3843 :
3844 : // Also, load values into the SCALAR dofs
3845 : // Note: We assume that all SCALAR dofs are on the
3846 : // processor with highest ID
3847 56836 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3848 : {
3849 567 : const auto & ics = _scalar_ics.getActiveObjects();
3850 1561 : for (const auto & ic : ics)
3851 : {
3852 994 : MooseVariableScalar & var = ic->variable();
3853 994 : var.reinit();
3854 :
3855 994 : DenseVector<Number> vals(var.order());
3856 994 : ic->compute(vals);
3857 :
3858 994 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3859 2323 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3860 : {
3861 1329 : const auto global_index = var.dofIndices()[i];
3862 1329 : var.sys().solution().set(global_index, vals(i));
3863 1329 : var.setValue(i, vals(i));
3864 : }
3865 994 : }
3866 : }
3867 :
3868 113933 : for (auto & sys : _solver_systems)
3869 : {
3870 57097 : sys->solution().close();
3871 57097 : sys->solution().localize(*sys->system().current_local_solution, sys->dofMap().get_send_list());
3872 : }
3873 :
3874 56836 : _aux->solution().close();
3875 56836 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3876 56836 : }
3877 :
3878 : void
3879 2041 : FEProblemBase::projectInitialConditionOnCustomRange(
3880 : ConstElemRange & elem_range,
3881 : ConstBndNodeRange & bnd_nodes,
3882 : const std::optional<std::set<VariableName>> & target_vars)
3883 : {
3884 2041 : if (target_vars)
3885 : {
3886 2041 : ComputeInitialConditionThread cic(*this, &(*target_vars));
3887 2041 : Threads::parallel_reduce(elem_range, cic);
3888 : }
3889 : else
3890 : {
3891 0 : ComputeInitialConditionThread cic(*this);
3892 0 : Threads::parallel_reduce(elem_range, cic);
3893 : }
3894 :
3895 : // Need to close the solution vector here so that boundary ICs take precendence
3896 4082 : for (auto & nl : _nl)
3897 2041 : nl->solution().close();
3898 2041 : _aux->solution().close();
3899 :
3900 2041 : if (target_vars)
3901 : {
3902 2041 : ComputeBoundaryInitialConditionThread cbic(*this, &(*target_vars));
3903 2041 : Threads::parallel_reduce(bnd_nodes, cbic);
3904 2041 : }
3905 : else
3906 : {
3907 0 : ComputeBoundaryInitialConditionThread cbic(*this);
3908 0 : Threads::parallel_reduce(bnd_nodes, cbic);
3909 0 : }
3910 :
3911 4082 : for (auto & nl : _nl)
3912 2041 : nl->solution().close();
3913 2041 : _aux->solution().close();
3914 :
3915 : // Also, load values into the SCALAR dofs
3916 : // Note: We assume that all SCALAR dofs are on the
3917 : // processor with highest ID
3918 2041 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3919 : {
3920 0 : const auto & ics = _scalar_ics.getActiveObjects();
3921 0 : for (const auto & ic : ics)
3922 : {
3923 0 : MooseVariableScalar & var = ic->variable();
3924 :
3925 0 : if (target_vars && !target_vars->count(var.name()))
3926 0 : continue;
3927 :
3928 0 : var.reinit();
3929 :
3930 0 : DenseVector<Number> vals(var.order());
3931 0 : ic->compute(vals);
3932 :
3933 0 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3934 0 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3935 : {
3936 0 : const auto global_index = var.dofIndices()[i];
3937 0 : var.sys().solution().set(global_index, vals(i));
3938 0 : var.setValue(i, vals(i));
3939 : }
3940 0 : }
3941 : }
3942 :
3943 4082 : for (auto & nl : _nl)
3944 : {
3945 2041 : nl->solution().close();
3946 2041 : nl->solution().localize(*nl->system().current_local_solution, nl->dofMap().get_send_list());
3947 : }
3948 :
3949 2041 : _aux->solution().close();
3950 2041 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3951 2041 : }
3952 :
3953 : void
3954 737 : FEProblemBase::projectFunctionOnCustomRange(ConstElemRange & elem_range,
3955 : Number (*func)(const Point &,
3956 : const libMesh::Parameters &,
3957 : const std::string &,
3958 : const std::string &),
3959 : Gradient (*func_grad)(const Point &,
3960 : const libMesh::Parameters &,
3961 : const std::string &,
3962 : const std::string &),
3963 : const libMesh::Parameters & params,
3964 : const std::vector<VariableName> & target_vars)
3965 : {
3966 : mooseAssert(!Threads::in_threads,
3967 : "We're performing a projection based on data from just the thread 0 variable, so any "
3968 : "modifications to the variable solution must have been thread joined already");
3969 :
3970 737 : std::unordered_map<unsigned int, std::vector<unsigned int>> sys_to_var_nums;
3971 :
3972 1474 : for (const auto & target_var : target_vars)
3973 : {
3974 737 : const auto sn = systemNumForVariable(target_var);
3975 737 : const auto & var = getStandardVariable(0, target_var);
3976 737 : sys_to_var_nums[sn].push_back(var.number());
3977 : }
3978 :
3979 1474 : for (const auto & [sys_num, var_nums] : sys_to_var_nums)
3980 : {
3981 737 : System & libmesh_sys = getSystemBase(sys_num).system();
3982 737 : libmesh_sys.project_solution(func, func_grad, params, elem_range, var_nums);
3983 : }
3984 737 : }
3985 :
3986 : std::shared_ptr<MaterialBase>
3987 252 : FEProblemBase::getMaterial(std::string name,
3988 : Moose::MaterialDataType type,
3989 : const THREAD_ID tid,
3990 : bool no_warn)
3991 : {
3992 252 : switch (type)
3993 : {
3994 65 : case Moose::NEIGHBOR_MATERIAL_DATA:
3995 65 : name += "_neighbor";
3996 65 : break;
3997 65 : case Moose::FACE_MATERIAL_DATA:
3998 65 : name += "_face";
3999 65 : break;
4000 122 : default:
4001 122 : break;
4002 : }
4003 :
4004 252 : std::shared_ptr<MaterialBase> material = _all_materials[type].getActiveObject(name, tid);
4005 657 : if (!no_warn && material->getParam<bool>("compute") && type == Moose::BLOCK_MATERIAL_DATA)
4006 3 : mooseWarning("You are retrieving a Material object (",
4007 3 : material->name(),
4008 : "), but its compute flag is set to true. This indicates that MOOSE is "
4009 : "computing this property which may not be desired and produce un-expected "
4010 : "results.");
4011 :
4012 246 : return material;
4013 : }
4014 :
4015 : MaterialData &
4016 33131760 : FEProblemBase::getMaterialData(Moose::MaterialDataType type,
4017 : const THREAD_ID tid,
4018 : const MooseObject * object) const
4019 : {
4020 33131760 : switch (type)
4021 : {
4022 853894 : case Moose::BLOCK_MATERIAL_DATA:
4023 853894 : if (object)
4024 261713 : _material_props.addConsumer(type, object);
4025 853894 : return _material_props.getMaterialData(tid);
4026 15162977 : case Moose::NEIGHBOR_MATERIAL_DATA:
4027 15162977 : if (object)
4028 22511 : _neighbor_material_props.addConsumer(type, object);
4029 15162977 : return _neighbor_material_props.getMaterialData(tid);
4030 17114889 : case Moose::BOUNDARY_MATERIAL_DATA:
4031 : case Moose::FACE_MATERIAL_DATA:
4032 : case Moose::INTERFACE_MATERIAL_DATA:
4033 17114889 : if (object)
4034 52369 : _bnd_material_props.addConsumer(type, object);
4035 17114889 : return _bnd_material_props.getMaterialData(tid);
4036 : }
4037 :
4038 0 : mooseError("FEProblemBase::getMaterialData(): Invalid MaterialDataType ", type);
4039 : }
4040 :
4041 : const std::set<const MooseObject *> &
4042 0 : FEProblemBase::getMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
4043 : {
4044 0 : switch (type)
4045 : {
4046 0 : case Moose::BLOCK_MATERIAL_DATA:
4047 0 : return _material_props.getConsumers(type);
4048 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
4049 0 : return _neighbor_material_props.getConsumers(type);
4050 0 : case Moose::BOUNDARY_MATERIAL_DATA:
4051 : case Moose::FACE_MATERIAL_DATA:
4052 : case Moose::INTERFACE_MATERIAL_DATA:
4053 0 : return _bnd_material_props.getConsumers(type);
4054 : }
4055 :
4056 0 : mooseError("FEProblemBase::getMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
4057 : type);
4058 : }
4059 :
4060 : void
4061 0 : FEProblemBase::setPreserveMatrixSparsityPattern(bool preserve)
4062 : {
4063 0 : if (_ignore_zeros_in_jacobian && preserve)
4064 0 : paramWarning(
4065 : "ignore_zeros_in_jacobian",
4066 : "We likely cannot preserve the sparsity pattern if ignoring zeros in the Jacobian, which "
4067 : "leads to removing those entries from the Jacobian sparsity pattern");
4068 0 : _preserve_matrix_sparsity_pattern = preserve;
4069 0 : }
4070 :
4071 : bool
4072 310229 : FEProblemBase::acceptInvalidSolution() const
4073 : {
4074 620373 : return allowInvalidSolution() || // invalid solutions are always allowed
4075 620373 : !_app.solutionInvalidity().hasInvalidSolutionError(); // if not allowed, check for errors
4076 : }
4077 :
4078 : void
4079 1060 : FEProblemBase::addFunctorMaterial(const std::string & functor_material_name,
4080 : const std::string & name,
4081 : InputParameters & parameters)
4082 : {
4083 : parallel_object_only();
4084 :
4085 1060 : auto add_functor_materials = [&](const auto & parameters, const auto & name)
4086 : {
4087 2211 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
4088 : {
4089 : // Create the general Block/Boundary MaterialBase object
4090 1151 : std::shared_ptr<MaterialBase> material =
4091 1151 : _factory.create<MaterialBase>(functor_material_name, name, parameters, tid);
4092 2302 : logAdd("FunctorMaterial", name, functor_material_name, parameters);
4093 1151 : _all_materials.addObject(material, tid);
4094 1151 : _materials.addObject(material, tid);
4095 : }
4096 1060 : };
4097 :
4098 2120 : parameters.set<SubProblem *>("_subproblem") = this;
4099 1060 : add_functor_materials(parameters, name);
4100 1060 : if (_displaced_problem)
4101 : {
4102 0 : auto disp_params = parameters;
4103 0 : disp_params.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4104 0 : add_functor_materials(disp_params, name + "_displaced");
4105 0 : }
4106 1060 : }
4107 :
4108 : void
4109 12169 : FEProblemBase::addMaterial(const std::string & mat_name,
4110 : const std::string & name,
4111 : InputParameters & parameters)
4112 : {
4113 24338 : addMaterialHelper({&_materials}, mat_name, name, parameters);
4114 12071 : }
4115 :
4116 : void
4117 340 : FEProblemBase::addInterfaceMaterial(const std::string & mat_name,
4118 : const std::string & name,
4119 : InputParameters & parameters)
4120 : {
4121 680 : addMaterialHelper({&_interface_materials}, mat_name, name, parameters);
4122 340 : }
4123 :
4124 : void
4125 13451 : FEProblemBase::addMaterialHelper(std::vector<MaterialWarehouse *> warehouses,
4126 : const std::string & mat_name,
4127 : const std::string & name,
4128 : InputParameters & parameters)
4129 : {
4130 : parallel_object_only();
4131 :
4132 13451 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4133 : {
4134 252 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4135 126 : _reinit_displaced_elem = _reinit_displaced_face = _reinit_displaced_neighbor = true;
4136 : }
4137 : else
4138 : {
4139 13325 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
4140 : {
4141 : // We allow Materials to request that they use_displaced_mesh,
4142 : // but then be overridden when no displacements variables are
4143 : // provided in the Mesh block. If that happened, update the value
4144 : // of use_displaced_mesh appropriately for this Material.
4145 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
4146 0 : parameters.set<bool>("use_displaced_mesh") = false;
4147 : }
4148 :
4149 39975 : parameters.set<SubProblem *>("_subproblem") = this;
4150 : }
4151 :
4152 13451 : unsigned int n_threads = libMesh::n_threads();
4153 :
4154 : #ifdef MOOSE_KOKKOS_ENABLED
4155 10030 : if (parameters.isKokkosObject())
4156 942 : n_threads = 1;
4157 : #endif
4158 :
4159 27988 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4160 : {
4161 : // Create the general Block/Boundary MaterialBase object
4162 : std::shared_ptr<MaterialBase> material =
4163 14643 : _factory.create<MaterialBase>(mat_name, name, parameters, tid);
4164 14537 : logAdd("Material", name, mat_name, parameters);
4165 29074 : bool discrete = !material->getParam<bool>("compute");
4166 :
4167 : // If the object is boundary restricted or if it is a functor material we do not create the
4168 : // neighbor and face objects
4169 14537 : if (material->boundaryRestricted() || dynamic_cast<FunctorMaterial *>(material.get()))
4170 : {
4171 2984 : _all_materials.addObject(material, tid);
4172 2984 : if (discrete)
4173 4 : _discrete_materials.addObject(material, tid);
4174 : else
4175 5960 : for (auto && warehouse : warehouses)
4176 2980 : warehouse->addObject(material, tid);
4177 : }
4178 :
4179 : // Non-boundary restricted require face and neighbor objects
4180 : else
4181 : {
4182 : // TODO: we only need to do this if we have needs for face materials (e.g.
4183 : // FV, DG, etc.) - but currently we always do it. Figure out how to fix
4184 : // this.
4185 :
4186 : // The name of the object being created, this is changed multiple times as objects are
4187 : // created below
4188 11553 : std::string object_name;
4189 :
4190 : // Create a copy of the supplied parameters to the setting for "_material_data_type" isn't
4191 : // used from a previous tid loop
4192 11553 : InputParameters current_parameters = parameters;
4193 :
4194 : // face material
4195 11553 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4196 : Moose::FACE_MATERIAL_DATA;
4197 11553 : object_name = name + "_face";
4198 : std::shared_ptr<MaterialBase> face_material =
4199 11553 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4200 :
4201 : // neighbor material
4202 23106 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4203 : Moose::NEIGHBOR_MATERIAL_DATA;
4204 11553 : current_parameters.set<bool>("_neighbor") = true;
4205 11553 : object_name = name + "_neighbor";
4206 : std::shared_ptr<MaterialBase> neighbor_material =
4207 11553 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4208 :
4209 : // Store the material objects
4210 11553 : _all_materials.addObjects(material, neighbor_material, face_material, tid);
4211 :
4212 11553 : if (discrete)
4213 73 : _discrete_materials.addObjects(material, neighbor_material, face_material, tid);
4214 : else
4215 22960 : for (auto && warehouse : warehouses)
4216 11480 : warehouse->addObjects(material, neighbor_material, face_material, tid);
4217 :
4218 : // Names of all controllable parameters for this Material object
4219 11553 : const std::string & base = parameters.getBase();
4220 34659 : MooseObjectParameterName name(MooseObjectName(base, material->name()), "*");
4221 : const auto param_names =
4222 11553 : _app.getInputParameterWarehouse().getControllableParameterNames(name);
4223 :
4224 : // Connect parameters of the primary Material object to those on the face and neighbor
4225 : // objects
4226 29392 : for (const auto & p_name : param_names)
4227 : {
4228 35678 : MooseObjectParameterName primary_name(MooseObjectName(base, material->name()),
4229 35678 : p_name.parameter());
4230 35678 : MooseObjectParameterName face_name(MooseObjectName(base, face_material->name()),
4231 35678 : p_name.parameter());
4232 35678 : MooseObjectParameterName neighbor_name(MooseObjectName(base, neighbor_material->name()),
4233 35678 : p_name.parameter());
4234 17839 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4235 : primary_name, face_name, false);
4236 17839 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4237 : primary_name, neighbor_name, false);
4238 17839 : }
4239 11553 : }
4240 14537 : }
4241 13345 : }
4242 :
4243 : void
4244 4680065 : FEProblemBase::prepareMaterials(const std::unordered_set<unsigned int> & consumer_needed_mat_props,
4245 : const SubdomainID blk_id,
4246 : const THREAD_ID tid)
4247 : {
4248 4680065 : std::set<MooseVariableFEBase *> needed_moose_vars;
4249 4680065 : std::unordered_set<unsigned int> needed_mat_props;
4250 :
4251 4680065 : if (_all_materials.hasActiveBlockObjects(blk_id, tid))
4252 : {
4253 588668 : _all_materials.updateVariableDependency(needed_moose_vars, tid);
4254 588668 : _all_materials.updateBlockMatPropDependency(blk_id, needed_mat_props, tid);
4255 : }
4256 :
4257 4680065 : const auto & ids = _mesh.getSubdomainBoundaryIds(blk_id);
4258 21596733 : for (const auto id : ids)
4259 : {
4260 16916668 : _materials.updateBoundaryVariableDependency(id, needed_moose_vars, tid);
4261 16916668 : _materials.updateBoundaryMatPropDependency(id, needed_mat_props, tid);
4262 : }
4263 :
4264 4680065 : const auto & current_active_elemental_moose_variables = getActiveElementalMooseVariables(tid);
4265 4680065 : needed_moose_vars.insert(current_active_elemental_moose_variables.begin(),
4266 : current_active_elemental_moose_variables.end());
4267 :
4268 4680065 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
4269 :
4270 4680065 : setActiveElementalMooseVariables(needed_moose_vars, tid);
4271 4680065 : setActiveMaterialProperties(needed_mat_props, tid);
4272 4680065 : }
4273 :
4274 : void
4275 364063159 : FEProblemBase::reinitMaterials(SubdomainID blk_id, const THREAD_ID tid, bool swap_stateful)
4276 : {
4277 364063159 : if (hasActiveMaterialProperties(tid))
4278 : {
4279 15061892 : auto && elem = _assembly[tid][0]->elem();
4280 15061892 : unsigned int n_points = _assembly[tid][0]->qRule()->n_points();
4281 :
4282 15061892 : auto & material_data = _material_props.getMaterialData(tid);
4283 15061892 : material_data.resize(n_points);
4284 :
4285 : // Only swap if requested
4286 15061892 : if (swap_stateful)
4287 15055350 : material_data.swap(*elem);
4288 :
4289 15061892 : if (_discrete_materials.hasActiveBlockObjects(blk_id, tid))
4290 2698 : material_data.reset(_discrete_materials.getActiveBlockObjects(blk_id, tid));
4291 :
4292 15061889 : if (_materials.hasActiveBlockObjects(blk_id, tid))
4293 15043350 : material_data.reinit(_materials.getActiveBlockObjects(blk_id, tid));
4294 : }
4295 364063104 : }
4296 :
4297 : void
4298 5327298 : FEProblemBase::reinitMaterialsFace(const SubdomainID blk_id,
4299 : const THREAD_ID tid,
4300 : const bool swap_stateful,
4301 : const std::deque<MaterialBase *> * const reinit_mats)
4302 : {
4303 : // we reinit more often than needed here because we dont have a way to check whether
4304 : // we need to compute the face materials on a particular (possibly external) face
4305 5327298 : if (hasActiveMaterialProperties(tid))
4306 : {
4307 1045875 : auto && elem = _assembly[tid][0]->elem();
4308 1045875 : unsigned int side = _assembly[tid][0]->side();
4309 1045875 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4310 :
4311 1045875 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4312 1045875 : bnd_material_data.resize(n_points);
4313 :
4314 1045875 : if (swap_stateful && !bnd_material_data.isSwapped())
4315 1024401 : bnd_material_data.swap(*elem, side);
4316 :
4317 1045875 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4318 0 : bnd_material_data.reset(
4319 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4320 :
4321 1045875 : if (reinit_mats)
4322 21474 : bnd_material_data.reinit(*reinit_mats);
4323 1024401 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4324 1024401 : bnd_material_data.reinit(
4325 1024401 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4326 : }
4327 5327298 : }
4328 :
4329 : void
4330 3877681 : FEProblemBase::reinitMaterialsFaceOnBoundary(const BoundaryID boundary_id,
4331 : const SubdomainID blk_id,
4332 : const THREAD_ID tid,
4333 : const bool swap_stateful,
4334 : const std::deque<MaterialBase *> * const reinit_mats)
4335 : {
4336 4208612 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4337 330931 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4338 330931 : needInternalNeighborSideMaterial(blk_id, tid)))
4339 : {
4340 407446 : const auto * const elem = _assembly[tid][0]->elem();
4341 407446 : unsigned int side = _assembly[tid][0]->side();
4342 407446 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4343 :
4344 407446 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4345 407446 : bnd_material_data.resize(n_points);
4346 :
4347 407446 : if (swap_stateful && !bnd_material_data.isSwapped())
4348 407446 : bnd_material_data.swap(*elem, side);
4349 :
4350 407446 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4351 0 : bnd_material_data.reset(
4352 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4353 :
4354 407446 : if (reinit_mats)
4355 0 : bnd_material_data.reinit(*reinit_mats);
4356 407446 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4357 393054 : bnd_material_data.reinit(
4358 393054 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4359 : }
4360 3877681 : }
4361 :
4362 : void
4363 43931 : FEProblemBase::reinitMaterialsNeighborOnBoundary(
4364 : const BoundaryID boundary_id,
4365 : const SubdomainID blk_id,
4366 : const THREAD_ID tid,
4367 : const bool swap_stateful,
4368 : const std::deque<MaterialBase *> * const reinit_mats)
4369 : {
4370 : // Since objects don't declare whether they need the face or neighbor (side) material properties,
4371 : // we use the same criteria for skipping material property computations as for face material
4372 : // properties This could be a future optimization.
4373 48141 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4374 4210 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4375 4210 : needInternalNeighborSideMaterial(blk_id, tid)))
4376 35986 : reinitMaterialsNeighbor(blk_id, tid, swap_stateful, reinit_mats);
4377 43931 : }
4378 :
4379 : void
4380 4195107 : FEProblemBase::reinitMaterialsNeighbor(const SubdomainID blk_id,
4381 : const THREAD_ID tid,
4382 : const bool swap_stateful,
4383 : const std::deque<MaterialBase *> * const reinit_mats)
4384 : {
4385 4195107 : if (hasActiveMaterialProperties(tid))
4386 : {
4387 : // NOTE: this will not work with h-adaptivity
4388 : // lindsayad: why not?
4389 :
4390 887183 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4391 887183 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(_assembly[tid][0]->elem());
4392 :
4393 : mooseAssert(neighbor, "neighbor should be non-null");
4394 : mooseAssert(blk_id == neighbor->subdomain_id(),
4395 : "The provided blk_id " << blk_id << " and neighbor subdomain ID "
4396 : << neighbor->subdomain_id() << " do not match.");
4397 :
4398 887183 : unsigned int n_points = _assembly[tid][0]->qRuleNeighbor()->n_points();
4399 :
4400 887183 : auto & neighbor_material_data = _neighbor_material_props.getMaterialData(tid);
4401 887183 : neighbor_material_data.resize(n_points);
4402 :
4403 : // Only swap if requested
4404 887183 : if (swap_stateful)
4405 865709 : neighbor_material_data.swap(*neighbor, neighbor_side);
4406 :
4407 887183 : if (_discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4408 0 : neighbor_material_data.reset(
4409 0 : _discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4410 :
4411 887183 : if (reinit_mats)
4412 21474 : neighbor_material_data.reinit(*reinit_mats);
4413 865709 : else if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4414 865446 : neighbor_material_data.reinit(
4415 865446 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4416 : }
4417 4195107 : }
4418 :
4419 : void
4420 4352631 : FEProblemBase::reinitMaterialsBoundary(const BoundaryID boundary_id,
4421 : const THREAD_ID tid,
4422 : const bool swap_stateful,
4423 : const std::deque<MaterialBase *> * const reinit_mats)
4424 : {
4425 4352631 : if (hasActiveMaterialProperties(tid) && needBoundaryMaterialOnSide(boundary_id, tid))
4426 : {
4427 285958 : auto && elem = _assembly[tid][0]->elem();
4428 285958 : unsigned int side = _assembly[tid][0]->side();
4429 285958 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4430 :
4431 285958 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4432 285958 : bnd_material_data.resize(n_points);
4433 :
4434 285958 : if (swap_stateful && !bnd_material_data.isSwapped())
4435 238273 : bnd_material_data.swap(*elem, side);
4436 :
4437 285958 : if (_discrete_materials.hasActiveBoundaryObjects(boundary_id, tid))
4438 0 : bnd_material_data.reset(_discrete_materials.getActiveBoundaryObjects(boundary_id, tid));
4439 :
4440 285958 : if (reinit_mats)
4441 21474 : bnd_material_data.reinit(*reinit_mats);
4442 264484 : else if (_materials.hasActiveBoundaryObjects(boundary_id, tid))
4443 23154 : bnd_material_data.reinit(_materials.getActiveBoundaryObjects(boundary_id, tid));
4444 : }
4445 4352631 : }
4446 :
4447 : void
4448 46788 : FEProblemBase::reinitMaterialsInterface(BoundaryID boundary_id,
4449 : const THREAD_ID tid,
4450 : bool swap_stateful)
4451 : {
4452 46788 : if (hasActiveMaterialProperties(tid) && needInterfaceMaterialOnSide(boundary_id, tid))
4453 : {
4454 37022 : const Elem * const & elem = _assembly[tid][0]->elem();
4455 37022 : unsigned int side = _assembly[tid][0]->side();
4456 37022 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4457 :
4458 37022 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4459 37022 : bnd_material_data.resize(n_points);
4460 :
4461 37022 : if (swap_stateful && !bnd_material_data.isSwapped())
4462 35527 : bnd_material_data.swap(*elem, side);
4463 :
4464 37022 : if (_interface_materials.hasActiveBoundaryObjects(boundary_id, tid))
4465 1171 : bnd_material_data.reinit(_interface_materials.getActiveBoundaryObjects(boundary_id, tid));
4466 : }
4467 46788 : }
4468 :
4469 : void
4470 362394214 : FEProblemBase::swapBackMaterials(const THREAD_ID tid)
4471 : {
4472 362394214 : auto && elem = _assembly[tid][0]->elem();
4473 362394214 : _material_props.getMaterialData(tid).swapBack(*elem);
4474 362394214 : }
4475 :
4476 : void
4477 8730018 : FEProblemBase::swapBackMaterialsFace(const THREAD_ID tid)
4478 : {
4479 8730018 : auto && elem = _assembly[tid][0]->elem();
4480 8730018 : unsigned int side = _assembly[tid][0]->side();
4481 8730018 : _bnd_material_props.getMaterialData(tid).swapBack(*elem, side);
4482 8730018 : }
4483 :
4484 : void
4485 3630872 : FEProblemBase::swapBackMaterialsNeighbor(const THREAD_ID tid)
4486 : {
4487 : // NOTE: this will not work with h-adaptivity
4488 3630872 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4489 : unsigned int neighbor_side =
4490 3630872 : neighbor ? neighbor->which_neighbor_am_i(_assembly[tid][0]->elem()) : libMesh::invalid_uint;
4491 :
4492 3630872 : if (!neighbor)
4493 : {
4494 0 : if (haveFV())
4495 : {
4496 : // If neighbor is null, then we're on the neighbor side of a mesh boundary, e.g. we're off
4497 : // the mesh in ghost-land. If we're using the finite volume method, then variable values and
4498 : // consequently material properties have well-defined values in this ghost region outside of
4499 : // the mesh and we really do want to reinit our neighbor materials in this case. Since we're
4500 : // off in ghost land it's safe to do swaps with `MaterialPropertyStorage` using the elem and
4501 : // elem_side keys
4502 0 : neighbor = _assembly[tid][0]->elem();
4503 0 : neighbor_side = _assembly[tid][0]->side();
4504 : mooseAssert(neighbor, "We should have an appropriate value for elem coming from Assembly");
4505 : }
4506 : else
4507 0 : mooseError("neighbor is null in Assembly!");
4508 : }
4509 :
4510 3630872 : _neighbor_material_props.getMaterialData(tid).swapBack(*neighbor, neighbor_side);
4511 3630872 : }
4512 :
4513 : void
4514 975673 : FEProblemBase::logAdd(const std::string & system,
4515 : const std::string & name,
4516 : const std::string & type,
4517 : const InputParameters & params) const
4518 : {
4519 975673 : if (_verbose_setup != "false")
4520 102 : _console << "[DBG] Adding " << system << " '" << name << "' of type " << type << std::endl;
4521 975673 : if (_verbose_setup == "extra")
4522 0 : _console << params << std::endl;
4523 975673 : }
4524 :
4525 : void
4526 137177 : FEProblemBase::addObjectParamsHelper(InputParameters & parameters,
4527 : const std::string & object_name,
4528 : const std::string & var_param_name)
4529 : {
4530 : // Due to objects like SolutionUserObject which manipulate libmesh objects
4531 : // and variables directly at the back end, we need a default option here
4532 : // which is going to be the pointer to the first solver system within this
4533 : // problem
4534 137177 : unsigned int sys_num = 0;
4535 137177 : if (parameters.isParamValid(var_param_name))
4536 : {
4537 57608 : const auto variable_name = parameters.varName(var_param_name, object_name);
4538 57608 : if (this->hasVariable(variable_name) || this->hasScalarVariable(variable_name))
4539 51294 : sys_num = getSystem(variable_name).number();
4540 57608 : }
4541 274354 : if (parameters.isParamValid("solver_sys"))
4542 : {
4543 3762 : const auto var_sys_num = sys_num;
4544 3762 : sys_num = getSystemBase(parameters.get<SolverSystemName>("solver_sys")).number();
4545 3762 : if (sys_num != var_sys_num && parameters.isParamValid(var_param_name))
4546 0 : mooseError("We dont support setting 'variable' to a variable that is not set to the same "
4547 : "system as the 'solver_sys' parameter");
4548 : }
4549 :
4550 138341 : if (_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4551 138341 : parameters.get<bool>("use_displaced_mesh"))
4552 : {
4553 1132 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4554 566 : if (sys_num == _aux->number())
4555 1104 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->systemBaseAuxiliary();
4556 : else
4557 594 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(sys_num);
4558 : }
4559 : else
4560 : {
4561 : // The object requested use_displaced_mesh, but it was overridden
4562 : // due to there being no displacements variables in the [Mesh] block.
4563 : // If that happened, update the value of use_displaced_mesh appropriately.
4564 221532 : if (!_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4565 221532 : parameters.get<bool>("use_displaced_mesh"))
4566 144 : parameters.set<bool>("use_displaced_mesh") = false;
4567 :
4568 273222 : parameters.set<SubProblem *>("_subproblem") = this;
4569 :
4570 136611 : if (sys_num == _aux->number())
4571 39975 : parameters.set<SystemBase *>("_sys") = _aux.get();
4572 : else
4573 369858 : parameters.set<SystemBase *>("_sys") = _solver_systems[sys_num].get();
4574 : }
4575 137177 : }
4576 :
4577 : void
4578 62212 : FEProblemBase::checkUserObjectNameCollision(const std::string & name,
4579 : const std::string & type) const
4580 : {
4581 62212 : if (hasUserObject(name))
4582 9 : mooseError("A ",
4583 9 : getUserObjectBase(name).typeAndName(),
4584 : " already exists. You may not add a ",
4585 : type,
4586 : " by the same name.");
4587 :
4588 : #ifdef MOOSE_KOKKOS_ENABLED
4589 46558 : if (hasKokkosUserObject(name))
4590 0 : mooseError("A ",
4591 0 : getKokkosUserObject<UserObjectBase>(name).typeAndName(),
4592 : " already exists. You may not add a ",
4593 : type,
4594 : " by the same name.");
4595 : #endif
4596 62203 : }
4597 :
4598 : void
4599 49502 : FEProblemBase::addPostprocessor(const std::string & pp_name,
4600 : const std::string & name,
4601 : InputParameters & parameters)
4602 : {
4603 49502 : checkUserObjectNameCollision(name, "Postprocessor");
4604 :
4605 49499 : addUserObject(pp_name, name, parameters);
4606 49470 : }
4607 :
4608 : void
4609 5693 : FEProblemBase::addVectorPostprocessor(const std::string & pp_name,
4610 : const std::string & name,
4611 : InputParameters & parameters)
4612 : {
4613 5693 : checkUserObjectNameCollision(name, "VectorPostprocessor");
4614 :
4615 5690 : addUserObject(pp_name, name, parameters);
4616 5660 : }
4617 :
4618 : void
4619 4719 : FEProblemBase::addReporter(const std::string & type,
4620 : const std::string & name,
4621 : InputParameters & parameters)
4622 : {
4623 4719 : checkUserObjectNameCollision(name, "Reporter");
4624 :
4625 4716 : addUserObject(type, name, parameters);
4626 4683 : }
4627 :
4628 : std::vector<std::shared_ptr<UserObject>>
4629 69989 : FEProblemBase::addUserObject(const std::string & user_object_name,
4630 : const std::string & name,
4631 : InputParameters & parameters)
4632 : {
4633 : parallel_object_only();
4634 :
4635 69989 : std::vector<std::shared_ptr<UserObject>> uos;
4636 :
4637 : // Add the _subproblem and _sys parameters depending on use_displaced_mesh
4638 69989 : addObjectParamsHelper(parameters, name);
4639 :
4640 113730 : for (const auto tid : make_range(libMesh::n_threads()))
4641 : {
4642 : // Create the UserObject
4643 : std::shared_ptr<UserObject> user_object =
4644 73787 : _factory.create<UserObject>(user_object_name, name, parameters, tid);
4645 73512 : logAdd("UserObject", name, user_object_name, parameters);
4646 73512 : uos.push_back(user_object);
4647 :
4648 73512 : if (tid != 0)
4649 3798 : user_object->setPrimaryThreadCopy(uos[0].get());
4650 :
4651 73512 : theWarehouse().add(user_object);
4652 :
4653 : // Attempt to create all the possible UserObject types
4654 73506 : auto euo = std::dynamic_pointer_cast<ElementUserObject>(user_object);
4655 73506 : auto suo = std::dynamic_pointer_cast<SideUserObject>(user_object);
4656 73506 : auto isuo = std::dynamic_pointer_cast<InternalSideUserObject>(user_object);
4657 73506 : auto iuo = std::dynamic_pointer_cast<InterfaceUserObjectBase>(user_object);
4658 73506 : auto nuo = std::dynamic_pointer_cast<NodalUserObject>(user_object);
4659 73506 : auto duo = std::dynamic_pointer_cast<DomainUserObject>(user_object);
4660 73506 : auto guo = std::dynamic_pointer_cast<GeneralUserObject>(user_object);
4661 73506 : auto tguo = std::dynamic_pointer_cast<ThreadedGeneralUserObject>(user_object);
4662 73506 : auto muo = std::dynamic_pointer_cast<MortarUserObject>(user_object);
4663 :
4664 : // Account for displaced mesh use
4665 73506 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4666 : {
4667 : // Whether to re-init or not depends on the attributes of the base classes.
4668 : // For example, InterfaceUOBase has "_current_side_elem" and "_neighbor_elem"
4669 : // so it needs to reinit on displaced neighbors and faces
4670 : // _reinit_displaced_elem -> _current_elem will be reinited
4671 : // _reinit_displaced_face -> _current_elem, lowerD if any and _current_side_elem to be
4672 : // reinited _reinit_displaced_neighbor -> _current_elem, lowerD if any and _current_neighbor
4673 : // to be reinited Note that as soon as you use materials on the displaced mesh, all three get
4674 : // turned on.
4675 615 : if (euo || nuo || duo)
4676 555 : _reinit_displaced_elem = true;
4677 615 : if (suo || duo || isuo || iuo)
4678 24 : _reinit_displaced_face = true;
4679 615 : if (iuo || duo || isuo)
4680 0 : _reinit_displaced_neighbor = true;
4681 : }
4682 :
4683 : // These objects only require one thread
4684 73506 : if ((guo && !tguo) || muo)
4685 29765 : break;
4686 341391 : }
4687 :
4688 : // Add as a Functor if it is one. We usually need to add the user object from thread 0 as the
4689 : // registered functor for all threads because when user objects are thread joined, generally only
4690 : // the primary thread copy ends up with all the data
4691 146698 : for (const auto tid : make_range(libMesh::n_threads()))
4692 : {
4693 76990 : const decltype(uos)::size_type uo_index = uos.front()->needThreadedCopy() ? tid : 0;
4694 76990 : if (const auto functor = dynamic_cast<Moose::FunctorBase<Real> *>(uos[uo_index].get()))
4695 : {
4696 60164 : this->addFunctor(name, *functor, tid);
4697 60164 : if (_displaced_problem)
4698 729 : _displaced_problem->addFunctor(name, *functor, tid);
4699 : }
4700 : }
4701 :
4702 69708 : return uos;
4703 2 : }
4704 :
4705 : void
4706 2002 : FEProblemBase::addFVInterpolationMethod(const std::string & method_type,
4707 : const std::string & name,
4708 : InputParameters & parameters)
4709 : {
4710 : parallel_object_only();
4711 :
4712 2002 : addObjectParamsHelper(parameters, name);
4713 :
4714 4004 : for (const auto tid : make_range(libMesh::n_threads()))
4715 : {
4716 2002 : auto method = _factory.create<FVInterpolationMethod>(method_type, name, parameters, tid);
4717 2002 : logAdd("FVInterpolationMethod", name, method_type, parameters);
4718 2002 : theWarehouse().add(method);
4719 2002 : }
4720 2002 : }
4721 :
4722 : const UserObject &
4723 285472 : FEProblemBase::getUserObjectBase(const std::string & name, const THREAD_ID tid /* = 0 */) const
4724 : {
4725 285472 : std::vector<UserObject *> objs;
4726 285472 : theWarehouse()
4727 570944 : .query()
4728 285472 : .condition<AttribSystem>("UserObject")
4729 285472 : .condition<AttribThread>(tid)
4730 285472 : .condition<AttribName>(name)
4731 285472 : .queryInto(objs);
4732 285472 : if (objs.empty())
4733 : {
4734 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_user_object"),
4735 : "A UserObject getter was called before UserObjects have been constructed. The "
4736 : "requested UserObject '" +
4737 : name + "' may exist in the input file, but UserObjects are not available yet.");
4738 :
4739 0 : mooseError("Unable to find user object with name '" + name + "'");
4740 : }
4741 : mooseAssert(objs.size() == 1, "Should only find one UO");
4742 570944 : return *(objs[0]);
4743 285472 : }
4744 :
4745 : const Positions &
4746 1142 : FEProblemBase::getPositionsObject(const std::string & name) const
4747 : {
4748 1142 : std::vector<Positions *> objs;
4749 1142 : theWarehouse()
4750 2284 : .query()
4751 1142 : .condition<AttribSystem>("UserObject")
4752 1142 : .condition<AttribName>(name)
4753 1142 : .queryInto(objs);
4754 1142 : if (objs.empty())
4755 0 : mooseError("Unable to find Positions object with name '" + name + "'");
4756 : mooseAssert(objs.size() == 1, "Should only find one Positions");
4757 2284 : return *(objs[0]);
4758 1142 : }
4759 :
4760 : bool
4761 90885 : FEProblemBase::hasUserObject(const std::string & name) const
4762 : {
4763 90885 : std::vector<UserObject *> objs;
4764 90885 : theWarehouse()
4765 90885 : .query()
4766 90885 : .condition<AttribSystem>("UserObject")
4767 181770 : .condition<AttribThread>(0)
4768 90885 : .condition<AttribName>(name)
4769 90885 : .queryInto(objs);
4770 181770 : return !objs.empty();
4771 90885 : }
4772 :
4773 : const FVInterpolationMethod &
4774 571 : FEProblemBase::getFVInterpolationMethod(const InterpolationMethodName & name,
4775 : const THREAD_ID tid) const
4776 : {
4777 571 : std::vector<FVInterpolationMethod *> methods;
4778 571 : theWarehouse()
4779 1142 : .query()
4780 571 : .condition<AttribSystem>("FVInterpolationMethod")
4781 571 : .condition<AttribThread>(tid)
4782 571 : .condition<AttribName>(name)
4783 571 : .queryInto(methods);
4784 :
4785 571 : if (methods.empty())
4786 : {
4787 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_interpolation_method"),
4788 : "An FVInterpolationMethod getter was called before FVInterpolationMethods have "
4789 : "been constructed. If you are attempting to access this object in the constructor "
4790 : "of another object then make sure that the FVInterpolationMethod is constructed "
4791 : "before the object using it.");
4792 :
4793 0 : mooseError("Unable to find FVInterpolationMethod with name '", name, "'");
4794 : }
4795 :
4796 : mooseAssert(methods.size() == 1, "Expected a single FVInterpolationMethod per thread");
4797 1142 : return *(methods[0]);
4798 571 : }
4799 :
4800 : const FVFaceInterpolationMethod &
4801 0 : FEProblemBase::getFVFaceInterpolationMethod(const InterpolationMethodName & name,
4802 : const THREAD_ID tid) const
4803 : {
4804 0 : const auto & method = getFVInterpolationMethod(name, tid);
4805 0 : const auto * face_method = dynamic_cast<const FVFaceInterpolationMethod *>(&method);
4806 :
4807 0 : if (!face_method)
4808 0 : mooseError("FVInterpolationMethod '",
4809 : name,
4810 : "' (",
4811 0 : method.type(),
4812 : ") is not a scalar face interpolation method.");
4813 :
4814 0 : return *face_method;
4815 : }
4816 :
4817 : const FVAdvectedInterpolationMethod &
4818 571 : FEProblemBase::getFVAdvectedInterpolationMethod(const InterpolationMethodName & name,
4819 : const THREAD_ID tid) const
4820 : {
4821 571 : const auto & method = getFVInterpolationMethod(name, tid);
4822 571 : const auto * advected_method = dynamic_cast<const FVAdvectedInterpolationMethod *>(&method);
4823 :
4824 571 : if (!advected_method)
4825 0 : mooseError("FVInterpolationMethod '",
4826 : name,
4827 : "' (",
4828 0 : method.type(),
4829 : ") is not an advected interpolation method.");
4830 :
4831 571 : return *advected_method;
4832 : }
4833 :
4834 : bool
4835 0 : FEProblemBase::hasFVInterpolationMethod(const InterpolationMethodName & name) const
4836 : {
4837 0 : std::vector<FVInterpolationMethod *> methods;
4838 0 : theWarehouse()
4839 0 : .query()
4840 0 : .condition<AttribSystem>("FVInterpolationMethod")
4841 0 : .condition<AttribThread>(0)
4842 0 : .condition<AttribName>(name)
4843 0 : .queryInto(methods);
4844 0 : return !methods.empty();
4845 0 : }
4846 :
4847 : bool
4848 308 : FEProblemBase::hasPostprocessorValueByName(const PostprocessorName & name) const
4849 : {
4850 308 : return _reporter_data.hasReporterValue<PostprocessorValue>(PostprocessorReporterName(name));
4851 : }
4852 :
4853 : const Postprocessor &
4854 2 : FEProblemBase::getPostprocessorObjectByName(const PostprocessorName & object_name,
4855 : const THREAD_ID tid) const
4856 : {
4857 2 : std::vector<Postprocessor *> objs;
4858 2 : theWarehouse()
4859 2 : .query()
4860 4 : .condition<AttribInterfaces>(Interfaces::Postprocessor)
4861 2 : .condition<AttribThread>(tid)
4862 2 : .condition<AttribName>(object_name)
4863 2 : .queryInto(objs);
4864 :
4865 2 : if (objs.empty())
4866 0 : mooseError("Unable to find Postprocessor with name '", object_name, "'");
4867 : mooseAssert(objs.size() == 1,
4868 : "We shouldn't find more than one postprocessor object for a given name");
4869 4 : return *(objs[0]);
4870 2 : }
4871 :
4872 : const PostprocessorValue &
4873 844535 : FEProblemBase::getPostprocessorValueByName(const PostprocessorName & name,
4874 : std::size_t t_index) const
4875 : {
4876 1689070 : return _reporter_data.getReporterValue<PostprocessorValue>(PostprocessorReporterName(name),
4877 1689070 : t_index);
4878 : }
4879 :
4880 : void
4881 577595 : FEProblemBase::setPostprocessorValueByName(const PostprocessorName & name,
4882 : const PostprocessorValue & value,
4883 : std::size_t t_index)
4884 : {
4885 577595 : _reporter_data.setReporterValue<PostprocessorValue>(
4886 1155190 : PostprocessorReporterName(name), value, t_index);
4887 577595 : }
4888 :
4889 : bool
4890 52 : FEProblemBase::hasPostprocessor(const std::string & name) const
4891 : {
4892 52 : mooseDeprecated("FEProblemBase::hasPostprocssor is being removed; use "
4893 : "hasPostprocessorValueByName instead.");
4894 52 : return hasPostprocessorValueByName(name);
4895 : }
4896 :
4897 : const VectorPostprocessorValue &
4898 50 : FEProblemBase::getVectorPostprocessorValueByName(const std::string & object_name,
4899 : const std::string & vector_name,
4900 : std::size_t t_index) const
4901 : {
4902 50 : return _reporter_data.getReporterValue<VectorPostprocessorValue>(
4903 100 : VectorPostprocessorReporterName(object_name, vector_name), t_index);
4904 : }
4905 :
4906 : void
4907 18 : FEProblemBase::setVectorPostprocessorValueByName(const std::string & object_name,
4908 : const std::string & vector_name,
4909 : const VectorPostprocessorValue & value,
4910 : std::size_t t_index)
4911 : {
4912 18 : _reporter_data.setReporterValue<VectorPostprocessorValue>(
4913 36 : VectorPostprocessorReporterName(object_name, vector_name), value, t_index);
4914 18 : }
4915 :
4916 : const VectorPostprocessor &
4917 9768 : FEProblemBase::getVectorPostprocessorObjectByName(const std::string & object_name,
4918 : const THREAD_ID tid) const
4919 : {
4920 9768 : std::vector<VectorPostprocessor *> objs;
4921 9768 : theWarehouse()
4922 9768 : .query()
4923 19536 : .condition<AttribInterfaces>(Interfaces::VectorPostprocessor)
4924 9768 : .condition<AttribThread>(tid)
4925 9768 : .condition<AttribName>(object_name)
4926 9768 : .queryInto(objs);
4927 :
4928 9768 : if (objs.empty())
4929 : {
4930 : mooseAssert(
4931 : getMooseApp().actionWarehouse().isTaskComplete("add_vector_postprocessor"),
4932 : "A VectorPostprocessor getter was called before VectorPostprocessors have been "
4933 : "constructed. The requested VectorPostprocessor '" +
4934 : object_name +
4935 : "' may exist in the input file, but VectorPostprocessors are not available yet.");
4936 :
4937 0 : mooseError("Unable to find VectorPostprocessor with name '", object_name, "'");
4938 : }
4939 : mooseAssert(objs.size() == 1,
4940 : "We shouldn't find more than one vector postprocessor object for a given name");
4941 19536 : return *(objs[0]);
4942 9768 : }
4943 :
4944 : void
4945 70 : FEProblemBase::parentOutputPositionChanged()
4946 : {
4947 2030 : for (const auto & it : _multi_apps)
4948 : {
4949 1960 : const auto & objects = it.second.getActiveObjects();
4950 1979 : for (const auto & obj : objects)
4951 19 : obj->parentOutputPositionChanged();
4952 : }
4953 70 : }
4954 :
4955 : void
4956 0 : FEProblemBase::computeIndicatorsAndMarkers()
4957 : {
4958 0 : computeIndicators();
4959 0 : computeMarkers();
4960 0 : }
4961 :
4962 : void
4963 224698 : FEProblemBase::computeIndicators()
4964 : {
4965 : // Initialize indicator aux variable fields
4966 224698 : if (_indicators.hasActiveObjects() || _internal_side_indicators.hasActiveObjects())
4967 : {
4968 11875 : TIME_SECTION("computeIndicators", 1, "Computing Indicators");
4969 :
4970 : // Internal side indicators may lead to creating a much larger sparsity pattern than dictated by
4971 : // the actual finite element scheme (e.g. CFEM)
4972 2375 : const auto old_do_derivatives = ADReal::do_derivatives;
4973 2375 : ADReal::do_derivatives = false;
4974 :
4975 2375 : std::vector<std::string> fields;
4976 :
4977 : // Indicator Fields
4978 2375 : const auto & indicators = _indicators.getActiveObjects();
4979 2521 : for (const auto & indicator : indicators)
4980 146 : fields.push_back(indicator->name());
4981 :
4982 : // InternalSideIndicator Fields
4983 2375 : const auto & internal_indicators = _internal_side_indicators.getActiveObjects();
4984 4706 : for (const auto & internal_indicator : internal_indicators)
4985 2331 : fields.push_back(internal_indicator->name());
4986 :
4987 2375 : _aux->zeroVariables(fields);
4988 :
4989 : // compute Indicators
4990 2375 : ComputeIndicatorThread cit(*this);
4991 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cit);
4992 2375 : _aux->solution().close();
4993 2375 : _aux->update();
4994 :
4995 2375 : ComputeIndicatorThread finalize_cit(*this, true);
4996 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), finalize_cit);
4997 2375 : _aux->solution().close();
4998 2375 : _aux->update();
4999 :
5000 2375 : ADReal::do_derivatives = old_do_derivatives;
5001 2375 : }
5002 224698 : }
5003 :
5004 : void
5005 224720 : FEProblemBase::computeMarkers()
5006 : {
5007 224720 : if (_markers.hasActiveObjects())
5008 : {
5009 32580 : TIME_SECTION("computeMarkers", 1, "Computing Markers");
5010 :
5011 6516 : std::vector<std::string> fields;
5012 :
5013 : // Marker Fields
5014 6516 : const auto & markers = _markers.getActiveObjects();
5015 14011 : for (const auto & marker : markers)
5016 7495 : fields.push_back(marker->name());
5017 :
5018 6516 : _aux->zeroVariables(fields);
5019 :
5020 6516 : _adaptivity.updateErrorVectors();
5021 :
5022 13660 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5023 : {
5024 7147 : const auto & markers = _markers.getActiveObjects(tid);
5025 15365 : for (const auto & marker : markers)
5026 8221 : marker->markerSetup();
5027 : }
5028 :
5029 6513 : ComputeMarkerThread cmt(*this);
5030 6513 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cmt);
5031 :
5032 6513 : _aux->solution().close();
5033 6513 : _aux->update();
5034 6513 : }
5035 224717 : }
5036 :
5037 : const ExecFlagType &
5038 6219493 : FEProblemBase::getCurrentExecuteOnFlag() const
5039 : {
5040 6219493 : return _current_execute_on_flag;
5041 : }
5042 :
5043 : void
5044 7010197 : FEProblemBase::setCurrentExecuteOnFlag(const ExecFlagType & flag)
5045 : {
5046 7010197 : _current_execute_on_flag = flag;
5047 7010197 : }
5048 :
5049 : void
5050 72 : FEProblemBase::executeAllObjects(const ExecFlagType & /*exec_type*/)
5051 : {
5052 72 : }
5053 :
5054 : void
5055 1798456 : FEProblemBase::customSetup(const ExecFlagType & exec_type)
5056 : {
5057 1798456 : SubProblem::customSetup(exec_type);
5058 :
5059 1798456 : if (_line_search)
5060 0 : _line_search->customSetup(exec_type);
5061 :
5062 1798456 : unsigned int n_threads = libMesh::n_threads();
5063 3774763 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5064 : {
5065 1976307 : _all_materials.customSetup(exec_type, tid);
5066 1976307 : _functions.customSetup(exec_type, tid);
5067 : }
5068 :
5069 : #ifdef MOOSE_KOKKOS_ENABLED
5070 1313945 : _kokkos_functions.customSetup(exec_type);
5071 : #endif
5072 :
5073 1798456 : _aux->customSetup(exec_type);
5074 3599882 : for (auto & nl : _nl)
5075 1801426 : nl->customSetup(exec_type);
5076 :
5077 1798456 : if (_displaced_problem)
5078 145157 : _displaced_problem->customSetup(exec_type);
5079 :
5080 3774763 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5081 : {
5082 1976307 : _internal_side_indicators.customSetup(exec_type, tid);
5083 1976307 : _indicators.customSetup(exec_type, tid);
5084 1976307 : _markers.customSetup(exec_type, tid);
5085 : }
5086 :
5087 1798456 : std::vector<UserObject *> userobjs;
5088 1798456 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
5089 4446318 : for (auto obj : userobjs)
5090 2647862 : obj->customSetup(exec_type);
5091 :
5092 : #ifdef MOOSE_KOKKOS_ENABLED
5093 : {
5094 1313945 : std::vector<UserObjectBase *> userobjs;
5095 1313945 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
5096 1327406 : for (auto obj : userobjs)
5097 13461 : obj->customSetup(exec_type);
5098 1313945 : }
5099 : #endif
5100 :
5101 1798456 : _app.getOutputWarehouse().customSetup(exec_type);
5102 1798456 : }
5103 :
5104 : void
5105 2171969 : FEProblemBase::execute(const ExecFlagType & exec_type)
5106 : {
5107 : // Set the current flag
5108 2171969 : setCurrentExecuteOnFlag(exec_type);
5109 :
5110 2171969 : if (exec_type != EXEC_INITIAL)
5111 2116034 : executeControls(exec_type);
5112 :
5113 : // intentially call this after executing controls because the setups may rely on the controls
5114 : // FIXME: we skip the following flags because they have dedicated setup functions in
5115 : // SetupInterface and it may not be appropriate to call them here.
5116 3972878 : if (!(exec_type == EXEC_INITIAL || exec_type == EXEC_TIMESTEP_BEGIN ||
5117 1800927 : exec_type == EXEC_SUBDOMAIN || exec_type == EXEC_NONLINEAR || exec_type == EXEC_LINEAR))
5118 1798456 : customSetup(exec_type);
5119 :
5120 2171951 : executeSamplers(exec_type);
5121 :
5122 : // Pre-aux UserObjects
5123 2171927 : computeUserObjects(exec_type, Moose::PRE_AUX);
5124 :
5125 : // Systems (includes system time derivative and aux kernel calculations)
5126 2171927 : computeSystems(exec_type);
5127 : // With the auxiliary system solution computed, sync the displaced problem auxiliary solution
5128 : // before computation of post-aux user objects. The undisplaced auxiliary system current local
5129 : // solution is updated (via System::update) within the AuxiliarySystem class's variable
5130 : // computation methods (e.g. computeElementalVarsHelper, computeNodalVarsHelper), so it is safe to
5131 : // use it here
5132 2171894 : if (_displaced_problem)
5133 179606 : _displaced_problem->syncAuxSolution(*getAuxiliarySystem().currentSolution());
5134 :
5135 : // Post-aux UserObjects
5136 2171894 : computeUserObjects(exec_type, Moose::POST_AUX);
5137 :
5138 : // Return the current flag to None
5139 2171805 : setCurrentExecuteOnFlag(EXEC_NONE);
5140 :
5141 2171805 : if (_uo_aux_state_check && !_checking_uo_aux_state)
5142 : {
5143 : // we will only check aux variables and postprocessors
5144 : // checking more reporter data can be added in the future if needed
5145 559 : std::unique_ptr<NumericVector<Number>> x = _aux->currentSolution()->clone();
5146 559 : DenseVector<Real> pp_values = getReporterData().getAllRealReporterValues();
5147 :
5148 : // call THIS execute one more time for checking the possible states
5149 559 : _checking_uo_aux_state = true;
5150 559 : FEProblemBase::execute(exec_type);
5151 559 : _checking_uo_aux_state = false;
5152 :
5153 559 : const Real check_tol = 1e-8;
5154 :
5155 559 : const Real xnorm = x->l2_norm();
5156 559 : *x -= *_aux->currentSolution();
5157 559 : if (x->l2_norm() > check_tol * xnorm)
5158 : {
5159 3 : const auto & sys = _aux->system();
5160 3 : const unsigned int n_vars = sys.n_vars();
5161 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5162 15 : for (const auto i : make_range(n_vars))
5163 : {
5164 12 : const Real vnorm = sys.calculate_norm(*x, i, DISCRETE_L2);
5165 12 : ordered_map.emplace(vnorm, sys.variable_name(i));
5166 : }
5167 :
5168 3 : std::ostringstream oss;
5169 15 : for (const auto & [error_norm, var_name] : ordered_map)
5170 12 : oss << " {" << var_name << ", " << error_norm << "},\n";
5171 :
5172 3 : mooseError("Aux kernels, user objects appear to have states for aux variables on ",
5173 : exec_type,
5174 : ".\nVariable error norms in descending order:\n",
5175 3 : oss.str());
5176 0 : }
5177 :
5178 556 : const DenseVector<Real> new_pp_values = getReporterData().getAllRealReporterValues();
5179 556 : if (pp_values.size() != new_pp_values.size())
5180 0 : mooseError("Second execution for uo/aux state check should not change the number of "
5181 : "real reporter values");
5182 :
5183 556 : const Real ppnorm = pp_values.l2_norm();
5184 556 : pp_values -= new_pp_values;
5185 556 : if (pp_values.l2_norm() > check_tol * ppnorm)
5186 : {
5187 3 : const auto pp_names = getReporterData().getAllRealReporterFullNames();
5188 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5189 12 : for (const auto i : index_range(pp_names))
5190 9 : ordered_map.emplace(std::abs(pp_values(i)), pp_names[i]);
5191 :
5192 3 : std::ostringstream oss;
5193 12 : for (const auto & [error_norm, pp_name] : ordered_map)
5194 9 : oss << " {" << pp_name << ", " << error_norm << "},\n";
5195 :
5196 3 : mooseError("Aux kernels, user objects appear to have states for real reporter values on ",
5197 : exec_type,
5198 : ".\nErrors of real reporter values in descending order:\n",
5199 3 : oss.str());
5200 0 : }
5201 553 : }
5202 2171799 : }
5203 :
5204 : // Finalize, threadJoin, and update PP values of Elemental/Nodal/Side/InternalSideUserObjects
5205 : void
5206 1343137 : FEProblemBase::joinAndFinalize(TheWarehouse::Query query, bool isgen)
5207 : {
5208 1343137 : std::vector<UserObject *> objs;
5209 1343137 : query.queryInto(objs);
5210 1343137 : if (!isgen)
5211 : {
5212 : // join all threaded user objects (i.e. not regular general user objects) to the primary
5213 : // thread
5214 1422422 : for (auto obj : objs)
5215 371125 : if (obj->primaryThreadCopy())
5216 30744 : obj->primaryThreadCopy()->threadJoin(*obj);
5217 : }
5218 :
5219 1343137 : query.condition<AttribThread>(0).queryInto(objs);
5220 :
5221 : // finalize objects and retrieve/store any postprocessor values
5222 1914920 : for (auto obj : objs)
5223 : {
5224 571863 : if (isgen && dynamic_cast<ThreadedGeneralUserObject *>(obj))
5225 133 : continue;
5226 571730 : if (isgen)
5227 : {
5228 : // general user objects are not run in their own threaded loop object - so run them here
5229 231349 : if (shouldPrintExecution(0))
5230 724 : _console << "[DBG] Initializing, executing & finalizing general UO '" << obj->name()
5231 724 : << "' on " << _current_execute_on_flag.name() << std::endl;
5232 231349 : obj->initialize();
5233 231349 : obj->execute();
5234 : }
5235 :
5236 571680 : obj->finalize();
5237 :
5238 : // These have to be stored piecemeal (with every call to this function) because general
5239 : // postprocessors (which run last after other userobjects have been completed) might depend on
5240 : // them being stored. This wouldn't be a problem if all userobjects satisfied the dependency
5241 : // resolver interface and could be sorted appropriately with the general userobjects, but they
5242 : // don't.
5243 571665 : auto pp = dynamic_cast<const Postprocessor *>(obj);
5244 571665 : if (pp)
5245 : {
5246 499255 : _reporter_data.finalize(obj->name());
5247 499255 : setPostprocessorValueByName(obj->name(), pp->getValue());
5248 : }
5249 :
5250 571656 : auto vpp = dynamic_cast<VectorPostprocessor *>(obj);
5251 571656 : if (vpp)
5252 13523 : _reporter_data.finalize(obj->name());
5253 :
5254 : // Update Reporter data
5255 571656 : auto reporter = dynamic_cast<Reporter *>(obj);
5256 571656 : if (reporter)
5257 5313 : _reporter_data.finalize(obj->name());
5258 : }
5259 1343057 : }
5260 :
5261 : TheWarehouse::Query
5262 20004755 : FEProblemBase::getUOQuery(const std::string & system,
5263 : const ExecFlagType & type,
5264 : const Moose::AuxGroup & group) const
5265 : {
5266 : TheWarehouse::Query query =
5267 20004755 : theWarehouse().query().condition<AttribSystem>(system).condition<AttribExecOns>(type);
5268 :
5269 20004755 : if (group == Moose::PRE_IC)
5270 97387 : query.condition<AttribPreIC>(true);
5271 19907368 : else if (group == Moose::PRE_AUX)
5272 9953305 : query.condition<AttribPreAux>(type);
5273 9954063 : else if (group == Moose::POST_AUX)
5274 9953995 : query.condition<AttribPostAux>(type);
5275 :
5276 20004755 : return query;
5277 0 : }
5278 :
5279 : void
5280 20004755 : FEProblemBase::getUOExecutionGroups(TheWarehouse::Query & query,
5281 : std::set<int> & execution_groups) const
5282 : {
5283 20004755 : std::vector<UserObjectBase *> uos;
5284 20004755 : query.queryIntoUnsorted(uos);
5285 20609790 : for (const auto & uo : uos)
5286 1815105 : execution_groups.insert(uo->getParam<int>("execution_order_group"));
5287 20004755 : }
5288 :
5289 : void
5290 56897 : FEProblemBase::computeUserObjectByName(const ExecFlagType & type,
5291 : const Moose::AuxGroup & group,
5292 : const std::string & name)
5293 : {
5294 56897 : const auto old_exec_flag = _current_execute_on_flag;
5295 56897 : _current_execute_on_flag = type;
5296 :
5297 56897 : std::set<int> execution_groups;
5298 :
5299 : #ifdef MOOSE_KOKKOS_ENABLED
5300 : TheWarehouse::Query kokkos_query =
5301 41026 : getUOQuery("KokkosUserObject", type, group).condition<AttribName>(name);
5302 41026 : getUOExecutionGroups(kokkos_query, execution_groups);
5303 : #endif
5304 :
5305 56897 : TheWarehouse::Query query = getUOQuery("UserObject", type, group).condition<AttribName>(name);
5306 56897 : getUOExecutionGroups(query, execution_groups);
5307 :
5308 82848 : for (const auto execution_group : execution_groups)
5309 : {
5310 : #ifdef MOOSE_KOKKOS_ENABLED
5311 18721 : computeKokkosUserObjectsInternal(
5312 18721 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5313 : #endif
5314 :
5315 25951 : computeUserObjectsInternal(type,
5316 25951 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5317 : }
5318 :
5319 56897 : _current_execute_on_flag = old_exec_flag;
5320 56897 : }
5321 :
5322 : void
5323 11514958 : FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGroup & group)
5324 : {
5325 11514958 : std::set<int> execution_groups;
5326 :
5327 : #ifdef MOOSE_KOKKOS_ENABLED
5328 8391874 : TheWarehouse::Query kokkos_query = getUOQuery("KokkosUserObject", type, group);
5329 8391874 : getUOExecutionGroups(kokkos_query, execution_groups);
5330 : #endif
5331 :
5332 11514958 : TheWarehouse::Query query = getUOQuery("UserObject", type, group);
5333 11514958 : getUOExecutionGroups(query, execution_groups);
5334 :
5335 11782483 : for (const auto execution_group : execution_groups)
5336 : {
5337 : #ifdef MOOSE_KOKKOS_ENABLED
5338 194886 : computeKokkosUserObjectsInternal(
5339 194886 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5340 : #endif
5341 :
5342 267614 : computeUserObjectsInternal(type,
5343 267614 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5344 : }
5345 11514869 : }
5346 :
5347 : void
5348 293565 : FEProblemBase::computeUserObjectsInternal(const ExecFlagType & type, TheWarehouse::Query & query)
5349 : {
5350 : try
5351 : {
5352 1467825 : TIME_SECTION("computeUserObjects", 1, "Computing User Objects");
5353 :
5354 293565 : std::vector<GeneralUserObject *> genobjs;
5355 293565 : query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject).queryInto(genobjs);
5356 :
5357 293565 : std::vector<UserObject *> userobjs;
5358 293565 : query.clone()
5359 587130 : .condition<AttribInterfaces>(Interfaces::ElementUserObject | Interfaces::SideUserObject |
5360 : Interfaces::InternalSideUserObject |
5361 587130 : Interfaces::InterfaceUserObject | Interfaces::DomainUserObject)
5362 293565 : .queryInto(userobjs);
5363 :
5364 293565 : std::vector<UserObject *> tgobjs;
5365 293565 : query.clone()
5366 587130 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5367 293565 : .queryInto(tgobjs);
5368 :
5369 293565 : std::vector<UserObject *> nodal;
5370 293565 : query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).queryInto(nodal);
5371 :
5372 293565 : std::vector<MortarUserObject *> mortar;
5373 293565 : query.clone().condition<AttribInterfaces>(Interfaces::MortarUserObject).queryInto(mortar);
5374 :
5375 293565 : if (userobjs.empty() && genobjs.empty() && tgobjs.empty() && nodal.empty() && mortar.empty())
5376 1710 : return;
5377 :
5378 : // Start the timer here since we have at least one active user object
5379 291855 : std::string compute_uo_tag = "computeUserObjects(" + Moose::stringify(type) + ")";
5380 :
5381 : // Perform Residual/Jacobian setups
5382 291855 : if (type == EXEC_LINEAR)
5383 : {
5384 125261 : for (auto obj : userobjs)
5385 71388 : obj->residualSetup();
5386 58331 : for (auto obj : nodal)
5387 4458 : obj->residualSetup();
5388 53873 : for (auto obj : mortar)
5389 0 : obj->residualSetup();
5390 53882 : for (auto obj : tgobjs)
5391 9 : obj->residualSetup();
5392 65556 : for (auto obj : genobjs)
5393 11683 : obj->residualSetup();
5394 : }
5395 237982 : else if (type == EXEC_NONLINEAR)
5396 : {
5397 14080 : for (auto obj : userobjs)
5398 4420 : obj->jacobianSetup();
5399 10052 : for (auto obj : nodal)
5400 392 : obj->jacobianSetup();
5401 9660 : for (auto obj : mortar)
5402 0 : obj->jacobianSetup();
5403 9663 : for (auto obj : tgobjs)
5404 3 : obj->jacobianSetup();
5405 26470 : for (auto obj : genobjs)
5406 16810 : obj->jacobianSetup();
5407 : }
5408 :
5409 642269 : for (auto obj : userobjs)
5410 350414 : obj->initialize();
5411 :
5412 : // Execute Side/InternalSide/Interface/Elemental/DomainUserObjects
5413 291855 : if (!userobjs.empty())
5414 : {
5415 : // non-nodal user objects have to be run separately before the nodal user objects run
5416 : // because some nodal user objects (NodalNormal related) depend on elemental user objects
5417 : // :-(
5418 206969 : ComputeUserObjectsThread cppt(*this, query);
5419 206969 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cppt);
5420 :
5421 : // There is one instance in rattlesnake where an elemental user object's finalize depends
5422 : // on a side user object having been finalized first :-(
5423 206960 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::SideUserObject));
5424 206957 : joinAndFinalize(
5425 413914 : query.clone().condition<AttribInterfaces>(Interfaces::InternalSideUserObject));
5426 206957 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::InterfaceUserObject));
5427 206957 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::ElementUserObject));
5428 206954 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::DomainUserObject));
5429 206954 : }
5430 :
5431 : // if any elemental user object may have written to variables we need to close the aux solution
5432 642162 : for (const auto & uo : userobjs)
5433 350355 : if (auto euo = dynamic_cast<const ElementUserObject *>(uo);
5434 350355 : euo && euo->hasWritableCoupledVariables())
5435 : {
5436 33 : _aux->solution().close();
5437 33 : _aux->system().update();
5438 33 : break;
5439 : }
5440 :
5441 : // Execute NodalUserObjects
5442 : // BISON has an axial reloc elemental user object that has a finalize func that depends on a
5443 : // nodal user object's prev value. So we can't initialize this until after elemental objects
5444 : // have been finalized :-(
5445 312146 : for (auto obj : nodal)
5446 20306 : obj->initialize();
5447 291840 : if (query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).count() > 0)
5448 : {
5449 16379 : ComputeNodalUserObjectsThread cnppt(*this, query);
5450 16379 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), cnppt);
5451 16379 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject));
5452 16379 : }
5453 :
5454 : // if any nodal user object may have written to variables we need to close the aux solution
5455 312110 : for (const auto & uo : nodal)
5456 20292 : if (auto nuo = dynamic_cast<const NodalUserObject *>(uo);
5457 20292 : nuo && nuo->hasWritableCoupledVariables())
5458 : {
5459 22 : _aux->solution().close();
5460 22 : _aux->system().update();
5461 22 : break;
5462 : }
5463 :
5464 : // Execute MortarUserObjects
5465 : {
5466 291862 : for (auto obj : mortar)
5467 22 : obj->initialize();
5468 291840 : if (!mortar.empty())
5469 : {
5470 33 : auto create_and_run_mortar_functors = [this, type, &mortar](const bool displaced)
5471 : {
5472 : // go over mortar interfaces and construct functors
5473 33 : const auto & mortar_interfaces = getMortarInterfaces(displaced);
5474 55 : for (const auto & [primary_secondary_boundary_pair, interface_config] : mortar_interfaces)
5475 : {
5476 : auto mortar_uos_to_execute =
5477 22 : getMortarUserObjects(primary_secondary_boundary_pair.first,
5478 22 : primary_secondary_boundary_pair.second,
5479 : displaced,
5480 22 : mortar);
5481 :
5482 : auto * const subproblem = displaced
5483 22 : ? static_cast<SubProblem *>(_displaced_problem.get())
5484 22 : : static_cast<SubProblem *>(this);
5485 : MortarUserObjectThread muot(mortar_uos_to_execute,
5486 22 : *interface_config.amg,
5487 : *subproblem,
5488 : *this,
5489 : displaced,
5490 22 : subproblem->assembly(0, 0));
5491 :
5492 22 : muot();
5493 22 : }
5494 55 : };
5495 :
5496 22 : create_and_run_mortar_functors(false);
5497 22 : if (_displaced_problem)
5498 11 : create_and_run_mortar_functors(true);
5499 22 : }
5500 291862 : for (auto obj : mortar)
5501 22 : obj->finalize();
5502 : }
5503 :
5504 : // Execute threaded general user objects
5505 292281 : for (auto obj : tgobjs)
5506 441 : obj->initialize();
5507 291840 : std::vector<GeneralUserObject *> tguos_zero;
5508 291840 : query.clone()
5509 291840 : .condition<AttribThread>(0)
5510 583680 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5511 291840 : .queryInto(tguos_zero);
5512 291973 : for (auto obj : tguos_zero)
5513 : {
5514 133 : std::vector<GeneralUserObject *> tguos;
5515 133 : auto q = query.clone()
5516 133 : .condition<AttribName>(obj->name())
5517 133 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject);
5518 133 : q.queryInto(tguos);
5519 :
5520 133 : ComputeThreadedGeneralUserObjectsThread ctguot(*this);
5521 :
5522 : // Force one thread per ThreadedGeneralUserObject via grainsize
5523 266 : Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(),
5524 133 : tguos.end(),
5525 : /*grainsize=*/1),
5526 : ctguot);
5527 133 : joinAndFinalize(q);
5528 133 : }
5529 :
5530 : // Execute general user objects
5531 291840 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject), true);
5532 302026 : }
5533 0 : catch (...)
5534 : {
5535 0 : handleException("computeUserObjectsInternal");
5536 0 : }
5537 : }
5538 :
5539 : void
5540 5733312 : FEProblemBase::executeControls(const ExecFlagType & exec_type)
5541 : {
5542 5733312 : if (_control_warehouse[exec_type].hasActiveObjects())
5543 : {
5544 40205 : TIME_SECTION("executeControls", 1, "Executing Controls");
5545 :
5546 8041 : DependencyResolver<std::shared_ptr<Control>> resolver;
5547 :
5548 8041 : auto controls_wh = _control_warehouse[exec_type];
5549 : // Add all of the dependencies into the resolver and sort them
5550 20003 : for (const auto & it : controls_wh.getActiveObjects())
5551 : {
5552 : // Make sure an item with no dependencies comes out too!
5553 11965 : resolver.addItem(it);
5554 :
5555 11965 : std::vector<std::string> & dependent_controls = it->getDependencies();
5556 14857 : for (const auto & depend_name : dependent_controls)
5557 : {
5558 2895 : if (controls_wh.hasActiveObject(depend_name))
5559 : {
5560 2892 : auto dep_control = controls_wh.getActiveObject(depend_name);
5561 2892 : resolver.addEdge(dep_control, it);
5562 2892 : }
5563 : else
5564 3 : mooseError("The Control \"",
5565 : depend_name,
5566 : "\" was not created, did you make a "
5567 : "spelling mistake or forget to include it "
5568 : "in your input file?");
5569 : }
5570 : }
5571 :
5572 8038 : const auto & ordered_controls = resolver.getSortedValues();
5573 :
5574 8038 : if (!ordered_controls.empty())
5575 : {
5576 : // already called by initialSetup when exec_type == EXEC_INITIAL
5577 8038 : if (exec_type != EXEC_INITIAL)
5578 7130 : _control_warehouse.setup(exec_type);
5579 :
5580 : // Run the controls in the proper order
5581 19958 : for (const auto & control : ordered_controls)
5582 11962 : control->execute();
5583 : }
5584 7996 : }
5585 5733267 : }
5586 :
5587 : void
5588 2171951 : FEProblemBase::executeSamplers(const ExecFlagType & exec_type)
5589 : {
5590 : // TODO: This should be done in a threaded loop, but this should be super quick so for now
5591 : // do a serial loop.
5592 4558735 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5593 : {
5594 2386808 : std::vector<Sampler *> objects;
5595 2386808 : theWarehouse()
5596 4773616 : .query()
5597 2386808 : .condition<AttribSystem>("Sampler")
5598 2386808 : .condition<AttribThread>(tid)
5599 2386808 : .condition<AttribExecOns>(exec_type)
5600 2386808 : .queryInto(objects);
5601 :
5602 2386808 : if (!objects.empty())
5603 : {
5604 1475 : TIME_SECTION("executeSamplers", 1, "Executing Samplers");
5605 295 : FEProblemBase::objectSetupHelper<Sampler>(objects, exec_type);
5606 295 : FEProblemBase::objectExecuteHelper<Sampler>(objects);
5607 271 : }
5608 2386784 : }
5609 2171927 : }
5610 :
5611 : void
5612 312344 : FEProblemBase::updateActiveObjects()
5613 : {
5614 1561720 : TIME_SECTION("updateActiveObjects", 5, "Updating Active Objects");
5615 :
5616 655597 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5617 : {
5618 689056 : for (auto & nl : _nl)
5619 345803 : nl->updateActive(tid);
5620 343253 : _aux->updateActive(tid);
5621 343253 : _indicators.updateActive(tid);
5622 343253 : _internal_side_indicators.updateActive(tid);
5623 343253 : _markers.updateActive(tid);
5624 343253 : _all_materials.updateActive(tid);
5625 343253 : _materials.updateActive(tid);
5626 343253 : _discrete_materials.updateActive(tid);
5627 : }
5628 :
5629 312344 : _control_warehouse.updateActive();
5630 312344 : _multi_apps.updateActive();
5631 312344 : _transient_multi_apps.updateActive();
5632 312344 : _transfers.updateActive();
5633 312344 : _to_multi_app_transfers.updateActive();
5634 312344 : _from_multi_app_transfers.updateActive();
5635 312344 : _between_multi_app_transfers.updateActive();
5636 :
5637 : #ifdef MOOSE_KOKKOS_ENABLED
5638 227781 : _kokkos_materials.updateActive();
5639 : #endif
5640 312344 : }
5641 :
5642 : void
5643 0 : FEProblemBase::reportMooseObjectDependency(MooseObject * /*a*/, MooseObject * /*b*/)
5644 : {
5645 : //<< "Object " << a->name() << " -> " << b->name() << std::endl;
5646 0 : }
5647 :
5648 : void
5649 67753 : FEProblemBase::reinitBecauseOfGhostingOrNewGeomObjects(const bool mortar_changed)
5650 : {
5651 338765 : TIME_SECTION("reinitBecauseOfGhostingOrNewGeomObjects",
5652 : 3,
5653 : "Reinitializing Because of Geometric Search Objects");
5654 :
5655 : // Need to see if _any_ processor has ghosted elems or geometry objects.
5656 67753 : bool needs_reinit = !_ghosted_elems.empty();
5657 135085 : needs_reinit = needs_reinit || !_geometric_search_data._nearest_node_locators.empty() ||
5658 67332 : (_mortar_data->hasObjects() && mortar_changed);
5659 67753 : needs_reinit =
5660 136699 : needs_reinit || (_displaced_problem &&
5661 4423 : (!_displaced_problem->geomSearchData()._nearest_node_locators.empty() ||
5662 68425 : (_mortar_data->hasDisplacedObjects() && mortar_changed)));
5663 67753 : _communicator.max(needs_reinit);
5664 :
5665 67753 : if (needs_reinit)
5666 : {
5667 : // Call reinit to get the ghosted vectors correct now that some geometric search has been done
5668 2211 : es().reinit();
5669 :
5670 2211 : if (_displaced_mesh)
5671 1607 : _displaced_problem->es().reinit();
5672 : }
5673 67753 : }
5674 :
5675 : void
5676 177 : FEProblemBase::addDamper(const std::string & damper_name,
5677 : const std::string & name,
5678 : InputParameters & parameters)
5679 : {
5680 : parallel_object_only();
5681 :
5682 : const auto nl_sys_num =
5683 177 : parameters.isParamValid("variable")
5684 588 : ? determineSolverSystem(parameters.varName("variable", name), true).second
5685 174 : : (unsigned int)0;
5686 :
5687 174 : if (!isSolverSystemNonlinear(nl_sys_num))
5688 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
5689 : "supported at the moment!");
5690 :
5691 348 : parameters.set<SubProblem *>("_subproblem") = this;
5692 348 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
5693 :
5694 174 : _has_dampers = true;
5695 174 : logAdd("Damper", name, damper_name, parameters);
5696 174 : _nl[nl_sys_num]->addDamper(damper_name, name, parameters);
5697 174 : }
5698 :
5699 : void
5700 162 : FEProblemBase::setupDampers()
5701 : {
5702 324 : for (auto & nl : _nl)
5703 162 : nl->setupDampers();
5704 162 : }
5705 :
5706 : void
5707 659 : FEProblemBase::addIndicator(const std::string & indicator_name,
5708 : const std::string & name,
5709 : InputParameters & parameters)
5710 : {
5711 : parallel_object_only();
5712 :
5713 659 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5714 : {
5715 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5716 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5717 0 : _reinit_displaced_elem = true;
5718 : }
5719 : else
5720 : {
5721 659 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5722 : {
5723 : // We allow Indicators to request that they use_displaced_mesh,
5724 : // but then be overridden when no displacements variables are
5725 : // provided in the Mesh block. If that happened, update the value
5726 : // of use_displaced_mesh appropriately for this Indicator.
5727 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5728 0 : parameters.set<bool>("use_displaced_mesh") = false;
5729 : }
5730 :
5731 1318 : parameters.set<SubProblem *>("_subproblem") = this;
5732 1977 : parameters.set<SystemBase *>("_sys") = _aux.get();
5733 : }
5734 :
5735 1382 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5736 : {
5737 : std::shared_ptr<Indicator> indicator =
5738 723 : _factory.create<Indicator>(indicator_name, name, parameters, tid);
5739 723 : logAdd("Indicator", name, indicator_name, parameters);
5740 : std::shared_ptr<InternalSideIndicatorBase> isi =
5741 723 : std::dynamic_pointer_cast<InternalSideIndicatorBase>(indicator);
5742 723 : if (isi)
5743 622 : _internal_side_indicators.addObject(isi, tid);
5744 : else
5745 101 : _indicators.addObject(indicator, tid);
5746 723 : }
5747 659 : }
5748 :
5749 : void
5750 1966 : FEProblemBase::addMarker(const std::string & marker_name,
5751 : const std::string & name,
5752 : InputParameters & parameters)
5753 : {
5754 : parallel_object_only();
5755 :
5756 1966 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5757 : {
5758 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5759 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5760 0 : _reinit_displaced_elem = true;
5761 : }
5762 : else
5763 : {
5764 1966 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5765 : {
5766 : // We allow Markers to request that they use_displaced_mesh,
5767 : // but then be overridden when no displacements variables are
5768 : // provided in the Mesh block. If that happened, update the value
5769 : // of use_displaced_mesh appropriately for this Marker.
5770 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5771 0 : parameters.set<bool>("use_displaced_mesh") = false;
5772 : }
5773 :
5774 3932 : parameters.set<SubProblem *>("_subproblem") = this;
5775 5898 : parameters.set<SystemBase *>("_sys") = _aux.get();
5776 : }
5777 :
5778 4107 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5779 : {
5780 2144 : std::shared_ptr<Marker> marker = _factory.create<Marker>(marker_name, name, parameters, tid);
5781 2141 : logAdd("Marker", name, marker_name, parameters);
5782 2141 : _markers.addObject(marker, tid);
5783 2141 : }
5784 1963 : }
5785 :
5786 : void
5787 8182 : FEProblemBase::addMultiApp(const std::string & multi_app_name,
5788 : const std::string & name,
5789 : InputParameters & parameters)
5790 : {
5791 : parallel_object_only();
5792 :
5793 16364 : parameters.set<MPI_Comm>("_mpi_comm") = _communicator.get();
5794 :
5795 8182 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5796 : {
5797 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5798 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5799 0 : _reinit_displaced_elem = true;
5800 : }
5801 : else
5802 : {
5803 8182 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5804 : {
5805 : // We allow MultiApps to request that they use_displaced_mesh,
5806 : // but then be overridden when no displacements variables are
5807 : // provided in the Mesh block. If that happened, update the value
5808 : // of use_displaced_mesh appropriately for this MultiApp.
5809 42 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5810 84 : parameters.set<bool>("use_displaced_mesh") = false;
5811 : }
5812 :
5813 16364 : parameters.set<SubProblem *>("_subproblem") = this;
5814 24546 : parameters.set<SystemBase *>("_sys") = _aux.get();
5815 : }
5816 :
5817 8182 : std::shared_ptr<MultiApp> multi_app = _factory.create<MultiApp>(multi_app_name, name, parameters);
5818 8170 : logAdd("MultiApp", name, multi_app_name, parameters);
5819 8170 : multi_app->setupPositions();
5820 :
5821 8083 : _multi_apps.addObject(multi_app);
5822 :
5823 : // Store TransientMultiApp objects in another container, this is needed for calling computeDT
5824 : std::shared_ptr<TransientMultiApp> trans_multi_app =
5825 8083 : std::dynamic_pointer_cast<TransientMultiApp>(multi_app);
5826 8083 : if (trans_multi_app)
5827 5395 : _transient_multi_apps.addObject(trans_multi_app);
5828 8083 : }
5829 :
5830 : bool
5831 112600 : FEProblemBase::hasMultiApps(ExecFlagType type) const
5832 : {
5833 112600 : return _multi_apps[type].hasActiveObjects();
5834 : }
5835 :
5836 : bool
5837 26605 : FEProblemBase::hasMultiApp(const std::string & multi_app_name) const
5838 : {
5839 26605 : return _multi_apps.hasActiveObject(multi_app_name);
5840 : }
5841 :
5842 : std::shared_ptr<MultiApp>
5843 26605 : FEProblemBase::getMultiApp(const std::string & multi_app_name) const
5844 : {
5845 26605 : if (!hasMultiApp(multi_app_name))
5846 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_multi_app"),
5847 : "A MultiApp getter was called before MultiApps have been constructed. "
5848 : "If you are attempting to access this object in the constructor of another object "
5849 : "then make sure that the MultiApp is constructed before the object using it.");
5850 :
5851 26605 : return _multi_apps.getObject(multi_app_name);
5852 : }
5853 :
5854 : void
5855 14475087 : FEProblemBase::execMultiAppTransfers(ExecFlagType type, Transfer::DIRECTION direction)
5856 : {
5857 14475087 : bool to_multiapp = direction == MultiAppTransfer::TO_MULTIAPP;
5858 14475087 : bool from_multiapp = direction == MultiAppTransfer::FROM_MULTIAPP;
5859 14475087 : std::string string_direction;
5860 14475087 : if (to_multiapp)
5861 4825083 : string_direction = " To ";
5862 9650004 : else if (from_multiapp)
5863 4824957 : string_direction = " From ";
5864 : else
5865 4825047 : string_direction = " Between ";
5866 :
5867 14475249 : const MooseObjectWarehouse<Transfer> & wh = to_multiapp ? _to_multi_app_transfers[type]
5868 24125001 : : from_multiapp ? _from_multi_app_transfers[type]
5869 33775167 : : _between_multi_app_transfers[type];
5870 :
5871 14475087 : if (wh.hasActiveObjects())
5872 : {
5873 546485 : TIME_SECTION("execMultiAppTransfers", 1, "Executing Transfers");
5874 :
5875 109297 : const auto & transfers = wh.getActiveObjects();
5876 :
5877 109297 : if (_verbose_multiapps)
5878 : {
5879 1120 : _console << COLOR_CYAN << "\nTransfers on " << Moose::stringify(type) << string_direction
5880 1120 : << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
5881 :
5882 : VariadicTable<std::string, std::string, std::string, std::string> table(
5883 2240 : {"Name", "Type", "From", "To"});
5884 :
5885 : // Build Table of Transfer Info
5886 3060 : for (const auto & transfer : transfers)
5887 : {
5888 1940 : auto multiapp_transfer = dynamic_cast<MultiAppTransfer *>(transfer.get());
5889 :
5890 1940 : table.addRow(multiapp_transfer->name(),
5891 1940 : multiapp_transfer->type(),
5892 3880 : multiapp_transfer->getFromName(),
5893 3880 : multiapp_transfer->getToName());
5894 : }
5895 :
5896 : // Print it
5897 1120 : table.print(_console);
5898 1120 : }
5899 :
5900 223635 : for (const auto & transfer : transfers)
5901 : {
5902 114435 : transfer->setCurrentDirection(direction);
5903 114435 : transfer->execute();
5904 : }
5905 :
5906 109200 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5907 :
5908 109200 : if (_verbose_multiapps)
5909 2228 : _console << COLOR_CYAN << "Transfers on " << Moose::stringify(type) << " Are Finished\n"
5910 1114 : << COLOR_DEFAULT << std::endl;
5911 109200 : }
5912 14365790 : else if (_multi_apps[type].getActiveObjects().size())
5913 : {
5914 110649 : if (_verbose_multiapps)
5915 7076 : _console << COLOR_CYAN << "\nNo Transfers on " << Moose::stringify(type) << string_direction
5916 3538 : << "MultiApps\n"
5917 3538 : << COLOR_DEFAULT << std::endl;
5918 : }
5919 14474990 : }
5920 :
5921 : std::vector<std::shared_ptr<Transfer>>
5922 12 : FEProblemBase::getTransfers(ExecFlagType type, Transfer::DIRECTION direction) const
5923 : {
5924 12 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5925 12 : return _to_multi_app_transfers[type].getActiveObjects();
5926 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5927 0 : return _from_multi_app_transfers[type].getActiveObjects();
5928 : else
5929 0 : return _between_multi_app_transfers[type].getActiveObjects();
5930 : }
5931 :
5932 : std::vector<std::shared_ptr<Transfer>>
5933 0 : FEProblemBase::getTransfers(Transfer::DIRECTION direction) const
5934 : {
5935 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5936 0 : return _to_multi_app_transfers.getActiveObjects();
5937 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5938 0 : return _from_multi_app_transfers.getActiveObjects();
5939 : else
5940 0 : return _between_multi_app_transfers.getActiveObjects();
5941 : }
5942 :
5943 : const ExecuteMooseObjectWarehouse<Transfer> &
5944 0 : FEProblemBase::getMultiAppTransferWarehouse(Transfer::DIRECTION direction) const
5945 : {
5946 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5947 0 : return _to_multi_app_transfers;
5948 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5949 0 : return _from_multi_app_transfers;
5950 : else
5951 0 : return _between_multi_app_transfers;
5952 : }
5953 :
5954 : bool
5955 4825086 : FEProblemBase::execMultiApps(ExecFlagType type, bool auto_advance)
5956 : {
5957 : // Active MultiApps
5958 : const std::vector<MooseSharedPointer<MultiApp>> & multi_apps =
5959 4825086 : _multi_apps[type].getActiveObjects();
5960 :
5961 : // Do anything that needs to be done to Apps before transfers
5962 4890359 : for (const auto & multi_app : multi_apps)
5963 65276 : multi_app->preTransfer(_dt, _time);
5964 :
5965 : // Execute Transfers _to_ MultiApps
5966 4825083 : execMultiAppTransfers(type, MultiAppTransfer::TO_MULTIAPP);
5967 :
5968 : // Execute Transfers _between_ Multiapps
5969 4825047 : execMultiAppTransfers(type, MultiAppTransfer::BETWEEN_MULTIAPP);
5970 :
5971 : // Execute MultiApps
5972 4825047 : if (multi_apps.size())
5973 : {
5974 320080 : TIME_SECTION("execMultiApps", 1, "Executing MultiApps", false);
5975 :
5976 64016 : if (_verbose_multiapps)
5977 2988 : _console << COLOR_CYAN << "\nExecuting MultiApps on " << Moose::stringify(type)
5978 1494 : << COLOR_DEFAULT << std::endl;
5979 :
5980 64016 : bool success = true;
5981 :
5982 129164 : for (const auto & multi_app : multi_apps)
5983 : {
5984 65231 : success = multi_app->solveStep(_dt, _time, auto_advance);
5985 : // no need to finish executing the subapps if one fails
5986 65222 : if (!success)
5987 74 : break;
5988 : }
5989 :
5990 64007 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5991 :
5992 64007 : _communicator.min(success);
5993 :
5994 64007 : if (!success)
5995 81 : return false;
5996 :
5997 63926 : if (_verbose_multiapps)
5998 2988 : _console << COLOR_CYAN << "Finished Executing MultiApps on " << Moose::stringify(type) << "\n"
5999 1494 : << COLOR_DEFAULT << std::endl;
6000 64007 : }
6001 :
6002 : // Execute Transfers _from_ MultiApps
6003 4824957 : execMultiAppTransfers(type, MultiAppTransfer::FROM_MULTIAPP);
6004 :
6005 : // If we made it here then everything passed
6006 4824896 : return true;
6007 : }
6008 :
6009 : void
6010 49422 : FEProblemBase::finalizeMultiApps()
6011 : {
6012 49422 : const auto & multi_apps = _multi_apps.getActiveObjects();
6013 :
6014 56488 : for (const auto & multi_app : multi_apps)
6015 7066 : multi_app->finalize();
6016 49422 : }
6017 :
6018 : void
6019 50962 : FEProblemBase::postExecute()
6020 : {
6021 50962 : const auto & multi_apps = _multi_apps.getActiveObjects();
6022 :
6023 58415 : for (const auto & multi_app : multi_apps)
6024 7453 : multi_app->postExecute();
6025 50962 : }
6026 :
6027 : void
6028 668220 : FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
6029 : {
6030 668220 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6031 :
6032 668220 : if (multi_apps.size())
6033 27809 : for (const auto & multi_app : multi_apps)
6034 14076 : multi_app->incrementTStep(_time);
6035 668220 : }
6036 :
6037 : void
6038 36370 : FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
6039 : {
6040 36370 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6041 :
6042 36370 : if (multi_apps.size())
6043 : {
6044 8584 : if (_verbose_multiapps)
6045 270 : _console << COLOR_CYAN << "\nAdvancing MultiApps on " << type.name() << COLOR_DEFAULT
6046 270 : << std::endl;
6047 :
6048 17174 : for (const auto & multi_app : multi_apps)
6049 8590 : multi_app->finishStep(recurse_through_multiapp_levels);
6050 :
6051 8584 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6052 :
6053 8584 : if (_verbose_multiapps)
6054 270 : _console << COLOR_CYAN << "Finished Advancing MultiApps on " << type.name() << "\n"
6055 270 : << COLOR_DEFAULT << std::endl;
6056 : }
6057 36370 : }
6058 :
6059 : void
6060 1121027 : FEProblemBase::backupMultiApps(ExecFlagType type)
6061 : {
6062 1121027 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6063 :
6064 1121027 : if (multi_apps.size())
6065 : {
6066 105995 : TIME_SECTION("backupMultiApps", 5, "Backing Up MultiApp");
6067 :
6068 21199 : if (_verbose_multiapps)
6069 798 : _console << COLOR_CYAN << "\nBacking Up MultiApps on " << type.name() << COLOR_DEFAULT
6070 798 : << std::endl;
6071 :
6072 43547 : for (const auto & multi_app : multi_apps)
6073 22348 : multi_app->backup();
6074 :
6075 21199 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6076 :
6077 21199 : if (_verbose_multiapps)
6078 798 : _console << COLOR_CYAN << "Finished Backing Up MultiApps on " << type.name() << "\n"
6079 798 : << COLOR_DEFAULT << std::endl;
6080 21199 : }
6081 1121027 : }
6082 :
6083 : void
6084 104738 : FEProblemBase::restoreMultiApps(ExecFlagType type, bool force)
6085 : {
6086 104738 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6087 :
6088 104738 : if (multi_apps.size())
6089 : {
6090 43065 : if (_verbose_multiapps)
6091 : {
6092 528 : if (force)
6093 0 : _console << COLOR_CYAN << "\nRestoring Multiapps on " << type.name()
6094 0 : << " because of solve failure!" << COLOR_DEFAULT << std::endl;
6095 : else
6096 528 : _console << COLOR_CYAN << "\nRestoring MultiApps on " << type.name() << COLOR_DEFAULT
6097 528 : << std::endl;
6098 : }
6099 :
6100 86172 : for (const auto & multi_app : multi_apps)
6101 43110 : multi_app->restore(force);
6102 :
6103 43062 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6104 :
6105 43062 : if (_verbose_multiapps)
6106 528 : _console << COLOR_CYAN << "Finished Restoring MultiApps on " << type.name() << "\n"
6107 528 : << COLOR_DEFAULT << std::endl;
6108 : }
6109 104735 : }
6110 :
6111 : Real
6112 692804 : FEProblemBase::computeMultiAppsDT(ExecFlagType type)
6113 : {
6114 692804 : const auto & multi_apps = _transient_multi_apps[type].getActiveObjects();
6115 :
6116 692804 : Real smallest_dt = std::numeric_limits<Real>::max();
6117 :
6118 710599 : for (const auto & multi_app : multi_apps)
6119 17795 : smallest_dt = std::min(smallest_dt, multi_app->computeDT());
6120 :
6121 692804 : return smallest_dt;
6122 : }
6123 :
6124 : void
6125 4770987 : FEProblemBase::execTransfers(ExecFlagType type)
6126 : {
6127 4770987 : if (_transfers[type].hasActiveObjects())
6128 : {
6129 0 : TIME_SECTION("execTransfers", 3, "Executing Transfers");
6130 :
6131 0 : const auto & transfers = _transfers[type].getActiveObjects();
6132 :
6133 0 : for (const auto & transfer : transfers)
6134 0 : transfer->execute();
6135 0 : }
6136 4770987 : }
6137 :
6138 : void
6139 13223 : FEProblemBase::addTransfer(const std::string & transfer_name,
6140 : const std::string & name,
6141 : InputParameters & parameters)
6142 : {
6143 : parallel_object_only();
6144 :
6145 13223 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
6146 : {
6147 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
6148 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
6149 0 : _reinit_displaced_elem = true;
6150 : }
6151 : else
6152 : {
6153 13223 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
6154 : {
6155 : // We allow Transfers to request that they use_displaced_mesh,
6156 : // but then be overridden when no displacements variables are
6157 : // provided in the Mesh block. If that happened, update the value
6158 : // of use_displaced_mesh appropriately for this Transfer.
6159 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
6160 0 : parameters.set<bool>("use_displaced_mesh") = false;
6161 : }
6162 :
6163 26446 : parameters.set<SubProblem *>("_subproblem") = this;
6164 39669 : parameters.set<SystemBase *>("_sys") = _aux.get();
6165 : }
6166 :
6167 : // Handle the "SAME_AS_MULTIAPP" execute option. The get method is used to test for the
6168 : // flag so the set by user flag is not reset, calling set with the true flag causes the set
6169 : // by user status to be reset, which should only be done if the EXEC_SAME_AS_MULTIAPP is
6170 : // being applied to the object.
6171 13223 : if (parameters.get<ExecFlagEnum>("execute_on").isValueSet(EXEC_SAME_AS_MULTIAPP))
6172 : {
6173 11084 : ExecFlagEnum & exec_enum = parameters.set<ExecFlagEnum>("execute_on", true);
6174 11084 : std::shared_ptr<MultiApp> multiapp;
6175 22168 : if (parameters.isParamValid("multi_app"))
6176 50 : multiapp = getMultiApp(parameters.get<MultiAppName>("multi_app"));
6177 : // This catches the sibling transfer case, where we want to be executing only as often as the
6178 : // receiving application. A transfer 'to' a multiapp is executed before that multiapp
6179 22068 : else if (parameters.isParamValid("to_multi_app"))
6180 5746 : multiapp = getMultiApp(parameters.get<MultiAppName>("to_multi_app"));
6181 10576 : else if (parameters.isParamValid("from_multi_app"))
6182 5282 : multiapp = getMultiApp(parameters.get<MultiAppName>("from_multi_app"));
6183 : // else do nothing because the user has provided invalid input. They should get a nice error
6184 : // about this during transfer construction. This necessitates checking for null in this next
6185 : // line, however
6186 11084 : if (multiapp)
6187 33234 : exec_enum = multiapp->getParam<ExecFlagEnum>("execute_on");
6188 11084 : }
6189 :
6190 : // Create the Transfer objects
6191 13223 : std::shared_ptr<Transfer> transfer = _factory.create<Transfer>(transfer_name, name, parameters);
6192 13166 : logAdd("Transfer", name, transfer_name, parameters);
6193 :
6194 : // Add MultiAppTransfer object
6195 : std::shared_ptr<MultiAppTransfer> multi_app_transfer =
6196 13166 : std::dynamic_pointer_cast<MultiAppTransfer>(transfer);
6197 13166 : if (multi_app_transfer)
6198 : {
6199 13166 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::TO_MULTIAPP))
6200 5277 : _to_multi_app_transfers.addObject(multi_app_transfer);
6201 13166 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::FROM_MULTIAPP))
6202 6419 : _from_multi_app_transfers.addObject(multi_app_transfer);
6203 13166 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::BETWEEN_MULTIAPP))
6204 1470 : _between_multi_app_transfers.addObject(multi_app_transfer);
6205 : }
6206 : else
6207 0 : _transfers.addObject(transfer);
6208 13166 : }
6209 :
6210 : bool
6211 1432073 : FEProblemBase::hasVariable(const std::string & var_name) const
6212 : {
6213 2337632 : for (auto & sys : _solver_systems)
6214 1437617 : if (sys->hasVariable(var_name))
6215 532058 : return true;
6216 900015 : if (_aux->hasVariable(var_name))
6217 830164 : return true;
6218 :
6219 69851 : return false;
6220 : }
6221 :
6222 : bool
6223 65 : FEProblemBase::hasSolverVariable(const std::string & var_name) const
6224 : {
6225 78 : for (auto & sys : _solver_systems)
6226 65 : if (sys->hasVariable(var_name))
6227 52 : return true;
6228 :
6229 13 : return false;
6230 : }
6231 :
6232 : const MooseVariableFieldBase &
6233 4300379 : FEProblemBase::getVariable(const THREAD_ID tid,
6234 : const std::string & var_name,
6235 : Moose::VarKindType expected_var_type,
6236 : Moose::VarFieldType expected_var_field_type) const
6237 : {
6238 12901131 : return getVariableHelper(
6239 4300379 : tid, var_name, expected_var_type, expected_var_field_type, _solver_systems, *_aux);
6240 : }
6241 :
6242 : MooseVariable &
6243 7640 : FEProblemBase::getStandardVariable(const THREAD_ID tid, const std::string & var_name)
6244 : {
6245 11522 : for (auto & sys : _solver_systems)
6246 7640 : if (sys->hasVariable(var_name))
6247 3758 : return sys->getFieldVariable<Real>(tid, var_name);
6248 3882 : if (_aux->hasVariable(var_name))
6249 3879 : return _aux->getFieldVariable<Real>(tid, var_name);
6250 :
6251 3 : mooseError("Unknown variable " + var_name);
6252 : }
6253 :
6254 : MooseVariableFieldBase &
6255 581 : FEProblemBase::getActualFieldVariable(const THREAD_ID tid, const std::string & var_name)
6256 : {
6257 771 : for (auto & sys : _solver_systems)
6258 581 : if (sys->hasVariable(var_name))
6259 391 : return sys->getActualFieldVariable<Real>(tid, var_name);
6260 190 : if (_aux->hasVariable(var_name))
6261 190 : return _aux->getActualFieldVariable<Real>(tid, var_name);
6262 :
6263 0 : mooseError("Unknown variable " + var_name);
6264 : }
6265 :
6266 : VectorMooseVariable &
6267 0 : FEProblemBase::getVectorVariable(const THREAD_ID tid, const std::string & var_name)
6268 : {
6269 0 : for (auto & sys : _solver_systems)
6270 0 : if (sys->hasVariable(var_name))
6271 0 : return sys->getFieldVariable<RealVectorValue>(tid, var_name);
6272 0 : if (_aux->hasVariable(var_name))
6273 0 : return _aux->getFieldVariable<RealVectorValue>(tid, var_name);
6274 :
6275 0 : mooseError("Unknown variable " + var_name);
6276 : }
6277 :
6278 : ArrayMooseVariable &
6279 351 : FEProblemBase::getArrayVariable(const THREAD_ID tid, const std::string & var_name)
6280 : {
6281 611 : for (auto & sys : _solver_systems)
6282 351 : if (sys->hasVariable(var_name))
6283 91 : return sys->getFieldVariable<RealEigenVector>(tid, var_name);
6284 260 : if (_aux->hasVariable(var_name))
6285 260 : return _aux->getFieldVariable<RealEigenVector>(tid, var_name);
6286 :
6287 0 : mooseError("Unknown variable " + var_name);
6288 : }
6289 :
6290 : bool
6291 180081 : FEProblemBase::hasScalarVariable(const std::string & var_name) const
6292 : {
6293 339127 : for (auto & sys : _solver_systems)
6294 180807 : if (sys->hasScalarVariable(var_name))
6295 21761 : return true;
6296 158320 : if (_aux->hasScalarVariable(var_name))
6297 10613 : return true;
6298 :
6299 147707 : return false;
6300 : }
6301 :
6302 : MooseVariableScalar &
6303 44342 : FEProblemBase::getScalarVariable(const THREAD_ID tid, const std::string & var_name)
6304 : {
6305 56683 : for (auto & sys : _solver_systems)
6306 44342 : if (sys->hasScalarVariable(var_name))
6307 32001 : return sys->getScalarVariable(tid, var_name);
6308 12341 : if (_aux->hasScalarVariable(var_name))
6309 12341 : return _aux->getScalarVariable(tid, var_name);
6310 :
6311 0 : mooseError("Unknown variable " + var_name);
6312 : }
6313 :
6314 : System &
6315 58761 : FEProblemBase::getSystem(const std::string & var_name)
6316 : {
6317 58761 : const auto [var_in_sys, sys_num] = determineSolverSystem(var_name);
6318 58761 : if (var_in_sys)
6319 39972 : return _solver_systems[sys_num]->system();
6320 18789 : else if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
6321 18789 : return _aux->system();
6322 : else
6323 0 : mooseError("Unable to find a system containing the variable " + var_name);
6324 : }
6325 :
6326 : const RestartableEquationSystems &
6327 0 : FEProblemBase::getRestartableEquationSystems() const
6328 : {
6329 0 : return _req.get();
6330 : }
6331 :
6332 : void
6333 464555 : FEProblemBase::setActiveFEVariableCoupleableMatrixTags(std::set<TagID> & mtags, const THREAD_ID tid)
6334 : {
6335 464555 : SubProblem::setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6336 :
6337 464555 : if (_displaced_problem)
6338 98260 : _displaced_problem->setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6339 464555 : }
6340 :
6341 : void
6342 6242236 : FEProblemBase::setActiveFEVariableCoupleableVectorTags(std::set<TagID> & vtags, const THREAD_ID tid)
6343 : {
6344 6242236 : SubProblem::setActiveFEVariableCoupleableVectorTags(vtags, tid);
6345 :
6346 6242236 : if (_displaced_problem)
6347 340076 : _displaced_problem->setActiveFEVariableCoupleableVectorTags(vtags, tid);
6348 6242236 : }
6349 :
6350 : void
6351 47814 : FEProblemBase::setActiveScalarVariableCoupleableMatrixTags(std::set<TagID> & mtags,
6352 : const THREAD_ID tid)
6353 : {
6354 47814 : SubProblem::setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6355 :
6356 47814 : if (_displaced_problem)
6357 0 : _displaced_problem->setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6358 47814 : }
6359 :
6360 : void
6361 47814 : FEProblemBase::setActiveScalarVariableCoupleableVectorTags(std::set<TagID> & vtags,
6362 : const THREAD_ID tid)
6363 : {
6364 47814 : SubProblem::setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6365 :
6366 47814 : if (_displaced_problem)
6367 0 : _displaced_problem->setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6368 47814 : }
6369 :
6370 : void
6371 9524041 : FEProblemBase::setActiveElementalMooseVariables(const std::set<MooseVariableFEBase *> & moose_vars,
6372 : const THREAD_ID tid)
6373 : {
6374 9524041 : SubProblem::setActiveElementalMooseVariables(moose_vars, tid);
6375 :
6376 9524041 : if (_displaced_problem)
6377 479387 : _displaced_problem->setActiveElementalMooseVariables(moose_vars, tid);
6378 9524041 : }
6379 :
6380 : void
6381 3837292 : FEProblemBase::clearActiveElementalMooseVariables(const THREAD_ID tid)
6382 : {
6383 3837292 : SubProblem::clearActiveElementalMooseVariables(tid);
6384 :
6385 3837292 : if (_displaced_problem)
6386 170463 : _displaced_problem->clearActiveElementalMooseVariables(tid);
6387 3837292 : }
6388 :
6389 : void
6390 206679 : FEProblemBase::clearActiveFEVariableCoupleableMatrixTags(const THREAD_ID tid)
6391 : {
6392 206679 : SubProblem::clearActiveFEVariableCoupleableMatrixTags(tid);
6393 :
6394 206679 : if (_displaced_problem)
6395 41849 : _displaced_problem->clearActiveFEVariableCoupleableMatrixTags(tid);
6396 206679 : }
6397 :
6398 : void
6399 206679 : FEProblemBase::clearActiveFEVariableCoupleableVectorTags(const THREAD_ID tid)
6400 : {
6401 206679 : SubProblem::clearActiveFEVariableCoupleableVectorTags(tid);
6402 :
6403 206679 : if (_displaced_problem)
6404 41849 : _displaced_problem->clearActiveFEVariableCoupleableVectorTags(tid);
6405 206679 : }
6406 :
6407 : void
6408 47814 : FEProblemBase::clearActiveScalarVariableCoupleableMatrixTags(const THREAD_ID tid)
6409 : {
6410 47814 : SubProblem::clearActiveScalarVariableCoupleableMatrixTags(tid);
6411 :
6412 47814 : if (_displaced_problem)
6413 0 : _displaced_problem->clearActiveScalarVariableCoupleableMatrixTags(tid);
6414 47814 : }
6415 :
6416 : void
6417 47814 : FEProblemBase::clearActiveScalarVariableCoupleableVectorTags(const THREAD_ID tid)
6418 : {
6419 47814 : SubProblem::clearActiveScalarVariableCoupleableVectorTags(tid);
6420 :
6421 47814 : if (_displaced_problem)
6422 0 : _displaced_problem->clearActiveScalarVariableCoupleableVectorTags(tid);
6423 47814 : }
6424 :
6425 : void
6426 4860360 : FEProblemBase::setActiveMaterialProperties(const std::unordered_set<unsigned int> & mat_prop_ids,
6427 : const THREAD_ID tid)
6428 : {
6429 : // mark active properties in every material
6430 5975981 : for (auto & mat : _all_materials.getObjects(tid))
6431 1115621 : mat->setActiveProperties(mat_prop_ids);
6432 5738964 : for (auto & mat : _all_materials[Moose::FACE_MATERIAL_DATA].getObjects(tid))
6433 878604 : mat->setActiveProperties(mat_prop_ids);
6434 5738964 : for (auto & mat : _all_materials[Moose::NEIGHBOR_MATERIAL_DATA].getObjects(tid))
6435 878604 : mat->setActiveProperties(mat_prop_ids);
6436 :
6437 4860360 : _has_active_material_properties[tid] = !mat_prop_ids.empty();
6438 4860360 : }
6439 :
6440 : bool
6441 383592115 : FEProblemBase::hasActiveMaterialProperties(const THREAD_ID tid) const
6442 : {
6443 383592115 : return _has_active_material_properties[tid];
6444 : }
6445 :
6446 : void
6447 3980045 : FEProblemBase::clearActiveMaterialProperties(const THREAD_ID tid)
6448 : {
6449 3980045 : _has_active_material_properties[tid] = 0;
6450 3980045 : }
6451 :
6452 : void
6453 60178 : FEProblemBase::addAnyRedistributers()
6454 : {
6455 : #ifdef LIBMESH_ENABLE_AMR
6456 62447 : if ((_adaptivity.isOn() || _num_grid_steps) &&
6457 2269 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
6458 2206 : _neighbor_material_props.hasStatefulProperties()))
6459 : {
6460 : // Even on a serialized Mesh, we don't keep our material
6461 : // properties serialized, so we'll rely on the callback to
6462 : // redistribute() to redistribute properties at the same time
6463 : // libMesh is redistributing elements.
6464 63 : auto add_redistributer = [this](MooseMesh & mesh,
6465 : const std::string & redistributer_name,
6466 : const bool use_displaced_mesh)
6467 : {
6468 63 : InputParameters redistribute_params = RedistributeProperties::validParams();
6469 63 : redistribute_params.set<MooseApp *>(MooseBase::app_param) = &_app;
6470 126 : redistribute_params.set<std::string>("for_whom") = this->name();
6471 189 : redistribute_params.set<MooseMesh *>("mesh") = &mesh;
6472 63 : redistribute_params.set<Moose::RelationshipManagerType>("rm_type") =
6473 : Moose::RelationshipManagerType::GEOMETRIC;
6474 126 : redistribute_params.set<bool>("use_displaced_mesh") = use_displaced_mesh;
6475 63 : redistribute_params.setHitNode(*parameters().getHitNode(), {});
6476 :
6477 : std::shared_ptr<RedistributeProperties> redistributer =
6478 63 : _factory.create<RedistributeProperties>(
6479 126 : "RedistributeProperties", redistributer_name, redistribute_params);
6480 :
6481 63 : if (_material_props.hasStatefulProperties())
6482 63 : redistributer->addMaterialPropertyStorage(_material_props);
6483 :
6484 63 : if (_bnd_material_props.hasStatefulProperties())
6485 63 : redistributer->addMaterialPropertyStorage(_bnd_material_props);
6486 :
6487 63 : if (_neighbor_material_props.hasStatefulProperties())
6488 63 : redistributer->addMaterialPropertyStorage(_neighbor_material_props);
6489 :
6490 63 : mesh.getMesh().add_ghosting_functor(redistributer);
6491 126 : };
6492 :
6493 63 : add_redistributer(_mesh, "mesh_property_redistributer", false);
6494 63 : if (_displaced_problem)
6495 0 : add_redistributer(_displaced_problem->mesh(), "displaced_mesh_property_redistributer", true);
6496 : }
6497 : #endif // LIBMESH_ENABLE_AMR
6498 60178 : }
6499 :
6500 : void
6501 62057 : FEProblemBase::updateMaxQps()
6502 : {
6503 : // Find the maximum number of quadrature points
6504 : {
6505 62057 : MaxQpsThread mqt(*this);
6506 62057 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mqt);
6507 62057 : _max_qps = mqt.max();
6508 :
6509 : // If we have more shape functions or more quadrature points on
6510 : // another processor, then we may need to handle those elements
6511 : // ourselves later after repartitioning.
6512 62057 : _communicator.max(_max_qps);
6513 : }
6514 :
6515 62057 : unsigned int max_qpts = getMaxQps();
6516 62057 : if (max_qpts > Moose::constMaxQpsPerElem)
6517 0 : mooseError("Max quadrature points per element assumptions made in some code (e.g. Coupleable ",
6518 : "and MaterialPropertyInterface classes) have been violated.\n",
6519 : "Complain to Moose developers to have constMaxQpsPerElem increased from ",
6520 : Moose::constMaxQpsPerElem,
6521 : " to ",
6522 : max_qpts);
6523 130736 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6524 : {
6525 : // the highest available order in libMesh is 43
6526 68679 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
6527 68679 : _zero[tid].resize(max_qpts, 0);
6528 68679 : _ad_zero[tid].resize(max_qpts, 0);
6529 68679 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
6530 68679 : _ad_grad_zero[tid].resize(max_qpts, ADRealGradient(0));
6531 68679 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
6532 68679 : _ad_second_zero[tid].resize(max_qpts, ADRealTensorValue(0));
6533 68679 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
6534 68679 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
6535 : }
6536 62057 : }
6537 :
6538 : void
6539 78 : FEProblemBase::bumpVolumeQRuleOrder(Order order, SubdomainID block)
6540 : {
6541 168 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6542 180 : for (const auto i : index_range(_nl))
6543 90 : _assembly[tid][i]->bumpVolumeQRuleOrder(order, block);
6544 :
6545 78 : if (_displaced_problem)
6546 0 : _displaced_problem->bumpVolumeQRuleOrder(order, block);
6547 :
6548 78 : updateMaxQps();
6549 78 : }
6550 :
6551 : void
6552 13 : FEProblemBase::bumpAllQRuleOrder(Order order, SubdomainID block)
6553 : {
6554 28 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6555 30 : for (const auto i : index_range(_nl))
6556 15 : _assembly[tid][i]->bumpAllQRuleOrder(order, block);
6557 :
6558 13 : if (_displaced_problem)
6559 0 : _displaced_problem->bumpAllQRuleOrder(order, block);
6560 :
6561 13 : updateMaxQps();
6562 13 : }
6563 :
6564 : void
6565 61966 : FEProblemBase::createQRules(QuadratureType type,
6566 : Order order,
6567 : Order volume_order,
6568 : Order face_order,
6569 : SubdomainID block,
6570 : const bool allow_negative_qweights)
6571 : {
6572 61966 : if (order == INVALID_ORDER)
6573 : {
6574 : // automatically determine the integration order
6575 61385 : order = _solver_systems[0]->getMinQuadratureOrder();
6576 61671 : for (const auto i : make_range(std::size_t(1), _solver_systems.size()))
6577 286 : if (order < _solver_systems[i]->getMinQuadratureOrder())
6578 0 : order = _solver_systems[i]->getMinQuadratureOrder();
6579 61385 : if (order < _aux->getMinQuadratureOrder())
6580 6093 : order = _aux->getMinQuadratureOrder();
6581 : }
6582 :
6583 61966 : if (volume_order == INVALID_ORDER)
6584 61814 : volume_order = order;
6585 :
6586 61966 : if (face_order == INVALID_ORDER)
6587 61814 : face_order = order;
6588 :
6589 130540 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6590 137456 : for (const auto i : index_range(_solver_systems))
6591 68882 : _assembly[tid][i]->createQRules(
6592 : type, order, volume_order, face_order, block, allow_negative_qweights);
6593 :
6594 61966 : if (_displaced_problem)
6595 2022 : _displaced_problem->createQRules(
6596 : type, order, volume_order, face_order, block, allow_negative_qweights);
6597 :
6598 61966 : updateMaxQps();
6599 61966 : }
6600 :
6601 : void
6602 19743 : FEProblemBase::setCoupling(Moose::CouplingType type)
6603 : {
6604 19743 : if (_trust_user_coupling_matrix)
6605 : {
6606 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6607 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6608 : "haven't been provided a coupling matrix!");
6609 :
6610 : // We've been told to trust the user coupling matrix, so we're going to leave things alone
6611 3 : return;
6612 : }
6613 :
6614 19740 : _coupling = type;
6615 : }
6616 :
6617 : void
6618 0 : FEProblemBase::setCouplingMatrix(CouplingMatrix * cm, const unsigned int i)
6619 : {
6620 : // TODO: Deprecate method
6621 0 : setCoupling(Moose::COUPLING_CUSTOM);
6622 0 : _cm[i].reset(cm);
6623 0 : }
6624 :
6625 : void
6626 13315 : FEProblemBase::setCouplingMatrix(std::unique_ptr<CouplingMatrix> cm, const unsigned int i)
6627 : {
6628 13315 : setCoupling(Moose::COUPLING_CUSTOM);
6629 13315 : _cm[i] = std::move(cm);
6630 13315 : }
6631 :
6632 : void
6633 3 : FEProblemBase::trustUserCouplingMatrix()
6634 : {
6635 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6636 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6637 : "haven't been provided a coupling matrix!");
6638 :
6639 3 : _trust_user_coupling_matrix = true;
6640 3 : }
6641 :
6642 : void
6643 63 : FEProblemBase::setNonlocalCouplingMatrix()
6644 : {
6645 315 : TIME_SECTION("setNonlocalCouplingMatrix", 5, "Setting Nonlocal Coupling Matrix");
6646 :
6647 63 : if (_nl.size() > 1)
6648 0 : mooseError("Nonlocal kernels are weirdly stored on the FEProblem so we don't currently support "
6649 : "multiple nonlinear systems with nonlocal kernels.");
6650 :
6651 126 : for (const auto nl_sys_num : index_range(_nl))
6652 : {
6653 63 : auto & nl = _nl[nl_sys_num];
6654 63 : auto & nonlocal_cm = _nonlocal_cm[nl_sys_num];
6655 63 : unsigned int n_vars = nl->nVariables();
6656 63 : nonlocal_cm.resize(n_vars);
6657 63 : const auto & vars = nl->getVariables(0);
6658 63 : const auto & nonlocal_kernel = _nonlocal_kernels.getObjects();
6659 63 : const auto & nonlocal_integrated_bc = _nonlocal_integrated_bcs.getObjects();
6660 189 : for (const auto & ivar : vars)
6661 : {
6662 196 : for (const auto & kernel : nonlocal_kernel)
6663 : {
6664 140 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6665 70 : if (i == kernel->variable().number())
6666 105 : for (const auto & jvar : vars)
6667 : {
6668 70 : const auto it = _var_dof_map.find(jvar->name());
6669 70 : if (it != _var_dof_map.end())
6670 : {
6671 62 : unsigned int j = jvar->number();
6672 62 : nonlocal_cm(i, j) = 1;
6673 : }
6674 : }
6675 : }
6676 182 : for (const auto & integrated_bc : nonlocal_integrated_bc)
6677 : {
6678 112 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6679 56 : if (i == integrated_bc->variable().number())
6680 84 : for (const auto & jvar : vars)
6681 : {
6682 56 : const auto it = _var_dof_map.find(jvar->name());
6683 56 : if (it != _var_dof_map.end())
6684 : {
6685 28 : unsigned int j = jvar->number();
6686 28 : nonlocal_cm(i, j) = 1;
6687 : }
6688 : }
6689 : }
6690 : }
6691 : }
6692 63 : }
6693 :
6694 : bool
6695 624 : FEProblemBase::areCoupled(const unsigned int ivar,
6696 : const unsigned int jvar,
6697 : const unsigned int nl_sys) const
6698 : {
6699 624 : return (*_cm[nl_sys])(ivar, jvar);
6700 : }
6701 :
6702 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6703 15227410 : FEProblemBase::couplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6704 : {
6705 15227410 : return _assembly[tid][nl_sys]->couplingEntries();
6706 : }
6707 :
6708 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6709 4162 : FEProblemBase::nonlocalCouplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6710 : {
6711 4162 : return _assembly[tid][nl_sys]->nonlocalCouplingEntries();
6712 : }
6713 :
6714 : void
6715 60672 : FEProblemBase::init()
6716 : {
6717 60672 : if (_initialized)
6718 0 : return;
6719 :
6720 303360 : TIME_SECTION("init", 2, "Initializing");
6721 :
6722 : // call executioner's preProblemInit so that it can do some setups before problem init
6723 60672 : _app.getExecutioner()->preProblemInit();
6724 :
6725 : // If we have AD and we are doing global AD indexing, then we should by default set the matrix
6726 : // coupling to full. If the user has told us to trust their coupling matrix, then this call will
6727 : // not do anything
6728 60672 : if (haveADObjects() && Moose::globalADIndexing())
6729 6335 : setCoupling(Moose::COUPLING_FULL);
6730 :
6731 120592 : for (const auto i : index_range(_nl))
6732 : {
6733 59920 : auto & nl = _nl[i];
6734 59920 : auto & cm = _cm[i];
6735 :
6736 59920 : unsigned int n_vars = nl->nVariables();
6737 : {
6738 299600 : TIME_SECTION("fillCouplingMatrix", 3, "Filling Coupling Matrix");
6739 :
6740 59920 : switch (_coupling)
6741 : {
6742 44898 : case Moose::COUPLING_DIAG:
6743 44898 : cm = std::make_unique<CouplingMatrix>(n_vars);
6744 81657 : for (unsigned int i = 0; i < n_vars; i++)
6745 36759 : (*cm)(i, i) = 1;
6746 44898 : break;
6747 :
6748 : // for full jacobian
6749 6479 : case Moose::COUPLING_FULL:
6750 6479 : cm = std::make_unique<CouplingMatrix>(n_vars);
6751 16586 : for (unsigned int i = 0; i < n_vars; i++)
6752 29284 : for (unsigned int j = 0; j < n_vars; j++)
6753 19177 : (*cm)(i, j) = 1;
6754 6479 : break;
6755 :
6756 8543 : case Moose::COUPLING_CUSTOM:
6757 : // do nothing, _cm was already set through couplingMatrix() call
6758 8543 : break;
6759 : }
6760 59920 : }
6761 :
6762 59920 : nl->dofMap()._dof_coupling = cm.get();
6763 :
6764 : // If there are no variables, make sure to pass a nullptr coupling
6765 : // matrix, to avoid warnings about non-nullptr yet empty
6766 : // CouplingMatrices.
6767 59920 : if (n_vars == 0)
6768 13889 : nl->dofMap()._dof_coupling = nullptr;
6769 :
6770 59920 : nl->dofMap().attach_extra_sparsity_function(&extraSparsity, nl.get());
6771 59920 : nl->dofMap().attach_extra_send_list_function(&extraSendList, nl.get());
6772 59920 : _aux->dofMap().attach_extra_send_list_function(&extraSendList, _aux.get());
6773 :
6774 59920 : if (!_skip_nl_system_check && _solve && n_vars == 0)
6775 0 : mooseError("No variables specified in nonlinear system '", nl->name(), "'.");
6776 : }
6777 :
6778 60672 : ghostGhostedBoundaries(); // We do this again right here in case new boundaries have been added
6779 :
6780 : // We may have added element/nodes to the mesh in ghostGhostedBoundaries so we need to update
6781 : // all of our mesh information. We need to make sure that mesh information is up-to-date before
6782 : // EquationSystems::init because that will call through to updateGeomSearch (for sparsity
6783 : // augmentation) and if we haven't added back boundary node information before that latter call,
6784 : // then we're screwed. We'll get things like "Unable to find closest node!"
6785 60672 : _mesh.meshChanged();
6786 60672 : if (_displaced_problem)
6787 2022 : _displaced_mesh->meshChanged();
6788 :
6789 60672 : if (_mesh.doingPRefinement())
6790 : {
6791 240 : preparePRefinement();
6792 240 : if (_displaced_problem)
6793 0 : _displaced_problem->preparePRefinement();
6794 : }
6795 :
6796 : // do not assemble system matrix for JFNK solve
6797 120592 : for (auto & nl : _nl)
6798 59920 : if (solverParams(nl->number())._type == Moose::ST_JFNK)
6799 120 : nl->turnOffJacobian();
6800 :
6801 121624 : for (auto & sys : _solver_systems)
6802 60952 : sys->preInit();
6803 60672 : _aux->preInit();
6804 :
6805 : // Build the mortar segment meshes, if they haven't been already, for a couple reasons:
6806 : // 1) Get the ghosting correct for both static and dynamic meshes
6807 : // 2) Make sure the mortar mesh is built for mortar constraints that live on the static mesh
6808 : //
6809 : // It is worth-while to note that mortar meshes that live on a dynamic mesh will be built
6810 : // during residual and Jacobian evaluation because when displacements are solution variables
6811 : // the mortar mesh will move and change during the course of a non-linear solve. We DO NOT
6812 : // redo ghosting during non-linear solve, so for purpose 1) the below call has to be made
6813 60672 : if (!_mortar_data->initialized())
6814 45225 : updateMortarMesh();
6815 :
6816 : {
6817 303360 : TIME_SECTION("EquationSystems::Init", 2, "Initializing Equation Systems");
6818 60672 : es().init();
6819 60672 : }
6820 :
6821 121624 : for (auto & sys : _solver_systems)
6822 60952 : sys->postInit();
6823 60672 : _aux->postInit();
6824 :
6825 : // Now that the equation system and the dof distribution is done, we can generate the
6826 : // finite volume-related parts if needed.
6827 60672 : if (haveFV())
6828 4129 : _mesh.setupFiniteVolumeMeshData();
6829 :
6830 121624 : for (auto & sys : _solver_systems)
6831 60952 : sys->update();
6832 60672 : _aux->update();
6833 :
6834 127519 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
6835 132962 : for (const auto i : index_range(_nl))
6836 : {
6837 : mooseAssert(
6838 : _cm[i],
6839 : "Coupling matrix not set for system "
6840 : << i
6841 : << ". This should only happen if a preconditioner was not setup for this system");
6842 66115 : _assembly[tid][i]->init(_cm[i].get());
6843 : }
6844 :
6845 60672 : if (_displaced_problem)
6846 2022 : _displaced_problem->init();
6847 :
6848 : #ifdef MOOSE_KOKKOS_ENABLED
6849 45869 : if (_has_kokkos_objects)
6850 2265 : initKokkos();
6851 : #endif
6852 :
6853 60672 : _initialized = true;
6854 60672 : }
6855 :
6856 : unsigned int
6857 12043 : FEProblemBase::nlSysNum(const NonlinearSystemName & nl_sys_name) const
6858 : {
6859 12043 : std::istringstream ss(nl_sys_name);
6860 : unsigned int nl_sys_num;
6861 12043 : if (!(ss >> nl_sys_num) || !ss.eof())
6862 12043 : nl_sys_num = libmesh_map_find(_nl_sys_name_to_num, nl_sys_name);
6863 :
6864 12043 : return nl_sys_num;
6865 12043 : }
6866 :
6867 : unsigned int
6868 78396 : FEProblemBase::linearSysNum(const LinearSystemName & linear_sys_name) const
6869 : {
6870 78396 : std::istringstream ss(linear_sys_name);
6871 : unsigned int linear_sys_num;
6872 78396 : if (!(ss >> linear_sys_num) || !ss.eof())
6873 78396 : linear_sys_num = libmesh_map_find(_linear_sys_name_to_num, linear_sys_name);
6874 :
6875 78396 : return linear_sys_num;
6876 78396 : }
6877 :
6878 : unsigned int
6879 125998 : FEProblemBase::solverSysNum(const SolverSystemName & solver_sys_name) const
6880 : {
6881 125998 : std::istringstream ss(solver_sys_name);
6882 : unsigned int solver_sys_num;
6883 125998 : if (!(ss >> solver_sys_num) || !ss.eof())
6884 : {
6885 125998 : const auto & search = _solver_sys_name_to_num.find(solver_sys_name);
6886 125998 : if (search == _solver_sys_name_to_num.end())
6887 0 : mooseError("The solver system number was requested for system '" + solver_sys_name,
6888 : "' but this system does not exist in the Problem. Systems can be added to the "
6889 : "problem using the 'nl_sys_names'/'linear_sys_names' parameter.\nSystems in the "
6890 0 : "Problem: " +
6891 0 : Moose::stringify(_solver_sys_names));
6892 125998 : solver_sys_num = search->second;
6893 : }
6894 :
6895 125998 : return solver_sys_num;
6896 125998 : }
6897 :
6898 : unsigned int
6899 1644 : FEProblemBase::systemNumForVariable(const VariableName & variable_name) const
6900 : {
6901 1746 : for (const auto & solver_sys : _solver_systems)
6902 1644 : if (solver_sys->hasVariable(variable_name))
6903 1542 : return solver_sys->number();
6904 : mooseAssert(_aux, "Should have an auxiliary system");
6905 102 : if (_aux->hasVariable(variable_name))
6906 102 : return _aux->number();
6907 :
6908 0 : mooseError("Variable '",
6909 : variable_name,
6910 : "' was not found in any solver (nonlinear/linear) or auxiliary system");
6911 : }
6912 :
6913 : void
6914 322308 : FEProblemBase::solve(const unsigned int nl_sys_num)
6915 : {
6916 1611540 : TIME_SECTION("solve", 1, "Solving", false);
6917 :
6918 322308 : setCurrentNonlinearSystem(nl_sys_num);
6919 :
6920 : // This prevents stale dof indices from lingering around and possibly leading to invalid reads
6921 : // and writes. Dof indices may be made stale through operations like mesh adaptivity
6922 322308 : clearAllDofIndices();
6923 322308 : if (_displaced_problem)
6924 32837 : _displaced_problem->clearAllDofIndices();
6925 :
6926 : // Setup the output system for printing linear/nonlinear iteration information and some solver
6927 : // settings, including setting matrix prefixes. This must occur before petscSetOptions
6928 322308 : initPetscOutputAndSomeSolverSettings();
6929 :
6930 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6931 : Moose::PetscSupport::petscSetOptions(
6932 : _petsc_options, _solver_params); // Make sure the PETSc options are setup for this app
6933 : #else
6934 : // Now this database will be the default
6935 : // Each app should have only one database
6936 322308 : if (!_app.isUltimateMaster())
6937 85566 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6938 : // We did not add PETSc options to database yet
6939 322308 : if (!_is_petsc_options_inserted)
6940 : {
6941 : // Insert options for all systems all at once
6942 39269 : Moose::PetscSupport::petscSetOptions(_petsc_options, _solver_params, this);
6943 39266 : _is_petsc_options_inserted = true;
6944 : }
6945 : #endif
6946 :
6947 : // set up DM which is required if use a field split preconditioner
6948 : // We need to setup DM every "solve()" because libMesh destroy SNES after solve()
6949 : // Do not worry, DM setup is very cheap
6950 322305 : _current_nl_sys->setupDM();
6951 :
6952 322305 : possiblyRebuildGeomSearchPatches();
6953 :
6954 : // reset flag so that residual evaluation does not get skipped
6955 : // and the next non-linear iteration does not automatically fail with
6956 : // "DIVERGED_NANORINF", when we throw an exception and stop solve
6957 322305 : _fail_next_system_convergence_check = false;
6958 :
6959 322305 : if (_solve)
6960 : {
6961 288667 : _current_nl_sys->solve();
6962 288612 : _current_nl_sys->update();
6963 : }
6964 :
6965 : // sync solutions in displaced problem
6966 322250 : if (_displaced_problem)
6967 32834 : _displaced_problem->syncSolutions();
6968 :
6969 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6970 322250 : if (!_app.isUltimateMaster())
6971 85566 : LibmeshPetscCall(PetscOptionsPop());
6972 : #endif
6973 322250 : }
6974 :
6975 : void
6976 199 : FEProblemBase::setException(const std::string & message)
6977 : {
6978 199 : _has_exception = true;
6979 199 : _exception_message = message;
6980 199 : }
6981 :
6982 : void
6983 19936088 : FEProblemBase::checkExceptionAndStopSolve(bool print_message)
6984 : {
6985 19936088 : if (_skip_exception_check)
6986 216 : return;
6987 :
6988 59807616 : TIME_SECTION("checkExceptionAndStopSolve", 5);
6989 :
6990 : // See if any processor had an exception. If it did, get back the
6991 : // processor that the exception occurred on.
6992 : unsigned int processor_id;
6993 :
6994 19935872 : _communicator.maxloc(_has_exception, processor_id);
6995 :
6996 19935872 : if (_has_exception)
6997 : {
6998 302 : _communicator.broadcast(_exception_message, processor_id);
6999 :
7000 432 : if (_current_execute_on_flag == EXEC_LINEAR || _current_execute_on_flag == EXEC_NONLINEAR ||
7001 130 : _current_execute_on_flag == EXEC_POSTCHECK)
7002 : {
7003 : // Print the message
7004 302 : if (_communicator.rank() == 0 && print_message)
7005 : {
7006 184 : _console << "\n" << _exception_message << "\n";
7007 184 : if (isTransient())
7008 : _console
7009 : << "To recover, the solution will fail and then be re-attempted with a reduced time "
7010 172 : "step.\n"
7011 172 : << std::endl;
7012 : }
7013 :
7014 : // Stop the solve -- this entails setting
7015 : // SNESSetFunctionDomainError() or directly inserting NaNs in the
7016 : // residual vector to let PETSc >= 3.6 return DIVERGED_NANORINF.
7017 302 : if (_current_nl_sys)
7018 302 : _current_nl_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7019 :
7020 302 : if (_current_linear_sys)
7021 0 : _current_linear_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7022 :
7023 : // and close Aux system (we MUST do this here; see #11525)
7024 302 : _aux->solution().close();
7025 :
7026 : // We've handled this exception, so we no longer have one.
7027 302 : _has_exception = false;
7028 :
7029 : // Force the next non-linear convergence check to fail (and all further residual evaluation
7030 : // to be skipped).
7031 302 : _fail_next_system_convergence_check = true;
7032 :
7033 : // Repropagate the exception, so it can be caught at a higher level, typically
7034 : // this is NonlinearSystem::computeResidual().
7035 302 : throw MooseException(_exception_message);
7036 : }
7037 : else
7038 0 : mooseError("The following parallel-communicated exception was detected during " +
7039 0 : Moose::stringify(_current_execute_on_flag) + " evaluation:\n" +
7040 0 : _exception_message +
7041 : "\nBecause this did not occur during residual evaluation, there"
7042 : " is no way to handle this, so the solution is aborting.\n");
7043 : }
7044 19935872 : }
7045 :
7046 : void
7047 3537819 : FEProblemBase::resetState()
7048 : {
7049 : // Our default state is to allow computing derivatives
7050 3537819 : ADReal::do_derivatives = true;
7051 3537819 : _current_execute_on_flag = EXEC_NONE;
7052 :
7053 : // Clear the VectorTags and MatrixTags
7054 3537819 : clearCurrentResidualVectorTags();
7055 3537819 : clearCurrentJacobianMatrixTags();
7056 :
7057 3537819 : _safe_access_tagged_vectors = true;
7058 3537819 : _safe_access_tagged_matrices = true;
7059 :
7060 3537819 : setCurrentlyComputingResidual(false);
7061 3537819 : setCurrentlyComputingJacobian(false);
7062 3537819 : setCurrentlyComputingResidualAndJacobian(false);
7063 3537819 : if (_displaced_problem)
7064 : {
7065 145291 : _displaced_problem->setCurrentlyComputingResidual(false);
7066 145291 : _displaced_problem->setCurrentlyComputingJacobian(false);
7067 145291 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(false);
7068 : }
7069 3537819 : }
7070 :
7071 : void
7072 26086 : FEProblemBase::solveLinearSystem(const unsigned int linear_sys_num,
7073 : const Moose::PetscSupport::PetscOptions * po)
7074 : {
7075 130430 : TIME_SECTION("solve", 1, "Solving", false);
7076 :
7077 26086 : setCurrentLinearSystem(linear_sys_num);
7078 :
7079 26086 : const Moose::PetscSupport::PetscOptions & options = po ? *po : _petsc_options;
7080 26086 : auto & solver_params = _solver_params[numNonlinearSystems() + linear_sys_num];
7081 :
7082 : // Set custom convergence criteria
7083 26086 : Moose::PetscSupport::petscSetDefaults(*this);
7084 :
7085 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
7086 : LibmeshPetscCall(Moose::PetscSupport::petscSetOptions(
7087 : options, solver_params)); // Make sure the PETSc options are setup for this app
7088 : #else
7089 : // Now this database will be the default
7090 : // Each app should have only one database
7091 26086 : if (!_app.isUltimateMaster())
7092 195 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
7093 :
7094 : // We did not add PETSc options to database yet
7095 26086 : if (!_is_petsc_options_inserted)
7096 : {
7097 942 : Moose::PetscSupport::petscSetOptions(options, solver_params, this);
7098 942 : _is_petsc_options_inserted = true;
7099 : }
7100 : #endif
7101 :
7102 26086 : if (_solve)
7103 26076 : _current_linear_sys->solve();
7104 :
7105 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
7106 26086 : if (!_app.isUltimateMaster())
7107 195 : LibmeshPetscCall(PetscOptionsPop());
7108 : #endif
7109 26086 : }
7110 :
7111 : bool
7112 335873 : FEProblemBase::solverSystemConverged(const unsigned int sys_num)
7113 : {
7114 335873 : if (_solve)
7115 335851 : return _solver_systems[sys_num]->converged();
7116 : else
7117 22 : return true;
7118 : }
7119 :
7120 : unsigned int
7121 4901 : FEProblemBase::nNonlinearIterations(const unsigned int nl_sys_num) const
7122 : {
7123 4901 : return _nl[nl_sys_num]->nNonlinearIterations();
7124 : }
7125 :
7126 : unsigned int
7127 4051 : FEProblemBase::nLinearIterations(const unsigned int nl_sys_num) const
7128 : {
7129 4051 : return _nl[nl_sys_num]->nLinearIterations();
7130 : }
7131 :
7132 : Real
7133 242 : FEProblemBase::finalNonlinearResidual(const unsigned int nl_sys_num) const
7134 : {
7135 242 : return _nl[nl_sys_num]->finalNonlinearResidual();
7136 : }
7137 :
7138 : bool
7139 760320 : FEProblemBase::computingPreSMOResidual(const unsigned int nl_sys_num) const
7140 : {
7141 760320 : return _nl[nl_sys_num]->computingPreSMOResidual();
7142 : }
7143 :
7144 : void
7145 55432 : FEProblemBase::copySolutionsBackwards()
7146 : {
7147 277160 : TIME_SECTION("copySolutionsBackwards", 3, "Copying Solutions Backward");
7148 :
7149 111125 : for (auto & sys : _solver_systems)
7150 55693 : sys->copySolutionsBackwards();
7151 55432 : _aux->copySolutionsBackwards();
7152 55432 : }
7153 :
7154 : void
7155 152 : FEProblemBase::skipNextForwardSolutionCopyToOld()
7156 : {
7157 324 : for (auto & sys : _solver_systems)
7158 172 : sys->skipNextSolutionToOldCopy();
7159 152 : _aux->skipNextSolutionToOldCopy();
7160 152 : }
7161 :
7162 : void
7163 233327 : FEProblemBase::advanceState()
7164 : {
7165 1166635 : TIME_SECTION("advanceState", 5, "Advancing State");
7166 :
7167 470021 : for (auto & sys : _solver_systems)
7168 236694 : sys->copyOldSolutions();
7169 233327 : _aux->copyOldSolutions();
7170 :
7171 233327 : if (_displaced_problem)
7172 : {
7173 61858 : for (const auto i : index_range(_solver_systems))
7174 30929 : _displaced_problem->solverSys(i).copyOldSolutions();
7175 30929 : _displaced_problem->auxSys().copyOldSolutions();
7176 : }
7177 :
7178 233327 : _reporter_data.copyValuesBack();
7179 :
7180 233327 : getMooseApp().getChainControlDataSystem().copyValuesBack();
7181 :
7182 233327 : if (_material_props.hasStatefulProperties())
7183 1760 : _material_props.shift();
7184 :
7185 233327 : if (_bnd_material_props.hasStatefulProperties())
7186 1612 : _bnd_material_props.shift();
7187 :
7188 233327 : if (_neighbor_material_props.hasStatefulProperties())
7189 1496 : _neighbor_material_props.shift();
7190 :
7191 : #ifdef MOOSE_KOKKOS_ENABLED
7192 171259 : if (_kokkos_material_props.hasStatefulProperties())
7193 566 : _kokkos_material_props.shift();
7194 :
7195 171259 : if (_kokkos_bnd_material_props.hasStatefulProperties())
7196 641 : _kokkos_bnd_material_props.shift();
7197 :
7198 171259 : if (_kokkos_neighbor_material_props.hasStatefulProperties())
7199 566 : _kokkos_neighbor_material_props.shift();
7200 : #endif
7201 233327 : }
7202 :
7203 : void
7204 3341 : FEProblemBase::restoreSolutions()
7205 : {
7206 16705 : TIME_SECTION("restoreSolutions", 5, "Restoring Solutions");
7207 :
7208 3341 : if (!_not_zeroed_tagged_vectors.empty())
7209 0 : paramError("not_zeroed_tag_vectors",
7210 : "There is currently no way to restore not-zeroed vectors.");
7211 :
7212 6684 : for (auto & sys : _solver_systems)
7213 : {
7214 3343 : if (_verbose_restore)
7215 18 : _console << "Restoring solutions on system " << sys->name() << "..." << std::endl;
7216 3343 : sys->restoreSolutions();
7217 : }
7218 :
7219 3341 : if (_verbose_restore)
7220 18 : _console << "Restoring solutions on Auxiliary system..." << std::endl;
7221 3341 : _aux->restoreSolutions();
7222 :
7223 3341 : if (_verbose_restore)
7224 18 : _console << "Restoring postprocessor, vector-postprocessor, and reporter data..." << std::endl;
7225 3341 : _reporter_data.restoreState(_verbose_restore);
7226 :
7227 3341 : if (_displaced_problem)
7228 132 : _displaced_problem->updateMesh();
7229 3341 : }
7230 :
7231 : void
7232 100 : FEProblemBase::saveOldSolutions()
7233 : {
7234 500 : TIME_SECTION("saveOldSolutions", 5, "Saving Old Solutions");
7235 :
7236 200 : for (auto & sys : _solver_systems)
7237 100 : sys->saveOldSolutions();
7238 100 : _aux->saveOldSolutions();
7239 100 : }
7240 :
7241 : void
7242 100 : FEProblemBase::restoreOldSolutions()
7243 : {
7244 500 : TIME_SECTION("restoreOldSolutions", 5, "Restoring Old Solutions");
7245 :
7246 200 : for (auto & sys : _solver_systems)
7247 100 : sys->restoreOldSolutions();
7248 100 : _aux->restoreOldSolutions();
7249 100 : }
7250 :
7251 : void
7252 1240331 : FEProblemBase::outputStep(ExecFlagType type)
7253 : {
7254 6201655 : TIME_SECTION("outputStep", 1, "Outputting");
7255 :
7256 1240331 : setCurrentExecuteOnFlag(type);
7257 :
7258 2494936 : for (auto & sys : _solver_systems)
7259 1254605 : sys->update();
7260 1240331 : _aux->update();
7261 :
7262 1240331 : if (_displaced_problem)
7263 126253 : _displaced_problem->syncSolutions();
7264 1240331 : _app.getOutputWarehouse().outputStep(type);
7265 :
7266 1240322 : setCurrentExecuteOnFlag(EXEC_NONE);
7267 1240322 : }
7268 :
7269 : void
7270 83891 : FEProblemBase::allowOutput(bool state)
7271 : {
7272 83891 : _app.getOutputWarehouse().allowOutput(state);
7273 83891 : }
7274 :
7275 : void
7276 23 : FEProblemBase::forceOutput()
7277 : {
7278 23 : _app.getOutputWarehouse().forceOutput();
7279 23 : }
7280 :
7281 : void
7282 341333 : FEProblemBase::initPetscOutputAndSomeSolverSettings()
7283 : {
7284 341333 : _app.getOutputWarehouse().solveSetup();
7285 341333 : Moose::PetscSupport::petscSetDefaults(*this);
7286 341333 : }
7287 :
7288 : void
7289 239546 : FEProblemBase::onTimestepBegin()
7290 : {
7291 718638 : TIME_SECTION("onTimestepBegin", 2);
7292 :
7293 482142 : for (auto & nl : _nl)
7294 242596 : nl->onTimestepBegin();
7295 239546 : }
7296 :
7297 : void
7298 310481 : FEProblemBase::onTimestepEnd()
7299 : {
7300 310481 : }
7301 :
7302 : Real
7303 6607231 : FEProblemBase::getTimeFromStateArg(const Moose::StateArg & state) const
7304 : {
7305 6607231 : if (state.iteration_type != Moose::SolutionIterationType::Time)
7306 : // If we are any iteration type other than time (e.g. nonlinear), then temporally we are still
7307 : // in the present time
7308 0 : return time();
7309 :
7310 6607231 : switch (state.state)
7311 : {
7312 6607231 : case 0:
7313 6607231 : return time();
7314 :
7315 0 : case 1:
7316 0 : return timeOld();
7317 :
7318 0 : default:
7319 0 : mooseError("Unhandled state ", state.state, " in FEProblemBase::getTimeFromStateArg");
7320 : }
7321 : }
7322 :
7323 : void
7324 30437 : FEProblemBase::addTimeIntegrator(const std::string & type,
7325 : const std::string & name,
7326 : InputParameters & parameters)
7327 : {
7328 : parallel_object_only();
7329 :
7330 91311 : parameters.set<SubProblem *>("_subproblem") = this;
7331 30437 : logAdd("TimeIntegrator", name, type, parameters);
7332 30437 : _aux->addTimeIntegrator(type, name + ":aux", parameters);
7333 60910 : for (auto & sys : _solver_systems)
7334 30473 : sys->addTimeIntegrator(type, name + ":" + sys->name(), parameters);
7335 30437 : _has_time_integrator = true;
7336 :
7337 : // add vectors to store u_dot, u_dotdot, udot_old, u_dotdot_old and
7338 : // solution vectors older than 2 time steps, if requested by the time
7339 : // integrator
7340 30437 : _aux->addDotVectors();
7341 60879 : for (auto & nl : _nl)
7342 : {
7343 30442 : nl->addDotVectors();
7344 :
7345 30442 : auto tag_udot = nl->getTimeIntegrators()[0]->uDotFactorTag();
7346 30442 : if (!nl->hasVector(tag_udot))
7347 30406 : nl->associateVectorToTag(*nl->solutionUDot(), tag_udot);
7348 30442 : auto tag_udotdot = nl->getTimeIntegrators()[0]->uDotDotFactorTag();
7349 30442 : if (!nl->hasVector(tag_udotdot) && uDotDotRequested())
7350 150 : nl->associateVectorToTag(*nl->solutionUDotDot(), tag_udotdot);
7351 : }
7352 :
7353 30437 : if (_displaced_problem)
7354 : // Time integrator does not exist when displaced problem is created.
7355 1728 : _displaced_problem->addTimeIntegrator();
7356 30437 : }
7357 :
7358 : void
7359 22 : FEProblemBase::addPredictor(const std::string & type,
7360 : const std::string & name,
7361 : InputParameters & parameters)
7362 : {
7363 : parallel_object_only();
7364 :
7365 22 : if (!numNonlinearSystems() && numLinearSystems())
7366 0 : mooseError("Vector bounds cannot be used with LinearSystems!");
7367 :
7368 44 : parameters.set<SubProblem *>("_subproblem") = this;
7369 22 : std::shared_ptr<Predictor> predictor = _factory.create<Predictor>(type, name, parameters);
7370 22 : logAdd("Predictor", name, type, parameters);
7371 :
7372 44 : for (auto & nl : _nl)
7373 22 : nl->setPredictor(predictor);
7374 22 : }
7375 :
7376 : Real
7377 62865 : FEProblemBase::computeResidualL2Norm(NonlinearSystemBase & sys)
7378 : {
7379 62865 : _current_nl_sys = &sys;
7380 62865 : computeResidual(*sys.currentSolution(), sys.RHS(), sys.number());
7381 62865 : return sys.RHS().l2_norm();
7382 : }
7383 :
7384 : Real
7385 84 : FEProblemBase::computeResidualL2Norm(LinearSystem & sys)
7386 : {
7387 84 : _current_linear_sys = &sys;
7388 :
7389 : // We assemble the current system to check the current residual
7390 84 : computeLinearSystemSys(sys.linearImplicitSystem(),
7391 84 : *sys.linearImplicitSystem().matrix,
7392 84 : *sys.linearImplicitSystem().rhs,
7393 : /*compute fresh gradients*/ true);
7394 :
7395 : // Unfortunate, but we have to allocate a new vector for the residual
7396 84 : auto residual = sys.linearImplicitSystem().rhs->clone();
7397 84 : residual->scale(-1.0);
7398 84 : residual->add_vector(*sys.currentSolution(), *sys.linearImplicitSystem().matrix);
7399 168 : return residual->l2_norm();
7400 84 : }
7401 :
7402 : Real
7403 62798 : FEProblemBase::computeResidualL2Norm()
7404 : {
7405 313990 : TIME_SECTION("computeResidualL2Norm", 2, "Computing L2 Norm of Residual");
7406 :
7407 : // We use sum the squared norms of the individual systems and then take the square root of it
7408 62798 : Real l2_norm = 0.0;
7409 125663 : for (auto sys : _nl)
7410 : {
7411 62865 : const auto norm = computeResidualL2Norm(*sys);
7412 62865 : l2_norm += norm * norm;
7413 62865 : }
7414 :
7415 62882 : for (auto sys : _linear_systems)
7416 : {
7417 84 : const auto norm = computeResidualL2Norm(*sys);
7418 84 : l2_norm += norm * norm;
7419 84 : }
7420 :
7421 125596 : return std::sqrt(l2_norm);
7422 62798 : }
7423 :
7424 : void
7425 2948029 : FEProblemBase::computeResidualSys(NonlinearImplicitSystem & sys,
7426 : const NumericVector<Number> & soln,
7427 : NumericVector<Number> & residual)
7428 : {
7429 : parallel_object_only();
7430 :
7431 8844087 : TIME_SECTION("computeResidualSys", 5);
7432 : // Reset before residual setup, calculation & execution
7433 2948029 : _app.solutionInvalidity().resetIterationOccurences();
7434 :
7435 2948029 : computeResidual(soln, residual, sys.number());
7436 2947999 : }
7437 :
7438 : void
7439 0 : FEProblemBase::computeResidual(NonlinearImplicitSystem & sys,
7440 : const NumericVector<Number> & soln,
7441 : NumericVector<Number> & residual)
7442 : {
7443 0 : mooseDeprecated("Please use computeResidualSys");
7444 :
7445 0 : computeResidualSys(sys, soln, residual);
7446 0 : }
7447 :
7448 : void
7449 3007716 : FEProblemBase::computeResidual(const NumericVector<Number> & soln,
7450 : NumericVector<Number> & residual,
7451 : const unsigned int nl_sys_num)
7452 : {
7453 3007716 : setCurrentNonlinearSystem(nl_sys_num);
7454 :
7455 : // We associate the residual tag with the given residual vector to make sure we
7456 : // don't filter it out below
7457 3007716 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7458 3007716 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7459 :
7460 : mooseAssert(_fe_vector_tags.empty(), "This should be empty indicating a clean starting state");
7461 : // We filter out tags which do not have associated vectors in the current nonlinear
7462 : // system. This is essential to be able to use system-dependent residual tags.
7463 3007716 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7464 :
7465 3007716 : computeResidualInternal(soln, residual, _fe_vector_tags);
7466 3007686 : _fe_vector_tags.clear();
7467 3007686 : }
7468 :
7469 : void
7470 9899 : FEProblemBase::computeResidualAndJacobian(const NumericVector<Number> & soln,
7471 : NumericVector<Number> & residual,
7472 : SparseMatrix<Number> & jacobian)
7473 : {
7474 : try
7475 : {
7476 : try
7477 : {
7478 : // vector tags
7479 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7480 9899 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7481 :
7482 : mooseAssert(_fe_vector_tags.empty(),
7483 : "This should be empty indicating a clean starting state");
7484 : // We filter out tags which do not have associated vectors in the current nonlinear
7485 : // system. This is essential to be able to use system-dependent residual tags.
7486 9899 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7487 :
7488 9899 : setCurrentResidualVectorTags(_fe_vector_tags);
7489 :
7490 : // matrix tags
7491 : {
7492 9899 : _fe_matrix_tags.clear();
7493 :
7494 9899 : auto & tags = getMatrixTags();
7495 29697 : for (auto & tag : tags)
7496 19798 : _fe_matrix_tags.insert(tag.second);
7497 : }
7498 :
7499 9899 : _current_nl_sys->setSolution(soln);
7500 :
7501 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7502 9899 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7503 :
7504 29697 : for (const auto tag : _fe_matrix_tags)
7505 19798 : if (_current_nl_sys->hasMatrix(tag))
7506 : {
7507 9899 : auto & matrix = _current_nl_sys->getMatrix(tag);
7508 9899 : matrix.zero();
7509 9899 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7510 : // PETSc algorithms require diagonal allocations regardless of whether there is non-zero
7511 : // diagonal dependence. With global AD indexing we only add non-zero
7512 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7513 123018 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7514 120004 : matrix.add(index, index, 0);
7515 : }
7516 :
7517 9899 : _aux->zeroVariablesForResidual();
7518 :
7519 9899 : unsigned int n_threads = libMesh::n_threads();
7520 :
7521 9899 : _current_execute_on_flag = EXEC_LINEAR;
7522 :
7523 : // Random interface objects
7524 9899 : for (const auto & it : _random_data_objects)
7525 0 : it.second->updateSeeds(EXEC_LINEAR);
7526 :
7527 9899 : setCurrentlyComputingResidual(true);
7528 9899 : setCurrentlyComputingJacobian(true);
7529 9899 : setCurrentlyComputingResidualAndJacobian(true);
7530 9899 : if (_displaced_problem)
7531 : {
7532 1057 : _displaced_problem->setCurrentlyComputingResidual(true);
7533 1057 : _displaced_problem->setCurrentlyComputingJacobian(true);
7534 1057 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(true);
7535 : }
7536 :
7537 9899 : execTransfers(EXEC_LINEAR);
7538 :
7539 9899 : execMultiApps(EXEC_LINEAR);
7540 :
7541 21234 : for (unsigned int tid = 0; tid < n_threads; tid++)
7542 11335 : reinitScalars(tid);
7543 :
7544 9899 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7545 :
7546 9899 : _aux->residualSetup();
7547 :
7548 9899 : if (_displaced_problem)
7549 : {
7550 1057 : computeSystems(EXEC_PRE_DISPLACE);
7551 1057 : _displaced_problem->updateMesh();
7552 1057 : if (_mortar_data->hasDisplacedObjects())
7553 1057 : updateMortarMesh();
7554 : }
7555 :
7556 21234 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7557 : {
7558 11335 : _all_materials.residualSetup(tid);
7559 11335 : _functions.residualSetup(tid);
7560 : }
7561 :
7562 : #ifdef MOOSE_KOKKOS_ENABLED
7563 8839 : _kokkos_functions.residualSetup();
7564 : #endif
7565 :
7566 9899 : computeSystems(EXEC_LINEAR);
7567 :
7568 9899 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7569 :
7570 9899 : executeControls(EXEC_LINEAR);
7571 :
7572 9899 : _app.getOutputWarehouse().residualSetup();
7573 :
7574 9899 : _safe_access_tagged_vectors = false;
7575 9899 : _safe_access_tagged_matrices = false;
7576 :
7577 9899 : _current_nl_sys->computeResidualAndJacobianTags(_fe_vector_tags, _fe_matrix_tags);
7578 :
7579 9899 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7580 9899 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7581 : }
7582 0 : catch (...)
7583 : {
7584 0 : handleException("computeResidualAndJacobian");
7585 0 : }
7586 : }
7587 0 : catch (const MooseException &)
7588 : {
7589 : // The buck stops here, we have already handled the exception by
7590 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7591 : // "diverged" reason during the next solve.
7592 0 : }
7593 0 : catch (...)
7594 : {
7595 0 : mooseError("Unexpected exception type");
7596 0 : }
7597 :
7598 9899 : resetState();
7599 9899 : _fe_vector_tags.clear();
7600 9899 : _fe_matrix_tags.clear();
7601 9899 : }
7602 :
7603 : void
7604 0 : FEProblemBase::computeResidualTag(const NumericVector<Number> & soln,
7605 : NumericVector<Number> & residual,
7606 : TagID tag)
7607 : {
7608 : try
7609 : {
7610 0 : _current_nl_sys->setSolution(soln);
7611 :
7612 0 : _current_nl_sys->associateVectorToTag(residual, tag);
7613 :
7614 0 : computeResidualTags({tag});
7615 :
7616 0 : _current_nl_sys->disassociateVectorFromTag(residual, tag);
7617 : }
7618 0 : catch (MooseException & e)
7619 : {
7620 : // If a MooseException propagates all the way to here, it means
7621 : // that it was thrown from a MOOSE system where we do not
7622 : // (currently) properly support the throwing of exceptions, and
7623 : // therefore we have no choice but to error out. It may be
7624 : // *possible* to handle exceptions from other systems, but in the
7625 : // meantime, we don't want to silently swallow any unhandled
7626 : // exceptions here.
7627 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7628 : "contact the MOOSE team for assistance.");
7629 0 : }
7630 0 : }
7631 :
7632 : void
7633 3011300 : FEProblemBase::computeResidualInternal(const NumericVector<Number> & soln,
7634 : NumericVector<Number> & residual,
7635 : const std::set<TagID> & tags)
7636 : {
7637 : parallel_object_only();
7638 :
7639 9033900 : TIME_SECTION("computeResidualInternal", 1);
7640 :
7641 : try
7642 : {
7643 3011300 : _current_nl_sys->setSolution(soln);
7644 :
7645 3011300 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7646 :
7647 3011300 : computeResidualTags(tags);
7648 :
7649 3011270 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7650 : }
7651 0 : catch (MooseException & e)
7652 : {
7653 : // If a MooseException propagates all the way to here, it means
7654 : // that it was thrown from a MOOSE system where we do not
7655 : // (currently) properly support the throwing of exceptions, and
7656 : // therefore we have no choice but to error out. It may be
7657 : // *possible* to handle exceptions from other systems, but in the
7658 : // meantime, we don't want to silently swallow any unhandled
7659 : // exceptions here.
7660 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7661 : "contact the MOOSE team for assistance.");
7662 0 : }
7663 3011270 : }
7664 :
7665 : void
7666 0 : FEProblemBase::computeResidualType(const NumericVector<Number> & soln,
7667 : NumericVector<Number> & residual,
7668 : TagID tag)
7669 : {
7670 0 : TIME_SECTION("computeResidualType", 5);
7671 :
7672 : try
7673 : {
7674 0 : _current_nl_sys->setSolution(soln);
7675 :
7676 0 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7677 :
7678 0 : computeResidualTags({tag, _current_nl_sys->residualVectorTag()});
7679 :
7680 0 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7681 : }
7682 0 : catch (MooseException & e)
7683 : {
7684 : // If a MooseException propagates all the way to here, it means
7685 : // that it was thrown from a MOOSE system where we do not
7686 : // (currently) properly support the throwing of exceptions, and
7687 : // therefore we have no choice but to error out. It may be
7688 : // *possible* to handle exceptions from other systems, but in the
7689 : // meantime, we don't want to silently swallow any unhandled
7690 : // exceptions here.
7691 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7692 : "contact the MOOSE team for assistance.");
7693 0 : }
7694 0 : }
7695 :
7696 : void
7697 3 : FEProblemBase::handleException(const std::string & calling_method)
7698 : {
7699 : auto create_exception_message =
7700 3 : [&calling_method](const std::string & exception_type, const auto & exception)
7701 : {
7702 : return std::string("A " + exception_type + " was raised during FEProblemBase::" +
7703 6 : calling_method + "\n" + std::string(exception.what()));
7704 3 : };
7705 :
7706 : try
7707 : {
7708 3 : throw;
7709 : }
7710 3 : catch (const MooseException & e)
7711 : {
7712 0 : setException(create_exception_message("MooseException", e));
7713 0 : }
7714 0 : catch (const MetaPhysicL::LogicError & e)
7715 : {
7716 0 : moose::translateMetaPhysicLError(e);
7717 0 : }
7718 3 : catch (const libMesh::PetscSolverException & e)
7719 : {
7720 : // One PETSc solver exception that we cannot currently recover from are new nonzero errors. In
7721 : // particular I have observed the following scenario in a parallel test:
7722 : // - Both processes throw because of a new nonzero during MOOSE's computeJacobianTags
7723 : // - We potentially handle the exceptions nicely here
7724 : // - When the matrix is closed in libMesh's libmesh_petsc_snes_solver, there is a new nonzero
7725 : // throw which we do not catch here in MOOSE and the simulation terminates. This only appears
7726 : // in parallel (and not all the time; a test I was examining threw with distributed mesh, but
7727 : // not with replicated). In serial there are no new throws from libmesh_petsc_snes_solver.
7728 : // So for uniformity of behavior across serial/parallel, we will choose to abort here and always
7729 : // produce a non-zero exit code
7730 6 : mooseError(create_exception_message("libMesh::PetscSolverException", e));
7731 0 : }
7732 0 : catch (const std::exception & e)
7733 : {
7734 : // This might be libMesh detecting a degenerate Jacobian or matrix
7735 0 : if (strstr(e.what(), "Jacobian") || strstr(e.what(), "singular") ||
7736 0 : strstr(e.what(), "det != 0"))
7737 0 : setException(create_exception_message("libMesh DegenerateMap", e));
7738 : else
7739 : {
7740 0 : const auto message = create_exception_message("std::exception", e);
7741 0 : if (_regard_general_exceptions_as_errors)
7742 0 : mooseError(message);
7743 : else
7744 0 : setException(message);
7745 0 : }
7746 0 : }
7747 :
7748 0 : checkExceptionAndStopSolve();
7749 0 : }
7750 :
7751 : void
7752 3046703 : FEProblemBase::computeResidualTags(const std::set<TagID> & tags)
7753 : {
7754 : parallel_object_only();
7755 :
7756 : try
7757 : {
7758 : try
7759 : {
7760 15233515 : TIME_SECTION("computeResidualTags", 5, "Computing Residual");
7761 :
7762 3046703 : ADReal::do_derivatives = false;
7763 :
7764 3046703 : setCurrentResidualVectorTags(tags);
7765 :
7766 3046703 : _aux->zeroVariablesForResidual();
7767 :
7768 3046703 : unsigned int n_threads = libMesh::n_threads();
7769 :
7770 3046703 : _current_execute_on_flag = EXEC_LINEAR;
7771 :
7772 : // Random interface objects
7773 3059585 : for (const auto & it : _random_data_objects)
7774 12882 : it.second->updateSeeds(EXEC_LINEAR);
7775 :
7776 3046703 : execTransfers(EXEC_LINEAR);
7777 :
7778 3046703 : execMultiApps(EXEC_LINEAR);
7779 :
7780 6401514 : for (unsigned int tid = 0; tid < n_threads; tid++)
7781 3354811 : reinitScalars(tid);
7782 :
7783 3046703 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7784 :
7785 3046703 : _aux->residualSetup();
7786 :
7787 3046703 : if (_displaced_problem)
7788 : {
7789 123090 : computeSystems(EXEC_PRE_DISPLACE);
7790 123090 : _displaced_problem->updateMesh();
7791 123090 : if (_mortar_data->hasDisplacedObjects())
7792 2211 : updateMortarMesh();
7793 : }
7794 :
7795 6401514 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7796 : {
7797 3354811 : _all_materials.residualSetup(tid);
7798 3354811 : _functions.residualSetup(tid);
7799 : }
7800 :
7801 : #ifdef MOOSE_KOKKOS_ENABLED
7802 2214644 : _kokkos_functions.residualSetup();
7803 : #endif
7804 :
7805 3046703 : computeSystems(EXEC_LINEAR);
7806 :
7807 3046703 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7808 :
7809 3046703 : executeControls(EXEC_LINEAR);
7810 :
7811 3046703 : _app.getOutputWarehouse().residualSetup();
7812 :
7813 3046703 : _safe_access_tagged_vectors = false;
7814 3046703 : _current_nl_sys->computeResidualTags(tags);
7815 3046673 : }
7816 0 : catch (...)
7817 : {
7818 0 : handleException("computeResidualTags");
7819 0 : }
7820 : }
7821 0 : catch (const MooseException &)
7822 : {
7823 : // The buck stops here, we have already handled the exception by
7824 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7825 : // "diverged" reason during the next solve.
7826 0 : }
7827 0 : catch (...)
7828 : {
7829 0 : mooseError("Unexpected exception type");
7830 0 : }
7831 :
7832 3046673 : resetState();
7833 3046673 : }
7834 :
7835 : void
7836 472856 : FEProblemBase::computeJacobianSys(NonlinearImplicitSystem & sys,
7837 : const NumericVector<Number> & soln,
7838 : SparseMatrix<Number> & jacobian)
7839 : {
7840 : // Reset before Jacobian setup, calculation & execution
7841 472856 : _app.solutionInvalidity().resetIterationOccurences();
7842 472856 : computeJacobian(soln, jacobian, sys.number());
7843 472843 : }
7844 :
7845 : void
7846 4467 : FEProblemBase::computeJacobianTag(const NumericVector<Number> & soln,
7847 : SparseMatrix<Number> & jacobian,
7848 : TagID tag)
7849 : {
7850 4467 : _current_nl_sys->setSolution(soln);
7851 :
7852 4467 : _current_nl_sys->associateMatrixToTag(jacobian, tag);
7853 :
7854 8934 : computeJacobianTags({tag});
7855 :
7856 4467 : _current_nl_sys->disassociateMatrixFromTag(jacobian, tag);
7857 4467 : }
7858 :
7859 : void
7860 472012 : FEProblemBase::computeJacobian(const NumericVector<Number> & soln,
7861 : SparseMatrix<Number> & jacobian,
7862 : const unsigned int nl_sys_num)
7863 : {
7864 472012 : setCurrentNonlinearSystem(nl_sys_num);
7865 :
7866 472012 : _fe_matrix_tags.clear();
7867 :
7868 472012 : auto & tags = getMatrixTags();
7869 1416298 : for (auto & tag : tags)
7870 944286 : _fe_matrix_tags.insert(tag.second);
7871 :
7872 472012 : computeJacobianInternal(soln, jacobian, _fe_matrix_tags);
7873 471999 : }
7874 :
7875 : void
7876 472012 : FEProblemBase::computeJacobianInternal(const NumericVector<Number> & soln,
7877 : SparseMatrix<Number> & jacobian,
7878 : const std::set<TagID> & tags)
7879 : {
7880 1416036 : TIME_SECTION("computeJacobianInternal", 1);
7881 :
7882 472012 : _current_nl_sys->setSolution(soln);
7883 :
7884 472012 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7885 :
7886 472012 : computeJacobianTags(tags);
7887 :
7888 471999 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7889 471999 : }
7890 :
7891 : void
7892 481260 : FEProblemBase::computeJacobianTags(const std::set<TagID> & tags)
7893 : {
7894 : try
7895 : {
7896 : try
7897 : {
7898 481260 : if (!_has_jacobian || !_const_jacobian)
7899 : {
7900 2374025 : TIME_SECTION("computeJacobianTags", 5, "Computing Jacobian");
7901 :
7902 1417085 : for (auto tag : tags)
7903 942280 : if (_current_nl_sys->hasMatrix(tag))
7904 : {
7905 476723 : auto & matrix = _current_nl_sys->getMatrix(tag);
7906 476723 : if (_restore_original_nonzero_pattern)
7907 7033 : matrix.restore_original_nonzero_pattern();
7908 : else
7909 469690 : matrix.zero();
7910 476723 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7911 : // PETSc algorithms require diagonal allocations regardless of whether there is
7912 : // non-zero diagonal dependence. With global AD indexing we only add non-zero
7913 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7914 4829926 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7915 4786133 : matrix.add(index, index, 0);
7916 : }
7917 :
7918 474805 : _aux->zeroVariablesForJacobian();
7919 :
7920 474805 : unsigned int n_threads = libMesh::n_threads();
7921 :
7922 : // Random interface objects
7923 476827 : for (const auto & it : _random_data_objects)
7924 2022 : it.second->updateSeeds(EXEC_NONLINEAR);
7925 :
7926 474805 : _current_execute_on_flag = EXEC_NONLINEAR;
7927 474805 : _currently_computing_jacobian = true;
7928 474805 : if (_displaced_problem)
7929 21147 : _displaced_problem->setCurrentlyComputingJacobian(true);
7930 :
7931 474805 : execTransfers(EXEC_NONLINEAR);
7932 474805 : execMultiApps(EXEC_NONLINEAR);
7933 :
7934 999789 : for (unsigned int tid = 0; tid < n_threads; tid++)
7935 524984 : reinitScalars(tid);
7936 :
7937 474805 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7938 :
7939 474805 : _aux->jacobianSetup();
7940 :
7941 474805 : if (_displaced_problem)
7942 : {
7943 21147 : computeSystems(EXEC_PRE_DISPLACE);
7944 21147 : _displaced_problem->updateMesh();
7945 : }
7946 :
7947 999782 : for (unsigned int tid = 0; tid < n_threads; tid++)
7948 : {
7949 524980 : _all_materials.jacobianSetup(tid);
7950 524980 : _functions.jacobianSetup(tid);
7951 : }
7952 :
7953 : #ifdef MOOSE_KOKKOS_ENABLED
7954 346150 : _kokkos_functions.jacobianSetup();
7955 : #endif
7956 :
7957 474802 : computeSystems(EXEC_NONLINEAR);
7958 :
7959 474802 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7960 :
7961 474802 : executeControls(EXEC_NONLINEAR);
7962 :
7963 474802 : _app.getOutputWarehouse().jacobianSetup();
7964 :
7965 474802 : _safe_access_tagged_matrices = false;
7966 :
7967 474802 : _current_nl_sys->computeJacobianTags(tags);
7968 :
7969 : // For explicit Euler calculations for example we often compute the Jacobian one time and
7970 : // then re-use it over and over. If we're performing automatic scaling, we don't want to
7971 : // use that kernel, diagonal-block only Jacobian for our actual matrix when performing
7972 : // solves!
7973 474792 : if (!_current_nl_sys->computingScalingJacobian())
7974 474299 : _has_jacobian = true;
7975 474795 : }
7976 : }
7977 3 : catch (...)
7978 : {
7979 3 : handleException("computeJacobianTags");
7980 0 : }
7981 : }
7982 0 : catch (const MooseException &)
7983 : {
7984 : // The buck stops here, we have already handled the exception by
7985 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7986 : // "diverged" reason during the next solve.
7987 0 : }
7988 0 : catch (...)
7989 : {
7990 0 : mooseError("Unexpected exception type");
7991 0 : }
7992 :
7993 481247 : resetState();
7994 481247 : }
7995 :
7996 : void
7997 263 : FEProblemBase::computeJacobianBlocks(std::vector<JacobianBlock *> & blocks,
7998 : const unsigned int nl_sys_num)
7999 : {
8000 789 : TIME_SECTION("computeTransientImplicitJacobian", 2);
8001 263 : setCurrentNonlinearSystem(nl_sys_num);
8002 :
8003 263 : if (_displaced_problem)
8004 : {
8005 0 : computeSystems(EXEC_PRE_DISPLACE);
8006 0 : _displaced_problem->updateMesh();
8007 : }
8008 :
8009 263 : computeSystems(EXEC_NONLINEAR);
8010 :
8011 263 : _currently_computing_jacobian = true;
8012 263 : _current_nl_sys->computeJacobianBlocks(blocks);
8013 263 : _currently_computing_jacobian = false;
8014 263 : }
8015 :
8016 : void
8017 0 : FEProblemBase::computeJacobianBlock(SparseMatrix<Number> & jacobian,
8018 : libMesh::System & precond_system,
8019 : unsigned int ivar,
8020 : unsigned int jvar)
8021 : {
8022 0 : JacobianBlock jac_block(precond_system, jacobian, ivar, jvar);
8023 0 : std::vector<JacobianBlock *> blocks = {&jac_block};
8024 : mooseAssert(_current_nl_sys, "This should be non-null");
8025 0 : computeJacobianBlocks(blocks, _current_nl_sys->number());
8026 0 : }
8027 :
8028 : void
8029 714 : FEProblemBase::computeBounds(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8030 : NumericVector<Number> & lower,
8031 : NumericVector<Number> & upper)
8032 : {
8033 : try
8034 : {
8035 : try
8036 : {
8037 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8038 : "I expect these system numbers to be the same");
8039 :
8040 3570 : if (!_current_nl_sys->hasVector("lower_bound") || !_current_nl_sys->hasVector("upper_bound"))
8041 0 : return;
8042 :
8043 3570 : TIME_SECTION("computeBounds", 1, "Computing Bounds");
8044 :
8045 1428 : NumericVector<Number> & _lower = _current_nl_sys->getVector("lower_bound");
8046 1428 : NumericVector<Number> & _upper = _current_nl_sys->getVector("upper_bound");
8047 714 : _lower.swap(lower);
8048 714 : _upper.swap(upper);
8049 1492 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
8050 778 : _all_materials.residualSetup(tid);
8051 :
8052 714 : _aux->residualSetup();
8053 714 : computeSystems(EXEC_LINEAR);
8054 714 : _lower.swap(lower);
8055 714 : _upper.swap(upper);
8056 714 : }
8057 0 : catch (...)
8058 : {
8059 0 : handleException("computeBounds");
8060 0 : }
8061 : }
8062 0 : catch (MooseException & e)
8063 : {
8064 0 : mooseError("Irrecoverable exception: " + std::string(e.what()));
8065 0 : }
8066 0 : catch (...)
8067 : {
8068 0 : mooseError("Unexpected exception type");
8069 0 : }
8070 : }
8071 :
8072 : void
8073 26160 : FEProblemBase::computeLinearSystemSys(LinearImplicitSystem & sys,
8074 : SparseMatrix<Number> & system_matrix,
8075 : NumericVector<Number> & rhs,
8076 : const bool compute_gradients)
8077 : {
8078 78480 : TIME_SECTION("computeLinearSystemSys", 5);
8079 :
8080 26160 : setCurrentLinearSystem(linearSysNum(sys.name()));
8081 :
8082 26160 : _current_linear_sys->associateVectorToTag(rhs, _current_linear_sys->rightHandSideVectorTag());
8083 26160 : _current_linear_sys->associateMatrixToTag(system_matrix, _current_linear_sys->systemMatrixTag());
8084 :
8085 : // We are using the residual tag system for right hand sides so we fetch everything
8086 26160 : const auto & vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
8087 :
8088 : // We filter out tags which do not have associated vectors in the current
8089 : // system. This is essential to be able to use system-dependent vector tags.
8090 26160 : selectVectorTagsFromSystem(*_current_linear_sys, vector_tags, _linear_vector_tags);
8091 26160 : selectMatrixTagsFromSystem(*_current_linear_sys, getMatrixTags(), _linear_matrix_tags);
8092 :
8093 26160 : computeLinearSystemTags(*(_current_linear_sys->currentSolution()),
8094 26160 : _linear_vector_tags,
8095 26160 : _linear_matrix_tags,
8096 : compute_gradients);
8097 :
8098 26160 : _current_linear_sys->disassociateMatrixFromTag(system_matrix,
8099 26160 : _current_linear_sys->systemMatrixTag());
8100 26160 : _current_linear_sys->disassociateVectorFromTag(rhs,
8101 26160 : _current_linear_sys->rightHandSideVectorTag());
8102 : // We reset the tags to the default containers for further operations
8103 26160 : _current_linear_sys->associateVectorToTag(_current_linear_sys->getRightHandSideVector(),
8104 26160 : _current_linear_sys->rightHandSideVectorTag());
8105 26160 : _current_linear_sys->associateMatrixToTag(_current_linear_sys->getSystemMatrix(),
8106 26160 : _current_linear_sys->systemMatrixTag());
8107 26160 : }
8108 :
8109 : void
8110 26160 : FEProblemBase::computeLinearSystemTags(const NumericVector<Number> & soln,
8111 : const std::set<TagID> & vector_tags,
8112 : const std::set<TagID> & matrix_tags,
8113 : const bool compute_gradients)
8114 : {
8115 130800 : TIME_SECTION("computeLinearSystemTags", 5, "Computing Linear System");
8116 :
8117 26160 : _current_linear_sys->setSolution(soln);
8118 :
8119 52340 : for (auto tag : matrix_tags)
8120 : {
8121 26180 : auto & matrix = _current_linear_sys->getMatrix(tag);
8122 26180 : matrix.zero();
8123 : }
8124 :
8125 26160 : unsigned int n_threads = libMesh::n_threads();
8126 :
8127 26160 : _current_execute_on_flag = EXEC_NONLINEAR;
8128 :
8129 : // Random interface objects
8130 26160 : for (const auto & it : _random_data_objects)
8131 0 : it.second->updateSeeds(EXEC_NONLINEAR);
8132 :
8133 26160 : execTransfers(EXEC_NONLINEAR);
8134 26160 : execMultiApps(EXEC_NONLINEAR);
8135 :
8136 26160 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
8137 :
8138 26160 : _aux->jacobianSetup();
8139 :
8140 52320 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
8141 : {
8142 26160 : _functions.jacobianSetup(tid);
8143 : }
8144 :
8145 : #ifdef MOOSE_KOKKOS_ENABLED
8146 17995 : _kokkos_functions.jacobianSetup();
8147 : #endif
8148 :
8149 : try
8150 : {
8151 26160 : computeSystems(EXEC_NONLINEAR);
8152 : }
8153 0 : catch (MooseException & e)
8154 : {
8155 0 : _console << "\nA MooseException was raised during Auxiliary variable computation.\n"
8156 0 : << "The next solve will fail, the timestep will be reduced, and we will try again.\n"
8157 0 : << std::endl;
8158 :
8159 : // We know the next solve is going to fail, so there's no point in
8160 : // computing anything else after this. Plus, using incompletely
8161 : // computed AuxVariables in subsequent calculations could lead to
8162 : // other errors or unhandled exceptions being thrown.
8163 0 : return;
8164 0 : }
8165 :
8166 26160 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
8167 26160 : executeControls(EXEC_NONLINEAR);
8168 :
8169 26160 : _app.getOutputWarehouse().jacobianSetup();
8170 :
8171 26160 : _current_linear_sys->computeLinearSystemTags(vector_tags, matrix_tags, compute_gradients);
8172 :
8173 : // Reset execution flag as after this point we are no longer on LINEAR
8174 26160 : _current_execute_on_flag = EXEC_NONE;
8175 :
8176 : // These are the relevant parts of resetState()
8177 26160 : _safe_access_tagged_vectors = true;
8178 26160 : _safe_access_tagged_matrices = true;
8179 26160 : }
8180 :
8181 : void
8182 295796 : FEProblemBase::computeNearNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8183 : std::vector<NumericVector<Number> *> & sp)
8184 : {
8185 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8186 : "I expect these system numbers to be the same");
8187 :
8188 295796 : sp.clear();
8189 887388 : for (unsigned int i = 0; i < subspaceDim("NearNullSpace"); ++i)
8190 : {
8191 0 : std::stringstream postfix;
8192 0 : postfix << "_" << i;
8193 0 : std::string modename = "NearNullSpace" + postfix.str();
8194 0 : sp.push_back(&_current_nl_sys->getVector(modename));
8195 0 : }
8196 295796 : }
8197 :
8198 : void
8199 295796 : FEProblemBase::computeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8200 : std::vector<NumericVector<Number> *> & sp)
8201 : {
8202 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8203 : "I expect these system numbers to be the same");
8204 295796 : sp.clear();
8205 887454 : for (unsigned int i = 0; i < subspaceDim("NullSpace"); ++i)
8206 : {
8207 22 : std::stringstream postfix;
8208 22 : postfix << "_" << i;
8209 22 : sp.push_back(&_current_nl_sys->getVector("NullSpace" + postfix.str()));
8210 22 : }
8211 295796 : }
8212 :
8213 : void
8214 295796 : FEProblemBase::computeTransposeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8215 : std::vector<NumericVector<Number> *> & sp)
8216 : {
8217 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8218 : "I expect these system numbers to be the same");
8219 295796 : sp.clear();
8220 887421 : for (unsigned int i = 0; i < subspaceDim("TransposeNullSpace"); ++i)
8221 : {
8222 11 : std::stringstream postfix;
8223 11 : postfix << "_" << i;
8224 11 : sp.push_back(&_current_nl_sys->getVector("TransposeNullSpace" + postfix.str()));
8225 11 : }
8226 295796 : }
8227 :
8228 : void
8229 2128 : FEProblemBase::computePostCheck(NonlinearImplicitSystem & sys,
8230 : const NumericVector<Number> & old_soln,
8231 : NumericVector<Number> & search_direction,
8232 : NumericVector<Number> & new_soln,
8233 : bool & changed_search_direction,
8234 : bool & changed_new_soln)
8235 : {
8236 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8237 : "I expect these system numbers to be the same");
8238 :
8239 : // This function replaces the old PetscSupport::dampedCheck() function.
8240 : //
8241 : // 1.) Recreate code in PetscSupport::dampedCheck() for constructing
8242 : // ghosted "soln" and "update" vectors.
8243 : // 2.) Call FEProblemBase::computeDamping() with these ghost vectors.
8244 : // 3.) Recreate the code in PetscSupport::dampedCheck() to actually update
8245 : // the solution vector based on the damping, and set the "changed" flags
8246 : // appropriately.
8247 :
8248 10640 : TIME_SECTION("computePostCheck", 2, "Computing Post Check");
8249 :
8250 2128 : _current_execute_on_flag = EXEC_POSTCHECK;
8251 :
8252 : // MOOSE's FEProblemBase doesn't update the solution during the
8253 : // postcheck, but FEProblemBase-derived classes might.
8254 2128 : if (_has_dampers || shouldUpdateSolution())
8255 : {
8256 : // We need ghosted versions of new_soln and search_direction (the
8257 : // ones we get from libmesh/PETSc are PARALLEL vectors. To make
8258 : // our lives simpler, we use the same ghosting pattern as the
8259 : // system's current_local_solution to create new ghosted vectors.
8260 :
8261 : // Construct zeroed-out clones with the same ghosted dofs as the
8262 : // System's current_local_solution.
8263 : std::unique_ptr<NumericVector<Number>> ghosted_solution =
8264 1625 : sys.current_local_solution->zero_clone(),
8265 : ghosted_search_direction =
8266 1625 : sys.current_local_solution->zero_clone();
8267 :
8268 : // Copy values from input vectors into clones with ghosted values.
8269 1625 : *ghosted_solution = new_soln;
8270 1625 : *ghosted_search_direction = search_direction;
8271 :
8272 1625 : if (_has_dampers)
8273 : {
8274 : // Compute the damping coefficient using the ghosted vectors
8275 1625 : Real damping = computeDamping(*ghosted_solution, *ghosted_search_direction);
8276 :
8277 : // If some non-trivial damping was computed, update the new_soln
8278 : // vector accordingly.
8279 1625 : if (damping < 1.0)
8280 : {
8281 1221 : new_soln = old_soln;
8282 1221 : new_soln.add(-damping, search_direction);
8283 1221 : changed_new_soln = true;
8284 : }
8285 : }
8286 :
8287 1625 : if (shouldUpdateSolution())
8288 : {
8289 : // Update the ghosted copy of the new solution, if necessary.
8290 0 : if (changed_new_soln)
8291 0 : *ghosted_solution = new_soln;
8292 :
8293 0 : bool updated_solution = updateSolution(new_soln, *ghosted_solution);
8294 0 : if (updated_solution)
8295 0 : changed_new_soln = true;
8296 : }
8297 1625 : }
8298 :
8299 2128 : if (vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
8300 : {
8301 503 : _current_nl_sys->setPreviousNewtonSolution(old_soln);
8302 503 : _aux->copyCurrentIntoPreviousNL();
8303 : }
8304 :
8305 : // MOOSE doesn't change the search_direction
8306 2128 : changed_search_direction = false;
8307 :
8308 2128 : _current_execute_on_flag = EXEC_NONE;
8309 2128 : }
8310 :
8311 : Real
8312 1625 : FEProblemBase::computeDamping(const NumericVector<Number> & soln,
8313 : const NumericVector<Number> & update)
8314 : {
8315 : // Default to no damping
8316 1625 : Real damping = 1.0;
8317 :
8318 1625 : if (_has_dampers)
8319 : {
8320 8125 : TIME_SECTION("computeDamping", 1, "Computing Damping");
8321 :
8322 : // Save pointer to the current solution
8323 1625 : const NumericVector<Number> * _saved_current_solution = _current_nl_sys->currentSolution();
8324 :
8325 1625 : _current_nl_sys->setSolution(soln);
8326 : // For now, do not re-compute auxiliary variables. Doing so allows a wild solution increment
8327 : // to get to the material models, which may not be able to cope with drastically different
8328 : // values. Once more complete dependency checking is in place, auxiliary variables (and
8329 : // material properties) will be computed as needed by dampers.
8330 : // _aux.compute();
8331 1625 : damping = _current_nl_sys->computeDamping(soln, update);
8332 :
8333 : // restore saved solution
8334 1625 : _current_nl_sys->setSolution(*_saved_current_solution);
8335 1625 : }
8336 :
8337 1625 : return damping;
8338 : }
8339 :
8340 : bool
8341 290392 : FEProblemBase::shouldUpdateSolution()
8342 : {
8343 290392 : return false;
8344 : }
8345 :
8346 : bool
8347 0 : FEProblemBase::updateSolution(NumericVector<Number> & /*vec_solution*/,
8348 : NumericVector<Number> & /*ghosted_solution*/)
8349 : {
8350 0 : return false;
8351 : }
8352 :
8353 : void
8354 203 : FEProblemBase::predictorCleanup(NumericVector<Number> & /*ghosted_solution*/)
8355 : {
8356 203 : }
8357 :
8358 : void
8359 2022 : FEProblemBase::addDisplacedProblem(std::shared_ptr<DisplacedProblem> displaced_problem)
8360 : {
8361 : parallel_object_only();
8362 :
8363 2022 : _displaced_mesh = &displaced_problem->mesh();
8364 2022 : _displaced_problem = displaced_problem;
8365 2022 : }
8366 :
8367 : void
8368 121144 : FEProblemBase::updateGeomSearch(GeometricSearchData::GeometricSearchType type)
8369 : {
8370 605720 : TIME_SECTION("updateGeometricSearch", 3, "Updating Geometric Search");
8371 :
8372 121144 : _geometric_search_data.update(type);
8373 :
8374 121144 : if (_displaced_problem)
8375 4209 : _displaced_problem->updateGeomSearch(type);
8376 121144 : }
8377 :
8378 : void
8379 64057 : FEProblemBase::updateMortarMesh()
8380 : {
8381 320285 : TIME_SECTION("updateMortarMesh", 5, "Updating Mortar Mesh");
8382 :
8383 64057 : FloatingPointExceptionGuard fpe_guard(_app);
8384 :
8385 64057 : _mortar_data->update();
8386 64054 : }
8387 :
8388 : void
8389 1414 : FEProblemBase::createMortarInterface(
8390 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8391 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8392 : bool on_displaced,
8393 : bool periodic,
8394 : const bool debug,
8395 : const bool correct_edge_dropping,
8396 : const Real minimum_projection_angle,
8397 : const MooseEnum & triangulation,
8398 : const bool triangulate_triangles)
8399 : {
8400 1414 : _has_mortar = true;
8401 :
8402 1414 : if (on_displaced)
8403 254 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8404 : primary_secondary_subdomain_pair,
8405 127 : *_displaced_problem,
8406 : on_displaced,
8407 : periodic,
8408 : debug,
8409 : correct_edge_dropping,
8410 : minimum_projection_angle,
8411 : triangulation,
8412 127 : triangulate_triangles);
8413 : else
8414 1287 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8415 : primary_secondary_subdomain_pair,
8416 : *this,
8417 : on_displaced,
8418 : periodic,
8419 : debug,
8420 : correct_edge_dropping,
8421 : minimum_projection_angle,
8422 : triangulation,
8423 1287 : triangulate_triangles);
8424 : }
8425 :
8426 : const AutomaticMortarGeneration &
8427 0 : FEProblemBase::getMortarInterface(
8428 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8429 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8430 : bool on_displaced) const
8431 : {
8432 0 : return _mortar_data->getMortarInterface(
8433 0 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8434 : }
8435 :
8436 : AutomaticMortarGeneration &
8437 221135 : FEProblemBase::getMortarInterface(
8438 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8439 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8440 : bool on_displaced)
8441 : {
8442 221135 : return _mortar_data->getMortarInterface(
8443 221135 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8444 : }
8445 :
8446 : void
8447 322305 : FEProblemBase::possiblyRebuildGeomSearchPatches()
8448 : {
8449 322305 : if (_displaced_problem) // Only need to do this if things are moving...
8450 : {
8451 164185 : TIME_SECTION("possiblyRebuildGeomSearchPatches", 5, "Rebuilding Geometric Search Patches");
8452 :
8453 32837 : switch (_mesh.getPatchUpdateStrategy())
8454 : {
8455 31742 : case Moose::Never:
8456 31742 : break;
8457 365 : case Moose::Iteration:
8458 : // Update the list of ghosted elements at the start of the time step
8459 365 : _geometric_search_data.updateGhostedElems();
8460 365 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8461 :
8462 365 : _displaced_problem->geomSearchData().updateGhostedElems();
8463 365 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8464 :
8465 : // The commands below ensure that the sparsity of the Jacobian matrix is
8466 : // augmented at the start of the time step using neighbor nodes from the end
8467 : // of the previous time step.
8468 :
8469 365 : reinitBecauseOfGhostingOrNewGeomObjects();
8470 :
8471 : // This is needed to reinitialize PETSc output
8472 365 : initPetscOutputAndSomeSolverSettings();
8473 :
8474 365 : break;
8475 :
8476 331 : case Moose::Auto:
8477 : {
8478 331 : Real max = _displaced_problem->geomSearchData().maxPatchPercentage();
8479 331 : _communicator.max(max);
8480 :
8481 : // If we haven't moved very far through the patch
8482 331 : if (max < 0.4)
8483 298 : break;
8484 : }
8485 : libmesh_fallthrough();
8486 :
8487 : // Let this fall through if things do need to be updated...
8488 : case Moose::Always:
8489 : // Flush output here to see the message before the reinitialization, which could take a
8490 : // while
8491 432 : _console << "\n\nUpdating geometric search patches\n" << std::endl;
8492 :
8493 432 : _geometric_search_data.clearNearestNodeLocators();
8494 432 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8495 :
8496 432 : _displaced_problem->geomSearchData().clearNearestNodeLocators();
8497 432 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8498 :
8499 432 : reinitBecauseOfGhostingOrNewGeomObjects();
8500 :
8501 : // This is needed to reinitialize PETSc output
8502 432 : initPetscOutputAndSomeSolverSettings();
8503 : }
8504 32837 : }
8505 322305 : }
8506 :
8507 : #ifdef LIBMESH_ENABLE_AMR
8508 : void
8509 55932 : FEProblemBase::initialAdaptMesh()
8510 : {
8511 55932 : unsigned int n = adaptivity().getInitialSteps();
8512 55932 : _cycles_completed = 0;
8513 55932 : if (n)
8514 : {
8515 631 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8516 3 : mooseError("HFEM does not support mesh adaptivity currently.");
8517 :
8518 3140 : TIME_SECTION("initialAdaptMesh", 2, "Performing Initial Adaptivity");
8519 :
8520 1510 : for (unsigned int i = 0; i < n; i++)
8521 : {
8522 1057 : computeIndicators();
8523 1057 : computeMarkers();
8524 :
8525 1057 : if (_adaptivity.initialAdaptMesh())
8526 : {
8527 882 : meshChanged(
8528 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8529 :
8530 : // reproject the initial condition
8531 882 : projectSolution();
8532 :
8533 882 : _cycles_completed++;
8534 : }
8535 : else
8536 : {
8537 175 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8538 175 : return;
8539 : }
8540 : }
8541 628 : }
8542 : }
8543 :
8544 : bool
8545 167971 : FEProblemBase::adaptMesh()
8546 : {
8547 : // reset cycle counter
8548 167971 : _cycles_completed = 0;
8549 :
8550 167971 : if (!_adaptivity.isAdaptivityDue())
8551 163394 : return false;
8552 :
8553 22885 : TIME_SECTION("adaptMesh", 3, "Adapting Mesh");
8554 :
8555 4577 : unsigned int cycles_per_step = _adaptivity.getCyclesPerStep();
8556 :
8557 4577 : bool mesh_changed = false;
8558 :
8559 8056 : for (unsigned int i = 0; i < cycles_per_step; ++i)
8560 : {
8561 4729 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8562 0 : mooseError("HFEM does not support mesh adaptivity currently.");
8563 :
8564 : // Markers were already computed once by Executioner
8565 4729 : if (_adaptivity.getRecomputeMarkersFlag() && i > 0)
8566 22 : computeMarkers();
8567 :
8568 : bool mesh_changed_this_step;
8569 4729 : mesh_changed_this_step = _adaptivity.adaptMesh();
8570 :
8571 4729 : if (mesh_changed_this_step)
8572 : {
8573 3479 : mesh_changed = true;
8574 :
8575 3479 : meshChanged(
8576 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8577 3479 : _cycles_completed++;
8578 : }
8579 : else
8580 : {
8581 : // If the mesh didn't change, we still need to update the displaced mesh
8582 : // to undo the undisplacement performed in Adaptivity::adaptMesh
8583 1250 : if (_displaced_problem)
8584 44 : _displaced_problem->updateMesh();
8585 :
8586 1250 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8587 1250 : break;
8588 : }
8589 :
8590 : // Show adaptivity progress
8591 3479 : _console << std::flush;
8592 : }
8593 :
8594 : // We're done with all intermediate changes; now get systems ready
8595 : // for real if necessary.
8596 4577 : if (mesh_changed)
8597 3338 : es().reinit_systems();
8598 :
8599 : // Execute multi-apps that need to run after adaptivity, but before the next timestep.
8600 4577 : execMultiApps(EXEC_POST_ADAPTIVITY);
8601 :
8602 4577 : return mesh_changed;
8603 4577 : }
8604 : #endif // LIBMESH_ENABLE_AMR
8605 :
8606 : void
8607 0 : FEProblemBase::initXFEM(std::shared_ptr<XFEMInterface> xfem)
8608 : {
8609 0 : _xfem = xfem;
8610 0 : _xfem->setMesh(&_mesh);
8611 0 : if (_displaced_mesh)
8612 0 : _xfem->setDisplacedMesh(_displaced_mesh);
8613 :
8614 0 : auto fill_data = [](auto & storage)
8615 : {
8616 0 : std::vector<MaterialData *> data(libMesh::n_threads());
8617 0 : for (const auto tid : make_range(libMesh::n_threads()))
8618 0 : data[tid] = &storage.getMaterialData(tid);
8619 0 : return data;
8620 0 : };
8621 0 : _xfem->setMaterialData(fill_data(_material_props));
8622 0 : _xfem->setBoundaryMaterialData(fill_data(_bnd_material_props));
8623 :
8624 0 : unsigned int n_threads = libMesh::n_threads();
8625 0 : for (unsigned int i = 0; i < n_threads; ++i)
8626 0 : for (const auto nl_sys_num : index_range(_nl))
8627 : {
8628 0 : _assembly[i][nl_sys_num]->setXFEM(_xfem);
8629 0 : if (_displaced_problem)
8630 0 : _displaced_problem->assembly(i, nl_sys_num).setXFEM(_xfem);
8631 : }
8632 0 : }
8633 :
8634 : bool
8635 0 : FEProblemBase::updateMeshXFEM()
8636 : {
8637 0 : TIME_SECTION("updateMeshXFEM", 5, "Updating XFEM");
8638 :
8639 0 : bool updated = false;
8640 0 : if (haveXFEM())
8641 : {
8642 0 : if (_xfem->updateHeal())
8643 : // XFEM exodiff tests rely on a given numbering because they cannot use map = true due to
8644 : // having coincident elements. While conceptually speaking we do not need to contract the
8645 : // mesh, we need its call to renumber_nodes_and_elements in order to preserve these tests
8646 0 : meshChanged(
8647 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8648 :
8649 0 : updated = _xfem->update(_time, _nl, *_aux);
8650 0 : if (updated)
8651 : {
8652 0 : meshChanged(
8653 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8654 0 : _xfem->initSolution(_nl, *_aux);
8655 0 : restoreSolutions();
8656 0 : _console << "\nXFEM update complete: Mesh modified" << std::endl;
8657 : }
8658 : else
8659 0 : _console << "\nXFEM update complete: Mesh not modified" << std::endl;
8660 : }
8661 0 : return updated;
8662 0 : }
8663 :
8664 : void
8665 7026 : FEProblemBase::meshChanged(const bool intermediate_change,
8666 : const bool contract_mesh,
8667 : const bool clean_refinement_flags)
8668 : {
8669 35130 : TIME_SECTION("meshChanged", 3, "Handling Mesh Changes");
8670 :
8671 7026 : _app.markMeshChangedForBackup();
8672 :
8673 13844 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8674 6818 : _neighbor_material_props.hasStatefulProperties())
8675 208 : _mesh.cacheChangedLists(); // Currently only used with adaptivity and stateful material
8676 : // properties
8677 :
8678 : // Clear these out because they corresponded to the old mesh
8679 7026 : _ghosted_elems.clear();
8680 7026 : ghostGhostedBoundaries();
8681 :
8682 : // The mesh changed. We notify the MooseMesh first, because
8683 : // callbacks (e.g. for sparsity calculations) triggered by the
8684 : // EquationSystems reinit may require up-to-date MooseMesh caches.
8685 7026 : _mesh.meshChanged();
8686 :
8687 : // If we're just going to alter the mesh again, all we need to
8688 : // handle here is AMR and projections, not full system reinit
8689 7026 : if (intermediate_change)
8690 3510 : es().reinit_solutions();
8691 : else
8692 3516 : es().reinit();
8693 :
8694 7026 : if (contract_mesh)
8695 : // Once vectors are restricted, we can delete children of coarsened elements
8696 4452 : _mesh.getMesh().contract();
8697 7026 : if (clean_refinement_flags)
8698 : {
8699 : // Finally clear refinement flags so that if someone tries to project vectors again without
8700 : // an intervening mesh refinement to clear flags they won't run into trouble
8701 4452 : MeshRefinement refinement(_mesh.getMesh());
8702 4452 : refinement.clean_refinement_flags();
8703 4452 : }
8704 :
8705 7026 : if (!intermediate_change)
8706 : {
8707 : // Since the mesh has changed, we need to make sure that we update any of our
8708 : // MOOSE-system specific data.
8709 7032 : for (auto & sys : _solver_systems)
8710 3516 : sys->reinit();
8711 3516 : _aux->reinit();
8712 : }
8713 :
8714 : // Updating MooseMesh first breaks other adaptivity code, unless we
8715 : // then *again* update the MooseMesh caches. E.g. the definition of
8716 : // "active" and "local" may have been *changed* by refinement and
8717 : // repartitioning done in EquationSystems::reinit().
8718 7026 : _mesh.meshChanged();
8719 :
8720 : // If we have finite volume variables, we will need to recompute additional elemental/face
8721 : // quantities
8722 7026 : if (haveFV() && _mesh.isFiniteVolumeInfoDirty())
8723 351 : _mesh.setupFiniteVolumeMeshData();
8724 :
8725 : // Let the meshChangedInterface notify the mesh changed event before we update the active
8726 : // semilocal nodes, because the set of ghosted elements may potentially be updated during a mesh
8727 : // changed event.
8728 125503 : for (const auto & mci : _notify_when_mesh_changes)
8729 118477 : mci->meshChanged();
8730 :
8731 : // Since the Mesh changed, update the PointLocator object used by DiracKernels.
8732 7026 : _dirac_kernel_info.updatePointLocator(_mesh);
8733 :
8734 : // Need to redo ghosting
8735 7026 : _geometric_search_data.reinit();
8736 :
8737 7026 : if (_displaced_problem)
8738 : {
8739 569 : _displaced_problem->meshChanged(contract_mesh, clean_refinement_flags);
8740 569 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8741 : }
8742 :
8743 7026 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8744 :
8745 7026 : _evaluable_local_elem_range.reset();
8746 7026 : _nl_evaluable_local_elem_range.reset();
8747 :
8748 : // Just like we reinitialized our geometric search objects, we also need to reinitialize our
8749 : // mortar meshes. Note that this needs to happen after DisplacedProblem::meshChanged because the
8750 : // mortar mesh discretization will depend necessarily on the displaced mesh being re-displaced
8751 7026 : _mortar_data->meshChanged();
8752 :
8753 : // Nonlinear systems hold the mortar mesh functors. The domains of definition of the mortar
8754 : // functors might have changed when the mesh changed.
8755 14042 : for (auto & nl_sys : _nl)
8756 7016 : nl_sys->reinitMortarFunctors();
8757 :
8758 7026 : reinitBecauseOfGhostingOrNewGeomObjects(/*mortar_changed=*/true);
8759 :
8760 : // We need to create new storage for newly active elements, and copy
8761 : // stateful properties from the old elements.
8762 7234 : if (_has_initialized_stateful &&
8763 208 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties()))
8764 : {
8765 208 : if (havePRefinement())
8766 66 : _mesh.buildPRefinementAndCoarseningMaps(_assembly[0][0].get());
8767 :
8768 : // Prolong properties onto newly refined elements' children
8769 : {
8770 : ProjectMaterialProperties pmp(
8771 208 : /* refine = */ true, *this, _material_props, _bnd_material_props, _assembly);
8772 208 : const auto & range = *_mesh.refinedElementRange();
8773 208 : Threads::parallel_reduce(range, pmp);
8774 :
8775 : // Concurrent erasure from the shared hash map is not safe while we are reading from it in
8776 : // ProjectMaterialProperties, so we handle erasure here. Moreover, erasure based on key is
8777 : // not thread safe in and of itself because it is a read-write operation. Note that we do not
8778 : // do the erasure for p-refinement because the coarse level element is the same as our active
8779 : // refined level element
8780 208 : if (!doingPRefinement())
8781 3272 : for (const auto & elem : range)
8782 : {
8783 3130 : _material_props.eraseProperty(elem);
8784 3130 : _bnd_material_props.eraseProperty(elem);
8785 3130 : _neighbor_material_props.eraseProperty(elem);
8786 : }
8787 208 : }
8788 :
8789 : // Restrict properties onto newly coarsened elements
8790 : {
8791 : ProjectMaterialProperties pmp(
8792 208 : /* refine = */ false, *this, _material_props, _bnd_material_props, _assembly);
8793 208 : const auto & range = *_mesh.coarsenedElementRange();
8794 208 : Threads::parallel_reduce(range, pmp);
8795 : // Note that we do not do the erasure for p-refinement because the coarse level element is the
8796 : // same as our active refined level element
8797 208 : if (!doingPRefinement())
8798 1322 : for (const auto & elem : range)
8799 : {
8800 1180 : auto && coarsened_children = _mesh.coarsenedElementChildren(elem);
8801 7732 : for (auto && child : coarsened_children)
8802 : {
8803 6552 : _material_props.eraseProperty(child);
8804 6552 : _bnd_material_props.eraseProperty(child);
8805 6552 : _neighbor_material_props.eraseProperty(child);
8806 : }
8807 : }
8808 208 : }
8809 : }
8810 :
8811 7026 : if (_calculate_jacobian_in_uo)
8812 0 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
8813 :
8814 7026 : _has_jacobian = false; // we have to recompute jacobian when mesh changed
8815 :
8816 : // Now for backwards compatibility with user code that overrode the old no-arg meshChanged we must
8817 : // call it here
8818 7026 : meshChanged();
8819 7026 : }
8820 :
8821 : void
8822 920931 : FEProblemBase::notifyWhenMeshChanges(MeshChangedInterface * mci)
8823 : {
8824 920931 : _notify_when_mesh_changes.push_back(mci);
8825 920931 : }
8826 :
8827 : void
8828 76515 : FEProblemBase::notifyWhenMeshDisplaces(MeshDisplacedInterface * mdi)
8829 : {
8830 76515 : _notify_when_mesh_displaces.push_back(mdi);
8831 76515 : }
8832 :
8833 : void
8834 63048 : FEProblemBase::meshDisplaced()
8835 : {
8836 91725 : for (const auto & mdi : _notify_when_mesh_displaces)
8837 28677 : mdi->meshDisplaced();
8838 63048 : }
8839 :
8840 : void
8841 9715 : FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const bool threaded)
8842 : {
8843 : ComputeMaterialsObjectThread cmt(
8844 9715 : *this, _material_props, _bnd_material_props, _neighbor_material_props, _assembly);
8845 9715 : if (threaded)
8846 9715 : Threads::parallel_reduce(elem_range, cmt);
8847 : else
8848 0 : cmt(elem_range, true);
8849 :
8850 : #ifdef MOOSE_KOKKOS_ENABLED
8851 7317 : if (_has_kokkos_objects)
8852 781 : initKokkosStatefulProps();
8853 : #endif
8854 9715 : }
8855 :
8856 : void
8857 60620 : FEProblemBase::checkProblemIntegrity()
8858 : {
8859 181860 : TIME_SECTION("checkProblemIntegrity", 5);
8860 :
8861 : // Subdomains specified by the "Problem/block" parameter
8862 121240 : const auto & subdomain_names = getParam<std::vector<SubdomainName>>("block");
8863 60620 : auto mesh_subdomains_vec = MooseMeshUtils::getSubdomainIDs(_mesh, subdomain_names);
8864 60620 : std::set<SubdomainID> mesh_subdomains(mesh_subdomains_vec.begin(), mesh_subdomains_vec.end());
8865 :
8866 : // Check kernel coverage of subdomains (blocks) in the mesh
8867 60620 : if (!_skip_nl_system_check && _solve && _kernel_coverage_check != CoverageCheckMode::FALSE &&
8868 41493 : _kernel_coverage_check != CoverageCheckMode::OFF)
8869 : {
8870 41481 : std::set<SubdomainID> blocks;
8871 41481 : if (_kernel_coverage_check == CoverageCheckMode::TRUE ||
8872 231 : _kernel_coverage_check == CoverageCheckMode::ON)
8873 41250 : blocks = mesh_subdomains;
8874 231 : else if (_kernel_coverage_check == CoverageCheckMode::SKIP_LIST)
8875 : {
8876 12 : blocks = mesh_subdomains;
8877 24 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8878 : {
8879 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8880 12 : if (id == Moose::INVALID_BLOCK_ID)
8881 0 : paramError("kernel_coverage_block_list",
8882 : "Subdomain \"",
8883 : subdomain_name,
8884 : "\" not found in mesh.");
8885 12 : blocks.erase(id);
8886 : }
8887 : }
8888 219 : else if (_kernel_coverage_check == CoverageCheckMode::ONLY_LIST)
8889 438 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8890 : {
8891 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8892 219 : if (id == Moose::INVALID_BLOCK_ID)
8893 0 : paramError("kernel_coverage_block_list",
8894 : "Subdomain \"",
8895 : subdomain_name,
8896 : "\" not found in mesh.");
8897 219 : blocks.insert(id);
8898 : }
8899 41481 : if (!blocks.empty())
8900 82185 : for (auto & nl : _nl)
8901 40716 : nl->checkKernelCoverage(blocks);
8902 41469 : }
8903 :
8904 : // Check materials
8905 : {
8906 : #ifdef LIBMESH_ENABLE_AMR
8907 62877 : if ((_adaptivity.isOn() || _num_grid_steps) &&
8908 2269 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8909 2206 : _neighbor_material_props.hasStatefulProperties()))
8910 : {
8911 63 : _console << "Using EXPERIMENTAL Stateful Material Property projection with Adaptivity!\n"
8912 63 : << std::flush;
8913 : }
8914 : #endif
8915 :
8916 60608 : std::set<SubdomainID> local_mesh_subs(mesh_subdomains);
8917 :
8918 60608 : if (_material_coverage_check != CoverageCheckMode::FALSE &&
8919 60524 : _material_coverage_check != CoverageCheckMode::OFF)
8920 : {
8921 : /**
8922 : * If a material is specified for any block in the simulation, then all blocks must
8923 : * have a material specified.
8924 : */
8925 60524 : bool check_material_coverage = false;
8926 60524 : std::set<SubdomainID> ids = _all_materials.getActiveBlocks();
8927 73109 : for (const auto & id : ids)
8928 : {
8929 12585 : local_mesh_subs.erase(id);
8930 12585 : check_material_coverage = true;
8931 : }
8932 :
8933 : // did the user limit the subdomains to be checked?
8934 60524 : if (_material_coverage_check == CoverageCheckMode::SKIP_LIST)
8935 : {
8936 24 : for (const auto & subdomain_name : _material_coverage_blocks)
8937 : {
8938 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8939 12 : if (id == Moose::INVALID_BLOCK_ID)
8940 0 : paramError("material_coverage_block_list",
8941 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8942 12 : local_mesh_subs.erase(id);
8943 : }
8944 : }
8945 60512 : else if (_material_coverage_check == CoverageCheckMode::ONLY_LIST)
8946 : {
8947 219 : std::set<SubdomainID> blocks(local_mesh_subs);
8948 438 : for (const auto & subdomain_name : _material_coverage_blocks)
8949 : {
8950 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8951 219 : if (id == Moose::INVALID_BLOCK_ID)
8952 0 : paramError("material_coverage_block_list",
8953 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8954 219 : blocks.erase(id);
8955 : }
8956 231 : for (const auto id : blocks)
8957 12 : local_mesh_subs.erase(id);
8958 219 : }
8959 :
8960 : // also exclude mortar spaces from the material check
8961 60524 : auto && mortar_subdomain_ids = _mortar_data->getMortarSubdomainIDs();
8962 62548 : for (auto subdomain_id : mortar_subdomain_ids)
8963 2024 : local_mesh_subs.erase(subdomain_id);
8964 :
8965 : // Check Material Coverage
8966 60524 : if (check_material_coverage && !local_mesh_subs.empty())
8967 : {
8968 6 : std::stringstream extra_subdomain_ids;
8969 : /// unsigned int is necessary to print SubdomainIDs in the statement below
8970 6 : std::copy(local_mesh_subs.begin(),
8971 : local_mesh_subs.end(),
8972 12 : std::ostream_iterator<unsigned int>(extra_subdomain_ids, " "));
8973 : /// vector is necessary to get the subdomain names
8974 : std::vector<SubdomainID> local_mesh_subs_vec(local_mesh_subs.begin(),
8975 6 : local_mesh_subs.end());
8976 :
8977 18 : mooseError("The following blocks from your input mesh do not contain an active material: " +
8978 12 : extra_subdomain_ids.str() +
8979 18 : "(names: " + Moose::stringify(_mesh.getSubdomainNames(local_mesh_subs_vec)) +
8980 : ")\nWhen ANY mesh block contains a Material object, "
8981 : "all blocks must contain a Material object.\n");
8982 0 : }
8983 60518 : }
8984 :
8985 : // Check material properties on blocks and boundaries
8986 60602 : checkBlockMatProps();
8987 60570 : checkBoundaryMatProps();
8988 :
8989 : // Check that material properties exist when requested by other properties on a given block
8990 60561 : const auto & materials = _all_materials.getActiveObjects();
8991 74645 : for (const auto & material : materials)
8992 14084 : material->checkStatefulSanity();
8993 :
8994 : // auto mats_to_check = _materials.getActiveBlockObjects();
8995 : // const auto & discrete_materials = _discrete_materials.getActiveBlockObjects();
8996 : // for (const auto & map_it : discrete_materials)
8997 : // for (const auto & container_element : map_it.second)
8998 : // mats_to_check[map_it.first].push_back(container_element);
8999 60561 : if (_material_dependency_check)
9000 60537 : checkDependMaterialsHelper(_all_materials.getActiveBlockObjects());
9001 60548 : }
9002 :
9003 60548 : checkUserObjects();
9004 :
9005 : // Verify that we don't have any Element type/Coordinate Type conflicts
9006 60548 : checkCoordinateSystems();
9007 :
9008 : // Coordinate transforms are only intended for use with MultiApps at this time. If you are not
9009 : // using multiapps but still require these, contact a moose developer
9010 60685 : if (_mesh.coordTransform().hasScalingOrRotationTransformation() && _app.isUltimateMaster() &&
9011 140 : !hasMultiApps())
9012 3 : mooseError("Coordinate transformation parameters, listed below, are only to be used in the "
9013 : "context of application to application field transfers at this time. The mesh is "
9014 : "not modified by these parameters within an application.\n"
9015 : "You should likely use a 'TransformGenerator' in the [Mesh] block to achieve the "
9016 : "desired mesh modification.\n\n",
9017 3 : Moose::stringify(MooseAppCoordTransform::validParams()));
9018 :
9019 : // If using displacements, verify that the order of the displacement
9020 : // variables matches the order of the elements in the displaced
9021 : // mesh.
9022 60542 : checkDisplacementOrders();
9023 :
9024 : // Check for postprocessor names with same name as a scalar variable
9025 60539 : checkDuplicatePostprocessorVariableNames();
9026 60539 : }
9027 :
9028 : void
9029 60542 : FEProblemBase::checkDisplacementOrders()
9030 : {
9031 60542 : if (_displaced_problem)
9032 : {
9033 2022 : bool mesh_has_second_order_elements = false;
9034 4044 : for (const auto & elem : as_range(_displaced_mesh->activeLocalElementsBegin(),
9035 464948 : _displaced_mesh->activeLocalElementsEnd()))
9036 : {
9037 229785 : if (elem->default_order() == SECOND)
9038 : {
9039 344 : mesh_has_second_order_elements = true;
9040 344 : break;
9041 : }
9042 2022 : }
9043 :
9044 : // We checked our local elements, so take the max over all processors.
9045 2022 : _displaced_mesh->comm().max(mesh_has_second_order_elements);
9046 :
9047 : // If the Mesh has second order elements, make sure the
9048 : // displacement variables are second-order.
9049 2022 : if (mesh_has_second_order_elements)
9050 : {
9051 : const std::vector<std::string> & displacement_variables =
9052 344 : _displaced_problem->getDisplacementVarNames();
9053 :
9054 1133 : for (const auto & var_name : displacement_variables)
9055 : {
9056 : MooseVariableFEBase & mv =
9057 792 : _displaced_problem->getVariable(/*tid=*/0,
9058 : var_name,
9059 : Moose::VarKindType::VAR_ANY,
9060 : Moose::VarFieldType::VAR_FIELD_STANDARD);
9061 792 : if (mv.order() != SECOND)
9062 3 : mooseError("Error: mesh has SECOND order elements, so all displacement variables must be "
9063 : "SECOND order.");
9064 : }
9065 : }
9066 : }
9067 60539 : }
9068 :
9069 : void
9070 60548 : FEProblemBase::checkUserObjects()
9071 : {
9072 : // Check user_objects block coverage
9073 60548 : std::set<SubdomainID> mesh_subdomains = _mesh.meshSubdomains();
9074 60548 : std::set<SubdomainID> user_objects_blocks;
9075 :
9076 : // gather names of all user_objects that were defined in the input file
9077 : // and the blocks that they are defined on
9078 60548 : std::set<std::string> names;
9079 :
9080 60548 : std::vector<UserObjectBase *> objects;
9081 60548 : theWarehouse().query().condition<AttribInterfaces>(Interfaces::UserObject).queryInto(objects);
9082 :
9083 134769 : for (const auto & obj : objects)
9084 74221 : names.insert(obj->name());
9085 :
9086 : // See if all referenced blocks are covered
9087 60548 : std::set<SubdomainID> difference;
9088 60548 : std::set_difference(user_objects_blocks.begin(),
9089 : user_objects_blocks.end(),
9090 : mesh_subdomains.begin(),
9091 : mesh_subdomains.end(),
9092 : std::inserter(difference, difference.end()));
9093 :
9094 60548 : if (!difference.empty())
9095 : {
9096 0 : std::ostringstream oss;
9097 0 : oss << "One or more UserObjects is referencing a nonexistent block:\n";
9098 0 : for (const auto & id : difference)
9099 0 : oss << id << "\n";
9100 0 : mooseError(oss.str());
9101 0 : }
9102 60548 : }
9103 :
9104 : void
9105 60537 : FEProblemBase::checkDependMaterialsHelper(
9106 : const std::map<SubdomainID, std::vector<std::shared_ptr<MaterialBase>>> & materials_map)
9107 : {
9108 73064 : for (const auto & it : materials_map)
9109 : {
9110 : /// These two sets are used to make sure that all dependent props on a block are actually supplied
9111 12535 : std::set<std::string> block_depend_props, block_supplied_props;
9112 :
9113 31183 : for (const auto & mat1 : it.second)
9114 : {
9115 18648 : auto & alldeps = mat1->getMatPropDependencies(); // includes requested stateful props
9116 20967 : for (auto & dep : alldeps)
9117 2319 : block_depend_props.insert(_material_prop_registry.getName(dep));
9118 :
9119 : // See if any of the active materials supply this property
9120 57628 : for (const auto & mat2 : it.second)
9121 : {
9122 38980 : const std::set<std::string> & supplied_props = mat2->MaterialBase::getSuppliedItems();
9123 38980 : block_supplied_props.insert(supplied_props.begin(), supplied_props.end());
9124 : }
9125 : }
9126 :
9127 : // Add zero material properties specific to this block and unrestricted
9128 12535 : block_supplied_props.insert(_zero_block_material_props[it.first].begin(),
9129 12535 : _zero_block_material_props[it.first].end());
9130 :
9131 : // Error check to make sure all properties consumed by materials are supplied on this block
9132 12535 : std::set<std::string> difference;
9133 12535 : std::set_difference(block_depend_props.begin(),
9134 : block_depend_props.end(),
9135 : block_supplied_props.begin(),
9136 : block_supplied_props.end(),
9137 : std::inserter(difference, difference.end()));
9138 :
9139 12535 : if (!difference.empty())
9140 : {
9141 8 : std::ostringstream oss;
9142 8 : oss << "One or more Material Properties were not supplied on block ";
9143 8 : const std::string & subdomain_name = _mesh.getSubdomainName(it.first);
9144 8 : if (subdomain_name.length() > 0)
9145 0 : oss << subdomain_name << " (" << it.first << ")";
9146 : else
9147 8 : oss << it.first;
9148 8 : oss << ":\n";
9149 16 : for (const auto & name : difference)
9150 8 : oss << name << "\n";
9151 8 : mooseError(oss.str());
9152 0 : }
9153 12527 : }
9154 :
9155 : // This loop checks that materials are not supplied by multiple Material objects
9156 73051 : for (const auto & it : materials_map)
9157 : {
9158 12527 : const auto & materials = it.second;
9159 12527 : std::set<std::string> inner_supplied, outer_supplied;
9160 :
9161 31155 : for (const auto & outer_mat : materials)
9162 : {
9163 : // Storage for properties for this material (outer) and all other materials (inner)
9164 18633 : outer_supplied = outer_mat->getSuppliedItems();
9165 18633 : inner_supplied.clear();
9166 :
9167 : // Property to material map for error reporting
9168 18633 : std::map<std::string, std::set<std::string>> prop_to_mat;
9169 39443 : for (const auto & name : outer_supplied)
9170 20810 : prop_to_mat[name].insert(outer_mat->name());
9171 :
9172 57589 : for (const auto & inner_mat : materials)
9173 : {
9174 38956 : if (outer_mat == inner_mat)
9175 18633 : continue;
9176 :
9177 : // Check whether these materials are an AD pair
9178 20323 : auto outer_mat_type = outer_mat->type();
9179 20323 : auto inner_mat_type = inner_mat->type();
9180 40646 : removeSubstring(outer_mat_type, "<RESIDUAL>");
9181 40646 : removeSubstring(outer_mat_type, "<JACOBIAN>");
9182 40646 : removeSubstring(inner_mat_type, "<RESIDUAL>");
9183 20323 : removeSubstring(inner_mat_type, "<JACOBIAN>");
9184 20323 : if (outer_mat_type == inner_mat_type && outer_mat_type != outer_mat->type() &&
9185 0 : inner_mat_type != inner_mat->type())
9186 0 : continue;
9187 :
9188 20323 : inner_supplied.insert(inner_mat->getSuppliedItems().begin(),
9189 20323 : inner_mat->getSuppliedItems().end());
9190 :
9191 115393 : for (const auto & inner_supplied_name : inner_supplied)
9192 95070 : prop_to_mat[inner_supplied_name].insert(inner_mat->name());
9193 20323 : }
9194 :
9195 : // Test that a property isn't supplied on multiple blocks
9196 18633 : std::set<std::string> intersection;
9197 18633 : std::set_intersection(outer_supplied.begin(),
9198 : outer_supplied.end(),
9199 : inner_supplied.begin(),
9200 : inner_supplied.end(),
9201 : std::inserter(intersection, intersection.end()));
9202 :
9203 18633 : if (!intersection.empty())
9204 : {
9205 5 : std::ostringstream oss;
9206 5 : oss << "The following material properties are declared on block " << it.first
9207 5 : << " by multiple materials:\n";
9208 10 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << "Material Property"
9209 5 : << "Material Objects\n";
9210 20 : for (const auto & outer_name : intersection)
9211 : {
9212 15 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << outer_name;
9213 45 : for (const auto & inner_name : prop_to_mat[outer_name])
9214 30 : oss << inner_name << " ";
9215 15 : oss << '\n';
9216 : }
9217 :
9218 5 : mooseError(oss.str());
9219 : break;
9220 0 : }
9221 18628 : }
9222 12522 : }
9223 60524 : }
9224 :
9225 : void
9226 60548 : FEProblemBase::checkCoordinateSystems()
9227 : {
9228 60548 : _mesh.checkCoordinateSystems();
9229 60545 : }
9230 :
9231 : void
9232 469 : FEProblemBase::setRestartFile(const std::string & file_name)
9233 : {
9234 469 : if (_app.isRecovering())
9235 : {
9236 23 : mooseInfo("Restart file ", file_name, " is NOT being used since we are performing recovery.");
9237 : }
9238 : else
9239 : {
9240 446 : _app.setRestart(true);
9241 446 : _app.setRestartRecoverFileBase(file_name);
9242 446 : mooseInfo("Using ", file_name, " for restart.");
9243 : }
9244 469 : }
9245 :
9246 : std::vector<VariableName>
9247 358231 : FEProblemBase::getVariableNames()
9248 : {
9249 358231 : std::vector<VariableName> names;
9250 :
9251 720702 : for (auto & sys : _solver_systems)
9252 : {
9253 362471 : const std::vector<VariableName> & var_names = sys->getVariableNames();
9254 362471 : names.insert(names.end(), var_names.begin(), var_names.end());
9255 : }
9256 :
9257 358231 : const std::vector<VariableName> & aux_var_names = _aux->getVariableNames();
9258 358231 : names.insert(names.end(), aux_var_names.begin(), aux_var_names.end());
9259 :
9260 358231 : return names;
9261 0 : }
9262 :
9263 : SolverParams &
9264 1627966 : FEProblemBase::solverParams(const unsigned int solver_sys_num)
9265 : {
9266 : mooseAssert(solver_sys_num < numSolverSystems(),
9267 : "Solver system number '" << solver_sys_num << "' is out of bounds. We have '"
9268 : << numSolverSystems() << "' solver systems");
9269 1627966 : return _solver_params[solver_sys_num];
9270 : }
9271 :
9272 : const SolverParams &
9273 14652 : FEProblemBase::solverParams(const unsigned int solver_sys_num) const
9274 : {
9275 14652 : return const_cast<FEProblemBase *>(this)->solverParams(solver_sys_num);
9276 : }
9277 :
9278 : void
9279 371 : FEProblemBase::registerRandomInterface(RandomInterface & random_interface, const std::string & name)
9280 : {
9281 371 : auto insert_pair = moose_try_emplace(
9282 371 : _random_data_objects, name, std::make_unique<RandomData>(*this, random_interface));
9283 :
9284 371 : auto random_data_ptr = insert_pair.first->second.get();
9285 371 : random_interface.setRandomDataPointer(random_data_ptr);
9286 371 : }
9287 :
9288 : bool
9289 1601501 : FEProblemBase::needBoundaryMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9290 : {
9291 1601501 : if (_bnd_mat_side_cache[tid].find(bnd_id) == _bnd_mat_side_cache[tid].end())
9292 : {
9293 28122 : auto & bnd_mat_side_cache = _bnd_mat_side_cache[tid][bnd_id];
9294 28122 : bnd_mat_side_cache = false;
9295 :
9296 : // Check systems
9297 28122 : if (_aux->needMaterialOnSide(bnd_id))
9298 : {
9299 506 : bnd_mat_side_cache = true;
9300 506 : return true;
9301 : }
9302 53008 : for (auto & nl : _nl)
9303 27476 : if (nl->needBoundaryMaterialOnSide(bnd_id, tid))
9304 : {
9305 2084 : bnd_mat_side_cache = true;
9306 2084 : return true;
9307 : }
9308 :
9309 : // TODO: these objects should be checked for whether they actually consume materials
9310 : // NOTE: InterfaceUO can use use boundary properties too
9311 25532 : if (theWarehouse()
9312 51064 : .query()
9313 25532 : .condition<AttribThread>(tid)
9314 25532 : .condition<AttribInterfaces>(Interfaces::SideUserObject | Interfaces::DomainUserObject |
9315 : Interfaces::InterfaceUserObject)
9316 25532 : .condition<AttribBoundaries>(bnd_id)
9317 25532 : .count() > 0)
9318 : {
9319 560 : bnd_mat_side_cache = true;
9320 560 : return true;
9321 : }
9322 : }
9323 :
9324 1598351 : return _bnd_mat_side_cache[tid][bnd_id];
9325 : }
9326 :
9327 : bool
9328 386695 : FEProblemBase::needInterfaceMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9329 : {
9330 386695 : if (_interface_mat_side_cache[tid].find(bnd_id) == _interface_mat_side_cache[tid].end())
9331 : {
9332 2700 : auto & interface_mat_side_cache = _interface_mat_side_cache[tid][bnd_id];
9333 2700 : interface_mat_side_cache = false;
9334 :
9335 : // Aux-system has not needed interface materials so far
9336 5184 : for (auto & nl : _nl)
9337 2700 : if (nl->needInterfaceMaterialOnSide(bnd_id, tid))
9338 : {
9339 216 : interface_mat_side_cache = true;
9340 216 : return true;
9341 : }
9342 :
9343 : // TODO: these objects should be checked for whether they actually consume materials
9344 2484 : if (theWarehouse()
9345 4968 : .query()
9346 2484 : .condition<AttribThread>(tid)
9347 2484 : .condition<AttribInterfaces>(Interfaces::InterfaceUserObject |
9348 : Interfaces::DomainUserObject)
9349 2484 : .condition<AttribBoundaries>(bnd_id)
9350 2484 : .count() > 0)
9351 : {
9352 79 : interface_mat_side_cache = true;
9353 79 : return true;
9354 : }
9355 2405 : else if (_interface_materials.hasActiveBoundaryObjects(bnd_id, tid))
9356 : {
9357 9 : interface_mat_side_cache = true;
9358 9 : return true;
9359 : }
9360 : }
9361 386391 : return _interface_mat_side_cache[tid][bnd_id];
9362 : }
9363 :
9364 : bool
9365 422848 : FEProblemBase::needInternalNeighborSideMaterial(SubdomainID subdomain_id, const THREAD_ID tid)
9366 : {
9367 422848 : if (_block_mat_side_cache[tid].find(subdomain_id) == _block_mat_side_cache[tid].end())
9368 : {
9369 12169 : _block_mat_side_cache[tid][subdomain_id] = false;
9370 :
9371 23899 : for (auto & nl : _nl)
9372 12140 : if (nl->needInternalNeighborSideMaterial(subdomain_id, tid))
9373 : {
9374 410 : _block_mat_side_cache[tid][subdomain_id] = true;
9375 410 : return true;
9376 : }
9377 :
9378 : // TODO: these objects should be checked for whether they actually consume materials
9379 11759 : if (theWarehouse()
9380 23518 : .query()
9381 11759 : .condition<AttribThread>(tid)
9382 11759 : .condition<AttribInterfaces>(Interfaces::InternalSideUserObject |
9383 : Interfaces::DomainUserObject)
9384 11759 : .condition<AttribSubdomains>(subdomain_id)
9385 11759 : .count() > 0)
9386 : {
9387 33 : _block_mat_side_cache[tid][subdomain_id] = true;
9388 33 : return true;
9389 : }
9390 : }
9391 :
9392 422405 : return _block_mat_side_cache[tid][subdomain_id];
9393 : }
9394 :
9395 : bool
9396 288264 : FEProblemBase::needsPreviousNewtonIteration() const
9397 : {
9398 288264 : return vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG);
9399 : }
9400 :
9401 : void
9402 76 : FEProblemBase::needsPreviousNewtonIteration(bool state)
9403 : {
9404 76 : if (state && !vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
9405 0 : mooseError("Previous nonlinear solution is required but not added through "
9406 : "Problem/previous_nl_solution_required=true");
9407 76 : }
9408 :
9409 : void
9410 52 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(bool needed,
9411 : const unsigned int solver_sys_num)
9412 : {
9413 52 : _previous_multiapp_fp_nl_solution_required[solver_sys_num] = needed;
9414 52 : }
9415 :
9416 : bool
9417 58479 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(
9418 : const unsigned int solver_sys_num) const
9419 : {
9420 58479 : return _previous_multiapp_fp_nl_solution_required[solver_sys_num];
9421 : }
9422 :
9423 : void
9424 13 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary(bool state)
9425 : {
9426 13 : _previous_multiapp_fp_aux_solution_required = state;
9427 13 : }
9428 :
9429 : bool
9430 58322 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary() const
9431 : {
9432 58322 : return _previous_multiapp_fp_aux_solution_required;
9433 : }
9434 :
9435 : bool
9436 7561278 : FEProblemBase::hasJacobian() const
9437 : {
9438 7561278 : return _has_jacobian;
9439 : }
9440 :
9441 : bool
9442 7076750 : FEProblemBase::constJacobian() const
9443 : {
9444 7076750 : return _const_jacobian;
9445 : }
9446 :
9447 : void
9448 288709 : FEProblemBase::addOutput(const std::string & object_type,
9449 : const std::string & object_name,
9450 : InputParameters & parameters)
9451 : {
9452 : parallel_object_only();
9453 :
9454 : // Get a reference to the OutputWarehouse
9455 288709 : OutputWarehouse & output_warehouse = _app.getOutputWarehouse();
9456 :
9457 : // Reject the reserved names for objects not built by MOOSE
9458 288709 : if (!parameters.get<bool>("_built_by_moose") && output_warehouse.isReservedName(object_name))
9459 6 : mooseError("The name '", object_name, "' is a reserved name for output objects");
9460 :
9461 : // Check that an object by the same name does not already exist; this must be done before the
9462 : // object is created to avoid getting misleading errors from the Parser
9463 288703 : if (output_warehouse.hasOutput(object_name))
9464 3 : mooseError("An output object named '", object_name, "' already exists");
9465 :
9466 : // Add a pointer to the FEProblemBase class
9467 577400 : parameters.addPrivateParam<FEProblemBase *>("_fe_problem_base", this);
9468 :
9469 : // --show-input should enable the display of the input file on the screen
9470 699709 : if (object_type == "Console" && _app.getParam<bool>("show_input") &&
9471 288727 : parameters.get<bool>("output_screen"))
9472 54 : parameters.set<ExecFlagEnum>("execute_input_on") = EXEC_INITIAL;
9473 :
9474 : // Apply only user-set parameters from the common [Outputs] block so that
9475 : // each output type's own defaults are not overridden by common defaults.
9476 288700 : const InputParameters * common = output_warehouse.getCommonParameters();
9477 288700 : if (common)
9478 288700 : parameters.applyCommonUserSetParameters(*common);
9479 :
9480 : // Set the correct value for the binary flag for XDA/XDR output
9481 288700 : if (object_type == "XDR")
9482 120 : parameters.set<bool>("_binary") = true;
9483 288640 : else if (object_type == "XDA")
9484 244 : parameters.set<bool>("_binary") = false;
9485 :
9486 : // Adjust the checkpoint suffix if auto recovery was enabled
9487 288700 : if (object_name == "auto_recovery_checkpoint")
9488 0 : parameters.set<std::string>("suffix") = "auto_recovery";
9489 :
9490 : // Create the object and add it to the warehouse
9491 288700 : std::shared_ptr<Output> output = _factory.create<Output>(object_type, object_name, parameters);
9492 288688 : logAdd("Output", object_name, object_type, parameters);
9493 288688 : output_warehouse.addOutput(output);
9494 288688 : }
9495 :
9496 : void
9497 22925 : FEProblemBase::haveADObjects(const bool have_ad_objects)
9498 : {
9499 22925 : _have_ad_objects = have_ad_objects;
9500 22925 : if (_displaced_problem)
9501 227 : _displaced_problem->SubProblem::haveADObjects(have_ad_objects);
9502 22925 : }
9503 :
9504 : const SystemBase &
9505 0 : FEProblemBase::getSystemBase(const unsigned int sys_num) const
9506 : {
9507 0 : if (sys_num < _solver_systems.size())
9508 0 : return *_solver_systems[sys_num];
9509 :
9510 0 : return *_aux;
9511 : }
9512 :
9513 : SystemBase &
9514 3762 : FEProblemBase::getSystemBase(const std::string & sys_name)
9515 : {
9516 3762 : if (std::find(_solver_sys_names.begin(), _solver_sys_names.end(), sys_name) !=
9517 7524 : _solver_sys_names.end())
9518 3762 : return getSystemBase(solverSysNum(sys_name));
9519 0 : else if (sys_name == "aux0")
9520 0 : return *_aux;
9521 : else
9522 0 : mooseError("System '" + sys_name + "' was requested from problem but does not exist.");
9523 : }
9524 :
9525 : SystemBase &
9526 5406 : FEProblemBase::getSystemBase(const unsigned int sys_num)
9527 : {
9528 5406 : if (sys_num < _solver_systems.size())
9529 5304 : return *_solver_systems[sys_num];
9530 :
9531 102 : return *_aux;
9532 : }
9533 :
9534 : const SystemBase &
9535 12660 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num) const
9536 : {
9537 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9538 12660 : return *_nl[sys_num];
9539 : }
9540 :
9541 : SystemBase &
9542 1925041 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num)
9543 : {
9544 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9545 1925041 : return *_nl[sys_num];
9546 : }
9547 :
9548 : const SystemBase &
9549 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num) const
9550 : {
9551 : mooseAssert(sys_num < _linear_systems.size(),
9552 : "System number greater than the number of linear systems");
9553 0 : return *_linear_systems[sys_num];
9554 : }
9555 :
9556 : SystemBase &
9557 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num)
9558 : {
9559 : mooseAssert(sys_num < _linear_systems.size(),
9560 : "System number greater than the number of linear systems");
9561 0 : return *_linear_systems[sys_num];
9562 : }
9563 :
9564 : const SystemBase &
9565 0 : FEProblemBase::systemBaseSolver(const unsigned int sys_num) const
9566 : {
9567 : mooseAssert(sys_num < _solver_systems.size(),
9568 : "System number greater than the number of solver systems");
9569 0 : return *_solver_systems[sys_num];
9570 : }
9571 :
9572 : SystemBase &
9573 6360191 : FEProblemBase::systemBaseSolver(const unsigned int sys_num)
9574 : {
9575 : mooseAssert(sys_num < _solver_systems.size(),
9576 : "System number greater than the number of solver systems");
9577 6360191 : return *_solver_systems[sys_num];
9578 : }
9579 :
9580 : const SystemBase &
9581 417 : FEProblemBase::systemBaseAuxiliary() const
9582 : {
9583 417 : return *_aux;
9584 : }
9585 :
9586 : SystemBase &
9587 9490984 : FEProblemBase::systemBaseAuxiliary()
9588 : {
9589 9490984 : return *_aux;
9590 : }
9591 :
9592 : void
9593 3898758 : FEProblemBase::computingNonlinearResid(bool computing_nonlinear_residual)
9594 : {
9595 : parallel_object_only();
9596 :
9597 3898758 : if (_displaced_problem)
9598 192292 : _displaced_problem->computingNonlinearResid(computing_nonlinear_residual);
9599 3898758 : _computing_nonlinear_residual = computing_nonlinear_residual;
9600 3898758 : }
9601 :
9602 : void
9603 9641241 : FEProblemBase::setCurrentlyComputingResidual(bool currently_computing_residual)
9604 : {
9605 9641241 : if (_displaced_problem)
9606 392528 : _displaced_problem->setCurrentlyComputingResidual(currently_computing_residual);
9607 9641241 : _currently_computing_residual = currently_computing_residual;
9608 9641241 : }
9609 :
9610 : void
9611 50 : FEProblemBase::uniformRefine()
9612 : {
9613 : // ResetDisplacedMeshThread::onNode looks up the reference mesh by ID, so we need to make sure
9614 : // we undisplace before adapting the reference mesh
9615 50 : if (_displaced_problem)
9616 34 : _displaced_problem->undisplaceMesh();
9617 :
9618 50 : Adaptivity::uniformRefine(&_mesh, 1);
9619 50 : if (_displaced_problem)
9620 34 : Adaptivity::uniformRefine(&_displaced_problem->mesh(), 1);
9621 :
9622 50 : meshChanged(
9623 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
9624 50 : }
9625 :
9626 : void
9627 60385 : FEProblemBase::automaticScaling(bool automatic_scaling)
9628 : {
9629 60385 : if (_displaced_problem)
9630 2022 : _displaced_problem->automaticScaling(automatic_scaling);
9631 :
9632 60385 : SubProblem::automaticScaling(automatic_scaling);
9633 60385 : }
9634 :
9635 : void
9636 474950 : FEProblemBase::reinitElemFaceRef(const Elem * elem,
9637 : unsigned int side,
9638 : Real tolerance,
9639 : const std::vector<Point> * const pts,
9640 : const std::vector<Real> * const weights,
9641 : const THREAD_ID tid)
9642 : {
9643 474950 : SubProblem::reinitElemFaceRef(elem, side, tolerance, pts, weights, tid);
9644 :
9645 474950 : if (_displaced_problem)
9646 20096 : _displaced_problem->reinitElemFaceRef(
9647 20096 : _displaced_mesh->elemPtr(elem->id()), side, tolerance, pts, weights, tid);
9648 474950 : }
9649 :
9650 : void
9651 474950 : FEProblemBase::reinitNeighborFaceRef(const Elem * neighbor_elem,
9652 : unsigned int neighbor_side,
9653 : Real tolerance,
9654 : const std::vector<Point> * const pts,
9655 : const std::vector<Real> * const weights,
9656 : const THREAD_ID tid)
9657 : {
9658 474950 : SubProblem::reinitNeighborFaceRef(neighbor_elem, neighbor_side, tolerance, pts, weights, tid);
9659 :
9660 474950 : if (_displaced_problem)
9661 20096 : _displaced_problem->reinitNeighborFaceRef(
9662 20096 : _displaced_mesh->elemPtr(neighbor_elem->id()), neighbor_side, tolerance, pts, weights, tid);
9663 474950 : }
9664 :
9665 : void
9666 3035869 : FEProblemBase::getFVMatsAndDependencies(
9667 : const SubdomainID blk_id,
9668 : std::vector<std::shared_ptr<MaterialBase>> & face_materials,
9669 : std::vector<std::shared_ptr<MaterialBase>> & neighbor_materials,
9670 : std::set<MooseVariableFieldBase *> & variables,
9671 : const THREAD_ID tid)
9672 : {
9673 3035869 : if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9674 : {
9675 : auto & this_face_mats =
9676 3544 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9677 7268 : for (std::shared_ptr<MaterialBase> face_mat : this_face_mats)
9678 3724 : if (face_mat->ghostable())
9679 : {
9680 3724 : face_materials.push_back(face_mat);
9681 3724 : auto & var_deps = face_mat->getMooseVariableDependencies();
9682 4084 : for (auto * var : var_deps)
9683 : {
9684 360 : if (!var->isFV())
9685 0 : mooseError(
9686 : "Ghostable materials should only have finite volume variables coupled into them.");
9687 360 : else if (face_mat->hasStatefulProperties())
9688 0 : mooseError("Finite volume materials do not currently support stateful properties.");
9689 360 : variables.insert(var);
9690 : }
9691 3724 : }
9692 : }
9693 :
9694 3035869 : if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9695 : {
9696 : auto & this_neighbor_mats =
9697 3544 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9698 7268 : for (std::shared_ptr<MaterialBase> neighbor_mat : this_neighbor_mats)
9699 3724 : if (neighbor_mat->ghostable())
9700 : {
9701 3724 : neighbor_materials.push_back(neighbor_mat);
9702 : #ifndef NDEBUG
9703 : auto & var_deps = neighbor_mat->getMooseVariableDependencies();
9704 : for (auto * var : var_deps)
9705 : {
9706 : if (!var->isFV())
9707 : mooseError(
9708 : "Ghostable materials should only have finite volume variables coupled into them.");
9709 : else if (neighbor_mat->hasStatefulProperties())
9710 : mooseError("Finite volume materials do not currently support stateful properties.");
9711 : auto pr = variables.insert(var);
9712 : mooseAssert(!pr.second,
9713 : "We should not have inserted any new variables dependencies from our "
9714 : "neighbor materials that didn't exist for our face materials");
9715 : }
9716 : #endif
9717 3724 : }
9718 : }
9719 3035869 : }
9720 :
9721 : void
9722 31842697 : FEProblemBase::resizeMaterialData(const Moose::MaterialDataType data_type,
9723 : const unsigned int nqp,
9724 : const THREAD_ID tid)
9725 : {
9726 31842697 : getMaterialData(data_type, tid).resize(nqp);
9727 31842697 : }
9728 :
9729 : void
9730 60325 : FEProblemBase::setNonlinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9731 : {
9732 60325 : if (convergence_names.size() != numNonlinearSystems())
9733 0 : paramError("nonlinear_convergence",
9734 : "There must be one convergence object per nonlinear system");
9735 60325 : _nonlinear_convergence_names = convergence_names;
9736 60325 : }
9737 :
9738 : void
9739 61787 : FEProblemBase::setMultiAppFixedPointConvergenceName(const ConvergenceName & convergence_name)
9740 : {
9741 61787 : _multiapp_fixed_point_convergence_name = convergence_name;
9742 61787 : }
9743 :
9744 : void
9745 30371 : FEProblemBase::setSteadyStateConvergenceName(const ConvergenceName & convergence_name)
9746 : {
9747 30371 : _steady_state_convergence_name = convergence_name;
9748 30371 : }
9749 :
9750 : const std::vector<ConvergenceName> &
9751 987227 : FEProblemBase::getNonlinearConvergenceNames() const
9752 : {
9753 987227 : if (_nonlinear_convergence_names)
9754 987227 : return *_nonlinear_convergence_names;
9755 0 : mooseError("The nonlinear system convergence name(s) have not been set.");
9756 : }
9757 :
9758 : bool
9759 26452 : FEProblemBase::hasLinearConvergenceObjects() const
9760 : {
9761 : // If false,this means we have not set one, not that we are querying this too early
9762 : // TODO: once there is a default linear CV object, error on the 'not set' case
9763 26452 : return _linear_convergence_names.has_value();
9764 : }
9765 :
9766 : void
9767 134 : FEProblemBase::setLinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9768 : {
9769 134 : if (convergence_names.size() != numLinearSystems())
9770 0 : paramError("linear_convergence", "There must be one convergence object per linear system");
9771 134 : _linear_convergence_names = convergence_names;
9772 134 : }
9773 :
9774 : const std::vector<ConvergenceName> &
9775 4635 : FEProblemBase::getLinearConvergenceNames() const
9776 : {
9777 4635 : if (_linear_convergence_names)
9778 4635 : return *_linear_convergence_names;
9779 0 : mooseError("The linear convergence name(s) have not been set.");
9780 : }
9781 :
9782 : const ConvergenceName &
9783 250752 : FEProblemBase::getMultiAppFixedPointConvergenceName() const
9784 : {
9785 250752 : if (_multiapp_fixed_point_convergence_name)
9786 250752 : return _multiapp_fixed_point_convergence_name.value();
9787 : else
9788 0 : mooseError("The fixed point convergence name has not been set.");
9789 : }
9790 :
9791 : const ConvergenceName &
9792 103205 : FEProblemBase::getSteadyStateConvergenceName() const
9793 : {
9794 103205 : if (_steady_state_convergence_name)
9795 103205 : return _steady_state_convergence_name.value();
9796 : else
9797 0 : mooseError("The steady convergence name has not been set.");
9798 : }
9799 :
9800 : void
9801 3056698 : FEProblemBase::residualSetup()
9802 : {
9803 3056698 : SubProblem::residualSetup();
9804 : // We need to setup all the nonlinear systems other than our current one which actually called
9805 : // this method (so we have to make sure we don't go in a circle)
9806 6202010 : for (const auto i : make_range(numNonlinearSystems()))
9807 3145312 : if (i != currentNlSysNum())
9808 88614 : _nl[i]->residualSetup();
9809 : // We don't setup the aux sys because that's been done elsewhere
9810 3056698 : if (_displaced_problem)
9811 124147 : _displaced_problem->residualSetup();
9812 3056698 : }
9813 :
9814 : void
9815 474802 : FEProblemBase::jacobianSetup()
9816 : {
9817 474802 : SubProblem::jacobianSetup();
9818 : // We need to setup all the nonlinear systems other than our current one which actually called
9819 : // this method (so we have to make sure we don't go in a circle)
9820 964858 : for (const auto i : make_range(numNonlinearSystems()))
9821 490056 : if (i != currentNlSysNum())
9822 15254 : _nl[i]->jacobianSetup();
9823 : // We don't setup the aux sys because that's been done elsewhere
9824 474802 : if (_displaced_problem)
9825 21144 : _displaced_problem->jacobianSetup();
9826 474802 : }
9827 :
9828 : MooseAppCoordTransform &
9829 96036 : FEProblemBase::coordTransform()
9830 : {
9831 96036 : return mesh().coordTransform();
9832 : }
9833 :
9834 : unsigned int
9835 478088599 : FEProblemBase::currentNlSysNum() const
9836 : {
9837 : // If we don't have nonlinear systems this should be an invalid number
9838 478088599 : unsigned int current_nl_sys_num = libMesh::invalid_uint;
9839 478088599 : if (_nl.size())
9840 478086487 : current_nl_sys_num = currentNonlinearSystem().number();
9841 :
9842 478088599 : return current_nl_sys_num;
9843 : }
9844 :
9845 : unsigned int
9846 0 : FEProblemBase::currentLinearSysNum() const
9847 : {
9848 : // If we don't have linear systems this should be an invalid number
9849 0 : unsigned int current_linear_sys_num = libMesh::invalid_uint;
9850 0 : if (_linear_systems.size())
9851 0 : current_linear_sys_num = currentLinearSystem().number();
9852 :
9853 0 : return current_linear_sys_num;
9854 : }
9855 :
9856 : bool
9857 123349233 : FEProblemBase::shouldPrintExecution(const THREAD_ID tid) const
9858 : {
9859 : // For now, only support printing from thread 0
9860 123349233 : if (tid != 0)
9861 519480 : return false;
9862 :
9863 245422124 : if (_print_execution_on.isValueSet(_current_execute_on_flag) ||
9864 122592371 : _print_execution_on.isValueSet(EXEC_ALWAYS))
9865 337814 : return true;
9866 : else
9867 122491939 : return false;
9868 : }
9869 :
9870 : std::vector<MortarUserObject *>
9871 473776 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9872 : const BoundaryID secondary_boundary_id,
9873 : const bool displaced,
9874 : const std::vector<MortarUserObject *> & mortar_uo_superset)
9875 : {
9876 473776 : std::vector<MortarUserObject *> mortar_uos;
9877 473776 : auto * const subproblem = displaced ? static_cast<SubProblem *>(_displaced_problem.get())
9878 473776 : : static_cast<SubProblem *>(this);
9879 473798 : for (auto * const obj : mortar_uo_superset)
9880 44 : if (obj->onInterface(primary_boundary_id, secondary_boundary_id) &&
9881 22 : (&obj->getSubProblem() == subproblem))
9882 22 : mortar_uos.push_back(obj);
9883 :
9884 473776 : return mortar_uos;
9885 0 : }
9886 :
9887 : std::vector<MortarUserObject *>
9888 473754 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9889 : const BoundaryID secondary_boundary_id,
9890 : const bool displaced)
9891 : {
9892 473754 : std::vector<MortarUserObject *> mortar_uos;
9893 473754 : theWarehouse()
9894 473754 : .query()
9895 947508 : .condition<AttribInterfaces>(Interfaces::MortarUserObject)
9896 473754 : .queryInto(mortar_uos);
9897 947508 : return getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced, mortar_uos);
9898 473754 : }
9899 :
9900 : void
9901 473754 : FEProblemBase::reinitMortarUserObjects(const BoundaryID primary_boundary_id,
9902 : const BoundaryID secondary_boundary_id,
9903 : const bool displaced)
9904 : {
9905 : const auto mortar_uos =
9906 473754 : getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced);
9907 473754 : for (auto * const mortar_uo : mortar_uos)
9908 : {
9909 0 : mortar_uo->setNormals();
9910 0 : mortar_uo->reinit();
9911 : }
9912 473754 : }
9913 :
9914 : void
9915 0 : FEProblemBase::setVerboseProblem(bool verbose)
9916 : {
9917 0 : _verbose_setup = verbose ? "true" : "false";
9918 0 : _verbose_multiapps = verbose;
9919 0 : _verbose_restore = verbose;
9920 0 : }
9921 :
9922 : void
9923 111832 : FEProblemBase::setCurrentLowerDElem(const Elem * const lower_d_elem, const THREAD_ID tid)
9924 : {
9925 111832 : SubProblem::setCurrentLowerDElem(lower_d_elem, tid);
9926 111832 : if (_displaced_problem)
9927 27619 : _displaced_problem->setCurrentLowerDElem(
9928 0 : lower_d_elem ? _displaced_mesh->elemPtr(lower_d_elem->id()) : nullptr, tid);
9929 111832 : }
9930 :
9931 : void
9932 121782539 : FEProblemBase::setCurrentBoundaryID(BoundaryID bid, const THREAD_ID tid)
9933 : {
9934 121782539 : SubProblem::setCurrentBoundaryID(bid, tid);
9935 121782539 : if (_displaced_problem)
9936 8992635 : _displaced_problem->setCurrentBoundaryID(bid, tid);
9937 121782539 : }
9938 :
9939 : void
9940 7420874 : FEProblemBase::setCurrentNonlinearSystem(const unsigned int nl_sys_num)
9941 : {
9942 : mooseAssert(nl_sys_num < _nl.size(),
9943 : "System number greater than the number of nonlinear systems");
9944 7420874 : _current_nl_sys = _nl[nl_sys_num].get();
9945 7420874 : _current_solver_sys = _current_nl_sys;
9946 7420874 : }
9947 :
9948 : void
9949 78406 : FEProblemBase::setCurrentLinearSystem(const unsigned int sys_num)
9950 : {
9951 : mooseAssert(sys_num < _linear_systems.size(),
9952 : "System number greater than the number of linear systems");
9953 78406 : _current_linear_sys = _linear_systems[sys_num].get();
9954 78406 : _current_solver_sys = _current_linear_sys;
9955 78406 : }
9956 :
9957 : void
9958 5876086 : FEProblemBase::computeSystems(const ExecFlagType & type)
9959 : {
9960 : // When performing an adjoint solve in the optimization module, the current solver system is the
9961 : // adjoint. However, the adjoint solve requires having accurate time derivative calculations for
9962 : // the forward system. The cleanest way to handle such uses is just to compute the time
9963 : // derivatives for all solver systems instead of trying to guess which ones we need and don't need
9964 11900793 : for (auto & solver_sys : _solver_systems)
9965 6024707 : solver_sys->compute(type);
9966 :
9967 5876086 : _aux->compute(type);
9968 5876053 : }
9969 :
9970 : const ConstElemRange &
9971 3939828 : FEProblemBase::getCurrentAlgebraicElementRange()
9972 : {
9973 3939828 : if (!_current_algebraic_elem_range)
9974 3939828 : return *_mesh.getActiveLocalElementRange();
9975 :
9976 0 : return *_current_algebraic_elem_range;
9977 : }
9978 : const ConstNodeRange &
9979 96264 : FEProblemBase::getCurrentAlgebraicNodeRange()
9980 : {
9981 96264 : if (!_current_algebraic_node_range)
9982 96264 : return *_mesh.getLocalNodeRange();
9983 :
9984 0 : return *_current_algebraic_node_range;
9985 : }
9986 : const ConstBndNodeRange &
9987 3488110 : FEProblemBase::getCurrentAlgebraicBndNodeRange()
9988 : {
9989 3488110 : if (!_current_algebraic_bnd_node_range)
9990 3488110 : return *_mesh.getBoundaryNodeRange();
9991 :
9992 0 : return *_current_algebraic_bnd_node_range;
9993 : }
9994 :
9995 : void
9996 0 : FEProblemBase::setCurrentAlgebraicElementRange(ConstElemRange * range)
9997 : {
9998 0 : if (!range)
9999 : {
10000 0 : _current_algebraic_elem_range = nullptr;
10001 0 : return;
10002 : }
10003 :
10004 0 : _current_algebraic_elem_range = std::make_unique<ConstElemRange>(*range);
10005 : }
10006 : void
10007 0 : FEProblemBase::setCurrentAlgebraicNodeRange(ConstNodeRange * range)
10008 : {
10009 0 : if (!range)
10010 : {
10011 0 : _current_algebraic_node_range = nullptr;
10012 0 : return;
10013 : }
10014 :
10015 0 : _current_algebraic_node_range = std::make_unique<ConstNodeRange>(*range);
10016 : }
10017 : void
10018 0 : FEProblemBase::setCurrentAlgebraicBndNodeRange(ConstBndNodeRange * range)
10019 : {
10020 0 : if (!range)
10021 : {
10022 0 : _current_algebraic_bnd_node_range = nullptr;
10023 0 : return;
10024 : }
10025 :
10026 0 : _current_algebraic_bnd_node_range = std::make_unique<ConstBndNodeRange>(*range);
10027 : }
10028 :
10029 : unsigned short
10030 58982 : FEProblemBase::getCurrentICState()
10031 : {
10032 58982 : return _current_ic_state;
10033 : }
10034 :
10035 : std::string
10036 53696 : FEProblemBase::solverTypeString(const unsigned int solver_sys_num)
10037 : {
10038 53696 : return Moose::stringify(solverParams(solver_sys_num)._type);
10039 : }
10040 :
10041 : SolverParams
10042 1032 : FEProblemBase::makeLinearSolverParams()
10043 : {
10044 1032 : SolverParams solver_params;
10045 1032 : solver_params._type = Moose::SolveType::ST_LINEAR;
10046 1032 : solver_params._line_search = Moose::LineSearchType::LS_NONE;
10047 1032 : return solver_params;
10048 : }
10049 :
10050 : const libMesh::CouplingMatrix &
10051 71902 : FEProblemBase::nonlocalCouplingMatrix(const unsigned i) const
10052 : {
10053 71902 : return _nonlocal_cm[i];
10054 : }
10055 :
10056 : bool
10057 90669757 : FEProblemBase::checkNonlocalCouplingRequirement() const
10058 : {
10059 90669757 : return _requires_nonlocal_coupling;
10060 : }
10061 :
10062 : const std::unordered_map<std::pair<BoundaryID, BoundaryID>, MortarInterfaceConfig> &
10063 118427 : FEProblemBase::getMortarInterfaces(bool on_displaced) const
10064 : {
10065 118427 : return _mortar_data->getMortarInterfaces(on_displaced);
10066 : }
|
