Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #ifdef MOOSE_KOKKOS_ENABLED
11 : #include "KokkosMaterialPropertyStorage.h"
12 : #endif
13 :
14 : #include "FEProblemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialPropertyStorage.h"
17 : #include "MooseEnum.h"
18 : #include "Factory.h"
19 : #include "MooseUtils.h"
20 : #include "DisplacedProblem.h"
21 : #include "SystemBase.h"
22 : #include "MaterialData.h"
23 : #include "ComputeUserObjectsThread.h"
24 : #include "ComputeNodalUserObjectsThread.h"
25 : #include "ComputeThreadedGeneralUserObjectsThread.h"
26 : #include "ComputeMaterialsObjectThread.h"
27 : #include "ProjectMaterialProperties.h"
28 : #include "ComputeIndicatorThread.h"
29 : #include "ComputeMarkerThread.h"
30 : #include "ComputeInitialConditionThread.h"
31 : #include "ComputeFVInitialConditionThread.h"
32 : #include "ComputeBoundaryInitialConditionThread.h"
33 : #include "MaxQpsThread.h"
34 : #include "ActionWarehouse.h"
35 : #include "Conversion.h"
36 : #include "Material.h"
37 : #include "FunctorMaterial.h"
38 : #include "ConstantIC.h"
39 : #include "Parser.h"
40 : #include "ElementH1Error.h"
41 : #include "Function.h"
42 : #include "Convergence.h"
43 : #include "NonlinearSystem.h"
44 : #include "LinearSystem.h"
45 : #include "SolverSystem.h"
46 : #include "Distribution.h"
47 : #include "Sampler.h"
48 : #include "FVAdvectedInterpolationMethod.h"
49 : #include "FVFaceInterpolationMethod.h"
50 : #include "FVInterpolationMethod.h"
51 : #include "PetscSupport.h"
52 : #include "RandomInterface.h"
53 : #include "RandomData.h"
54 : #include "MooseEigenSystem.h"
55 : #include "MooseParsedFunction.h"
56 : #include "MeshChangedInterface.h"
57 : #include "MeshDisplacedInterface.h"
58 : #include "ComputeJacobianBlocksThread.h"
59 : #include "ScalarInitialCondition.h"
60 : #include "FVInitialConditionTempl.h"
61 : #include "ElementPostprocessor.h"
62 : #include "NodalPostprocessor.h"
63 : #include "SidePostprocessor.h"
64 : #include "InternalSidePostprocessor.h"
65 : #include "InterfacePostprocessor.h"
66 : #include "GeneralPostprocessor.h"
67 : #include "ElementVectorPostprocessor.h"
68 : #include "NodalVectorPostprocessor.h"
69 : #include "SideVectorPostprocessor.h"
70 : #include "InternalSideVectorPostprocessor.h"
71 : #include "GeneralVectorPostprocessor.h"
72 : #include "Positions.h"
73 : #include "Indicator.h"
74 : #include "Marker.h"
75 : #include "MultiApp.h"
76 : #include "MultiAppTransfer.h"
77 : #include "TransientMultiApp.h"
78 : #include "ElementUserObject.h"
79 : #include "DomainUserObject.h"
80 : #include "NodalUserObject.h"
81 : #include "SideUserObject.h"
82 : #include "InternalSideUserObject.h"
83 : #include "InterfaceUserObject.h"
84 : #include "GeneralUserObject.h"
85 : #include "ThreadedGeneralUserObject.h"
86 : #include "InternalSideIndicatorBase.h"
87 : #include "Transfer.h"
88 : #include "MultiAppTransfer.h"
89 : #include "MultiMooseEnum.h"
90 : #include "Predictor.h"
91 : #include "Assembly.h"
92 : #include "Control.h"
93 : #include "XFEMInterface.h"
94 : #include "ConsoleUtils.h"
95 : #include "NonlocalKernel.h"
96 : #include "NonlocalIntegratedBC.h"
97 : #include "ShapeElementUserObject.h"
98 : #include "ShapeSideUserObject.h"
99 : #include "MooseVariableFE.h"
100 : #include "MooseVariableScalar.h"
101 : #include "InputParameterWarehouse.h"
102 : #include "TimeIntegrator.h"
103 : #include "LineSearch.h"
104 : #include "FloatingPointExceptionGuard.h"
105 : #include "MaxVarNDofsPerElem.h"
106 : #include "MaxVarNDofsPerNode.h"
107 : #include "FVKernel.h"
108 : #include "LinearFVKernel.h"
109 : #include "FVTimeKernel.h"
110 : #include "MooseVariableFV.h"
111 : #include "MooseLinearVariableFV.h"
112 : #include "FVBoundaryCondition.h"
113 : #include "LinearFVBoundaryCondition.h"
114 : #include "FVInterfaceKernel.h"
115 : #include "Reporter.h"
116 : #include "ADUtils.h"
117 : #include "Executioner.h"
118 : #include "VariadicTable.h"
119 : #include "BoundaryNodeIntegrityCheckThread.h"
120 : #include "BoundaryElemIntegrityCheckThread.h"
121 : #include "NodalBCBase.h"
122 : #include "MortarUserObject.h"
123 : #include "MortarUserObjectThread.h"
124 : #include "RedistributeProperties.h"
125 : #include "Checkpoint.h"
126 : #include "MortarInterfaceWarehouse.h"
127 : #include "AutomaticMortarGeneration.h"
128 :
129 : #include "libmesh/exodusII_io.h"
130 : #include "libmesh/quadrature.h"
131 : #include "libmesh/coupling_matrix.h"
132 : #include "libmesh/nonlinear_solver.h"
133 : #include "libmesh/sparse_matrix.h"
134 : #include "libmesh/string_to_enum.h"
135 : #include "libmesh/fe_interface.h"
136 : #include "libmesh/enum_norm_type.h"
137 : #include "libmesh/petsc_solver_exception.h"
138 :
139 : #include "metaphysicl/dualnumber.h"
140 :
141 : // C++
142 : #include <cstring> // for "Jacobian" exception test
143 :
144 : using namespace libMesh;
145 :
146 : // Anonymous namespace for helper function
147 : namespace
148 : {
149 : /**
150 : * Method for sorting the MooseVariableFEBases based on variable numbers
151 : */
152 : bool
153 53 : sortMooseVariables(const MooseVariableFEBase * a, const MooseVariableFEBase * b)
154 : {
155 53 : return a->number() < b->number();
156 : }
157 : } // namespace
158 :
159 : Threads::spin_mutex get_function_mutex;
160 :
161 : InputParameters
162 194512 : FEProblemBase::validParams()
163 : {
164 194512 : InputParameters params = SubProblem::validParams();
165 778048 : params.addParam<unsigned int>("null_space_dimension", 0, "The dimension of the nullspace");
166 583536 : params.addParam<unsigned int>(
167 389024 : "transpose_null_space_dimension", 0, "The dimension of the transpose nullspace");
168 583536 : params.addParam<unsigned int>(
169 389024 : "near_null_space_dimension", 0, "The dimension of the near nullspace");
170 583536 : params.addParam<bool>("solve",
171 389024 : true,
172 : "Whether or not to actually solve the Nonlinear system. "
173 : "This is handy in the case that all you want to do is "
174 : "execute AuxKernels, Transfers, etc. without actually "
175 : "solving anything");
176 583536 : params.addParam<bool>("use_nonlinear",
177 389024 : true,
178 : "Determines whether to use a Nonlinear vs a "
179 : "Eigenvalue system (Automatically determined based "
180 : "on executioner)");
181 778048 : params.addParam<bool>("error_on_jacobian_nonzero_reallocation",
182 : "This causes PETSc to error if it had to reallocate memory in the Jacobian "
183 : "matrix due to not having enough nonzeros");
184 583536 : params.addParam<bool>("ignore_zeros_in_jacobian",
185 389024 : false,
186 : "Do not explicitly store zero values in "
187 : "the Jacobian matrix if true");
188 583536 : params.addParam<bool>("force_restart",
189 389024 : false,
190 : "EXPERIMENTAL: If true, a sub_app may use a "
191 : "restart file instead of using of using the master "
192 : "backup file");
193 972560 : params.addDeprecatedParam<bool>("skip_additional_restart_data",
194 389024 : false,
195 : "True to skip additional data in equation system for restart.",
196 : "This parameter is no longer used, as we do not load additional "
197 : "vectors by default with restart");
198 583536 : params.addParam<bool>("skip_nl_system_check",
199 389024 : false,
200 : "True to skip the NonlinearSystem check for work to do (e.g. Make sure "
201 : "that there are variables to solve for).");
202 583536 : params.addParam<bool>("allow_initial_conditions_with_restart",
203 389024 : false,
204 : "True to allow the user to specify initial conditions when restarting. "
205 : "Initial conditions can override any restarted field");
206 :
207 389024 : auto coverage_check_description = [](std::string scope, std::string list_param_name)
208 : {
209 778048 : return "Controls, if and how a " + scope +
210 : " subdomain coverage check is performed. "
211 : "With 'TRUE' or 'ON' all subdomains are checked (the default). Setting 'FALSE' or 'OFF' "
212 : "will disable the check for all subdomains. "
213 : "To exclude a predefined set of subdomains 'SKIP_LIST' is to "
214 778048 : "be used, while the subdomains to skip are to be defined in the parameter '" +
215 778048 : list_param_name +
216 : "'. To limit the check to a list of subdomains, 'ONLY_LIST' is to "
217 1167072 : "be used (again, using the parameter '" +
218 778048 : list_param_name + "').";
219 : };
220 :
221 972560 : params.addParam<std::vector<SubdomainName>>(
222 : "block",
223 : {"ANY_BLOCK_ID"},
224 : "List of subdomains for kernel coverage and material coverage checks. Setting this parameter "
225 : "is equivalent to setting 'kernel_coverage_block_list' and 'material_coverage_block_list' as "
226 : "well as using 'ONLY_LIST' as the coverage check mode.");
227 :
228 778048 : MooseEnum kernel_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
229 194512 : params.addParam<MooseEnum>("kernel_coverage_check",
230 : kernel_coverage_check_modes,
231 972560 : coverage_check_description("kernel", "kernel_coverage_block_list"));
232 778048 : params.addParam<std::vector<SubdomainName>>(
233 : "kernel_coverage_block_list",
234 : {},
235 : "List of subdomains for kernel coverage check. The meaning of this list is controlled by the "
236 : "parameter 'kernel_coverage_check' (whether this is the list of subdomains to be checked, "
237 : "not to be checked or not taken into account).");
238 583536 : params.addParam<bool>(
239 : "boundary_restricted_node_integrity_check",
240 389024 : true,
241 : "Set to false to disable checking of boundary restricted nodal object variable dependencies, "
242 : "e.g. are the variable dependencies defined on the selected boundaries?");
243 583536 : params.addParam<bool>("boundary_restricted_elem_integrity_check",
244 389024 : true,
245 : "Set to false to disable checking of boundary restricted elemental object "
246 : "variable dependencies, e.g. are the variable dependencies defined on the "
247 : "selected boundaries?");
248 778048 : MooseEnum material_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
249 194512 : params.addParam<MooseEnum>(
250 : "material_coverage_check",
251 : material_coverage_check_modes,
252 972560 : coverage_check_description("material", "material_coverage_block_list"));
253 778048 : params.addParam<std::vector<SubdomainName>>(
254 : "material_coverage_block_list",
255 : {},
256 : "List of subdomains for material coverage check. The meaning of this list is controlled by "
257 : "the parameter 'material_coverage_check' (whether this is the list of subdomains to be "
258 : "checked, not to be checked or not taken into account).");
259 :
260 583536 : params.addParam<bool>("fv_bcs_integrity_check",
261 389024 : true,
262 : "Set to false to disable checking of overlapping Dirichlet and Flux BCs "
263 : "and/or multiple DirichletBCs per sideset");
264 :
265 583536 : params.addParam<bool>(
266 389024 : "material_dependency_check", true, "Set to false to disable material dependency check");
267 583536 : params.addParam<bool>("parallel_barrier_messaging",
268 389024 : false,
269 : "Displays messaging from parallel "
270 : "barrier notifications when executing "
271 : "or transferring to/from Multiapps "
272 : "(default: false)");
273 :
274 778048 : MooseEnum verbosity("false true extra", "false");
275 778048 : params.addParam<MooseEnum>("verbose_setup",
276 : verbosity,
277 : "Set to 'true' to have the problem report on any object created. Set "
278 : "to 'extra' to also display all parameters.");
279 583536 : params.addParam<bool>("verbose_multiapps",
280 389024 : false,
281 : "Set to True to enable verbose screen printing related to MultiApps");
282 583536 : params.addParam<bool>(
283 : "verbose_restore",
284 389024 : false,
285 : "Set to True to enable verbose screen printing related to solution restoration");
286 :
287 778048 : params.addParam<FileNameNoExtension>("restart_file_base",
288 : "File base name used for restart (e.g. "
289 : "<path>/<filebase> or <path>/LATEST to "
290 : "grab the latest file available)");
291 :
292 778048 : params.addParam<std::vector<std::vector<TagName>>>(
293 : "extra_tag_vectors",
294 : {},
295 : "Extra vectors to add to the system that can be filled by objects which compute residuals "
296 : "and Jacobians (Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
297 : "nonlinear system the extra tag vectors should be added for");
298 :
299 778048 : params.addParam<std::vector<std::vector<TagName>>>(
300 : "not_zeroed_tag_vectors",
301 : {},
302 : "Extra vector tags which the sytem will not zero when other vector tags are zeroed. "
303 : "The outer index is for which nonlinear system the extra tag vectors should be added for");
304 :
305 778048 : params.addParam<std::vector<std::vector<TagName>>>(
306 : "extra_tag_matrices",
307 : {},
308 : "Extra matrices to add to the system that can be filled "
309 : "by objects which compute residuals and Jacobians "
310 : "(Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
311 : "nonlinear system the extra tag vectors should be added for");
312 :
313 778048 : params.addParam<std::vector<TagName>>(
314 : "extra_tag_solutions",
315 : {},
316 : "Extra solution vectors to add to the system that can be used by "
317 : "objects for coupling variable values stored in them.");
318 :
319 583536 : params.addParam<bool>("previous_nl_solution_required",
320 389024 : false,
321 : "True to indicate that this calculation requires a solution vector for "
322 : "storing the previous nonlinear iteration.");
323 :
324 583536 : params.addParam<std::vector<NonlinearSystemName>>(
325 778048 : "nl_sys_names", std::vector<NonlinearSystemName>{"nl0"}, "The nonlinear system names");
326 :
327 778048 : params.addParam<std::vector<LinearSystemName>>("linear_sys_names", {}, "The linear system names");
328 :
329 583536 : params.addParam<bool>("check_uo_aux_state",
330 389024 : false,
331 : "True to turn on a check that no state presents during the evaluation of "
332 : "user objects and aux kernels");
333 :
334 194512 : params.addPrivateParam<MooseMesh *>("mesh");
335 :
336 583536 : params.declareControllable("solve");
337 :
338 583536 : params.addParam<bool>(
339 : "allow_invalid_solution",
340 389024 : false,
341 : "Set to true to allow convergence even though the solution has been marked as 'invalid'");
342 583536 : params.addParam<bool>("show_invalid_solution_console",
343 389024 : true,
344 : "Set to true to show the invalid solution occurrence summary in console");
345 583536 : params.addParam<bool>("immediately_print_invalid_solution",
346 389024 : false,
347 : "Whether or not to report invalid solution warnings at the time the "
348 : "warning is produced instead of after the calculation");
349 :
350 583536 : params.addParam<bool>(
351 : "identify_variable_groups_in_nl",
352 389024 : true,
353 : "Whether to identify variable groups in nonlinear systems. This affects dof ordering");
354 :
355 583536 : params.addParam<bool>(
356 : "regard_general_exceptions_as_errors",
357 389024 : false,
358 : "If we catch an exception during residual/Jacobian evaluaton for which we don't have "
359 : "specific handling, immediately error instead of allowing the time step to be cut");
360 :
361 583536 : params.addParam<bool>("use_hash_table_matrix_assembly",
362 389024 : false,
363 : "Whether to assemble matrices using hash tables instead of preallocating "
364 : "matrix memory. This can be a good option if the sparsity pattern changes "
365 : "throughout the course of the simulation.");
366 778048 : params.addParam<bool>(
367 : "restore_original_nonzero_pattern",
368 : "Whether we should reset matrix memory for every Jacobian evaluation. This option is useful "
369 : "if the sparsity pattern is constantly changing and you are using hash table assembly or if "
370 : "you wish to continually restore the matrix to the originally preallocated sparsity pattern "
371 : "computed by relationship managers.");
372 :
373 778048 : params.addParamNamesToGroup(
374 : "skip_nl_system_check kernel_coverage_check kernel_coverage_block_list "
375 : "boundary_restricted_node_integrity_check "
376 : "boundary_restricted_elem_integrity_check material_coverage_check "
377 : "material_coverage_block_list fv_bcs_integrity_check "
378 : "material_dependency_check check_uo_aux_state error_on_jacobian_nonzero_reallocation",
379 : "Simulation checks");
380 778048 : params.addParamNamesToGroup("use_nonlinear previous_nl_solution_required nl_sys_names "
381 : "ignore_zeros_in_jacobian identify_variable_groups_in_nl "
382 : "use_hash_table_matrix_assembly restore_original_nonzero_pattern",
383 : "Nonlinear system(s)");
384 778048 : params.addParamNamesToGroup(
385 : "restart_file_base force_restart allow_initial_conditions_with_restart", "Restart");
386 778048 : params.addParamNamesToGroup(
387 : "verbose_setup verbose_multiapps verbose_restore parallel_barrier_messaging", "Verbosity");
388 778048 : params.addParamNamesToGroup(
389 : "null_space_dimension transpose_null_space_dimension near_null_space_dimension",
390 : "Null space removal");
391 778048 : params.addParamNamesToGroup(
392 : "extra_tag_vectors extra_tag_matrices extra_tag_solutions not_zeroed_tag_vectors",
393 : "Contribution to tagged field data");
394 583536 : params.addParamNamesToGroup(
395 : "allow_invalid_solution show_invalid_solution_console immediately_print_invalid_solution",
396 : "Solution validity control");
397 :
398 389024 : return params;
399 583536 : }
400 :
401 63243 : FEProblemBase::FEProblemBase(const InputParameters & parameters)
402 : : SubProblem(parameters),
403 : Restartable(this, "FEProblemBase"),
404 252972 : _mesh(*getCheckedPointerParam<MooseMesh *>("mesh")),
405 126486 : _req(declareManagedRestartableDataWithContext<RestartableEquationSystems>(
406 : "equation_systems", nullptr, _mesh)),
407 63243 : _initialized(false),
408 126486 : _solve(getParam<bool>("solve")),
409 63243 : _transient(false),
410 126486 : _time(declareRestartableData<Real>("time")),
411 126486 : _time_old(declareRestartableData<Real>("time_old")),
412 126486 : _t_step(declareRecoverableData<int>("t_step")),
413 126486 : _dt(declareRestartableData<Real>("dt")),
414 126486 : _dt_old(declareRestartableData<Real>("dt_old")),
415 63243 : _need_to_add_default_nonlinear_convergence(false),
416 63243 : _need_to_add_default_multiapp_fixed_point_convergence(false),
417 63243 : _need_to_add_default_steady_state_convergence(false),
418 126486 : _linear_sys_names(getParam<std::vector<LinearSystemName>>("linear_sys_names")),
419 63243 : _num_linear_sys(_linear_sys_names.size()),
420 126486 : _linear_systems(_num_linear_sys, nullptr),
421 63243 : _current_linear_sys(nullptr),
422 126486 : _using_default_nl(!isParamSetByUser("nl_sys_names")),
423 187407 : _nl_sys_names(!_using_default_nl || (_using_default_nl && !_linear_sys_names.size())
424 63243 : ? getParam<std::vector<NonlinearSystemName>>("nl_sys_names")
425 : : std::vector<NonlinearSystemName>()),
426 63243 : _num_nl_sys(_nl_sys_names.size()),
427 126486 : _nl(_num_nl_sys, nullptr),
428 63243 : _current_nl_sys(nullptr),
429 126486 : _solver_systems(_num_nl_sys + _num_linear_sys, nullptr),
430 63243 : _aux(nullptr),
431 63243 : _coupling(Moose::COUPLING_DIAG),
432 : #ifdef MOOSE_KOKKOS_ENABLED
433 47924 : _kokkos_assembly(*this),
434 : #endif
435 63243 : _mesh_divisions(/*threaded=*/true),
436 63243 : _material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
437 63243 : "material_props", &_mesh, _material_prop_registry, *this)),
438 63243 : _bnd_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
439 63243 : "bnd_material_props", &_mesh, _material_prop_registry, *this)),
440 63243 : _neighbor_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
441 63243 : "neighbor_material_props", &_mesh, _material_prop_registry, *this)),
442 : #ifdef MOOSE_KOKKOS_ENABLED
443 47924 : _kokkos_material_props(
444 47924 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
445 47924 : "kokkos_material_props", &_mesh, _material_prop_registry, *this)),
446 47924 : _kokkos_bnd_material_props(
447 47924 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
448 47924 : "kokkos_bnd_material_props", &_mesh, _material_prop_registry, *this)),
449 47924 : _kokkos_neighbor_material_props(
450 47924 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
451 47924 : "kokkos_neighbor_material_props", &_mesh, _material_prop_registry, *this)),
452 : #endif
453 63243 : _reporter_data(_app),
454 63243 : _multi_apps(_app.getExecuteOnEnum()),
455 63243 : _transient_multi_apps(_app.getExecuteOnEnum()),
456 63243 : _transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
457 63243 : _to_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
458 63243 : _from_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
459 63243 : _between_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
460 : #ifdef LIBMESH_ENABLE_AMR
461 63243 : _adaptivity(*this),
462 63243 : _cycles_completed(0),
463 : #endif
464 63243 : _displaced_mesh(nullptr),
465 63243 : _geometric_search_data(*this, _mesh),
466 63243 : _mortar_data(std::make_unique<MortarInterfaceWarehouse>(*this)),
467 63243 : _reinit_displaced_elem(false),
468 63243 : _reinit_displaced_face(false),
469 63243 : _reinit_displaced_neighbor(false),
470 63243 : _input_file_saved(false),
471 63243 : _has_dampers(false),
472 63243 : _has_constraints(false),
473 63243 : _snesmf_reuse_base(true),
474 63243 : _skip_exception_check(false),
475 63243 : _snesmf_reuse_base_set_by_user(false),
476 63243 : _has_initialized_stateful(false),
477 63243 : _const_jacobian(false),
478 63243 : _has_jacobian(false),
479 63243 : _needs_old_newton_iter(false),
480 126486 : _previous_nl_solution_required(getParam<bool>("previous_nl_solution_required")),
481 126486 : _previous_multiapp_fp_nl_solution_required(_num_nl_sys + _num_linear_sys, false),
482 63243 : _previous_multiapp_fp_aux_solution_required(false),
483 63243 : _has_nonlocal_coupling(false),
484 63243 : _calculate_jacobian_in_uo(false),
485 63243 : _kernel_coverage_check(
486 126486 : getParam<MooseEnum>("kernel_coverage_check").getEnum<CoverageCheckMode>()),
487 126486 : _kernel_coverage_blocks(getParam<std::vector<SubdomainName>>("kernel_coverage_block_list")),
488 63243 : _boundary_restricted_node_integrity_check(
489 126486 : getParam<bool>("boundary_restricted_node_integrity_check")),
490 63243 : _boundary_restricted_elem_integrity_check(
491 126486 : getParam<bool>("boundary_restricted_elem_integrity_check")),
492 63243 : _material_coverage_check(
493 126486 : getParam<MooseEnum>("material_coverage_check").getEnum<CoverageCheckMode>()),
494 126486 : _material_coverage_blocks(getParam<std::vector<SubdomainName>>("material_coverage_block_list")),
495 126486 : _fv_bcs_integrity_check(getParam<bool>("fv_bcs_integrity_check")),
496 126486 : _material_dependency_check(getParam<bool>("material_dependency_check")),
497 126486 : _uo_aux_state_check(getParam<bool>("check_uo_aux_state")),
498 : #ifndef NDEBUG
499 : _check_residual_for_nans(false),
500 : #endif
501 63243 : _max_qps(std::numeric_limits<unsigned int>::max()),
502 63243 : _max_scalar_order(INVALID_ORDER),
503 63243 : _has_time_integrator(false),
504 63243 : _has_exception(false),
505 126486 : _parallel_barrier_messaging(getParam<bool>("parallel_barrier_messaging")),
506 126486 : _verbose_setup(getParam<MooseEnum>("verbose_setup")),
507 126486 : _verbose_multiapps(getParam<bool>("verbose_multiapps")),
508 126486 : _verbose_restore(getParam<bool>("verbose_restore")),
509 63243 : _current_execute_on_flag(EXEC_NONE),
510 63243 : _control_warehouse(_app.getExecuteOnEnum(), /*threaded=*/false),
511 63243 : _is_petsc_options_inserted(false),
512 63243 : _line_search(nullptr),
513 63243 : _using_ad_mat_props(false),
514 63243 : _current_ic_state(0),
515 126486 : _use_hash_table_matrix_assembly(getParam<bool>("use_hash_table_matrix_assembly")),
516 63243 : _error_on_jacobian_nonzero_reallocation(
517 126486 : isParamValid("error_on_jacobian_nonzero_reallocation")
518 127013 : ? getParam<bool>("error_on_jacobian_nonzero_reallocation")
519 62716 : : _app.errorOnJacobianNonzeroReallocation()),
520 126486 : _restore_original_nonzero_pattern(isParamValid("restore_original_nonzero_pattern")
521 126486 : ? getParam<bool>("restore_original_nonzero_pattern")
522 63243 : : _use_hash_table_matrix_assembly),
523 126486 : _ignore_zeros_in_jacobian(getParam<bool>("ignore_zeros_in_jacobian")),
524 63243 : _preserve_matrix_sparsity_pattern(true),
525 126486 : _force_restart(getParam<bool>("force_restart")),
526 126486 : _allow_ics_during_restart(getParam<bool>("allow_initial_conditions_with_restart")),
527 126486 : _skip_nl_system_check(getParam<bool>("skip_nl_system_check")),
528 63243 : _fail_next_system_convergence_check(false),
529 126486 : _allow_invalid_solution(getParam<bool>("allow_invalid_solution")),
530 126486 : _show_invalid_solution_console(getParam<bool>("show_invalid_solution_console")),
531 126486 : _immediately_print_invalid_solution(getParam<bool>("immediately_print_invalid_solution")),
532 63243 : _started_initial_setup(false),
533 63243 : _has_internal_edge_residual_objects(false),
534 63243 : _u_dot_requested(false),
535 63243 : _u_dotdot_requested(false),
536 63243 : _u_dot_old_requested(false),
537 63243 : _u_dotdot_old_requested(false),
538 63243 : _has_mortar(false),
539 63243 : _num_grid_steps(0),
540 63243 : _print_execution_on(),
541 126486 : _identify_variable_groups_in_nl(getParam<bool>("identify_variable_groups_in_nl")),
542 63243 : _regard_general_exceptions_as_errors(getParam<bool>("regard_general_exceptions_as_errors")),
543 1201617 : _requires_nonlocal_coupling(false)
544 : {
545 : auto checkCoverageCheckConflict =
546 126486 : [this](const std::string & coverage_check,
547 : const CoverageCheckMode & coverage_check_mode,
548 : const std::vector<SubdomainName> & coverage_blocks) -> void
549 : {
550 126486 : if (coverage_check_mode != CoverageCheckMode::FALSE &&
551 122060 : coverage_check_mode != CoverageCheckMode::OFF)
552 122042 : if (coverage_blocks.size() > 1)
553 0 : if (std::find(coverage_blocks.begin(), coverage_blocks.end(), "ANY_BLOCK_ID") !=
554 0 : coverage_blocks.end())
555 0 : paramError(coverage_check,
556 : "The list of blocks used for ",
557 : coverage_check,
558 : " cannot contain 'ANY_BLOCK_ID' along with other blocks. ");
559 126486 : };
560 :
561 63243 : checkCoverageCheckConflict(
562 63243 : "kernel_coverage_check", _kernel_coverage_check, _kernel_coverage_blocks);
563 63243 : checkCoverageCheckConflict(
564 63243 : "material_coverage_check", _material_coverage_check, _material_coverage_blocks);
565 :
566 : // Initialize static do_derivatives member. We initialize this to true so that all the
567 : // default AD things that we setup early in the simulation actually get their derivative
568 : // vectors initalized. We will toggle this to false when doing residual evaluations
569 63243 : ADReal::do_derivatives = true;
570 :
571 : // Disable refinement/coarsening in EquationSystems::reinit because we already do this ourselves
572 63243 : es().disable_refine_in_reinit();
573 :
574 63243 : _solver_params.reserve(_num_nl_sys + _num_linear_sys);
575 : // Default constructor fine for nonlinear because it will be populated later by framework
576 : // executioner/solve object parameters
577 63243 : _solver_params.resize(_num_nl_sys);
578 125604 : for (const auto i : index_range(_nl_sys_names))
579 : {
580 62361 : const auto & name = _nl_sys_names[i];
581 62361 : _nl_sys_name_to_num[name] = i;
582 62361 : _solver_sys_name_to_num[name] = i;
583 62361 : _solver_sys_names.push_back(name);
584 : }
585 :
586 64507 : for (const auto i : index_range(_linear_sys_names))
587 : {
588 1264 : const auto & name = _linear_sys_names[i];
589 1264 : _linear_sys_name_to_num[name] = i;
590 1264 : _solver_sys_name_to_num[name] = i + _num_nl_sys;
591 1264 : _solver_sys_names.push_back(name);
592 : // Unlike for nonlinear these are basically dummy parameters
593 1264 : _solver_params.push_back(makeLinearSolverParams());
594 : }
595 :
596 63243 : _nonlocal_cm.resize(numSolverSystems());
597 63243 : _cm.resize(numSolverSystems());
598 :
599 63243 : _time = 0.0;
600 63243 : _time_old = 0.0;
601 63243 : _t_step = 0;
602 63243 : _dt = 0;
603 63243 : _dt_old = _dt;
604 :
605 63243 : unsigned int n_threads = libMesh::n_threads();
606 :
607 63243 : _real_zero.resize(n_threads, 0.);
608 63243 : _scalar_zero.resize(n_threads);
609 63243 : _zero.resize(n_threads);
610 63243 : _phi_zero.resize(n_threads);
611 63243 : _ad_zero.resize(n_threads);
612 63243 : _grad_zero.resize(n_threads);
613 63243 : _ad_grad_zero.resize(n_threads);
614 63243 : _grad_phi_zero.resize(n_threads);
615 63243 : _second_zero.resize(n_threads);
616 63243 : _ad_second_zero.resize(n_threads);
617 63243 : _second_phi_zero.resize(n_threads);
618 63243 : _point_zero.resize(n_threads);
619 63243 : _vector_zero.resize(n_threads);
620 63243 : _vector_curl_zero.resize(n_threads);
621 63243 : _uo_jacobian_moose_vars.resize(n_threads);
622 :
623 63243 : _has_active_material_properties.resize(n_threads, 0);
624 :
625 63243 : _block_mat_side_cache.resize(n_threads);
626 63243 : _bnd_mat_side_cache.resize(n_threads);
627 63243 : _interface_mat_side_cache.resize(n_threads);
628 :
629 126486 : es().parameters.set<FEProblemBase *>("_fe_problem_base") = this;
630 :
631 189729 : if (isParamValid("restart_file_base"))
632 : {
633 946 : std::string restart_file_base = getParam<FileNameNoExtension>("restart_file_base");
634 :
635 : // This check reverts to old behavior of providing "restart_file_base=" to mean
636 : // don't restart... BISON currently relies on this. It could probably be removed.
637 : // The new MooseUtils::convertLatestCheckpoint will error out if a checkpoint file
638 : // is not found, which I think makes sense. Which means, without this, if you
639 : // set "restart_file_base=", you'll get a "No checkpoint file found" error
640 473 : if (restart_file_base.size())
641 : {
642 473 : restart_file_base = MooseUtils::convertLatestCheckpoint(restart_file_base);
643 473 : setRestartFile(restart_file_base);
644 : }
645 473 : }
646 :
647 : // // Generally speaking, the mesh is prepared for use, and consequently remote elements are deleted
648 : // // well before our Problem(s) are constructed. Historically, in MooseMesh we have a bunch of
649 : // // needs_prepare type flags that make it so we never call prepare_for_use (and consequently
650 : // // delete_remote_elements) again. So the below line, historically, has had no impact. HOWEVER:
651 : // // I've added some code in SetupMeshCompleteAction for deleting remote elements post
652 : // // EquationSystems::init. If I execute that code without default ghosting, then I get > 40 MOOSE
653 : // // test failures, so we clearly have some simulations that are not yet covered properly by
654 : // // relationship managers. Until that is resolved, I am going to retain default geometric ghosting
655 : // if (!_default_ghosting)
656 : // _mesh.getMesh().remove_ghosting_functor(_mesh.getMesh().default_ghosting());
657 :
658 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
659 : // Main app should hold the default database to handle system petsc options
660 63243 : if (!_app.isUltimateMaster())
661 12245 : LibmeshPetscCall(PetscOptionsCreate(&_petsc_option_data_base));
662 : #endif
663 :
664 63243 : if (!_solve)
665 : {
666 : // If we are not solving, we do not care about seeing unused petsc options
667 50685 : Moose::PetscSupport::setSinglePetscOption("-options_left", "0");
668 : // We don't want petscSetOptions being called in solve and clearing the option that was just set
669 16895 : _is_petsc_options_inserted = true;
670 : }
671 63243 : }
672 :
673 : const MooseMesh &
674 0 : FEProblemBase::mesh(bool use_displaced) const
675 : {
676 0 : if (use_displaced && !_displaced_problem)
677 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
678 : "Regular mesh will be returned");
679 0 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
680 : }
681 :
682 : MooseMesh &
683 359630 : FEProblemBase::mesh(bool use_displaced)
684 : {
685 359630 : if (use_displaced && !_displaced_problem)
686 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
687 : "Regular mesh will be returned");
688 359630 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
689 : }
690 :
691 : void
692 63243 : FEProblemBase::createTagVectors()
693 : {
694 : // add vectors and their tags to system
695 126486 : auto & vectors = getParam<std::vector<std::vector<TagName>>>("extra_tag_vectors");
696 64043 : for (const auto sys_num : index_range(vectors))
697 1908 : for (auto & vector : vectors[sys_num])
698 : {
699 1108 : auto tag = addVectorTag(vector);
700 1108 : _solver_systems[sys_num]->addVector(tag, false, libMesh::GHOSTED);
701 : }
702 :
703 126486 : auto & not_zeroed_vectors = getParam<std::vector<std::vector<TagName>>>("not_zeroed_tag_vectors");
704 63254 : for (const auto sys_num : index_range(not_zeroed_vectors))
705 22 : for (auto & vector : not_zeroed_vectors[sys_num])
706 : {
707 11 : auto tag = addVectorTag(vector);
708 11 : _solver_systems[sys_num]->addVector(tag, false, GHOSTED);
709 11 : addNotZeroedVectorTag(tag);
710 : }
711 63243 : }
712 :
713 : void
714 62307 : FEProblemBase::createTagMatrices(CreateTaggedMatrixKey)
715 : {
716 124614 : auto & matrices = getParam<std::vector<std::vector<TagName>>>("extra_tag_matrices");
717 62560 : for (const auto sys_num : index_range(matrices))
718 706 : for (auto & matrix : matrices[sys_num])
719 : {
720 453 : auto tag = addMatrixTag(matrix);
721 453 : _solver_systems[sys_num]->addMatrix(tag);
722 : }
723 :
724 124936 : for (auto & sys : _solver_systems)
725 62629 : sys->sizeVariableMatrixData();
726 62307 : _aux->sizeVariableMatrixData();
727 62307 : }
728 :
729 : void
730 63243 : FEProblemBase::createTagSolutions()
731 : {
732 189754 : for (auto & vector : getParam<std::vector<TagName>>("extra_tag_solutions"))
733 : {
734 25 : auto tag = addVectorTag(vector, Moose::VECTOR_TAG_SOLUTION);
735 50 : for (auto & sys : _solver_systems)
736 25 : sys->addVector(tag, false, libMesh::GHOSTED);
737 25 : _aux->addVector(tag, false, libMesh::GHOSTED);
738 : }
739 :
740 63243 : if (_previous_nl_solution_required)
741 : {
742 : // We'll populate the zeroth state of the nonlinear iterations with the current solution for
743 : // ease of use in doing things like copying solutions backwards. We're just storing pointers in
744 : // the solution states containers so populating the zeroth state does not cost us the memory of
745 : // a new vector
746 83 : needSolutionState(1, Moose::SolutionIterationType::Nonlinear);
747 : }
748 :
749 63243 : auto tag = addVectorTag(Moose::SOLUTION_TAG, Moose::VECTOR_TAG_SOLUTION);
750 126868 : for (auto & sys : _solver_systems)
751 63625 : sys->associateVectorToTag(*sys->system().current_local_solution.get(), tag);
752 63243 : _aux->associateVectorToTag(*_aux->system().current_local_solution.get(), tag);
753 63243 : }
754 :
755 : void
756 135 : FEProblemBase::needSolutionState(unsigned int state, Moose::SolutionIterationType iteration_type)
757 : {
758 270 : for (auto & sys : _solver_systems)
759 135 : sys->needSolutionState(state, iteration_type);
760 135 : _aux->needSolutionState(state, iteration_type);
761 135 : }
762 :
763 : bool
764 48 : FEProblemBase::hasSolutionState(unsigned int state,
765 : Moose::SolutionIterationType iteration_type) const
766 : {
767 48 : bool has_solution_state = false;
768 107 : for (auto & sys : _solver_systems)
769 59 : has_solution_state |= sys->hasSolutionState(state, iteration_type);
770 48 : has_solution_state |= _aux->hasSolutionState(state, iteration_type);
771 48 : return has_solution_state;
772 : }
773 :
774 : void
775 63243 : FEProblemBase::newAssemblyArray(std::vector<std::shared_ptr<SolverSystem>> & solver_systems)
776 : {
777 63243 : unsigned int n_threads = libMesh::n_threads();
778 :
779 63243 : _assembly.resize(n_threads);
780 133460 : for (const auto i : make_range(n_threads))
781 : {
782 70217 : _assembly[i].resize(solver_systems.size());
783 140847 : for (const auto j : index_range(solver_systems))
784 70630 : _assembly[i][j] = std::make_unique<Assembly>(*solver_systems[j], i);
785 : }
786 63243 : }
787 :
788 : void
789 59960 : FEProblemBase::initNullSpaceVectors(const InputParameters & parameters,
790 : std::vector<std::shared_ptr<NonlinearSystemBase>> & nls)
791 : {
792 299800 : TIME_SECTION("initNullSpaceVectors", 5, "Initializing Null Space Vectors");
793 :
794 59960 : unsigned int dimNullSpace = parameters.get<unsigned int>("null_space_dimension");
795 : unsigned int dimTransposeNullSpace =
796 59960 : parameters.get<unsigned int>("transpose_null_space_dimension");
797 59960 : unsigned int dimNearNullSpace = parameters.get<unsigned int>("near_null_space_dimension");
798 59984 : for (unsigned int i = 0; i < dimNullSpace; ++i)
799 : {
800 24 : std::ostringstream oss;
801 24 : oss << "_" << i;
802 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
803 : // builder might march over all nodes
804 48 : for (auto & nl : nls)
805 24 : nl->addVector("NullSpace" + oss.str(), false, libMesh::GHOSTED);
806 24 : }
807 119920 : _subspace_dim["NullSpace"] = dimNullSpace;
808 59972 : for (unsigned int i = 0; i < dimTransposeNullSpace; ++i)
809 : {
810 12 : std::ostringstream oss;
811 12 : oss << "_" << i;
812 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
813 : // builder might march over all nodes
814 24 : for (auto & nl : nls)
815 12 : nl->addVector("TransposeNullSpace" + oss.str(), false, libMesh::GHOSTED);
816 12 : }
817 119920 : _subspace_dim["TransposeNullSpace"] = dimTransposeNullSpace;
818 59960 : for (unsigned int i = 0; i < dimNearNullSpace; ++i)
819 : {
820 0 : std::ostringstream oss;
821 0 : oss << "_" << i;
822 : // do not project, since this will be recomputed, but make it ghosted, since the near-nullspace
823 : // builder might march over all semilocal nodes
824 0 : for (auto & nl : nls)
825 0 : nl->addVector("NearNullSpace" + oss.str(), false, libMesh::GHOSTED);
826 0 : }
827 119920 : _subspace_dim["NearNullSpace"] = dimNearNullSpace;
828 59960 : }
829 :
830 180315 : FEProblemBase::~FEProblemBase()
831 : {
832 : // Flush the Console stream, the underlying call to Console::mooseConsole
833 : // relies on a call to Output::checkInterval that has references to
834 : // _time, etc. If it is not flushed here memory problems arise if you have
835 : // an unflushed stream and start destructing things.
836 60105 : _console << std::flush;
837 :
838 60105 : unsigned int n_threads = libMesh::n_threads();
839 125845 : for (unsigned int i = 0; i < n_threads; i++)
840 : {
841 65740 : _zero[i].release();
842 65740 : _phi_zero[i].release();
843 65740 : _scalar_zero[i].release();
844 65740 : _grad_zero[i].release();
845 65740 : _grad_phi_zero[i].release();
846 65740 : _second_zero[i].release();
847 65740 : _second_phi_zero[i].release();
848 65740 : _vector_zero[i].release();
849 65740 : _vector_curl_zero[i].release();
850 65740 : _ad_zero[i].release();
851 65740 : _ad_grad_zero[i].release();
852 65740 : _ad_second_zero[i].release();
853 : }
854 :
855 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
856 60105 : if (!_app.isUltimateMaster())
857 : {
858 11481 : auto ierr = PetscOptionsDestroy(&_petsc_option_data_base);
859 : // Don't throw on destruction
860 11481 : CHKERRABORT(this->comm().get(), ierr);
861 : }
862 : #endif
863 60105 : }
864 :
865 : void
866 0 : FEProblemBase::setCoordSystem(const std::vector<SubdomainName> & blocks,
867 : const MultiMooseEnum & coord_sys)
868 : {
869 0 : TIME_SECTION("setCoordSystem", 5, "Setting Coordinate System");
870 0 : _mesh.setCoordSystem(blocks, coord_sys);
871 0 : }
872 :
873 : void
874 0 : FEProblemBase::setAxisymmetricCoordAxis(const MooseEnum & rz_coord_axis)
875 : {
876 0 : _mesh.setAxisymmetricCoordAxis(rz_coord_axis);
877 0 : }
878 :
879 : const ConstElemRange &
880 1711 : FEProblemBase::getEvaluableElementRange()
881 : {
882 1711 : if (!_evaluable_local_elem_range)
883 : {
884 749 : std::vector<const DofMap *> dof_maps(es().n_systems());
885 2247 : for (const auto i : make_range(es().n_systems()))
886 : {
887 1498 : const auto & sys = es().get_system(i);
888 1498 : dof_maps[i] = &sys.get_dof_map();
889 : }
890 : _evaluable_local_elem_range =
891 1498 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
892 2247 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
893 749 : }
894 1711 : return *_evaluable_local_elem_range;
895 : }
896 :
897 : const ConstElemRange &
898 208 : FEProblemBase::getNonlinearEvaluableElementRange()
899 : {
900 208 : if (!_nl_evaluable_local_elem_range)
901 : {
902 208 : std::vector<const DofMap *> dof_maps(_nl.size());
903 416 : for (const auto i : index_range(dof_maps))
904 208 : dof_maps[i] = &_nl[i]->dofMap();
905 : _nl_evaluable_local_elem_range =
906 416 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
907 624 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
908 208 : }
909 :
910 208 : return *_nl_evaluable_local_elem_range;
911 : }
912 :
913 : void
914 60912 : FEProblemBase::initialSetup()
915 : {
916 304560 : TIME_SECTION("initialSetup", 2, "Performing Initial Setup");
917 :
918 60912 : SubProblem::initialSetup();
919 :
920 60912 : if (_app.isRecovering() + _app.isRestarting() + bool(_app.getExReaderForRestart()) > 1)
921 0 : mooseError("Checkpoint recovery and restart and exodus restart are all mutually exclusive.");
922 :
923 60912 : if (_skip_exception_check)
924 9 : mooseWarning("MOOSE may fail to catch an exception when the \"skip_exception_check\" parameter "
925 : "is used. If you receive a terse MPI error during execution, remove this "
926 : "parameter and rerun your simulation");
927 :
928 : // set state flag indicating that we are in or beyond initialSetup.
929 : // This can be used to throw errors in methods that _must_ be called at construction time.
930 60912 : _started_initial_setup = true;
931 60912 : setCurrentExecuteOnFlag(EXEC_INITIAL);
932 :
933 : // Setup the solution states (current, old, etc) in each system based on
934 : // its default and the states requested of each of its variables
935 122146 : for (const auto i : index_range(_solver_systems))
936 : {
937 61234 : _solver_systems[i]->initSolutionState();
938 61234 : if (getDisplacedProblem())
939 2013 : getDisplacedProblem()->solverSys(i).initSolutionState();
940 : }
941 60912 : _aux->initSolutionState();
942 60912 : if (getDisplacedProblem())
943 2013 : getDisplacedProblem()->auxSys().initSolutionState();
944 :
945 : // always execute to get the max number of DoF per element and node needed to initialize phi_zero
946 : // variables
947 60912 : dof_id_type global_max_var_n_dofs_per_elem = 0;
948 122146 : for (const auto i : index_range(_solver_systems))
949 : {
950 61234 : auto & sys = *_solver_systems[i];
951 : dof_id_type max_var_n_dofs_per_elem;
952 : dof_id_type max_var_n_dofs_per_node;
953 : {
954 306170 : TIME_SECTION("computingMaxDofs", 3, "Computing Max Dofs Per Element");
955 :
956 61234 : MaxVarNDofsPerElem mvndpe(*this, sys);
957 61234 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mvndpe);
958 61234 : max_var_n_dofs_per_elem = mvndpe.max();
959 61234 : _communicator.max(max_var_n_dofs_per_elem);
960 :
961 61234 : MaxVarNDofsPerNode mvndpn(*this, sys);
962 61234 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), mvndpn);
963 61234 : max_var_n_dofs_per_node = mvndpn.max();
964 61234 : _communicator.max(max_var_n_dofs_per_node);
965 61234 : global_max_var_n_dofs_per_elem =
966 61234 : std::max(global_max_var_n_dofs_per_elem, max_var_n_dofs_per_elem);
967 61234 : }
968 :
969 : {
970 306170 : TIME_SECTION("assignMaxDofs", 5, "Assigning Maximum Dofs Per Elem");
971 :
972 61234 : sys.assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
973 61234 : auto displaced_problem = getDisplacedProblem();
974 61234 : if (displaced_problem)
975 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
976 :
977 61234 : sys.assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
978 61234 : if (displaced_problem)
979 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
980 61234 : }
981 : }
982 :
983 : {
984 304560 : TIME_SECTION("resizingVarValues", 5, "Resizing Variable Values");
985 :
986 127894 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
987 : {
988 133964 : _phi_zero[tid].resize(global_max_var_n_dofs_per_elem, std::vector<Real>(getMaxQps(), 0.));
989 133964 : _grad_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
990 133964 : std::vector<RealGradient>(getMaxQps(), RealGradient(0.)));
991 133964 : _second_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
992 133964 : std::vector<RealTensor>(getMaxQps(), RealTensor(0.)));
993 : }
994 60912 : }
995 :
996 : // Set up stateful material property redistribution, if we suspect
997 : // it may be necessary later.
998 60912 : addAnyRedistributers();
999 :
1000 60912 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1001 : {
1002 : // Only load all of the vectors if we're recovering
1003 4579 : _req.set().setLoadAllVectors(_app.isRecovering());
1004 :
1005 : // This forces stateful material property loading to be an exact one-to-one match
1006 4579 : if (_app.isRecovering())
1007 : {
1008 16172 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1009 12129 : props->setRecovering();
1010 :
1011 : #ifdef MOOSE_KOKKOS_ENABLED
1012 16032 : for (auto props :
1013 20040 : {&_kokkos_material_props, &_kokkos_bnd_material_props, &_kokkos_neighbor_material_props})
1014 12024 : props->setRecovering();
1015 : #endif
1016 : }
1017 :
1018 22895 : TIME_SECTION("restore", 3, "Restoring from backup");
1019 :
1020 : // We could have a cached backup when this app is a sub-app and has been given a Backup
1021 4579 : if (!_app.hasInitialBackup())
1022 3798 : _app.restore(_app.restartFolderBase(_app.getRestartRecoverFileBase()), _app.isRestarting());
1023 : else
1024 781 : _app.restoreFromInitialBackup(_app.isRestarting());
1025 :
1026 : /**
1027 : * If this is a restart run, the user may want to override the start time, which we already set
1028 : * in the constructor. "_time" however will have been "restored" from the restart file. We need
1029 : * to honor the original request of the developer now that the restore has been completed.
1030 : */
1031 4546 : if (_app.isRestarting())
1032 : {
1033 503 : if (_app.hasStartTime())
1034 173 : _time = _time_old = _app.getStartTime();
1035 : else
1036 330 : _time_old = _time;
1037 : }
1038 4546 : }
1039 : else
1040 : {
1041 56333 : libMesh::ExodusII_IO * reader = _app.getExReaderForRestart();
1042 :
1043 56333 : if (reader)
1044 : {
1045 1890 : TIME_SECTION("copyingFromExodus", 3, "Copying Variables From Exodus");
1046 :
1047 764 : for (auto & sys : _solver_systems)
1048 389 : sys->copyVars(*reader);
1049 375 : _aux->copyVars(*reader);
1050 375 : }
1051 : else
1052 : {
1053 55955 : if (_solver_systems[0]->hasVarCopy() || _aux->hasVarCopy())
1054 0 : mooseError("Need Exodus reader to restart variables but the reader is not available\n"
1055 : "Use either FileMesh with an Exodus mesh file or FileMeshGenerator with an "
1056 : "Exodus mesh file and with use_for_exodus_restart equal to true");
1057 : }
1058 : }
1059 :
1060 : // Perform output related setups
1061 60876 : _app.getOutputWarehouse().initialSetup();
1062 :
1063 : // Flush all output to _console that occur during construction and initialization of objects
1064 60830 : _app.getOutputWarehouse().mooseConsole();
1065 :
1066 : // Build Refinement and Coarsening maps for stateful material projections if necessary
1067 63099 : if ((_adaptivity.isOn() || _num_grid_steps) &&
1068 2269 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1069 2206 : _neighbor_material_props.hasStatefulProperties()))
1070 : {
1071 63 : if (_has_internal_edge_residual_objects)
1072 6 : mooseError("Stateful neighbor material properties do not work with mesh adaptivity");
1073 :
1074 57 : _mesh.buildRefinementAndCoarseningMaps(_assembly[0][0].get());
1075 : }
1076 :
1077 60824 : if (!_app.isRecovering())
1078 : {
1079 : /**
1080 : * If we are not recovering but we are doing restart (_app.getExodusFileRestart() == true) with
1081 : * additional uniform refinements. We have to delay the refinement until this point
1082 : * in time so that the equation systems are initialized and projections can be performed.
1083 : */
1084 56781 : if (_mesh.uniformRefineLevel() > 0 && _app.getExodusFileRestart())
1085 : {
1086 10 : if (!_app.isUltimateMaster())
1087 0 : mooseError(
1088 : "Doing extra refinements when restarting is NOT supported for sub-apps of a MultiApp");
1089 :
1090 10 : adaptivity().uniformRefineWithProjection();
1091 : }
1092 : }
1093 :
1094 60824 : unsigned int n_threads = libMesh::n_threads();
1095 :
1096 : // Convergence initial setup
1097 : {
1098 304120 : TIME_SECTION("convergenceInitialSetup", 5, "Initializing Convergence objects");
1099 :
1100 127667 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1101 66858 : _convergences.initialSetup(tid);
1102 60809 : }
1103 :
1104 : // UserObject initialSetup
1105 60809 : std::set<std::string> depend_objects_ic = _ics.getDependObjects();
1106 60809 : std::set<std::string> depend_objects_aux = _aux->getDependObjects();
1107 :
1108 60809 : std::map<int, std::vector<UserObjectBase *>> group_userobjs;
1109 :
1110 : // This replaces all prior updateDependObjects calls on the old user object warehouses.
1111 60809 : TheWarehouse::Query uo_query = theWarehouse().query().condition<AttribSystem>("UserObject");
1112 60809 : std::vector<UserObjectBase *> userobjs;
1113 60809 : uo_query.queryInto(userobjs);
1114 60809 : groupUserObjects(
1115 60809 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1116 :
1117 134177 : for (auto obj : userobjs)
1118 220104 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1119 :
1120 : #ifdef MOOSE_KOKKOS_ENABLED
1121 : {
1122 : TheWarehouse::Query uo_query =
1123 46122 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject");
1124 46122 : std::vector<UserObjectBase *> userobjs;
1125 46122 : uo_query.queryInto(userobjs);
1126 46122 : groupUserObjects(
1127 46122 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1128 :
1129 47362 : for (auto obj : userobjs)
1130 3720 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1131 46122 : }
1132 : #endif
1133 :
1134 91754 : for (auto & [group, objs] : group_userobjs)
1135 105500 : for (auto obj : objs)
1136 74555 : obj->initialSetup();
1137 :
1138 : // check if jacobian calculation is done in userobject
1139 127466 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1140 66737 : checkUserObjectJacobianRequirement(tid);
1141 :
1142 : // Check whether nonlocal coupling is required or not
1143 60729 : checkNonlocalCoupling();
1144 60729 : if (_requires_nonlocal_coupling)
1145 63 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
1146 :
1147 : {
1148 303645 : TIME_SECTION("initializingFunctions", 5, "Initializing Functions");
1149 :
1150 : // Call the initialSetup methods for functions
1151 127434 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1152 : {
1153 66729 : reinitScalars(tid); // initialize scalars so they are properly sized for use as input into
1154 : // ParsedFunctions
1155 66729 : _functions.initialSetup(tid);
1156 : }
1157 :
1158 : #ifdef MOOSE_KOKKOS_ENABLED
1159 46053 : _kokkos_functions.initialSetup();
1160 : #endif
1161 60705 : }
1162 :
1163 : {
1164 303525 : TIME_SECTION("initializingRandomObjects", 5, "Initializing Random Objects");
1165 :
1166 : // Random interface objects
1167 61029 : for (const auto & it : _random_data_objects)
1168 324 : it.second->updateSeeds(EXEC_INITIAL);
1169 60705 : }
1170 :
1171 60705 : if (!_app.isRecovering())
1172 : {
1173 56662 : computeUserObjects(EXEC_INITIAL, Moose::PRE_IC);
1174 :
1175 : {
1176 283310 : TIME_SECTION("ICinitialSetup", 5, "Setting Up Initial Conditions");
1177 :
1178 119271 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1179 : {
1180 62615 : _ics.initialSetup(tid);
1181 62609 : _fv_ics.initialSetup(tid);
1182 : }
1183 :
1184 56656 : _scalar_ics.initialSetup();
1185 56656 : }
1186 :
1187 56656 : projectSolution();
1188 : }
1189 :
1190 : // Materials
1191 60693 : if (_all_materials.hasActiveObjects(0))
1192 : {
1193 42535 : TIME_SECTION("materialInitialSetup", 3, "Setting Up Materials");
1194 :
1195 17754 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1196 : {
1197 : // Sort the Material objects, these will be actually computed by MOOSE in reinit methods.
1198 9265 : _materials.sort(tid);
1199 9262 : _interface_materials.sort(tid);
1200 :
1201 : // Call initialSetup on all material objects
1202 9262 : _all_materials.initialSetup(tid);
1203 :
1204 : // Discrete materials may insert additional dependencies on materials during the initial
1205 : // setup. Therefore we resolve the dependencies once more, now with the additional
1206 : // dependencies due to discrete materials.
1207 9247 : if (_discrete_materials.hasActiveObjects())
1208 : {
1209 56 : _materials.sort(tid);
1210 56 : _interface_materials.sort(tid);
1211 : }
1212 : }
1213 :
1214 : #ifdef MOOSE_KOKKOS_ENABLED
1215 6383 : _kokkos_materials.sort(0, true);
1216 : #endif
1217 :
1218 : {
1219 42435 : TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");
1220 :
1221 8487 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1222 :
1223 16299 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1224 7812 : _neighbor_material_props.hasStatefulProperties())
1225 675 : _has_initialized_stateful = true;
1226 : #ifdef MOOSE_KOKKOS_ENABLED
1227 6381 : if (_kokkos_material_props.hasStatefulProperties() ||
1228 12545 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1229 6164 : _kokkos_neighbor_material_props.hasStatefulProperties())
1230 217 : _has_initialized_stateful = true;
1231 : #endif
1232 8487 : }
1233 8487 : }
1234 :
1235 : // setRestartInPlace() is set because the property maps have now been setup and we can
1236 : // dataLoad() them directly in place
1237 : // setRecovering() is set because from now on we require a one-to-one mapping of
1238 : // stateful properties because we shouldn't be declaring any more
1239 242692 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1240 : {
1241 182019 : props->setRestartInPlace();
1242 182019 : props->setRecovering();
1243 : }
1244 :
1245 127335 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1246 : {
1247 66662 : _internal_side_indicators.initialSetup(tid);
1248 66662 : _indicators.initialSetup(tid);
1249 66662 : _markers.sort(tid);
1250 66662 : _markers.initialSetup(tid);
1251 : }
1252 :
1253 : #ifdef LIBMESH_ENABLE_AMR
1254 :
1255 60673 : if (!_app.isRecovering() && !_app.restoredInitialBackupMesh())
1256 : {
1257 56628 : unsigned int n = adaptivity().getInitialSteps();
1258 56628 : if (n && !_app.isUltimateMaster() && _app.isRestarting())
1259 0 : mooseError("Cannot perform initial adaptivity during restart on sub-apps of a MultiApp!");
1260 :
1261 56628 : initialAdaptMesh();
1262 : }
1263 :
1264 : #endif // LIBMESH_ENABLE_AMR
1265 :
1266 60670 : if (!_app.isRecovering() && !_app.isRestarting())
1267 : {
1268 : // During initial setup the solution is copied to the older solution states (old, older, etc)
1269 56124 : copySolutionsBackwards();
1270 :
1271 : // Check if there are old state initial conditions
1272 56124 : auto ics = _ics.getActiveObjects();
1273 56124 : auto fv_ics = _fv_ics.getActiveObjects();
1274 56124 : auto scalar_ics = _scalar_ics.getActiveObjects();
1275 56124 : unsigned short ic_state_max = 0;
1276 :
1277 168372 : auto findMax = [&ic_state_max](const auto & obj_list)
1278 : {
1279 199174 : for (auto ic : obj_list.getActiveObjects())
1280 30802 : ic_state_max = std::max(ic_state_max, ic->getState());
1281 224496 : };
1282 56124 : findMax(_ics);
1283 56124 : findMax(_fv_ics);
1284 56124 : findMax(_scalar_ics);
1285 :
1286 : // if there are old state ICs, compute them and write to old states accordingly
1287 56124 : if (ic_state_max > 0)
1288 : {
1289 : // state 0 copy (we'll overwrite current state when evaluating ICs and need to restore it once
1290 : // we're done with the old/older state ICs)
1291 0 : std::vector<std::unique_ptr<NumericVector<Real>>> state0_sys_buffers(_solver_systems.size());
1292 0 : std::unique_ptr<NumericVector<Real>> state0_aux_buffer;
1293 :
1294 : // save state 0
1295 0 : for (const auto i : index_range(_solver_systems))
1296 0 : state0_sys_buffers[i] = _solver_systems[i]->solutionState(0).clone();
1297 :
1298 0 : state0_aux_buffer = _aux->solutionState(0).clone();
1299 :
1300 : // compute old state ICs
1301 0 : for (_current_ic_state = 1; _current_ic_state <= ic_state_max; _current_ic_state++)
1302 : {
1303 0 : projectSolution();
1304 :
1305 0 : for (auto & sys : _solver_systems)
1306 0 : sys->solutionState(_current_ic_state) = sys->solutionState(0);
1307 :
1308 0 : _aux->solutionState(_current_ic_state) = _aux->solutionState(0);
1309 : }
1310 0 : _current_ic_state = 0;
1311 :
1312 : // recover state 0
1313 0 : for (const auto i : index_range(_solver_systems))
1314 : {
1315 0 : _solver_systems[i]->solutionState(0) = *state0_sys_buffers[i];
1316 0 : _solver_systems[i]->solutionState(0).close();
1317 0 : _solver_systems[i]->update();
1318 : }
1319 0 : _aux->solutionState(0) = *state0_aux_buffer;
1320 0 : _aux->solutionState(0).close();
1321 0 : _aux->update();
1322 0 : }
1323 56124 : }
1324 :
1325 60670 : if (!_app.isRecovering())
1326 : {
1327 56627 : if (haveXFEM())
1328 0 : updateMeshXFEM();
1329 : }
1330 :
1331 : // Call initialSetup on the solver systems
1332 121662 : for (auto & sys : _solver_systems)
1333 60992 : sys->initialSetup();
1334 :
1335 : // Auxilary variable initialSetup calls
1336 60670 : _aux->initialSetup();
1337 :
1338 60664 : if (_displaced_problem)
1339 : // initialSetup for displaced systems
1340 2013 : _displaced_problem->initialSetup();
1341 :
1342 121650 : for (auto & sys : _solver_systems)
1343 60986 : sys->setSolution(*(sys->system().current_local_solution.get()));
1344 :
1345 : // Update the nearest node searches (has to be called after the problem is all set up)
1346 : // We do this here because this sets up the Element's DoFs to ghost
1347 60664 : updateGeomSearch(GeometricSearchData::NEAREST_NODE);
1348 :
1349 60664 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
1350 60664 : if (_displaced_mesh)
1351 2013 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
1352 :
1353 : // We need to move the mesh in order to build a map between mortar secondary and primary
1354 : // interfaces. This map will then be used by the AgumentSparsityOnInterface ghosting functor to
1355 : // know which dofs we need ghosted when we call EquationSystems::reinit
1356 60664 : if (_displaced_problem && _mortar_data->hasDisplacedObjects())
1357 : {
1358 114 : _displaced_problem->updateMesh();
1359 : // if displacements were applied to the mesh, the mortar mesh should be updated too
1360 114 : updateMortarMesh();
1361 : }
1362 :
1363 : // Possibly reinit one more time to get ghosting correct
1364 60664 : reinitBecauseOfGhostingOrNewGeomObjects();
1365 :
1366 60664 : if (_displaced_mesh)
1367 2013 : _displaced_problem->updateMesh();
1368 :
1369 60664 : updateGeomSearch(); // Call all of the rest of the geometric searches
1370 :
1371 121647 : for (auto & sys : _solver_systems)
1372 : {
1373 60986 : const auto & tis = sys->getTimeIntegrators();
1374 :
1375 : {
1376 304930 : TIME_SECTION("timeIntegratorInitialSetup", 5, "Initializing Time Integrator");
1377 90987 : for (auto & ti : tis)
1378 30004 : ti->initialSetup();
1379 60983 : }
1380 : }
1381 :
1382 : // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ main-app restartable data
1383 : // has been restored
1384 :
1385 : // Call initialSetup on the MultiApps
1386 60661 : if (_multi_apps.hasObjects())
1387 : {
1388 35310 : TIME_SECTION("initialSetupMultiApps", 2, "Initializing MultiApps", false);
1389 7062 : _multi_apps.initialSetup();
1390 7056 : }
1391 :
1392 : // Call initialSetup on the transfers
1393 : {
1394 303275 : TIME_SECTION("initialSetupTransfers", 2, "Initializing Transfers");
1395 :
1396 60655 : _transfers.initialSetup();
1397 :
1398 : // Call initialSetup on the MultiAppTransfers to be executed on TO_MULTIAPP
1399 60655 : const auto & to_multi_app_objects = _to_multi_app_transfers.getActiveObjects();
1400 65788 : for (const auto & transfer : to_multi_app_objects)
1401 : {
1402 5181 : transfer->setCurrentDirection(Transfer::DIRECTION::TO_MULTIAPP);
1403 5181 : transfer->initialSetup();
1404 : }
1405 :
1406 : // Call initialSetup on the MultiAppTransfers to be executed on FROM_MULTIAPP
1407 60607 : const auto & from_multi_app_objects = _from_multi_app_transfers.getActiveObjects();
1408 66844 : for (const auto & transfer : from_multi_app_objects)
1409 : {
1410 6279 : transfer->setCurrentDirection(Transfer::DIRECTION::FROM_MULTIAPP);
1411 6279 : transfer->initialSetup();
1412 : }
1413 :
1414 : // Call initialSetup on the MultiAppTransfers to be executed on BETWEEN_MULTIAPP
1415 60565 : const auto & between_multi_app_objects = _between_multi_app_transfers.getActiveObjects();
1416 62035 : for (const auto & transfer : between_multi_app_objects)
1417 : {
1418 1470 : transfer->setCurrentDirection(Transfer::DIRECTION::BETWEEN_MULTIAPP);
1419 1470 : transfer->initialSetup();
1420 : }
1421 60565 : }
1422 :
1423 60565 : if (_boundary_restricted_node_integrity_check)
1424 : {
1425 181659 : TIME_SECTION("BoundaryRestrictedNodeIntegrityCheck", 5);
1426 :
1427 : // check that variables are defined along boundaries of boundary restricted nodal objects
1428 60553 : const auto & bnd_nodes = getCurrentAlgebraicBndNodeRange();
1429 60553 : BoundaryNodeIntegrityCheckThread bnict(*this, uo_query);
1430 60553 : Threads::parallel_reduce(bnd_nodes, bnict);
1431 :
1432 : // Nodal bcs aren't threaded
1433 120175 : for (auto & nl : _nl)
1434 : {
1435 59640 : const auto & nodal_bcs = nl->getNodalBCWarehouse();
1436 59640 : if (!nodal_bcs.hasBoundaryObjects())
1437 28097 : continue;
1438 :
1439 2502675 : for (const auto & bnode : bnd_nodes)
1440 : {
1441 2471141 : const auto boundary_id = bnode->_bnd_id;
1442 2471141 : const Node * const node = bnode->_node;
1443 :
1444 2471141 : if (node->processor_id() != this->processor_id())
1445 582846 : continue;
1446 :
1447 1888295 : const auto & bnd_name = _mesh.getBoundaryName(boundary_id);
1448 :
1449 : // Avoid assertion in getBoundaryObjects that we have boundary objects for this boundary ID
1450 1888295 : if (!nodal_bcs.hasBoundaryObjects(boundary_id))
1451 968741 : continue;
1452 :
1453 919554 : const auto & bnd_objects = nodal_bcs.getBoundaryObjects(boundary_id);
1454 1930734 : for (const auto & bnd_object : bnd_objects)
1455 : {
1456 1011189 : const auto & bnd_variable = bnd_object->variable();
1457 : // Skip if this object uses geometric search because coupled variables may be defined on
1458 : // paired boundaries instead of the boundary this node is on. Also skip if this boundary
1459 : // condition isn't applicable to the current node, e.g. if the node doesn't have any
1460 : // degrees of freedom for the boundary condition's variable
1461 1011189 : if (!bnd_object->requiresGeometricSearch() &&
1462 2022378 : bnd_object->checkVariableBoundaryIntegrity() &&
1463 1011189 : node->n_dofs(nl->number(), bnd_variable.number()))
1464 : {
1465 : std::set<MooseVariableFieldBase *> vars_to_omit = {
1466 : &static_cast<MooseVariableFieldBase &>(
1467 1930910 : const_cast<MooseVariableBase &>(bnd_variable))};
1468 :
1469 1930910 : boundaryIntegrityCheckError(
1470 1930901 : *bnd_object, bnd_object->checkAllVariables(*node, vars_to_omit), bnd_name);
1471 965446 : }
1472 : }
1473 : }
1474 : }
1475 60535 : }
1476 :
1477 60547 : if (_boundary_restricted_elem_integrity_check)
1478 : {
1479 181551 : TIME_SECTION("BoundaryRestrictedElemIntegrityCheck", 5);
1480 :
1481 : // check that variables are defined along boundaries of boundary restricted elemental objects
1482 60517 : ConstBndElemRange & bnd_elems = *mesh().getBoundaryElementRange();
1483 60517 : BoundaryElemIntegrityCheckThread beict(*this, uo_query);
1484 60517 : Threads::parallel_reduce(bnd_elems, beict);
1485 60505 : }
1486 :
1487 60535 : if (!_app.isRecovering())
1488 : {
1489 56492 : execTransfers(EXEC_INITIAL);
1490 :
1491 56492 : bool converged = execMultiApps(EXEC_INITIAL);
1492 56486 : if (!converged)
1493 3 : mooseError("failed to converge initial MultiApp");
1494 :
1495 : // We'll backup the Multiapp here
1496 56483 : backupMultiApps(EXEC_INITIAL);
1497 :
1498 118861 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1499 62378 : reinitScalars(tid);
1500 :
1501 56483 : execute(EXEC_INITIAL);
1502 :
1503 : // The FEProblemBase::execute method doesn't call all the systems on EXEC_INITIAL, but it does
1504 : // set/unset the current flag. Therefore, this resets the current flag to EXEC_INITIAL so that
1505 : // subsequent calls (e.g., executeControls) have the proper flag.
1506 56405 : setCurrentExecuteOnFlag(EXEC_INITIAL);
1507 : }
1508 :
1509 : // Here we will initialize the stateful properties once more since they may have been updated
1510 : // during initialSetup by calls to computeProperties.
1511 : //
1512 : // It's really bad that we don't allow this during restart. It means that we can't add new
1513 : // stateful materials
1514 : // during restart. This is only happening because this _has_ to be below initial userobject
1515 : // execution.
1516 : // Otherwise this could be done up above... _before_ restoring restartable data... which would
1517 : // allow you to have
1518 : // this happen during restart. I honestly have no idea why this has to happen after initial user
1519 : // object computation.
1520 : // THAT is something we should fix... so I've opened this ticket: #5804
1521 60448 : if (!_app.isRecovering() && !_app.isRestarting())
1522 : {
1523 111232 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1524 55330 : _neighbor_material_props.hasStatefulProperties())
1525 : {
1526 2860 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1527 :
1528 572 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1529 572 : }
1530 : #ifdef MOOSE_KOKKOS_ENABLED
1531 41500 : if (_kokkos_material_props.hasStatefulProperties() ||
1532 82835 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1533 41335 : _kokkos_neighbor_material_props.hasStatefulProperties())
1534 : {
1535 825 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1536 :
1537 165 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1538 165 : }
1539 : #endif
1540 : }
1541 :
1542 : // Control Logic
1543 60448 : _control_warehouse.initialSetup();
1544 60448 : executeControls(EXEC_INITIAL);
1545 :
1546 : // Scalar variables need to reinited for the initial conditions to be available for output
1547 126748 : for (unsigned int tid = 0; tid < n_threads; tid++)
1548 66327 : reinitScalars(tid);
1549 :
1550 60421 : if (_displaced_mesh)
1551 2013 : _displaced_problem->syncSolutions();
1552 :
1553 : // Writes all calls to _console from initialSetup() methods
1554 60421 : _app.getOutputWarehouse().mooseConsole();
1555 :
1556 60421 : if (_requires_nonlocal_coupling)
1557 : {
1558 63 : setNonlocalCouplingMatrix();
1559 126 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1560 126 : for (auto & assembly : _assembly[tid])
1561 63 : assembly->initNonlocalCoupling();
1562 : }
1563 :
1564 : {
1565 302105 : TIME_SECTION("lineSearchInitialSetup", 5, "Initializing Line Search");
1566 :
1567 60421 : if (_line_search)
1568 0 : _line_search->initialSetup();
1569 60421 : }
1570 :
1571 : // Perform Reporter get/declare check
1572 60421 : _reporter_data.check();
1573 :
1574 : // We do this late to allow objects to get late restartable data
1575 60421 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1576 4546 : _app.finalizeRestore();
1577 :
1578 60421 : setCurrentExecuteOnFlag(EXEC_NONE);
1579 60421 : }
1580 :
1581 : void
1582 61273 : FEProblemBase::checkDuplicatePostprocessorVariableNames()
1583 : {
1584 113456 : for (const auto & pp : _reporter_data.getPostprocessorNames())
1585 52183 : if (hasScalarVariable(pp))
1586 0 : mooseError("Postprocessor \"" + pp +
1587 61273 : "\" has the same name as a scalar variable in the system.");
1588 61273 : }
1589 :
1590 : void
1591 270990 : FEProblemBase::timestepSetup()
1592 : {
1593 270990 : SubProblem::timestepSetup();
1594 :
1595 270990 : if (_t_step > 1 && _num_grid_steps)
1596 : {
1597 31 : libMesh::MeshRefinement mesh_refinement(_mesh);
1598 31 : std::unique_ptr<libMesh::MeshRefinement> displaced_mesh_refinement(nullptr);
1599 31 : if (_displaced_mesh)
1600 23 : displaced_mesh_refinement = std::make_unique<libMesh::MeshRefinement>(*_displaced_mesh);
1601 :
1602 62 : for (MooseIndex(_num_grid_steps) i = 0; i < _num_grid_steps; ++i)
1603 : {
1604 31 : if (_displaced_problem)
1605 : // If the DisplacedProblem is active, undisplace the DisplacedMesh in preparation for
1606 : // refinement. We can't safely refine the DisplacedMesh directly, since the Hilbert keys
1607 : // computed on the inconsistenly-displaced Mesh are different on different processors,
1608 : // leading to inconsistent Hilbert keys. We must do this before the undisplaced Mesh is
1609 : // coarsensed, so that the element and node numbering is still consistent. We also have to
1610 : // make sure this is done during every step of coarsening otherwise different partitions
1611 : // will be generated for the reference and displaced meshes (even for replicated)
1612 23 : _displaced_problem->undisplaceMesh();
1613 :
1614 31 : mesh_refinement.uniformly_coarsen();
1615 31 : if (_displaced_mesh)
1616 23 : displaced_mesh_refinement->uniformly_coarsen();
1617 :
1618 : // Mark this as an intermediate change because we do not yet want to reinit_systems. E.g. we
1619 : // need things to happen in the following order for the undisplaced problem:
1620 : // u1) EquationSystems::reinit_solutions. This will restrict the solution vectors and then
1621 : // contract the mesh
1622 : // u2) MooseMesh::meshChanged. This will update the node/side lists and other
1623 : // things which needs to happen after the contraction
1624 : // u3) GeometricSearchData::reinit. Once the node/side lists are updated we can perform our
1625 : // geometric searches which will aid in determining sparsity patterns
1626 : //
1627 : // We do these things for the displaced problem (if it exists)
1628 : // d1) EquationSystems::reinit. Restrict the displaced problem vector copies and then contract
1629 : // the mesh. It's safe to do a full reinit with the displaced because there are no
1630 : // matrices that sparsity pattern calculations will be conducted for
1631 : // d2) MooseMesh::meshChanged. This will update the node/side lists and other
1632 : // things which needs to happen after the contraction
1633 : // d3) UpdateDisplacedMeshThread::operator(). Re-displace the mesh using the *displaced*
1634 : // solution vector copy because we don't know the state of the reference solution vector.
1635 : // It's safe to use the displaced copy because we are outside of a non-linear solve,
1636 : // and there is no concern about differences between solution and current_local_solution
1637 : // d4) GeometricSearchData::reinit. With the node/side lists updated and the mesh
1638 : // re-displaced, we can perform our geometric searches, which will aid in determining the
1639 : // sparsity pattern of the matrix held by the libMesh::ImplicitSystem held by the
1640 : // NonlinearSystem held by this
1641 31 : meshChanged(
1642 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
1643 : }
1644 :
1645 : // u4) Now that all the geometric searches have been done (both undisplaced and displaced),
1646 : // we're ready to update the sparsity pattern
1647 31 : es().reinit_systems();
1648 31 : }
1649 :
1650 270990 : _control_warehouse.timestepSetup();
1651 270990 : if (_line_search)
1652 0 : _line_search->timestepSetup();
1653 :
1654 : // Random interface objects
1655 272370 : for (const auto & it : _random_data_objects)
1656 1380 : it.second->updateSeeds(EXEC_TIMESTEP_BEGIN);
1657 :
1658 270990 : unsigned int n_threads = libMesh::n_threads();
1659 568801 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1660 : {
1661 297811 : _all_materials.timestepSetup(tid);
1662 297811 : _functions.timestepSetup(tid);
1663 : }
1664 :
1665 : #ifdef MOOSE_KOKKOS_ENABLED
1666 199005 : _kokkos_functions.timestepSetup();
1667 : #endif
1668 :
1669 270990 : _aux->timestepSetup();
1670 545504 : for (auto & sys : _solver_systems)
1671 274514 : sys->timestepSetup();
1672 :
1673 270990 : if (_displaced_problem)
1674 : // timestepSetup for displaced systems
1675 30674 : _displaced_problem->timestepSetup();
1676 :
1677 568801 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1678 : {
1679 297811 : _internal_side_indicators.timestepSetup(tid);
1680 297811 : _indicators.timestepSetup(tid);
1681 297811 : _markers.timestepSetup(tid);
1682 : }
1683 :
1684 270990 : std::vector<UserObject *> userobjs;
1685 270990 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
1686 606947 : for (auto obj : userobjs)
1687 335957 : obj->timestepSetup();
1688 :
1689 : #ifdef MOOSE_KOKKOS_ENABLED
1690 : {
1691 199005 : std::vector<UserObjectBase *> userobjs;
1692 199005 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
1693 201197 : for (auto obj : userobjs)
1694 2192 : obj->timestepSetup();
1695 199005 : }
1696 : #endif
1697 :
1698 : // Timestep setup of output objects
1699 270990 : _app.getOutputWarehouse().timestepSetup();
1700 :
1701 270990 : if (_requires_nonlocal_coupling)
1702 97 : if (_nonlocal_kernels.hasActiveObjects() || _nonlocal_integrated_bcs.hasActiveObjects())
1703 97 : _has_nonlocal_coupling = true;
1704 270990 : }
1705 :
1706 : unsigned int
1707 760053 : FEProblemBase::getMaxQps() const
1708 : {
1709 760053 : if (_max_qps == std::numeric_limits<unsigned int>::max())
1710 0 : mooseError("Max QPS uninitialized");
1711 760053 : return _max_qps;
1712 : }
1713 :
1714 : Order
1715 52 : FEProblemBase::getMaxScalarOrder() const
1716 : {
1717 52 : return _max_scalar_order;
1718 : }
1719 :
1720 : void
1721 60729 : FEProblemBase::checkNonlocalCoupling()
1722 : {
1723 303645 : TIME_SECTION("checkNonlocalCoupling", 5, "Checking Nonlocal Coupling");
1724 :
1725 127466 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
1726 132593 : for (auto & nl : _nl)
1727 : {
1728 65856 : const auto & all_kernels = nl->getKernelWarehouse();
1729 65856 : const auto & kernels = all_kernels.getObjects(tid);
1730 151000 : for (const auto & kernel : kernels)
1731 : {
1732 : std::shared_ptr<NonlocalKernel> nonlocal_kernel =
1733 85144 : std::dynamic_pointer_cast<NonlocalKernel>(kernel);
1734 85144 : if (nonlocal_kernel)
1735 : {
1736 35 : if (_calculate_jacobian_in_uo)
1737 35 : _requires_nonlocal_coupling = true;
1738 35 : _nonlocal_kernels.addObject(kernel, tid);
1739 : }
1740 85144 : }
1741 : const MooseObjectWarehouse<IntegratedBCBase> & all_integrated_bcs =
1742 65856 : nl->getIntegratedBCWarehouse();
1743 65856 : const auto & integrated_bcs = all_integrated_bcs.getObjects(tid);
1744 74745 : for (const auto & integrated_bc : integrated_bcs)
1745 : {
1746 : std::shared_ptr<NonlocalIntegratedBC> nonlocal_integrated_bc =
1747 8889 : std::dynamic_pointer_cast<NonlocalIntegratedBC>(integrated_bc);
1748 8889 : if (nonlocal_integrated_bc)
1749 : {
1750 28 : if (_calculate_jacobian_in_uo)
1751 28 : _requires_nonlocal_coupling = true;
1752 28 : _nonlocal_integrated_bcs.addObject(integrated_bc, tid);
1753 : }
1754 8889 : }
1755 : }
1756 60729 : }
1757 :
1758 : void
1759 66737 : FEProblemBase::checkUserObjectJacobianRequirement(THREAD_ID tid)
1760 : {
1761 66737 : std::set<const MooseVariableFEBase *> uo_jacobian_moose_vars;
1762 : {
1763 66737 : std::vector<ShapeElementUserObject *> objs;
1764 66737 : theWarehouse()
1765 66737 : .query()
1766 133474 : .condition<AttribInterfaces>(Interfaces::ShapeElementUserObject)
1767 66737 : .condition<AttribThread>(tid)
1768 66737 : .queryInto(objs);
1769 :
1770 66781 : for (const auto & uo : objs)
1771 : {
1772 44 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1773 44 : const auto & mv_deps = uo->jacobianMooseVariables();
1774 44 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1775 : }
1776 66737 : }
1777 : {
1778 66737 : std::vector<ShapeSideUserObject *> objs;
1779 66737 : theWarehouse()
1780 66737 : .query()
1781 133474 : .condition<AttribInterfaces>(Interfaces::ShapeSideUserObject)
1782 66737 : .condition<AttribThread>(tid)
1783 66737 : .queryInto(objs);
1784 66793 : for (const auto & uo : objs)
1785 : {
1786 56 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1787 56 : const auto & mv_deps = uo->jacobianMooseVariables();
1788 56 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1789 : }
1790 66737 : }
1791 :
1792 66737 : _uo_jacobian_moose_vars[tid].assign(uo_jacobian_moose_vars.begin(), uo_jacobian_moose_vars.end());
1793 133474 : std::sort(
1794 133474 : _uo_jacobian_moose_vars[tid].begin(), _uo_jacobian_moose_vars[tid].end(), sortMooseVariables);
1795 66737 : }
1796 :
1797 : void
1798 63 : FEProblemBase::setVariableAllDoFMap(const std::vector<const MooseVariableFEBase *> & moose_vars)
1799 : {
1800 153 : for (unsigned int i = 0; i < moose_vars.size(); ++i)
1801 : {
1802 90 : VariableName var_name = moose_vars[i]->name();
1803 90 : auto & sys = _solver_systems[moose_vars[i]->sys().number()];
1804 90 : sys->setVariableGlobalDoFs(var_name);
1805 90 : _var_dof_map[var_name] = sys->getVariableGlobalDoFs();
1806 90 : }
1807 63 : }
1808 :
1809 : void
1810 374680803 : FEProblemBase::prepare(const Elem * elem, const THREAD_ID tid)
1811 : {
1812 752146632 : for (const auto i : index_range(_solver_systems))
1813 : {
1814 377465829 : _assembly[tid][i]->reinit(elem);
1815 377465829 : _solver_systems[i]->prepare(tid);
1816 :
1817 377465829 : if (i < _num_nl_sys)
1818 : {
1819 : // This method is called outside of residual/Jacobian callbacks during initial condition
1820 : // evaluation
1821 376869674 : if ((!_has_jacobian || !_const_jacobian) && currentlyComputingJacobian())
1822 47240558 : _assembly[tid][i]->prepareJacobianBlock();
1823 376869674 : _assembly[tid][i]->prepareResidual();
1824 376869674 : if (_has_nonlocal_coupling && currentlyComputingJacobian())
1825 8824 : _assembly[tid][i]->prepareNonlocal();
1826 : }
1827 : }
1828 374680803 : _aux->prepare(tid);
1829 :
1830 386325298 : if (_displaced_problem &&
1831 : // _reinit_displaced_neighbor applies to interface type objects which will do computations
1832 : // based on both elem and neighbor. Consequently, despite what you might think by its name, we
1833 : // must make sure we prepare the displaced elem
1834 386325298 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1835 : {
1836 7561302 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), tid);
1837 7561278 : if (_has_nonlocal_coupling)
1838 0 : _displaced_problem->prepareNonlocal(tid);
1839 : }
1840 374680779 : }
1841 :
1842 : void
1843 25443 : FEProblemBase::prepareFace(const Elem * elem, const THREAD_ID tid)
1844 : {
1845 50886 : for (auto & nl : _nl)
1846 25443 : nl->prepareFace(tid, true);
1847 25443 : _aux->prepareFace(tid, false);
1848 :
1849 25443 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1850 0 : _displaced_problem->prepareFace(_displaced_mesh->elemPtr(elem->id()), tid);
1851 25443 : }
1852 :
1853 : void
1854 0 : FEProblemBase::prepare(const Elem * elem,
1855 : unsigned int ivar,
1856 : unsigned int jvar,
1857 : const std::vector<dof_id_type> & dof_indices,
1858 : const THREAD_ID tid)
1859 : {
1860 0 : for (const auto i : index_range(_nl))
1861 : {
1862 0 : _assembly[tid][i]->reinit(elem);
1863 0 : _nl[i]->prepare(tid);
1864 : }
1865 :
1866 0 : _aux->prepare(tid);
1867 0 : const auto current_nl_sys_num = _current_nl_sys->number();
1868 0 : _assembly[tid][current_nl_sys_num]->prepareBlock(ivar, jvar, dof_indices);
1869 0 : if (_has_nonlocal_coupling)
1870 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1871 : {
1872 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1873 0 : _assembly[tid][current_nl_sys_num]->prepareBlockNonlocal(
1874 : ivar, jvar, dof_indices, jv.allDofIndices());
1875 : }
1876 :
1877 0 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1878 : {
1879 0 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), ivar, jvar, dof_indices, tid);
1880 0 : if (_has_nonlocal_coupling)
1881 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1882 : {
1883 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1884 0 : _displaced_problem->prepareBlockNonlocal(ivar, jvar, dof_indices, jv.allDofIndices(), tid);
1885 : }
1886 : }
1887 0 : }
1888 :
1889 : void
1890 406027636 : FEProblemBase::setCurrentSubdomainID(const Elem * elem, const THREAD_ID tid)
1891 : {
1892 406027636 : SubdomainID did = elem->subdomain_id();
1893 815851555 : for (const auto i : index_range(_solver_systems))
1894 : {
1895 409823919 : _assembly[tid][i]->setCurrentSubdomainID(did);
1896 422285455 : if (_displaced_problem &&
1897 422285455 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1898 7814055 : _displaced_problem->assembly(tid, i).setCurrentSubdomainID(did);
1899 : }
1900 406027636 : }
1901 :
1902 : void
1903 1456346275 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, unsigned int side, const THREAD_ID tid)
1904 : {
1905 1456346275 : SubdomainID did = elem->neighbor_ptr(side)->subdomain_id();
1906 2917860525 : for (const auto i : index_range(_nl))
1907 : {
1908 1461514250 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1909 1505090708 : if (_displaced_problem &&
1910 1505090708 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1911 25839139 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1912 : }
1913 1456346275 : }
1914 :
1915 : void
1916 15257920 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, const THREAD_ID tid)
1917 : {
1918 15257920 : SubdomainID did = elem->subdomain_id();
1919 31467152 : for (const auto i : index_range(_nl))
1920 : {
1921 16209232 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1922 16260528 : if (_displaced_problem &&
1923 16260528 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1924 51296 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1925 : }
1926 15257920 : }
1927 :
1928 : void
1929 131930 : FEProblemBase::prepareAssembly(const THREAD_ID tid)
1930 : {
1931 131930 : _assembly[tid][_current_nl_sys->number()]->prepare();
1932 131930 : if (_has_nonlocal_coupling)
1933 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
1934 :
1935 183226 : if (_displaced_problem &&
1936 183226 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1937 : {
1938 51296 : _displaced_problem->prepareAssembly(tid);
1939 51296 : if (_has_nonlocal_coupling)
1940 0 : _displaced_problem->prepareNonlocal(tid);
1941 : }
1942 131930 : }
1943 :
1944 : void
1945 9409 : FEProblemBase::prepareAssemblyNeighbor(const THREAD_ID tid)
1946 : {
1947 9409 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
1948 :
1949 9409 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
1950 0 : _displaced_problem->prepareAssemblyNeighbor(tid);
1951 9409 : }
1952 :
1953 : void
1954 266613 : FEProblemBase::addResidual(const THREAD_ID tid)
1955 : {
1956 533226 : _assembly[tid][_current_nl_sys->number()]->addResidual(Assembly::GlobalDataKey{},
1957 266613 : currentResidualVectorTags());
1958 :
1959 266613 : if (_displaced_problem)
1960 4776 : _displaced_problem->addResidual(tid);
1961 266613 : }
1962 :
1963 : void
1964 1984649 : FEProblemBase::addResidualNeighbor(const THREAD_ID tid)
1965 : {
1966 3969298 : _assembly[tid][_current_nl_sys->number()]->addResidualNeighbor(Assembly::GlobalDataKey{},
1967 1984649 : currentResidualVectorTags());
1968 :
1969 1984649 : if (_displaced_problem)
1970 61744 : _displaced_problem->addResidualNeighbor(tid);
1971 1984649 : }
1972 :
1973 : void
1974 1966980 : FEProblemBase::addResidualLower(const THREAD_ID tid)
1975 : {
1976 3933960 : _assembly[tid][_current_nl_sys->number()]->addResidualLower(Assembly::GlobalDataKey{},
1977 1966980 : currentResidualVectorTags());
1978 :
1979 1966980 : if (_displaced_problem)
1980 61956 : _displaced_problem->addResidualLower(tid);
1981 1966980 : }
1982 :
1983 : void
1984 48426 : FEProblemBase::addResidualScalar(const THREAD_ID tid /* = 0*/)
1985 : {
1986 96852 : _assembly[tid][_current_nl_sys->number()]->addResidualScalar(Assembly::GlobalDataKey{},
1987 48426 : currentResidualVectorTags());
1988 48426 : }
1989 :
1990 : void
1991 288290004 : FEProblemBase::cacheResidual(const THREAD_ID tid)
1992 : {
1993 288290004 : SubProblem::cacheResidual(tid);
1994 288290004 : if (_displaced_problem)
1995 7590898 : _displaced_problem->cacheResidual(tid);
1996 288290004 : }
1997 :
1998 : void
1999 50077 : FEProblemBase::cacheResidualNeighbor(const THREAD_ID tid)
2000 : {
2001 50077 : SubProblem::cacheResidualNeighbor(tid);
2002 50077 : if (_displaced_problem)
2003 32 : _displaced_problem->cacheResidualNeighbor(tid);
2004 50077 : }
2005 :
2006 : void
2007 17008404 : FEProblemBase::addCachedResidual(const THREAD_ID tid)
2008 : {
2009 17008404 : SubProblem::addCachedResidual(tid);
2010 17008404 : if (_displaced_problem)
2011 495407 : _displaced_problem->addCachedResidual(tid);
2012 17008404 : }
2013 :
2014 : void
2015 11332 : FEProblemBase::addCachedResidualDirectly(NumericVector<Number> & residual, const THREAD_ID tid)
2016 : {
2017 11332 : if (_current_nl_sys->hasVector(_current_nl_sys->timeVectorTag()))
2018 30252 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2019 30252 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->timeVectorTag()));
2020 :
2021 11332 : if (_current_nl_sys->hasVector(_current_nl_sys->nonTimeVectorTag()))
2022 33996 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2023 33996 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->nonTimeVectorTag()));
2024 :
2025 11332 : std::vector<VectorTag> extra_residual_vector_tags;
2026 11332 : extra_residual_vector_tags.reserve(currentResidualVectorTags().size());
2027 11332 : const auto time_tag = _current_nl_sys->timeVectorTag();
2028 11332 : const auto non_time_tag = _current_nl_sys->nonTimeVectorTag();
2029 44152 : for (const auto & vector_tag : currentResidualVectorTags())
2030 32820 : if (vector_tag._id != time_tag && vector_tag._id != non_time_tag)
2031 11404 : extra_residual_vector_tags.push_back(vector_tag);
2032 :
2033 : // Flush extra vector tag caches (e.g. from extra_vector_tags on NodalConstraints)
2034 : // to their respective system vectors after the standard TIME/NONTIME caches above.
2035 : // Without this, NodalConstraint contributions to extra vector tags are silently
2036 : // discarded by the blanket clearCachedResiduals.
2037 11332 : _assembly[tid][_current_nl_sys->number()]->addCachedResiduals(Assembly::GlobalDataKey{},
2038 : extra_residual_vector_tags);
2039 :
2040 : // We do this because by adding the cached residual directly, we cannot ensure that all of the
2041 : // cached residuals are emptied after only the two add calls above
2042 11332 : _assembly[tid][_current_nl_sys->number()]->clearCachedResiduals(Assembly::GlobalDataKey{});
2043 :
2044 11332 : if (_displaced_problem)
2045 35 : _displaced_problem->addCachedResidualDirectly(residual, tid);
2046 11332 : }
2047 :
2048 : void
2049 0 : FEProblemBase::setResidual(NumericVector<Number> & residual, const THREAD_ID tid)
2050 : {
2051 0 : _assembly[tid][_current_nl_sys->number()]->setResidual(
2052 : residual,
2053 0 : Assembly::GlobalDataKey{},
2054 0 : getVectorTag(_nl[_current_nl_sys->number()]->residualVectorTag()));
2055 0 : if (_displaced_problem)
2056 0 : _displaced_problem->setResidual(residual, tid);
2057 0 : }
2058 :
2059 : void
2060 0 : FEProblemBase::setResidualNeighbor(NumericVector<Number> & residual, const THREAD_ID tid)
2061 : {
2062 0 : _assembly[tid][_current_nl_sys->number()]->setResidualNeighbor(
2063 0 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->residualVectorTag()));
2064 0 : if (_displaced_problem)
2065 0 : _displaced_problem->setResidualNeighbor(residual, tid);
2066 0 : }
2067 :
2068 : void
2069 37415 : FEProblemBase::addJacobian(const THREAD_ID tid)
2070 : {
2071 37415 : _assembly[tid][_current_nl_sys->number()]->addJacobian(Assembly::GlobalDataKey{});
2072 37415 : if (_has_nonlocal_coupling)
2073 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianNonlocal(Assembly::GlobalDataKey{});
2074 37415 : if (_displaced_problem)
2075 : {
2076 200 : _displaced_problem->addJacobian(tid);
2077 200 : if (_has_nonlocal_coupling)
2078 0 : _displaced_problem->addJacobianNonlocal(tid);
2079 : }
2080 37415 : }
2081 :
2082 : void
2083 7852 : FEProblemBase::addJacobianNeighbor(const THREAD_ID tid)
2084 : {
2085 7852 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighbor(Assembly::GlobalDataKey{});
2086 7852 : if (_displaced_problem)
2087 44 : _displaced_problem->addJacobianNeighbor(tid);
2088 7852 : }
2089 :
2090 : void
2091 109104 : FEProblemBase::addJacobianNeighborLowerD(const THREAD_ID tid)
2092 : {
2093 109104 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborLowerD(Assembly::GlobalDataKey{});
2094 109104 : if (_displaced_problem)
2095 3072 : _displaced_problem->addJacobianNeighborLowerD(tid);
2096 109104 : }
2097 :
2098 : void
2099 4696 : FEProblemBase::addJacobianLowerD(const THREAD_ID tid)
2100 : {
2101 4696 : _assembly[tid][_current_nl_sys->number()]->addJacobianLowerD(Assembly::GlobalDataKey{});
2102 4696 : if (_displaced_problem)
2103 192 : _displaced_problem->addJacobianLowerD(tid);
2104 4696 : }
2105 :
2106 : void
2107 11603 : FEProblemBase::addJacobianScalar(const THREAD_ID tid /* = 0*/)
2108 : {
2109 11603 : _assembly[tid][_current_nl_sys->number()]->addJacobianScalar(Assembly::GlobalDataKey{});
2110 11603 : }
2111 :
2112 : void
2113 30002 : FEProblemBase::addJacobianOffDiagScalar(unsigned int ivar, const THREAD_ID tid /* = 0*/)
2114 : {
2115 60004 : _assembly[tid][_current_nl_sys->number()]->addJacobianOffDiagScalar(ivar,
2116 30002 : Assembly::GlobalDataKey{});
2117 30002 : }
2118 :
2119 : void
2120 46631996 : FEProblemBase::cacheJacobian(const THREAD_ID tid)
2121 : {
2122 46631996 : SubProblem::cacheJacobian(tid);
2123 46631996 : if (_displaced_problem)
2124 1544138 : _displaced_problem->cacheJacobian(tid);
2125 46631996 : }
2126 :
2127 : void
2128 9409 : FEProblemBase::cacheJacobianNeighbor(const THREAD_ID tid)
2129 : {
2130 9409 : SubProblem::cacheJacobianNeighbor(tid);
2131 9409 : if (_displaced_problem)
2132 0 : _displaced_problem->cacheJacobianNeighbor(tid);
2133 9409 : }
2134 :
2135 : void
2136 2886454 : FEProblemBase::addCachedJacobian(const THREAD_ID tid)
2137 : {
2138 2886454 : SubProblem::addCachedJacobian(tid);
2139 2886451 : if (_displaced_problem)
2140 96451 : _displaced_problem->addCachedJacobian(tid);
2141 2886451 : }
2142 :
2143 : void
2144 70832 : FEProblemBase::addJacobianBlockTags(SparseMatrix<Number> & jacobian,
2145 : unsigned int ivar,
2146 : unsigned int jvar,
2147 : const DofMap & dof_map,
2148 : std::vector<dof_id_type> & dof_indices,
2149 : const std::set<TagID> & tags,
2150 : const THREAD_ID tid)
2151 : {
2152 141664 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockTags(
2153 70832 : jacobian, ivar, jvar, dof_map, dof_indices, Assembly::GlobalDataKey{}, tags);
2154 :
2155 70832 : if (_has_nonlocal_coupling)
2156 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2157 : {
2158 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2159 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockNonlocalTags(
2160 : jacobian,
2161 : ivar,
2162 : jvar,
2163 : dof_map,
2164 : dof_indices,
2165 : jv.allDofIndices(),
2166 0 : Assembly::GlobalDataKey{},
2167 : tags);
2168 : }
2169 :
2170 70832 : if (_displaced_problem)
2171 : {
2172 0 : _displaced_problem->addJacobianBlockTags(jacobian, ivar, jvar, dof_map, dof_indices, tags, tid);
2173 0 : if (_has_nonlocal_coupling)
2174 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2175 : {
2176 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2177 0 : _displaced_problem->addJacobianBlockNonlocal(
2178 : jacobian, ivar, jvar, dof_map, dof_indices, jv.allDofIndices(), tags, tid);
2179 : }
2180 : }
2181 70832 : }
2182 :
2183 : void
2184 768 : FEProblemBase::addJacobianNeighbor(SparseMatrix<Number> & jacobian,
2185 : unsigned int ivar,
2186 : unsigned int jvar,
2187 : const DofMap & dof_map,
2188 : std::vector<dof_id_type> & dof_indices,
2189 : std::vector<dof_id_type> & neighbor_dof_indices,
2190 : const std::set<TagID> & tags,
2191 : const THREAD_ID tid)
2192 : {
2193 1536 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborTags(jacobian,
2194 : ivar,
2195 : jvar,
2196 : dof_map,
2197 : dof_indices,
2198 : neighbor_dof_indices,
2199 768 : Assembly::GlobalDataKey{},
2200 : tags);
2201 768 : if (_displaced_problem)
2202 0 : _displaced_problem->addJacobianNeighbor(
2203 : jacobian, ivar, jvar, dof_map, dof_indices, neighbor_dof_indices, tags, tid);
2204 768 : }
2205 :
2206 : void
2207 125434652 : FEProblemBase::prepareShapes(unsigned int var, const THREAD_ID tid)
2208 : {
2209 125434652 : _assembly[tid][_current_nl_sys->number()]->copyShapes(var);
2210 125434652 : }
2211 :
2212 : void
2213 602686 : FEProblemBase::prepareFaceShapes(unsigned int var, const THREAD_ID tid)
2214 : {
2215 602686 : _assembly[tid][_current_nl_sys->number()]->copyFaceShapes(var);
2216 602686 : }
2217 :
2218 : void
2219 184534 : FEProblemBase::prepareNeighborShapes(unsigned int var, const THREAD_ID tid)
2220 : {
2221 184534 : _assembly[tid][_current_nl_sys->number()]->copyNeighborShapes(var);
2222 184534 : }
2223 :
2224 : void
2225 874909 : FEProblemBase::addGhostedElem(dof_id_type elem_id)
2226 : {
2227 874909 : if (_mesh.elemPtr(elem_id)->processor_id() != processor_id())
2228 205736 : _ghosted_elems.insert(elem_id);
2229 874909 : }
2230 :
2231 : void
2232 29442 : FEProblemBase::addGhostedBoundary(BoundaryID boundary_id)
2233 : {
2234 29442 : _mesh.addGhostedBoundary(boundary_id);
2235 29442 : if (_displaced_problem)
2236 26702 : _displaced_mesh->addGhostedBoundary(boundary_id);
2237 29442 : }
2238 :
2239 : void
2240 68432 : FEProblemBase::ghostGhostedBoundaries()
2241 : {
2242 342160 : TIME_SECTION("ghostGhostedBoundaries", 3, "Ghosting Ghosted Boundaries");
2243 :
2244 68432 : _mesh.ghostGhostedBoundaries();
2245 :
2246 68432 : if (_displaced_problem)
2247 2591 : _displaced_mesh->ghostGhostedBoundaries();
2248 68432 : }
2249 :
2250 : void
2251 0 : FEProblemBase::sizeZeroes(unsigned int /*size*/, const THREAD_ID /*tid*/)
2252 : {
2253 0 : mooseDoOnce(mooseWarning(
2254 : "This function is deprecated and no longer performs any function. Please do not call it."));
2255 0 : }
2256 :
2257 : bool
2258 304028 : FEProblemBase::reinitDirac(const Elem * elem, const THREAD_ID tid)
2259 : {
2260 304028 : std::vector<Point> & points = _dirac_kernel_info.getPoints()[elem].first;
2261 :
2262 304028 : unsigned int n_points = points.size();
2263 :
2264 304028 : if (n_points)
2265 : {
2266 299052 : if (n_points > _max_qps)
2267 : {
2268 0 : _max_qps = n_points;
2269 :
2270 : /**
2271 : * The maximum number of qps can rise if several Dirac points are added to a single element.
2272 : * In that case we need to resize the zeros to compensate.
2273 : */
2274 0 : unsigned int max_qpts = getMaxQps();
2275 0 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
2276 : {
2277 : // the highest available order in libMesh is 43
2278 0 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
2279 0 : _zero[tid].resize(max_qpts, 0);
2280 0 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
2281 0 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
2282 0 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
2283 0 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
2284 : }
2285 : }
2286 :
2287 598104 : for (const auto i : index_range(_nl))
2288 : {
2289 299052 : _assembly[tid][i]->reinitAtPhysical(elem, points);
2290 299052 : _nl[i]->prepare(tid);
2291 : }
2292 299052 : _aux->prepare(tid);
2293 :
2294 299052 : reinitElem(elem, tid);
2295 : }
2296 :
2297 304028 : _assembly[tid][_current_nl_sys->number()]->prepare();
2298 304028 : if (_has_nonlocal_coupling)
2299 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
2300 :
2301 304028 : bool have_points = n_points > 0;
2302 304028 : if (_displaced_problem && (_reinit_displaced_elem))
2303 : {
2304 4976 : have_points |= _displaced_problem->reinitDirac(_displaced_mesh->elemPtr(elem->id()), tid);
2305 4976 : if (_has_nonlocal_coupling)
2306 0 : _displaced_problem->prepareNonlocal(tid);
2307 : }
2308 :
2309 304028 : return have_points;
2310 : }
2311 :
2312 : void
2313 374926995 : FEProblemBase::reinitElem(const Elem * elem, const THREAD_ID tid)
2314 : {
2315 752639016 : for (auto & sys : _solver_systems)
2316 377712021 : sys->reinitElem(elem, tid);
2317 374926995 : _aux->reinitElem(elem, tid);
2318 :
2319 374926995 : if (_displaced_problem && _reinit_displaced_elem)
2320 4423675 : _displaced_problem->reinitElem(_displaced_mesh->elemPtr(elem->id()), tid);
2321 374926995 : }
2322 :
2323 : void
2324 58516 : FEProblemBase::reinitElemPhys(const Elem * const elem,
2325 : const std::vector<Point> & phys_points_in_elem,
2326 : const THREAD_ID tid)
2327 : {
2328 : mooseAssert(_mesh.queryElemPtr(elem->id()) == elem,
2329 : "Are you calling this method with a displaced mesh element?");
2330 :
2331 117032 : for (const auto i : index_range(_solver_systems))
2332 : {
2333 58516 : _assembly[tid][i]->reinitAtPhysical(elem, phys_points_in_elem);
2334 58516 : _solver_systems[i]->prepare(tid);
2335 58516 : _assembly[tid][i]->prepare();
2336 58516 : if (_has_nonlocal_coupling)
2337 0 : _assembly[tid][i]->prepareNonlocal();
2338 : }
2339 58516 : _aux->prepare(tid);
2340 :
2341 58516 : reinitElem(elem, tid);
2342 58516 : }
2343 :
2344 : void
2345 0 : FEProblemBase::reinitElemFace(const Elem * const elem,
2346 : const unsigned int side,
2347 : const BoundaryID,
2348 : const THREAD_ID tid)
2349 : {
2350 0 : mooseDeprecated(
2351 : "reinitElemFace with a BoundaryID argument is deprecated because the boundary id was never "
2352 : "used. Please call reinitElemFace without the BoundaryID argument instead");
2353 :
2354 0 : reinitElemFace(elem, side, tid);
2355 0 : }
2356 :
2357 : void
2358 5210835 : FEProblemBase::reinitElemFace(const Elem * const elem, const unsigned int side, const THREAD_ID tid)
2359 : {
2360 10422018 : for (const auto i : index_range(_solver_systems))
2361 : {
2362 5211183 : _assembly[tid][i]->reinit(elem, side);
2363 5211183 : _solver_systems[i]->reinitElemFace(elem, side, tid);
2364 : }
2365 5210835 : _aux->reinitElemFace(elem, side, tid);
2366 :
2367 5210835 : if (_displaced_problem && _reinit_displaced_face)
2368 82248 : _displaced_problem->reinitElemFace(_displaced_mesh->elemPtr(elem->id()), side, tid);
2369 5210835 : }
2370 :
2371 : void
2372 578467 : FEProblemBase::reinitLowerDElem(const Elem * lower_d_elem,
2373 : const THREAD_ID tid,
2374 : const std::vector<Point> * const pts,
2375 : const std::vector<Real> * const weights)
2376 : {
2377 578467 : SubProblem::reinitLowerDElem(lower_d_elem, tid, pts, weights);
2378 :
2379 578467 : if (_displaced_problem && _displaced_mesh)
2380 960 : _displaced_problem->reinitLowerDElem(
2381 960 : _displaced_mesh->elemPtr(lower_d_elem->id()), tid, pts, weights);
2382 578467 : }
2383 :
2384 : void
2385 26043616 : FEProblemBase::reinitNode(const Node * node, const THREAD_ID tid)
2386 : {
2387 26043616 : if (_displaced_problem && _reinit_displaced_elem)
2388 1001711 : _displaced_problem->reinitNode(&_displaced_mesh->nodeRef(node->id()), tid);
2389 :
2390 52090148 : for (const auto i : index_range(_nl))
2391 : {
2392 26046532 : _assembly[tid][i]->reinit(node);
2393 26046532 : _nl[i]->reinitNode(node, tid);
2394 : }
2395 26043616 : _aux->reinitNode(node, tid);
2396 26043616 : }
2397 :
2398 : void
2399 65230881 : FEProblemBase::reinitNodeFace(const Node * node, BoundaryID bnd_id, const THREAD_ID tid)
2400 : {
2401 65230881 : if (_displaced_problem && _reinit_displaced_face)
2402 3388880 : _displaced_problem->reinitNodeFace(&_displaced_mesh->nodeRef(node->id()), bnd_id, tid);
2403 :
2404 131256806 : for (const auto i : index_range(_nl))
2405 : {
2406 66025925 : _assembly[tid][i]->reinit(node);
2407 66025925 : _nl[i]->reinitNodeFace(node, bnd_id, tid);
2408 : }
2409 65230881 : _aux->reinitNodeFace(node, bnd_id, tid);
2410 65230881 : }
2411 :
2412 : void
2413 5097 : FEProblemBase::reinitNodes(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2414 : {
2415 5097 : if (_displaced_problem && _reinit_displaced_elem)
2416 0 : _displaced_problem->reinitNodes(nodes, tid);
2417 :
2418 10194 : for (auto & nl : _nl)
2419 5097 : nl->reinitNodes(nodes, tid);
2420 5097 : _aux->reinitNodes(nodes, tid);
2421 5097 : }
2422 :
2423 : void
2424 1003 : FEProblemBase::reinitNodesNeighbor(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2425 : {
2426 1003 : if (_displaced_problem && _reinit_displaced_elem)
2427 0 : _displaced_problem->reinitNodesNeighbor(nodes, tid);
2428 :
2429 2006 : for (auto & nl : _nl)
2430 1003 : nl->reinitNodesNeighbor(nodes, tid);
2431 1003 : _aux->reinitNodesNeighbor(nodes, tid);
2432 1003 : }
2433 :
2434 : void
2435 8465208 : FEProblemBase::reinitScalars(const THREAD_ID tid, bool reinit_for_derivative_reordering /*=false*/)
2436 : {
2437 42326040 : TIME_SECTION("reinitScalars", 3, "Reinitializing Scalar Variables");
2438 :
2439 8465208 : if (_displaced_problem && _reinit_displaced_elem)
2440 113549 : _displaced_problem->reinitScalars(tid, reinit_for_derivative_reordering);
2441 :
2442 17169861 : for (auto & nl : _nl)
2443 8704653 : nl->reinitScalars(tid, reinit_for_derivative_reordering);
2444 8465208 : _aux->reinitScalars(tid, reinit_for_derivative_reordering);
2445 :
2446 : // This is called outside of residual/Jacobian call-backs
2447 17177105 : for (auto & assembly : _assembly[tid])
2448 8711897 : assembly->prepareScalar();
2449 8465208 : }
2450 :
2451 : void
2452 185551 : FEProblemBase::reinitOffDiagScalars(const THREAD_ID tid)
2453 : {
2454 185551 : _assembly[tid][_current_nl_sys->number()]->prepareOffDiagScalar();
2455 185551 : if (_displaced_problem)
2456 60 : _displaced_problem->reinitOffDiagScalars(tid);
2457 185551 : }
2458 :
2459 : void
2460 3642152 : FEProblemBase::reinitNeighbor(const Elem * elem, unsigned int side, const THREAD_ID tid)
2461 : {
2462 3642152 : setNeighborSubdomainID(elem, side, tid);
2463 :
2464 3642152 : const Elem * neighbor = elem->neighbor_ptr(side);
2465 3642152 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(elem);
2466 :
2467 7284331 : for (const auto i : index_range(_nl))
2468 : {
2469 3642179 : _assembly[tid][i]->reinitElemAndNeighbor(elem, side, neighbor, neighbor_side);
2470 3642179 : _nl[i]->prepareNeighbor(tid);
2471 : // Called during stateful material property evaluation outside of solve
2472 3642179 : _assembly[tid][i]->prepareNeighbor();
2473 : }
2474 3642152 : _aux->prepareNeighbor(tid);
2475 :
2476 7284331 : for (auto & nl : _nl)
2477 : {
2478 3642179 : nl->reinitElemFace(elem, side, tid);
2479 3642179 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2480 : }
2481 3642152 : _aux->reinitElemFace(elem, side, tid);
2482 3642152 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2483 :
2484 3642152 : if (_displaced_problem && _reinit_displaced_neighbor)
2485 : {
2486 : // There are cases like for cohesive zone modeling without significant sliding where we cannot
2487 : // use FEInterface::inverse_map in Assembly::reinitElemAndNeighbor in the displaced problem
2488 : // because the physical points coming from the element don't actually lie on the neighbor.
2489 : // Moreover, what's the point of doing another physical point inversion in other cases? We only
2490 : // care about the reference points which we can just take from the undisplaced computation
2491 64800 : const auto & displaced_ref_pts = _assembly[tid][0]->qRuleNeighbor()->get_points();
2492 :
2493 64800 : _displaced_problem->reinitNeighbor(
2494 64800 : _displaced_mesh->elemPtr(elem->id()), side, tid, &displaced_ref_pts);
2495 : }
2496 3642152 : }
2497 :
2498 : void
2499 2069317 : FEProblemBase::reinitElemNeighborAndLowerD(const Elem * elem,
2500 : unsigned int side,
2501 : const THREAD_ID tid)
2502 : {
2503 2069317 : reinitNeighbor(elem, side, tid);
2504 :
2505 2069317 : const Elem * lower_d_elem = _mesh.getLowerDElem(elem, side);
2506 2069317 : if (lower_d_elem && _mesh.interiorLowerDBlocks().count(lower_d_elem->subdomain_id()) > 0)
2507 10332 : reinitLowerDElem(lower_d_elem, tid);
2508 : else
2509 : {
2510 : // with mesh refinement, lower-dimensional element might be defined on neighbor side
2511 2058985 : auto & neighbor = _assembly[tid][0]->neighbor();
2512 2058985 : auto & neighbor_side = _assembly[tid][0]->neighborSide();
2513 2058985 : const Elem * lower_d_elem_neighbor = _mesh.getLowerDElem(neighbor, neighbor_side);
2514 2058985 : if (lower_d_elem_neighbor &&
2515 2058985 : _mesh.interiorLowerDBlocks().count(lower_d_elem_neighbor->subdomain_id()) > 0)
2516 : {
2517 0 : auto qps = _assembly[tid][0]->qPointsFaceNeighbor().stdVector();
2518 0 : std::vector<Point> reference_points;
2519 0 : FEMap::inverse_map(
2520 0 : lower_d_elem_neighbor->dim(), lower_d_elem_neighbor, qps, reference_points);
2521 0 : reinitLowerDElem(lower_d_elem_neighbor, tid, &reference_points);
2522 0 : }
2523 : }
2524 :
2525 2069317 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
2526 64740 : _displaced_problem->reinitElemNeighborAndLowerD(
2527 64740 : _displaced_mesh->elemPtr(elem->id()), side, tid);
2528 2069317 : }
2529 :
2530 : void
2531 97230 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2532 : unsigned int neighbor_side,
2533 : const std::vector<Point> & physical_points,
2534 : const THREAD_ID tid)
2535 : {
2536 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2537 : "Are you calling this method with a displaced mesh element?");
2538 :
2539 194460 : for (const auto i : index_range(_nl))
2540 : {
2541 : // Reinits shape the functions at the physical points
2542 97230 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, neighbor_side, physical_points);
2543 :
2544 : // Sets the neighbor dof indices
2545 97230 : _nl[i]->prepareNeighbor(tid);
2546 : }
2547 97230 : _aux->prepareNeighbor(tid);
2548 :
2549 : // Resizes Re and Ke
2550 97230 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2551 :
2552 : // Compute the values of each variable at the points
2553 194460 : for (auto & nl : _nl)
2554 97230 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2555 97230 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2556 97230 : }
2557 :
2558 : void
2559 19880 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2560 : const std::vector<Point> & physical_points,
2561 : const THREAD_ID tid)
2562 : {
2563 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2564 : "Are you calling this method with a displaced mesh element?");
2565 :
2566 39760 : for (const auto i : index_range(_nl))
2567 : {
2568 : // Reinits shape the functions at the physical points
2569 19880 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, physical_points);
2570 :
2571 : // Sets the neighbor dof indices
2572 19880 : _nl[i]->prepareNeighbor(tid);
2573 : }
2574 19880 : _aux->prepareNeighbor(tid);
2575 :
2576 : // Resizes Re and Ke
2577 19880 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2578 :
2579 : // Compute the values of each variable at the points
2580 39760 : for (auto & nl : _nl)
2581 19880 : nl->reinitNeighbor(neighbor, tid);
2582 19880 : _aux->reinitNeighbor(neighbor, tid);
2583 19880 : }
2584 :
2585 : void
2586 35518 : FEProblemBase::getDiracElements(std::set<const Elem *> & elems)
2587 : {
2588 : // First add in the undisplaced elements
2589 35518 : elems = _dirac_kernel_info.getElements();
2590 :
2591 35518 : if (_displaced_problem)
2592 : {
2593 2359 : std::set<const Elem *> displaced_elements;
2594 2359 : _displaced_problem->getDiracElements(displaced_elements);
2595 :
2596 : { // Use the ids from the displaced elements to get the undisplaced elements
2597 : // and add them to the list
2598 7335 : for (const auto & elem : displaced_elements)
2599 4976 : elems.insert(_mesh.elemPtr(elem->id()));
2600 : }
2601 2359 : }
2602 35518 : }
2603 :
2604 : void
2605 3530960 : FEProblemBase::clearDiracInfo()
2606 : {
2607 3530960 : _dirac_kernel_info.clearPoints();
2608 :
2609 3530960 : if (_displaced_problem)
2610 144138 : _displaced_problem->clearDiracInfo();
2611 3530960 : }
2612 :
2613 : void
2614 5099965 : FEProblemBase::subdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2615 : {
2616 5099965 : _all_materials.subdomainSetup(subdomain, tid);
2617 : // Call the subdomain methods of the output system, these are not threaded so only call it once
2618 5099965 : if (tid == 0)
2619 5087239 : _app.getOutputWarehouse().subdomainSetup();
2620 :
2621 10334999 : for (auto & nl : _nl)
2622 5235034 : nl->subdomainSetup(subdomain, tid);
2623 :
2624 : // FIXME: call displaced_problem->subdomainSetup() ?
2625 : // When adding possibility with materials being evaluated on displaced mesh
2626 5099965 : }
2627 :
2628 : void
2629 16562420 : FEProblemBase::neighborSubdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2630 : {
2631 16562420 : _all_materials.neighborSubdomainSetup(subdomain, tid);
2632 16562420 : }
2633 :
2634 : void
2635 50053 : FEProblemBase::addFunction(const std::string & type,
2636 : const std::string & name,
2637 : InputParameters & parameters)
2638 : {
2639 : parallel_object_only();
2640 :
2641 100106 : parameters.set<SubProblem *>("_subproblem") = this;
2642 :
2643 104450 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2644 : {
2645 54503 : std::shared_ptr<Function> func = _factory.create<Function>(type, name, parameters, tid);
2646 54397 : logAdd("Function", name, type, parameters);
2647 54397 : _functions.addObject(func, tid);
2648 :
2649 54397 : if (auto * const functor = dynamic_cast<Moose::FunctorBase<Real> *>(func.get()))
2650 : {
2651 54397 : this->addFunctor(name, *functor, tid);
2652 54397 : if (_displaced_problem)
2653 1780 : _displaced_problem->addFunctor(name, *functor, tid);
2654 : }
2655 : else
2656 0 : mooseError("Unrecognized function functor type");
2657 54397 : }
2658 49947 : }
2659 :
2660 : void
2661 154241 : FEProblemBase::addConvergence(const std::string & type,
2662 : const std::string & name,
2663 : InputParameters & parameters)
2664 : {
2665 : parallel_object_only();
2666 :
2667 325022 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2668 : {
2669 170808 : std::shared_ptr<Convergence> conv = _factory.create<Convergence>(type, name, parameters, tid);
2670 170781 : _convergences.addObject(conv, tid);
2671 170781 : }
2672 154214 : }
2673 :
2674 : void
2675 60293 : FEProblemBase::addDefaultNonlinearConvergence(const InputParameters & params_to_apply)
2676 : {
2677 60293 : const std::string class_name = "DefaultNonlinearConvergence";
2678 60293 : InputParameters params = _factory.getValidParams(class_name);
2679 60293 : params.applyParameters(params_to_apply);
2680 60293 : params.applyParameters(parameters());
2681 60293 : params.set<bool>("added_as_default") = true;
2682 119638 : for (const auto & conv_name : getNonlinearConvergenceNames())
2683 59345 : addConvergence(class_name, conv_name, params);
2684 60293 : }
2685 :
2686 : void
2687 62459 : FEProblemBase::addDefaultMultiAppFixedPointConvergence(const InputParameters & params_to_apply)
2688 : {
2689 62459 : const std::string class_name = "DefaultMultiAppFixedPointConvergence";
2690 62459 : InputParameters params = _factory.getValidParams(class_name);
2691 62459 : params.applyParameters(params_to_apply);
2692 62459 : params.applyParameters(parameters());
2693 62459 : params.set<bool>("added_as_default") = true;
2694 62459 : addConvergence(class_name, getMultiAppFixedPointConvergenceName(), params);
2695 62450 : }
2696 :
2697 : void
2698 30524 : FEProblemBase::addDefaultSteadyStateConvergence(const InputParameters & params_to_apply)
2699 : {
2700 30524 : const std::string class_name = "DefaultSteadyStateConvergence";
2701 30524 : InputParameters params = _factory.getValidParams(class_name);
2702 30524 : params.applyParameters(params_to_apply);
2703 30524 : params.applyParameters(parameters());
2704 30524 : params.set<bool>("added_as_default") = true;
2705 30524 : addConvergence(class_name, getSteadyStateConvergenceName(), params);
2706 30524 : }
2707 :
2708 : bool
2709 87479 : FEProblemBase::hasFunction(const std::string & name, const THREAD_ID tid)
2710 : {
2711 87479 : return _functions.hasActiveObject(name, tid);
2712 : }
2713 :
2714 : Function &
2715 63631 : FEProblemBase::getFunction(const std::string & name, const THREAD_ID tid)
2716 : {
2717 : // This thread lock is necessary since this method will create functions
2718 : // for all threads if one is missing.
2719 63631 : Threads::spin_mutex::scoped_lock lock(get_function_mutex);
2720 :
2721 63631 : if (!hasFunction(name, tid))
2722 : {
2723 : // If we didn't find a function, it might be a default function, attempt to construct one now
2724 20498 : std::istringstream ss(name);
2725 : Real real_value;
2726 :
2727 : // First see if it's just a constant. If it is, build a ConstantFunction
2728 20498 : if (ss >> real_value && ss.eof())
2729 : {
2730 12740 : InputParameters params = _factory.getValidParams("ConstantFunction");
2731 12740 : params.set<Real>("value") = real_value;
2732 19110 : addFunction("ConstantFunction", ss.str(), params);
2733 6370 : }
2734 : else
2735 : {
2736 14128 : FunctionParserBase<Real> fp;
2737 14128 : std::string vars = "x,y,z,t,NaN,pi,e";
2738 14128 : if (fp.Parse(name, vars) == -1) // -1 for success
2739 : {
2740 : // It parsed ok, so build a MooseParsedFunction
2741 42369 : InputParameters params = _factory.getValidParams("ParsedFunction");
2742 14123 : params.set<std::string>("expression") = name;
2743 28246 : addFunction("ParsedFunction", name, params);
2744 14123 : }
2745 14128 : }
2746 :
2747 : // Try once more
2748 20498 : if (!hasFunction(name, tid))
2749 : {
2750 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_function"),
2751 : "getFunction() was called before Functions have been constructed. The requested "
2752 : "Function '" +
2753 : name + "' may exist in the input file, but Functions are not available yet.");
2754 :
2755 7 : mooseError("Unable to find function " + name);
2756 : }
2757 20495 : }
2758 :
2759 63626 : auto * const ret = dynamic_cast<Function *>(_functions.getActiveObject(name, tid).get());
2760 63626 : if (!ret)
2761 0 : mooseError("No function named ", name, " of appropriate type");
2762 :
2763 63626 : return *ret;
2764 63628 : }
2765 :
2766 : bool
2767 213501 : FEProblemBase::hasConvergence(const std::string & name, const THREAD_ID tid) const
2768 : {
2769 213501 : return _convergences.hasActiveObject(name, tid);
2770 : }
2771 :
2772 : Convergence &
2773 1137772 : FEProblemBase::getConvergence(const std::string & name, const THREAD_ID tid) const
2774 : {
2775 1137772 : auto * const ret = dynamic_cast<Convergence *>(_convergences.getActiveObject(name, tid).get());
2776 1137772 : if (!ret)
2777 0 : mooseError("The Convergence object '", name, "' does not exist.");
2778 :
2779 1137772 : return *ret;
2780 : }
2781 :
2782 : const std::vector<std::shared_ptr<Convergence>> &
2783 196195 : FEProblemBase::getConvergenceObjects(const THREAD_ID tid) const
2784 : {
2785 196195 : return _convergences.getActiveObjects(tid);
2786 : }
2787 :
2788 : void
2789 687 : FEProblemBase::addMeshDivision(const std::string & type,
2790 : const std::string & name,
2791 : InputParameters & parameters)
2792 : {
2793 : parallel_object_only();
2794 1374 : parameters.set<FEProblemBase *>("_fe_problem_base") = this;
2795 1374 : parameters.set<SubProblem *>("_subproblem") = this;
2796 1462 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2797 : {
2798 784 : std::shared_ptr<MeshDivision> func = _factory.create<MeshDivision>(type, name, parameters, tid);
2799 775 : _mesh_divisions.addObject(func, tid);
2800 775 : }
2801 678 : }
2802 :
2803 : MeshDivision &
2804 1567 : FEProblemBase::getMeshDivision(const std::string & name, const THREAD_ID tid) const
2805 : {
2806 1567 : auto * const ret = dynamic_cast<MeshDivision *>(_mesh_divisions.getActiveObject(name, tid).get());
2807 1567 : if (!ret)
2808 0 : mooseError("No MeshDivision object named ", name, " of appropriate type");
2809 1567 : return *ret;
2810 : }
2811 :
2812 : void
2813 0 : FEProblemBase::lineSearch()
2814 : {
2815 0 : _line_search->lineSearch();
2816 0 : }
2817 :
2818 : NonlinearSystem &
2819 0 : FEProblemBase::getNonlinearSystem(const unsigned int sys_num)
2820 : {
2821 0 : mooseDeprecated("FEProblemBase::getNonlinearSystem() is deprecated, please use "
2822 : "FEProblemBase::getNonlinearSystemBase() \n");
2823 :
2824 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
2825 0 : auto nl_sys = std::dynamic_pointer_cast<NonlinearSystem>(_nl[sys_num]);
2826 :
2827 0 : if (!nl_sys)
2828 0 : mooseError("This is not a NonlinearSystem");
2829 :
2830 0 : return *nl_sys;
2831 0 : }
2832 :
2833 : void
2834 2 : FEProblemBase::addDistribution(const std::string & type,
2835 : const std::string & name,
2836 : InputParameters & parameters)
2837 : {
2838 4 : parameters.set<std::string>("type") = type;
2839 2 : addObject<Distribution>(type, name, parameters, /* threaded = */ false);
2840 2 : }
2841 :
2842 : bool
2843 4 : FEProblemBase::hasDistribution(const std::string & name) const
2844 : {
2845 4 : std::vector<Distribution *> objs;
2846 4 : theWarehouse()
2847 8 : .query()
2848 4 : .condition<AttribSystem>("Distribution")
2849 4 : .condition<AttribName>(name)
2850 4 : .queryInto(objs);
2851 8 : return !objs.empty();
2852 4 : }
2853 :
2854 : Distribution &
2855 4 : FEProblemBase::getDistribution(const std::string & name)
2856 : {
2857 4 : std::vector<Distribution *> objs;
2858 4 : theWarehouse()
2859 8 : .query()
2860 4 : .condition<AttribSystem>("Distribution")
2861 4 : .condition<AttribName>(name)
2862 4 : .queryInto(objs);
2863 4 : if (objs.empty())
2864 : {
2865 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_distribution"),
2866 : "A Distribution getter was called before Distributions have been constructed. "
2867 : "If you are attempting to access this object in the constructor of another object "
2868 : "then make sure that the Distribution is constructed before the object using it.");
2869 0 : mooseError("Unable to find Distribution with name '" + name + "'");
2870 : }
2871 8 : return *(objs[0]);
2872 4 : }
2873 :
2874 : void
2875 275 : FEProblemBase::addSampler(const std::string & type,
2876 : const std::string & name,
2877 : InputParameters & parameters)
2878 : {
2879 275 : const auto samplers = addObject<Sampler>(type, name, parameters);
2880 569 : for (auto & sampler : samplers)
2881 303 : sampler->init();
2882 266 : }
2883 :
2884 : Sampler &
2885 266 : FEProblemBase::getSampler(const std::string & name, const THREAD_ID tid)
2886 : {
2887 266 : std::vector<Sampler *> objs;
2888 266 : theWarehouse()
2889 532 : .query()
2890 266 : .condition<AttribSystem>("Sampler")
2891 266 : .condition<AttribThread>(tid)
2892 266 : .condition<AttribName>(name)
2893 266 : .queryInto(objs);
2894 266 : if (objs.empty())
2895 : {
2896 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_sampler"),
2897 : "A Sampler getter was called before Samplers have been constructed. "
2898 : "If you are attempting to access this object in the constructor of another object "
2899 : "then make sure that the Sampler is constructed before the object using it.");
2900 :
2901 0 : mooseError(
2902 0 : "Unable to find Sampler with name '" + name +
2903 : "', if you are attempting to access this object in the constructor of another object then "
2904 : "make sure that the Sampler is constructed before the object using it.");
2905 : }
2906 532 : return *(objs[0]);
2907 266 : }
2908 :
2909 : bool
2910 155377 : FEProblemBase::duplicateVariableCheck(const std::string & var_name,
2911 : const FEType & type,
2912 : bool is_aux,
2913 : const std::set<SubdomainID> * const active_subdomains)
2914 : {
2915 155377 : std::set<SubdomainID> subdomainIDs;
2916 155377 : if (active_subdomains->size() == 0)
2917 : {
2918 146930 : const auto subdomains = _mesh.meshSubdomains();
2919 146930 : subdomainIDs.insert(subdomains.begin(), subdomains.end());
2920 146930 : }
2921 : else
2922 8447 : subdomainIDs.insert(active_subdomains->begin(), active_subdomains->end());
2923 :
2924 311745 : for (auto & sys : _solver_systems)
2925 : {
2926 156380 : SystemBase * curr_sys_ptr = sys.get();
2927 156380 : SystemBase * other_sys_ptr = _aux.get();
2928 156380 : std::string error_prefix = "";
2929 156380 : if (is_aux)
2930 : {
2931 94446 : curr_sys_ptr = _aux.get();
2932 94446 : other_sys_ptr = sys.get();
2933 94446 : error_prefix = "aux";
2934 : }
2935 :
2936 156380 : if (other_sys_ptr->hasVariable(var_name))
2937 3 : mooseError("Cannot have an auxiliary variable and a solver variable with the same name: ",
2938 : var_name);
2939 :
2940 156377 : if (curr_sys_ptr->hasVariable(var_name))
2941 : {
2942 : const Variable & var =
2943 9 : curr_sys_ptr->system().variable(curr_sys_ptr->system().variable_number(var_name));
2944 :
2945 : // variable type
2946 9 : if (var.type() != type)
2947 : {
2948 12 : const auto stringifyType = [](FEType t)
2949 12 : { return Moose::stringify(t.family) + " of order " + Moose::stringify(t.order); };
2950 :
2951 6 : mooseError("Mismatching types are specified for ",
2952 : error_prefix,
2953 : "variable with name '",
2954 : var_name,
2955 : "': '",
2956 6 : stringifyType(var.type()),
2957 : "' and '",
2958 6 : stringifyType(type),
2959 : "'");
2960 : }
2961 :
2962 : // block-restriction
2963 3 : if (!(active_subdomains->size() == 0 && var.active_subdomains().size() == 0))
2964 : {
2965 3 : const auto varActiveSubdomains = var.active_subdomains();
2966 3 : std::set<SubdomainID> varSubdomainIDs;
2967 3 : if (varActiveSubdomains.size() == 0)
2968 : {
2969 0 : const auto subdomains = _mesh.meshSubdomains();
2970 0 : varSubdomainIDs.insert(subdomains.begin(), subdomains.end());
2971 0 : }
2972 : else
2973 3 : varSubdomainIDs.insert(varActiveSubdomains.begin(), varActiveSubdomains.end());
2974 :
2975 : // Is subdomainIDs a subset of varSubdomainIDs? With this we allow the case that the newly
2976 : // requested block restriction is only a subset of the existing one.
2977 3 : const auto isSubset = std::includes(varSubdomainIDs.begin(),
2978 : varSubdomainIDs.end(),
2979 : subdomainIDs.begin(),
2980 : subdomainIDs.end());
2981 :
2982 3 : if (!isSubset)
2983 : {
2984 : // helper function: make a string from a set of subdomain ids
2985 6 : const auto stringifySubdomains = [this](std::set<SubdomainID> subdomainIDs)
2986 : {
2987 6 : std::stringstream s;
2988 15 : for (auto const i : subdomainIDs)
2989 : {
2990 : // do we need to insert a comma?
2991 9 : if (s.tellp() != 0)
2992 3 : s << ", ";
2993 :
2994 : // insert subdomain name and id -or- only the id (if no name is given)
2995 9 : const auto subdomainName = _mesh.getSubdomainName(i);
2996 9 : if (subdomainName.empty())
2997 9 : s << i;
2998 : else
2999 0 : s << subdomainName << " (" << i << ")";
3000 9 : }
3001 12 : return s.str();
3002 6 : };
3003 :
3004 6 : const std::string msg = "Mismatching block-restrictions are specified for " +
3005 6 : error_prefix + "variable with name '" + var_name + "': {" +
3006 12 : stringifySubdomains(varSubdomainIDs) + "} and {" +
3007 9 : stringifySubdomains(subdomainIDs) + "}";
3008 :
3009 3 : mooseError(msg);
3010 0 : }
3011 0 : }
3012 :
3013 0 : return true;
3014 : }
3015 156368 : }
3016 :
3017 155365 : return false;
3018 155365 : }
3019 :
3020 : void
3021 61071 : FEProblemBase::addVariable(const std::string & var_type,
3022 : const std::string & var_name,
3023 : InputParameters & params)
3024 : {
3025 : parallel_object_only();
3026 :
3027 61071 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3028 61071 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3029 61071 : const auto fe_type = FEType(order, family);
3030 :
3031 : const auto active_subdomains_vector =
3032 61071 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3033 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3034 61071 : active_subdomains_vector.end());
3035 :
3036 61071 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ false, &active_subdomains))
3037 0 : return;
3038 :
3039 183204 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3040 61068 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_SOLVER;
3041 61068 : SolverSystemName sys_name = params.get<SolverSystemName>("solver_sys");
3042 :
3043 61068 : const auto solver_system_number = solverSysNum(sys_name);
3044 61068 : logAdd("Variable", var_name, var_type, params);
3045 61068 : _solver_systems[solver_system_number]->addVariable(var_type, var_name, params);
3046 61056 : if (_displaced_problem)
3047 : // MooseObjects need to be unique so change the name here
3048 3453 : _displaced_problem->addVariable(var_type, var_name, params, solver_system_number);
3049 :
3050 61056 : _solver_var_to_sys_num[var_name] = solver_system_number;
3051 :
3052 61056 : markFamilyPRefinement(params);
3053 61056 : if (_displaced_problem)
3054 3453 : _displaced_problem->markFamilyPRefinement(params);
3055 61056 : }
3056 :
3057 : std::pair<bool, unsigned int>
3058 4577904 : FEProblemBase::determineSolverSystem(const std::string & var_name,
3059 : const bool error_if_not_found) const
3060 : {
3061 4577904 : auto map_it = _solver_var_to_sys_num.find(var_name);
3062 4577904 : const bool var_in_sys = map_it != _solver_var_to_sys_num.end();
3063 4577904 : if (var_in_sys)
3064 : mooseAssert(_solver_systems[map_it->second]->hasVariable(var_name) ||
3065 : _solver_systems[map_it->second]->hasScalarVariable(var_name),
3066 : "If the variable is in our FEProblem solver system map, then it must be in the "
3067 : "solver system we expect");
3068 3106824 : else if (error_if_not_found)
3069 : {
3070 32 : if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
3071 21 : mooseError("No solver variable named ",
3072 : var_name,
3073 : " found. Did you specify an auxiliary variable when you meant to specify a "
3074 : "solver variable?");
3075 : else
3076 11 : mooseError("Unknown variable '",
3077 : var_name,
3078 : "'. It does not exist in the solver system(s) or auxiliary system");
3079 : }
3080 :
3081 9155744 : return std::make_pair(var_in_sys, var_in_sys ? map_it->second : libMesh::invalid_uint);
3082 : }
3083 :
3084 : void
3085 162107 : FEProblemBase::setResidualObjectParamsAndLog(const std::string & ro_name,
3086 : const std::string & name,
3087 : InputParameters & parameters,
3088 : const unsigned int nl_sys_num,
3089 : const std::string & base_name,
3090 : bool & reinit_displaced)
3091 : {
3092 162107 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3093 : {
3094 1920 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3095 1920 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3096 960 : reinit_displaced = true;
3097 : }
3098 : else
3099 : {
3100 161147 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3101 : {
3102 : // We allow Kernels to request that they use_displaced_mesh,
3103 : // but then be overridden when no displacements variables are
3104 : // provided in the Mesh block. If that happened, update the value
3105 : // of use_displaced_mesh appropriately for this Kernel.
3106 105 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3107 210 : parameters.set<bool>("use_displaced_mesh") = false;
3108 : }
3109 :
3110 322294 : parameters.set<SubProblem *>("_subproblem") = this;
3111 483441 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3112 : }
3113 :
3114 162107 : logAdd(base_name, name, ro_name, parameters);
3115 162107 : }
3116 :
3117 : void
3118 65424 : FEProblemBase::setAuxKernelParamsAndLog(const std::string & ak_name,
3119 : const std::string & name,
3120 : InputParameters & parameters,
3121 : const std::string & base_name)
3122 : {
3123 65424 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3124 : {
3125 22872 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3126 22872 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3127 22872 : parameters.set<SystemBase *>("_nl_sys") = &_displaced_problem->solverSys(0);
3128 11436 : if (!parameters.get<std::vector<BoundaryName>>("boundary").empty())
3129 10990 : _reinit_displaced_face = true;
3130 : else
3131 446 : _reinit_displaced_elem = true;
3132 : }
3133 : else
3134 : {
3135 53988 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3136 : {
3137 : // We allow AuxKernels to request that they use_displaced_mesh,
3138 : // but then be overridden when no displacements variables are
3139 : // provided in the Mesh block. If that happened, update the value
3140 : // of use_displaced_mesh appropriately for this AuxKernel.
3141 795 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3142 1590 : parameters.set<bool>("use_displaced_mesh") = false;
3143 : }
3144 :
3145 107976 : parameters.set<SubProblem *>("_subproblem") = this;
3146 107976 : parameters.set<SystemBase *>("_sys") = _aux.get();
3147 161964 : parameters.set<SystemBase *>("_nl_sys") = _solver_systems[0].get();
3148 : }
3149 :
3150 65424 : logAdd(base_name, name, ak_name, parameters);
3151 65424 : }
3152 :
3153 : void
3154 77758 : FEProblemBase::addKernel(const std::string & kernel_name,
3155 : const std::string & name,
3156 : InputParameters & parameters)
3157 : {
3158 : parallel_object_only();
3159 155516 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3160 77746 : if (!isSolverSystemNonlinear(nl_sys_num))
3161 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
3162 : "supported at the moment!");
3163 77746 : setResidualObjectParamsAndLog(
3164 77746 : kernel_name, name, parameters, nl_sys_num, "Kernel", _reinit_displaced_elem);
3165 :
3166 77746 : _nl[nl_sys_num]->addKernel(kernel_name, name, parameters);
3167 77593 : }
3168 :
3169 : void
3170 431 : FEProblemBase::addHDGKernel(const std::string & kernel_name,
3171 : const std::string & name,
3172 : InputParameters & parameters)
3173 : {
3174 : parallel_object_only();
3175 862 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3176 431 : if (!isSolverSystemNonlinear(nl_sys_num))
3177 0 : mooseError("You are trying to add a HDGKernel to a linear variable/system, which is not "
3178 : "supported at the moment!");
3179 431 : setResidualObjectParamsAndLog(
3180 431 : kernel_name, name, parameters, nl_sys_num, "HDGKernel", _reinit_displaced_elem);
3181 :
3182 431 : _nl[nl_sys_num]->addHDGKernel(kernel_name, name, parameters);
3183 431 : }
3184 :
3185 : void
3186 602 : FEProblemBase::addNodalKernel(const std::string & kernel_name,
3187 : const std::string & name,
3188 : InputParameters & parameters)
3189 : {
3190 : parallel_object_only();
3191 :
3192 1204 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3193 599 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3194 : {
3195 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3196 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3197 0 : _reinit_displaced_elem = true;
3198 : }
3199 : else
3200 : {
3201 599 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3202 : {
3203 : // We allow NodalKernels to request that they use_displaced_mesh,
3204 : // but then be overridden when no displacements variables are
3205 : // provided in the Mesh block. If that happened, update the value
3206 : // of use_displaced_mesh appropriately for this NodalKernel.
3207 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3208 0 : parameters.set<bool>("use_displaced_mesh") = false;
3209 : }
3210 :
3211 1198 : parameters.set<SubProblem *>("_subproblem") = this;
3212 1797 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3213 : }
3214 599 : logAdd("NodalKernel", name, kernel_name, parameters);
3215 599 : _nl[nl_sys_num]->addNodalKernel(kernel_name, name, parameters);
3216 599 : }
3217 :
3218 : void
3219 1319 : FEProblemBase::addScalarKernel(const std::string & kernel_name,
3220 : const std::string & name,
3221 : InputParameters & parameters)
3222 : {
3223 : parallel_object_only();
3224 :
3225 2638 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3226 1319 : if (!isSolverSystemNonlinear(nl_sys_num))
3227 0 : mooseError("You are trying to add a ScalarKernel to a linear variable/system, which is not "
3228 : "supported at the moment!");
3229 :
3230 1319 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3231 : {
3232 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3233 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3234 : }
3235 : else
3236 : {
3237 1319 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3238 : {
3239 : // We allow ScalarKernels to request that they use_displaced_mesh,
3240 : // but then be overridden when no displacements variables are
3241 : // provided in the Mesh block. If that happened, update the value
3242 : // of use_displaced_mesh appropriately for this ScalarKernel.
3243 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3244 0 : parameters.set<bool>("use_displaced_mesh") = false;
3245 : }
3246 :
3247 2638 : parameters.set<SubProblem *>("_subproblem") = this;
3248 3957 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3249 : }
3250 :
3251 1319 : logAdd("ScalarKernel", name, kernel_name, parameters);
3252 1319 : _nl[nl_sys_num]->addScalarKernel(kernel_name, name, parameters);
3253 1313 : }
3254 :
3255 : void
3256 75528 : FEProblemBase::addBoundaryCondition(const std::string & bc_name,
3257 : const std::string & name,
3258 : InputParameters & parameters)
3259 : {
3260 : parallel_object_only();
3261 :
3262 151060 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3263 75520 : if (!isSolverSystemNonlinear(nl_sys_num))
3264 0 : mooseError(
3265 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
3266 : "supported at the moment!");
3267 :
3268 75520 : setResidualObjectParamsAndLog(
3269 75520 : bc_name, name, parameters, nl_sys_num, "BoundaryCondition", _reinit_displaced_face);
3270 75520 : _nl[nl_sys_num]->addBoundaryCondition(bc_name, name, parameters);
3271 75476 : }
3272 :
3273 : void
3274 1783 : FEProblemBase::addConstraint(const std::string & c_name,
3275 : const std::string & name,
3276 : InputParameters & parameters)
3277 : {
3278 : parallel_object_only();
3279 :
3280 1783 : _has_constraints = true;
3281 :
3282 1783 : auto determine_var_param_name = [¶meters, this]()
3283 : {
3284 5349 : if (parameters.isParamValid("variable"))
3285 1222 : return "variable";
3286 : else
3287 : {
3288 : // must be a mortar constraint
3289 1122 : const bool has_secondary_var = parameters.isParamValid("secondary_variable");
3290 1122 : const bool has_primary_var = parameters.isParamValid("primary_variable");
3291 561 : if (!has_secondary_var && !has_primary_var)
3292 0 : mooseError(
3293 : "Either a 'secondary_variable' or 'primary_variable' parameter must be supplied for '",
3294 0 : parameters.getObjectName(),
3295 : "'");
3296 561 : return has_secondary_var ? "secondary_variable" : "primary_variable";
3297 : }
3298 1783 : };
3299 :
3300 : const auto nl_sys_num =
3301 3566 : determineSolverSystem(parameters.varName(determine_var_param_name(), name), true).second;
3302 1780 : if (!isSolverSystemNonlinear(nl_sys_num))
3303 0 : mooseError("You are trying to add a Constraint to a linear variable/system, which is not "
3304 : "supported at the moment!");
3305 :
3306 1780 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3307 : {
3308 266 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3309 266 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3310 133 : _reinit_displaced_face = true;
3311 : }
3312 : else
3313 : {
3314 : // It might _want_ to use a displaced mesh... but we're not so set it to false
3315 1647 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3316 3294 : parameters.set<bool>("use_displaced_mesh") = false;
3317 :
3318 3294 : parameters.set<SubProblem *>("_subproblem") = this;
3319 4941 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3320 : }
3321 :
3322 1780 : logAdd("Constraint", name, c_name, parameters);
3323 1780 : _nl[nl_sys_num]->addConstraint(c_name, name, parameters);
3324 1756 : }
3325 :
3326 : void
3327 94306 : FEProblemBase::addAuxVariable(const std::string & var_type,
3328 : const std::string & var_name,
3329 : InputParameters & params)
3330 : {
3331 : parallel_object_only();
3332 :
3333 94306 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3334 94306 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3335 94306 : const auto fe_type = FEType(order, family);
3336 :
3337 : const auto active_subdomains_vector =
3338 94306 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3339 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3340 94306 : active_subdomains_vector.end());
3341 :
3342 94306 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ true, &active_subdomains))
3343 0 : return;
3344 :
3345 282891 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3346 188594 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3347 :
3348 94297 : logAdd("AuxVariable", var_name, var_type, params);
3349 94297 : _aux->addVariable(var_type, var_name, params);
3350 94297 : if (_displaced_problem)
3351 : // MooseObjects need to be unique so change the name here
3352 10166 : _displaced_problem->addAuxVariable(var_type, var_name, params);
3353 :
3354 94297 : markFamilyPRefinement(params);
3355 94297 : if (_displaced_problem)
3356 10166 : _displaced_problem->markFamilyPRefinement(params);
3357 94297 : }
3358 :
3359 : void
3360 2628 : FEProblemBase::addElementalFieldVariable(const std::string & var_type,
3361 : const std::string & var_name,
3362 : InputParameters & params)
3363 : {
3364 2628 : addAuxVariable(var_type, var_name, params);
3365 2628 : }
3366 :
3367 : void
3368 0 : FEProblemBase::addAuxVariable(const std::string & var_name,
3369 : const FEType & type,
3370 : const std::set<SubdomainID> * const active_subdomains)
3371 : {
3372 : parallel_object_only();
3373 :
3374 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3375 :
3376 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3377 0 : return;
3378 :
3379 0 : std::string var_type;
3380 0 : if (type == FEType(0, MONOMIAL))
3381 0 : var_type = "MooseVariableConstMonomial";
3382 0 : else if (type.family == SCALAR)
3383 0 : var_type = "MooseVariableScalar";
3384 0 : else if (FEInterface::field_type(type) == TYPE_VECTOR)
3385 0 : var_type = "VectorMooseVariable";
3386 : else
3387 0 : var_type = "MooseVariable";
3388 :
3389 0 : InputParameters params = _factory.getValidParams(var_type);
3390 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3391 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3392 0 : params.set<MooseEnum>("order") = type.order.get_order();
3393 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3394 :
3395 0 : if (active_subdomains)
3396 0 : for (const SubdomainID & id : *active_subdomains)
3397 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3398 :
3399 0 : logAdd("AuxVariable", var_name, var_type, params);
3400 0 : _aux->addVariable(var_type, var_name, params);
3401 0 : if (_displaced_problem)
3402 0 : _displaced_problem->addAuxVariable("MooseVariable", var_name, params);
3403 :
3404 0 : markFamilyPRefinement(params);
3405 0 : if (_displaced_problem)
3406 0 : _displaced_problem->markFamilyPRefinement(params);
3407 0 : }
3408 :
3409 : void
3410 0 : FEProblemBase::addAuxArrayVariable(const std::string & var_name,
3411 : const FEType & type,
3412 : unsigned int components,
3413 : const std::set<SubdomainID> * const active_subdomains)
3414 : {
3415 : parallel_object_only();
3416 :
3417 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3418 :
3419 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3420 0 : return;
3421 :
3422 0 : InputParameters params = _factory.getValidParams("ArrayMooseVariable");
3423 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3424 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3425 0 : params.set<MooseEnum>("order") = type.order.get_order();
3426 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3427 0 : params.set<unsigned int>("components") = components;
3428 :
3429 0 : if (active_subdomains)
3430 0 : for (const SubdomainID & id : *active_subdomains)
3431 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3432 :
3433 0 : logAdd("Variable", var_name, "ArrayMooseVariable", params);
3434 0 : _aux->addVariable("ArrayMooseVariable", var_name, params);
3435 0 : if (_displaced_problem)
3436 0 : _displaced_problem->addAuxVariable("ArrayMooseVariable", var_name, params);
3437 :
3438 0 : markFamilyPRefinement(params);
3439 0 : if (_displaced_problem)
3440 0 : _displaced_problem->markFamilyPRefinement(params);
3441 0 : }
3442 :
3443 : void
3444 0 : FEProblemBase::addAuxScalarVariable(const std::string & var_name,
3445 : Order order,
3446 : Real /*scale_factor*/,
3447 : const std::set<SubdomainID> * const active_subdomains)
3448 : {
3449 : parallel_object_only();
3450 :
3451 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3452 :
3453 0 : if (order > _max_scalar_order)
3454 0 : _max_scalar_order = order;
3455 :
3456 0 : FEType type(order, SCALAR);
3457 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3458 0 : return;
3459 :
3460 0 : InputParameters params = _factory.getValidParams("MooseVariableScalar");
3461 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3462 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3463 :
3464 0 : params.set<MooseEnum>("order") = type.order.get_order();
3465 0 : params.set<MooseEnum>("family") = "SCALAR";
3466 0 : params.set<std::vector<Real>>("scaling") = std::vector<Real>{1};
3467 0 : if (active_subdomains)
3468 0 : for (const SubdomainID & id : *active_subdomains)
3469 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3470 :
3471 0 : logAdd("ScalarVariable", var_name, "MooseVariableScalar", params);
3472 0 : _aux->addVariable("MooseVariableScalar", var_name, params);
3473 0 : if (_displaced_problem)
3474 0 : _displaced_problem->addAuxVariable("MooseVariableScalar", var_name, params);
3475 0 : }
3476 :
3477 : void
3478 64788 : FEProblemBase::addAuxKernel(const std::string & kernel_name,
3479 : const std::string & name,
3480 : InputParameters & parameters)
3481 : {
3482 : parallel_object_only();
3483 :
3484 64788 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "AuxKernel");
3485 :
3486 64788 : _aux->addKernel(kernel_name, name, parameters);
3487 64674 : }
3488 :
3489 : void
3490 475 : FEProblemBase::addAuxScalarKernel(const std::string & kernel_name,
3491 : const std::string & name,
3492 : InputParameters & parameters)
3493 : {
3494 : parallel_object_only();
3495 :
3496 475 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3497 : {
3498 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3499 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3500 : }
3501 : else
3502 : {
3503 475 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3504 : {
3505 : // We allow AuxScalarKernels to request that they use_displaced_mesh,
3506 : // but then be overridden when no displacements variables are
3507 : // provided in the Mesh block. If that happened, update the value
3508 : // of use_displaced_mesh appropriately for this AuxScalarKernel.
3509 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3510 0 : parameters.set<bool>("use_displaced_mesh") = false;
3511 : }
3512 :
3513 950 : parameters.set<SubProblem *>("_subproblem") = this;
3514 1425 : parameters.set<SystemBase *>("_sys") = _aux.get();
3515 : }
3516 :
3517 475 : logAdd("AuxScalarKernel", name, kernel_name, parameters);
3518 475 : _aux->addScalarKernel(kernel_name, name, parameters);
3519 472 : }
3520 :
3521 : void
3522 871 : FEProblemBase::addDiracKernel(const std::string & kernel_name,
3523 : const std::string & name,
3524 : InputParameters & parameters)
3525 : {
3526 : parallel_object_only();
3527 :
3528 1742 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3529 868 : if (!isSolverSystemNonlinear(nl_sys_num))
3530 0 : mooseError("You are trying to add a DiracKernel to a linear variable/system, which is not "
3531 : "supported at the moment!");
3532 :
3533 868 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3534 : {
3535 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3536 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3537 12 : _reinit_displaced_elem = true;
3538 : }
3539 : else
3540 : {
3541 856 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3542 : {
3543 : // We allow DiracKernels to request that they use_displaced_mesh,
3544 : // but then be overridden when no displacements variables are
3545 : // provided in the Mesh block. If that happened, update the value
3546 : // of use_displaced_mesh appropriately for this DiracKernel.
3547 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3548 0 : parameters.set<bool>("use_displaced_mesh") = false;
3549 : }
3550 :
3551 1712 : parameters.set<SubProblem *>("_subproblem") = this;
3552 2568 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3553 : }
3554 :
3555 868 : logAdd("DiracKernel", name, kernel_name, parameters);
3556 868 : _nl[nl_sys_num]->addDiracKernel(kernel_name, name, parameters);
3557 862 : }
3558 :
3559 : // DGKernels ////
3560 :
3561 : void
3562 1250 : FEProblemBase::addDGKernel(const std::string & dg_kernel_name,
3563 : const std::string & name,
3564 : InputParameters & parameters)
3565 : {
3566 : parallel_object_only();
3567 :
3568 2500 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3569 1247 : if (!isSolverSystemNonlinear(nl_sys_num))
3570 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
3571 : "supported at the moment!");
3572 :
3573 1247 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3574 : {
3575 48 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3576 48 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3577 24 : _reinit_displaced_neighbor = true;
3578 : }
3579 : else
3580 : {
3581 1223 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3582 : {
3583 : // We allow DGKernels to request that they use_displaced_mesh,
3584 : // but then be overridden when no displacements variables are
3585 : // provided in the Mesh block. If that happened, update the value
3586 : // of use_displaced_mesh appropriately for this DGKernel.
3587 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3588 0 : parameters.set<bool>("use_displaced_mesh") = false;
3589 : }
3590 :
3591 2446 : parameters.set<SubProblem *>("_subproblem") = this;
3592 3669 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3593 : }
3594 :
3595 1247 : logAdd("DGKernel", name, dg_kernel_name, parameters);
3596 1247 : _nl[nl_sys_num]->addDGKernel(dg_kernel_name, name, parameters);
3597 :
3598 1247 : _has_internal_edge_residual_objects = true;
3599 1247 : }
3600 :
3601 : void
3602 6643 : FEProblemBase::addFVKernel(const std::string & fv_kernel_name,
3603 : const std::string & name,
3604 : InputParameters & parameters)
3605 : {
3606 6643 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3607 : // FVElementalKernels are computed in the historically finite element threaded loops. They rely
3608 : // on Assembly data like _current_elem. When we call reinit on the FEProblemBase we will only
3609 : // reinit the DisplacedProblem and its associated Assembly objects if we mark this boolean as
3610 : // true
3611 0 : _reinit_displaced_elem = true;
3612 6643 : addObject<FVKernel>(fv_kernel_name, name, parameters);
3613 6637 : }
3614 :
3615 : void
3616 5891 : FEProblemBase::addFVBC(const std::string & fv_bc_name,
3617 : const std::string & name,
3618 : InputParameters & parameters)
3619 : {
3620 5891 : addObject<FVBoundaryCondition>(fv_bc_name, name, parameters);
3621 5891 : }
3622 :
3623 : void
3624 246 : FEProblemBase::addFVInterfaceKernel(const std::string & fv_ik_name,
3625 : const std::string & name,
3626 : InputParameters & parameters)
3627 : {
3628 : /// We assume that variable1 and variable2 can live on different systems, in this case
3629 : /// the user needs to create two interface kernels with flipped variables and parameters
3630 246 : addObject<FVInterfaceKernel>(
3631 : fv_ik_name, name, parameters, /*threaded=*/true, /*variable_param_name=*/"variable1");
3632 237 : }
3633 :
3634 : void
3635 2202 : FEProblemBase::addLinearFVKernel(const std::string & kernel_name,
3636 : const std::string & name,
3637 : InputParameters & parameters)
3638 : {
3639 2202 : addObject<LinearFVKernel>(kernel_name, name, parameters);
3640 2202 : }
3641 :
3642 : void
3643 2079 : FEProblemBase::addLinearFVBC(const std::string & bc_name,
3644 : const std::string & name,
3645 : InputParameters & parameters)
3646 : {
3647 2079 : addObject<LinearFVBoundaryCondition>(bc_name, name, parameters);
3648 2079 : }
3649 :
3650 : // InterfaceKernels ////
3651 :
3652 : void
3653 800 : FEProblemBase::addInterfaceKernel(const std::string & interface_kernel_name,
3654 : const std::string & name,
3655 : InputParameters & parameters)
3656 : {
3657 : parallel_object_only();
3658 :
3659 1600 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3660 797 : if (!isSolverSystemNonlinear(nl_sys_num))
3661 0 : mooseError("You are trying to add a InterfaceKernel to a linear variable/system, which is not "
3662 : "supported at the moment!");
3663 :
3664 797 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3665 : {
3666 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3667 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3668 12 : _reinit_displaced_neighbor = true;
3669 : }
3670 : else
3671 : {
3672 785 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3673 : {
3674 : // We allow InterfaceKernels to request that they use_displaced_mesh,
3675 : // but then be overridden when no displacements variables are
3676 : // provided in the Mesh block. If that happened, update the value
3677 : // of use_displaced_mesh appropriately for this InterfaceKernel.
3678 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3679 0 : parameters.set<bool>("use_displaced_mesh") = false;
3680 : }
3681 :
3682 1570 : parameters.set<SubProblem *>("_subproblem") = this;
3683 2355 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3684 : }
3685 :
3686 797 : logAdd("InterfaceKernel", name, interface_kernel_name, parameters);
3687 797 : _nl[nl_sys_num]->addInterfaceKernel(interface_kernel_name, name, parameters);
3688 :
3689 797 : _has_internal_edge_residual_objects = true;
3690 797 : }
3691 :
3692 : void
3693 33916 : FEProblemBase::checkICRestartError(const std::string & ic_name,
3694 : const std::string & name,
3695 : const VariableName & var_name)
3696 : {
3697 33916 : if (!_allow_ics_during_restart)
3698 : {
3699 33837 : std::string restart_method = "";
3700 33837 : if (_app.isRestarting())
3701 : restart_method =
3702 0 : "a checkpoint restart, by IC object '" + ic_name + "' for variable '" + name + "'";
3703 33837 : else if (_app.getExReaderForRestart())
3704 : {
3705 3 : std::vector<std::string> restarted_vars = _app.getExReaderForRestart()->get_elem_var_names();
3706 3 : const auto nodal_vars = _app.getExReaderForRestart()->get_nodal_var_names();
3707 3 : const auto global_vars = _app.getExReaderForRestart()->get_global_var_names();
3708 3 : restarted_vars.insert(restarted_vars.end(), nodal_vars.begin(), nodal_vars.end());
3709 3 : restarted_vars.insert(restarted_vars.end(), global_vars.begin(), global_vars.end());
3710 :
3711 3 : if (std::find(restarted_vars.begin(), restarted_vars.end(), var_name) != restarted_vars.end())
3712 6 : restart_method = "an Exodus restart, by IC object '" + ic_name + "' for variable '" + name +
3713 3 : "' that is also being restarted";
3714 3 : }
3715 33837 : if (!restart_method.empty())
3716 3 : mooseError(
3717 : "Initial conditions have been specified during ",
3718 : restart_method,
3719 : ".\nThis is only allowed if you specify 'allow_initial_conditions_with_restart' to "
3720 : "the [Problem], as initial conditions can override restarted fields");
3721 33834 : }
3722 33913 : }
3723 :
3724 : void
3725 31565 : FEProblemBase::addInitialCondition(const std::string & ic_name,
3726 : const std::string & name,
3727 : InputParameters & parameters)
3728 : {
3729 : parallel_object_only();
3730 :
3731 : // before we start to mess with the initial condition, we need to check parameters for errors.
3732 31565 : parameters.checkParams(name);
3733 31562 : const std::string & var_name = parameters.get<VariableName>("variable");
3734 :
3735 : // Forbid initial conditions on a restarted problem, as they would override the restart
3736 31562 : checkICRestartError(ic_name, name, var_name);
3737 :
3738 63118 : parameters.set<SubProblem *>("_subproblem") = this;
3739 :
3740 : // field IC
3741 31559 : if (hasVariable(var_name))
3742 : {
3743 63458 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3744 : {
3745 33459 : MooseVariableFEBase & var = getVariable(
3746 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3747 66918 : parameters.set<SystemBase *>("_sys") = &var.sys();
3748 33459 : std::shared_ptr<InitialConditionBase> ic;
3749 33459 : if (dynamic_cast<MooseVariable *>(&var))
3750 31305 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3751 2154 : else if (dynamic_cast<VectorMooseVariable *>(&var))
3752 283 : ic = _factory.create<VectorInitialCondition>(ic_name, name, parameters, tid);
3753 1871 : else if (dynamic_cast<ArrayMooseVariable *>(&var))
3754 1207 : ic = _factory.create<ArrayInitialCondition>(ic_name, name, parameters, tid);
3755 664 : else if (dynamic_cast<MooseVariableFVReal *>(&var))
3756 664 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3757 0 : else if (dynamic_cast<MooseLinearVariableFVReal *>(&var))
3758 0 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3759 : else
3760 0 : mooseError("Your FE variable in initial condition ",
3761 : name,
3762 : " must be either of scalar or vector type");
3763 33438 : logAdd("IC", name, ic_name, parameters);
3764 33438 : _ics.addObject(ic, tid);
3765 33423 : }
3766 : }
3767 :
3768 : // scalar IC
3769 1524 : else if (hasScalarVariable(var_name))
3770 : {
3771 1524 : MooseVariableScalar & var = getScalarVariable(0, var_name);
3772 3048 : parameters.set<SystemBase *>("_sys") = &var.sys();
3773 : std::shared_ptr<ScalarInitialCondition> ic =
3774 1524 : _factory.create<ScalarInitialCondition>(ic_name, name, parameters);
3775 1524 : logAdd("ScalarIC", name, ic_name, parameters);
3776 1524 : _scalar_ics.addObject(ic);
3777 1524 : }
3778 :
3779 : else
3780 0 : mooseError(
3781 : "Variable '", var_name, "' requested in initial condition '", name, "' does not exist.");
3782 31523 : }
3783 :
3784 : void
3785 2354 : FEProblemBase::addFVInitialCondition(const std::string & ic_name,
3786 : const std::string & name,
3787 : InputParameters & parameters)
3788 : {
3789 : parallel_object_only();
3790 :
3791 : // before we start to mess with the initial condition, we need to check parameters for errors.
3792 2354 : parameters.checkParams(name);
3793 2354 : const std::string & var_name = parameters.get<VariableName>("variable");
3794 :
3795 : // Forbid initial conditions on a restarted problem, as they would override the restart
3796 2354 : checkICRestartError(ic_name, name, var_name);
3797 :
3798 4708 : parameters.set<SubProblem *>("_subproblem") = this;
3799 :
3800 : // field IC
3801 2354 : if (hasVariable(var_name))
3802 : {
3803 4790 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3804 : {
3805 2436 : auto & var = getVariable(
3806 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3807 4872 : parameters.set<SystemBase *>("_sys") = &var.sys();
3808 2436 : std::shared_ptr<FVInitialConditionBase> ic;
3809 2436 : if (var.isFV())
3810 2436 : ic = _factory.create<FVInitialCondition>(ic_name, name, parameters, tid);
3811 : else
3812 0 : mooseError(
3813 : "Your variable for an FVInitialCondition needs to be an a finite volume variable!");
3814 2436 : _fv_ics.addObject(ic, tid);
3815 2436 : }
3816 : }
3817 : else
3818 0 : mooseError("Variable '",
3819 : var_name,
3820 : "' requested in finite volume initial condition '",
3821 : name,
3822 : "' does not exist.");
3823 2354 : }
3824 :
3825 : void
3826 57538 : FEProblemBase::projectSolution()
3827 : {
3828 287690 : TIME_SECTION("projectSolution", 2, "Projecting Initial Solutions")
3829 :
3830 57538 : FloatingPointExceptionGuard fpe_guard(_app);
3831 :
3832 57538 : ComputeInitialConditionThread cic(*this);
3833 57538 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cic);
3834 :
3835 57532 : if (haveFV())
3836 : {
3837 : using ElemInfoRange = StoredRange<MooseMesh::const_elem_info_iterator, const ElemInfo *>;
3838 4139 : ElemInfoRange elem_info_range(_mesh.ownedElemInfoBegin(), _mesh.ownedElemInfoEnd());
3839 :
3840 4139 : ComputeFVInitialConditionThread cfvic(*this);
3841 4139 : Threads::parallel_reduce(elem_info_range, cfvic);
3842 4139 : }
3843 :
3844 : // Need to close the solution vector here so that boundary ICs take precendence
3845 114175 : for (auto & nl : _nl)
3846 56643 : nl->solution().close();
3847 57532 : _aux->solution().close();
3848 :
3849 : // now run boundary-restricted initial conditions
3850 57532 : ComputeBoundaryInitialConditionThread cbic(*this);
3851 57532 : Threads::parallel_reduce(getCurrentAlgebraicBndNodeRange(), cbic);
3852 :
3853 114175 : for (auto & nl : _nl)
3854 56643 : nl->solution().close();
3855 57532 : _aux->solution().close();
3856 :
3857 : // Also, load values into the SCALAR dofs
3858 : // Note: We assume that all SCALAR dofs are on the
3859 : // processor with highest ID
3860 57532 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3861 : {
3862 567 : const auto & ics = _scalar_ics.getActiveObjects();
3863 1561 : for (const auto & ic : ics)
3864 : {
3865 994 : MooseVariableScalar & var = ic->variable();
3866 994 : var.reinit();
3867 :
3868 994 : DenseVector<Number> vals(var.order());
3869 994 : ic->compute(vals);
3870 :
3871 994 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3872 2323 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3873 : {
3874 1329 : const auto global_index = var.dofIndices()[i];
3875 1329 : var.sys().solution().set(global_index, vals(i));
3876 1329 : var.setValue(i, vals(i));
3877 : }
3878 994 : }
3879 : }
3880 :
3881 115365 : for (auto & sys : _solver_systems)
3882 : {
3883 57833 : sys->solution().close();
3884 57833 : sys->solution().localize(*sys->system().current_local_solution, sys->dofMap().get_send_list());
3885 : }
3886 :
3887 57532 : _aux->solution().close();
3888 57532 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3889 57532 : }
3890 :
3891 : void
3892 2041 : FEProblemBase::projectInitialConditionOnCustomRange(
3893 : ConstElemRange & elem_range,
3894 : ConstBndNodeRange & bnd_nodes,
3895 : const std::optional<std::set<VariableName>> & target_vars)
3896 : {
3897 2041 : if (target_vars)
3898 : {
3899 2041 : ComputeInitialConditionThread cic(*this, &(*target_vars));
3900 2041 : Threads::parallel_reduce(elem_range, cic);
3901 : }
3902 : else
3903 : {
3904 0 : ComputeInitialConditionThread cic(*this);
3905 0 : Threads::parallel_reduce(elem_range, cic);
3906 : }
3907 :
3908 : // Need to close the solution vector here so that boundary ICs take precendence
3909 4082 : for (auto & nl : _nl)
3910 2041 : nl->solution().close();
3911 2041 : _aux->solution().close();
3912 :
3913 2041 : if (target_vars)
3914 : {
3915 2041 : ComputeBoundaryInitialConditionThread cbic(*this, &(*target_vars));
3916 2041 : Threads::parallel_reduce(bnd_nodes, cbic);
3917 2041 : }
3918 : else
3919 : {
3920 0 : ComputeBoundaryInitialConditionThread cbic(*this);
3921 0 : Threads::parallel_reduce(bnd_nodes, cbic);
3922 0 : }
3923 :
3924 4082 : for (auto & nl : _nl)
3925 2041 : nl->solution().close();
3926 2041 : _aux->solution().close();
3927 :
3928 : // Also, load values into the SCALAR dofs
3929 : // Note: We assume that all SCALAR dofs are on the
3930 : // processor with highest ID
3931 2041 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3932 : {
3933 0 : const auto & ics = _scalar_ics.getActiveObjects();
3934 0 : for (const auto & ic : ics)
3935 : {
3936 0 : MooseVariableScalar & var = ic->variable();
3937 :
3938 0 : if (target_vars && !target_vars->count(var.name()))
3939 0 : continue;
3940 :
3941 0 : var.reinit();
3942 :
3943 0 : DenseVector<Number> vals(var.order());
3944 0 : ic->compute(vals);
3945 :
3946 0 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3947 0 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3948 : {
3949 0 : const auto global_index = var.dofIndices()[i];
3950 0 : var.sys().solution().set(global_index, vals(i));
3951 0 : var.setValue(i, vals(i));
3952 : }
3953 0 : }
3954 : }
3955 :
3956 4082 : for (auto & nl : _nl)
3957 : {
3958 2041 : nl->solution().close();
3959 2041 : nl->solution().localize(*nl->system().current_local_solution, nl->dofMap().get_send_list());
3960 : }
3961 :
3962 2041 : _aux->solution().close();
3963 2041 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3964 2041 : }
3965 :
3966 : void
3967 737 : FEProblemBase::projectFunctionOnCustomRange(ConstElemRange & elem_range,
3968 : Number (*func)(const Point &,
3969 : const libMesh::Parameters &,
3970 : const std::string &,
3971 : const std::string &),
3972 : Gradient (*func_grad)(const Point &,
3973 : const libMesh::Parameters &,
3974 : const std::string &,
3975 : const std::string &),
3976 : const libMesh::Parameters & params,
3977 : const std::vector<VariableName> & target_vars)
3978 : {
3979 : mooseAssert(!Threads::in_threads,
3980 : "We're performing a projection based on data from just the thread 0 variable, so any "
3981 : "modifications to the variable solution must have been thread joined already");
3982 :
3983 737 : std::unordered_map<unsigned int, std::vector<unsigned int>> sys_to_var_nums;
3984 :
3985 1474 : for (const auto & target_var : target_vars)
3986 : {
3987 737 : const auto sn = systemNumForVariable(target_var);
3988 737 : const auto & var = getStandardVariable(0, target_var);
3989 737 : sys_to_var_nums[sn].push_back(var.number());
3990 : }
3991 :
3992 1474 : for (const auto & [sys_num, var_nums] : sys_to_var_nums)
3993 : {
3994 737 : System & libmesh_sys = getSystemBase(sys_num).system();
3995 737 : libmesh_sys.project_solution(func, func_grad, params, elem_range, var_nums);
3996 : }
3997 737 : }
3998 :
3999 : std::shared_ptr<MaterialBase>
4000 252 : FEProblemBase::getMaterial(std::string name,
4001 : Moose::MaterialDataType type,
4002 : const THREAD_ID tid,
4003 : bool no_warn)
4004 : {
4005 252 : switch (type)
4006 : {
4007 65 : case Moose::NEIGHBOR_MATERIAL_DATA:
4008 65 : name += "_neighbor";
4009 65 : break;
4010 65 : case Moose::FACE_MATERIAL_DATA:
4011 65 : name += "_face";
4012 65 : break;
4013 122 : default:
4014 122 : break;
4015 : }
4016 :
4017 252 : std::shared_ptr<MaterialBase> material = _all_materials[type].getActiveObject(name, tid);
4018 657 : if (!no_warn && material->getParam<bool>("compute") && type == Moose::BLOCK_MATERIAL_DATA)
4019 3 : mooseWarning("You are retrieving a Material object (",
4020 3 : material->name(),
4021 : "), but its compute flag is set to true. This indicates that MOOSE is "
4022 : "computing this property which may not be desired and produce un-expected "
4023 : "results.");
4024 :
4025 246 : return material;
4026 : }
4027 :
4028 : MaterialData &
4029 33141685 : FEProblemBase::getMaterialData(Moose::MaterialDataType type,
4030 : const THREAD_ID tid,
4031 : const MooseObject * object) const
4032 : {
4033 33141685 : switch (type)
4034 : {
4035 860506 : case Moose::BLOCK_MATERIAL_DATA:
4036 860506 : if (object)
4037 263664 : _material_props.addConsumer(type, object);
4038 860506 : return _material_props.getMaterialData(tid);
4039 15163426 : case Moose::NEIGHBOR_MATERIAL_DATA:
4040 15163426 : if (object)
4041 22616 : _neighbor_material_props.addConsumer(type, object);
4042 15163426 : return _neighbor_material_props.getMaterialData(tid);
4043 17117753 : case Moose::BOUNDARY_MATERIAL_DATA:
4044 : case Moose::FACE_MATERIAL_DATA:
4045 : case Moose::INTERFACE_MATERIAL_DATA:
4046 17117753 : if (object)
4047 52452 : _bnd_material_props.addConsumer(type, object);
4048 17117753 : return _bnd_material_props.getMaterialData(tid);
4049 : }
4050 :
4051 0 : mooseError("FEProblemBase::getMaterialData(): Invalid MaterialDataType ", type);
4052 : }
4053 :
4054 : const std::set<const MooseObject *> &
4055 0 : FEProblemBase::getMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
4056 : {
4057 0 : switch (type)
4058 : {
4059 0 : case Moose::BLOCK_MATERIAL_DATA:
4060 0 : return _material_props.getConsumers(type);
4061 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
4062 0 : return _neighbor_material_props.getConsumers(type);
4063 0 : case Moose::BOUNDARY_MATERIAL_DATA:
4064 : case Moose::FACE_MATERIAL_DATA:
4065 : case Moose::INTERFACE_MATERIAL_DATA:
4066 0 : return _bnd_material_props.getConsumers(type);
4067 : }
4068 :
4069 0 : mooseError("FEProblemBase::getMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
4070 : type);
4071 : }
4072 :
4073 : void
4074 0 : FEProblemBase::setPreserveMatrixSparsityPattern(bool preserve)
4075 : {
4076 0 : if (_ignore_zeros_in_jacobian && preserve)
4077 0 : paramWarning(
4078 : "ignore_zeros_in_jacobian",
4079 : "We likely cannot preserve the sparsity pattern if ignoring zeros in the Jacobian, which "
4080 : "leads to removing those entries from the Jacobian sparsity pattern");
4081 0 : _preserve_matrix_sparsity_pattern = preserve;
4082 0 : }
4083 :
4084 : bool
4085 311125 : FEProblemBase::acceptInvalidSolution() const
4086 : {
4087 622165 : return allowInvalidSolution() || // invalid solutions are always allowed
4088 622165 : !_app.solutionInvalidity().hasInvalidSolutionError(); // if not allowed, check for errors
4089 : }
4090 :
4091 : void
4092 1105 : FEProblemBase::addFunctorMaterial(const std::string & functor_material_name,
4093 : const std::string & name,
4094 : InputParameters & parameters)
4095 : {
4096 : parallel_object_only();
4097 :
4098 1105 : auto add_functor_materials = [&](const auto & parameters, const auto & name)
4099 : {
4100 2305 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
4101 : {
4102 : // Create the general Block/Boundary MaterialBase object
4103 1200 : std::shared_ptr<MaterialBase> material =
4104 1200 : _factory.create<MaterialBase>(functor_material_name, name, parameters, tid);
4105 2400 : logAdd("FunctorMaterial", name, functor_material_name, parameters);
4106 1200 : _all_materials.addObject(material, tid);
4107 1200 : _materials.addObject(material, tid);
4108 : }
4109 1105 : };
4110 :
4111 2210 : parameters.set<SubProblem *>("_subproblem") = this;
4112 1105 : add_functor_materials(parameters, name);
4113 1105 : if (_displaced_problem)
4114 : {
4115 0 : auto disp_params = parameters;
4116 0 : disp_params.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4117 0 : add_functor_materials(disp_params, name + "_displaced");
4118 0 : }
4119 1105 : }
4120 :
4121 : void
4122 12241 : FEProblemBase::addMaterial(const std::string & mat_name,
4123 : const std::string & name,
4124 : InputParameters & parameters)
4125 : {
4126 24482 : addMaterialHelper({&_materials}, mat_name, name, parameters);
4127 12143 : }
4128 :
4129 : void
4130 340 : FEProblemBase::addInterfaceMaterial(const std::string & mat_name,
4131 : const std::string & name,
4132 : InputParameters & parameters)
4133 : {
4134 680 : addMaterialHelper({&_interface_materials}, mat_name, name, parameters);
4135 340 : }
4136 :
4137 : void
4138 13523 : FEProblemBase::addMaterialHelper(std::vector<MaterialWarehouse *> warehouses,
4139 : const std::string & mat_name,
4140 : const std::string & name,
4141 : InputParameters & parameters)
4142 : {
4143 : parallel_object_only();
4144 :
4145 13523 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4146 : {
4147 252 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4148 126 : _reinit_displaced_elem = _reinit_displaced_face = _reinit_displaced_neighbor = true;
4149 : }
4150 : else
4151 : {
4152 13397 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
4153 : {
4154 : // We allow Materials to request that they use_displaced_mesh,
4155 : // but then be overridden when no displacements variables are
4156 : // provided in the Mesh block. If that happened, update the value
4157 : // of use_displaced_mesh appropriately for this Material.
4158 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
4159 0 : parameters.set<bool>("use_displaced_mesh") = false;
4160 : }
4161 :
4162 40191 : parameters.set<SubProblem *>("_subproblem") = this;
4163 : }
4164 :
4165 13523 : unsigned int n_threads = libMesh::n_threads();
4166 :
4167 : #ifdef MOOSE_KOKKOS_ENABLED
4168 10086 : if (parameters.isKokkosObject())
4169 942 : n_threads = 1;
4170 : #endif
4171 :
4172 28140 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4173 : {
4174 : // Create the general Block/Boundary MaterialBase object
4175 : std::shared_ptr<MaterialBase> material =
4176 14723 : _factory.create<MaterialBase>(mat_name, name, parameters, tid);
4177 14617 : logAdd("Material", name, mat_name, parameters);
4178 29234 : bool discrete = !material->getParam<bool>("compute");
4179 :
4180 : // If the object is boundary restricted or if it is a functor material we do not create the
4181 : // neighbor and face objects
4182 14617 : if (material->boundaryRestricted() || dynamic_cast<FunctorMaterial *>(material.get()))
4183 : {
4184 2984 : _all_materials.addObject(material, tid);
4185 2984 : if (discrete)
4186 4 : _discrete_materials.addObject(material, tid);
4187 : else
4188 5960 : for (auto && warehouse : warehouses)
4189 2980 : warehouse->addObject(material, tid);
4190 : }
4191 :
4192 : // Non-boundary restricted require face and neighbor objects
4193 : else
4194 : {
4195 : // TODO: we only need to do this if we have needs for face materials (e.g.
4196 : // FV, DG, etc.) - but currently we always do it. Figure out how to fix
4197 : // this.
4198 :
4199 : // The name of the object being created, this is changed multiple times as objects are
4200 : // created below
4201 11633 : std::string object_name;
4202 :
4203 : // Create a copy of the supplied parameters to the setting for "_material_data_type" isn't
4204 : // used from a previous tid loop
4205 11633 : InputParameters current_parameters = parameters;
4206 :
4207 : // face material
4208 11633 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4209 : Moose::FACE_MATERIAL_DATA;
4210 11633 : object_name = name + "_face";
4211 : std::shared_ptr<MaterialBase> face_material =
4212 11633 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4213 :
4214 : // neighbor material
4215 23266 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4216 : Moose::NEIGHBOR_MATERIAL_DATA;
4217 11633 : current_parameters.set<bool>("_neighbor") = true;
4218 11633 : object_name = name + "_neighbor";
4219 : std::shared_ptr<MaterialBase> neighbor_material =
4220 11633 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4221 :
4222 : // Store the material objects
4223 11633 : _all_materials.addObjects(material, neighbor_material, face_material, tid);
4224 :
4225 11633 : if (discrete)
4226 73 : _discrete_materials.addObjects(material, neighbor_material, face_material, tid);
4227 : else
4228 23120 : for (auto && warehouse : warehouses)
4229 11560 : warehouse->addObjects(material, neighbor_material, face_material, tid);
4230 :
4231 : // Names of all controllable parameters for this Material object
4232 11633 : const std::string & base = parameters.getBase();
4233 34899 : MooseObjectParameterName name(MooseObjectName(base, material->name()), "*");
4234 : const auto param_names =
4235 11633 : _app.getInputParameterWarehouse().getControllableParameterNames(name);
4236 :
4237 : // Connect parameters of the primary Material object to those on the face and neighbor
4238 : // objects
4239 29560 : for (const auto & p_name : param_names)
4240 : {
4241 35854 : MooseObjectParameterName primary_name(MooseObjectName(base, material->name()),
4242 35854 : p_name.parameter());
4243 35854 : MooseObjectParameterName face_name(MooseObjectName(base, face_material->name()),
4244 35854 : p_name.parameter());
4245 35854 : MooseObjectParameterName neighbor_name(MooseObjectName(base, neighbor_material->name()),
4246 35854 : p_name.parameter());
4247 17927 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4248 : primary_name, face_name, false);
4249 17927 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4250 : primary_name, neighbor_name, false);
4251 17927 : }
4252 11633 : }
4253 14617 : }
4254 13417 : }
4255 :
4256 : void
4257 4683441 : FEProblemBase::prepareMaterials(const std::unordered_set<unsigned int> & consumer_needed_mat_props,
4258 : const SubdomainID blk_id,
4259 : const THREAD_ID tid)
4260 : {
4261 4683441 : std::set<MooseVariableFEBase *> needed_moose_vars;
4262 4683441 : std::unordered_set<unsigned int> needed_mat_props;
4263 :
4264 4683441 : if (_all_materials.hasActiveBlockObjects(blk_id, tid))
4265 : {
4266 590556 : _all_materials.updateVariableDependency(needed_moose_vars, tid);
4267 590556 : _all_materials.updateBlockMatPropDependency(blk_id, needed_mat_props, tid);
4268 : }
4269 :
4270 4683441 : const auto & ids = _mesh.getSubdomainBoundaryIds(blk_id);
4271 21610508 : for (const auto id : ids)
4272 : {
4273 16927067 : _materials.updateBoundaryVariableDependency(id, needed_moose_vars, tid);
4274 16927067 : _materials.updateBoundaryMatPropDependency(id, needed_mat_props, tid);
4275 : }
4276 :
4277 4683441 : const auto & current_active_elemental_moose_variables = getActiveElementalMooseVariables(tid);
4278 4683441 : needed_moose_vars.insert(current_active_elemental_moose_variables.begin(),
4279 : current_active_elemental_moose_variables.end());
4280 :
4281 4683441 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
4282 :
4283 4683441 : setActiveElementalMooseVariables(needed_moose_vars, tid);
4284 4683441 : setActiveMaterialProperties(needed_mat_props, tid);
4285 4683441 : }
4286 :
4287 : void
4288 363782692 : FEProblemBase::reinitMaterials(SubdomainID blk_id, const THREAD_ID tid, bool swap_stateful)
4289 : {
4290 363782692 : if (hasActiveMaterialProperties(tid))
4291 : {
4292 14572509 : auto && elem = _assembly[tid][0]->elem();
4293 14572509 : unsigned int n_points = _assembly[tid][0]->qRule()->n_points();
4294 :
4295 14572509 : auto & material_data = _material_props.getMaterialData(tid);
4296 14572509 : material_data.resize(n_points);
4297 :
4298 : // Only swap if requested
4299 14572509 : if (swap_stateful)
4300 14565967 : material_data.swap(*elem);
4301 :
4302 14572509 : if (_discrete_materials.hasActiveBlockObjects(blk_id, tid))
4303 2698 : material_data.reset(_discrete_materials.getActiveBlockObjects(blk_id, tid));
4304 :
4305 14572506 : if (_materials.hasActiveBlockObjects(blk_id, tid))
4306 14553967 : material_data.reinit(_materials.getActiveBlockObjects(blk_id, tid));
4307 : }
4308 363782637 : }
4309 :
4310 : void
4311 5415488 : FEProblemBase::reinitMaterialsFace(const SubdomainID blk_id,
4312 : const THREAD_ID tid,
4313 : const bool swap_stateful,
4314 : const std::deque<MaterialBase *> * const reinit_mats)
4315 : {
4316 : // we reinit more often than needed here because we dont have a way to check whether
4317 : // we need to compute the face materials on a particular (possibly external) face
4318 5415488 : if (hasActiveMaterialProperties(tid))
4319 : {
4320 1045875 : auto && elem = _assembly[tid][0]->elem();
4321 1045875 : unsigned int side = _assembly[tid][0]->side();
4322 1045875 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4323 :
4324 1045875 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4325 1045875 : bnd_material_data.resize(n_points);
4326 :
4327 1045875 : if (swap_stateful && !bnd_material_data.isSwapped())
4328 1024401 : bnd_material_data.swap(*elem, side);
4329 :
4330 1045875 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4331 0 : bnd_material_data.reset(
4332 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4333 :
4334 1045875 : if (reinit_mats)
4335 21474 : bnd_material_data.reinit(*reinit_mats);
4336 1024401 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4337 1024401 : bnd_material_data.reinit(
4338 1024401 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4339 : }
4340 5415488 : }
4341 :
4342 : void
4343 3900193 : FEProblemBase::reinitMaterialsFaceOnBoundary(const BoundaryID boundary_id,
4344 : const SubdomainID blk_id,
4345 : const THREAD_ID tid,
4346 : const bool swap_stateful,
4347 : const std::deque<MaterialBase *> * const reinit_mats)
4348 : {
4349 4231124 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4350 330931 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4351 330931 : needInternalNeighborSideMaterial(blk_id, tid)))
4352 : {
4353 407446 : const auto * const elem = _assembly[tid][0]->elem();
4354 407446 : unsigned int side = _assembly[tid][0]->side();
4355 407446 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4356 :
4357 407446 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4358 407446 : bnd_material_data.resize(n_points);
4359 :
4360 407446 : if (swap_stateful && !bnd_material_data.isSwapped())
4361 407446 : bnd_material_data.swap(*elem, side);
4362 :
4363 407446 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4364 0 : bnd_material_data.reset(
4365 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4366 :
4367 407446 : if (reinit_mats)
4368 0 : bnd_material_data.reinit(*reinit_mats);
4369 407446 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4370 393054 : bnd_material_data.reinit(
4371 393054 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4372 : }
4373 3900193 : }
4374 :
4375 : void
4376 43931 : FEProblemBase::reinitMaterialsNeighborOnBoundary(
4377 : const BoundaryID boundary_id,
4378 : const SubdomainID blk_id,
4379 : const THREAD_ID tid,
4380 : const bool swap_stateful,
4381 : const std::deque<MaterialBase *> * const reinit_mats)
4382 : {
4383 : // Since objects don't declare whether they need the face or neighbor (side) material properties,
4384 : // we use the same criteria for skipping material property computations as for face material
4385 : // properties This could be a future optimization.
4386 48141 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4387 4210 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4388 4210 : needInternalNeighborSideMaterial(blk_id, tid)))
4389 35986 : reinitMaterialsNeighbor(blk_id, tid, swap_stateful, reinit_mats);
4390 43931 : }
4391 :
4392 : void
4393 4283297 : FEProblemBase::reinitMaterialsNeighbor(const SubdomainID blk_id,
4394 : const THREAD_ID tid,
4395 : const bool swap_stateful,
4396 : const std::deque<MaterialBase *> * const reinit_mats)
4397 : {
4398 4283297 : if (hasActiveMaterialProperties(tid))
4399 : {
4400 : // NOTE: this will not work with h-adaptivity
4401 : // lindsayad: why not?
4402 :
4403 887183 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4404 887183 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(_assembly[tid][0]->elem());
4405 :
4406 : mooseAssert(neighbor, "neighbor should be non-null");
4407 : mooseAssert(blk_id == neighbor->subdomain_id(),
4408 : "The provided blk_id " << blk_id << " and neighbor subdomain ID "
4409 : << neighbor->subdomain_id() << " do not match.");
4410 :
4411 887183 : unsigned int n_points = _assembly[tid][0]->qRuleNeighbor()->n_points();
4412 :
4413 887183 : auto & neighbor_material_data = _neighbor_material_props.getMaterialData(tid);
4414 887183 : neighbor_material_data.resize(n_points);
4415 :
4416 : // Only swap if requested
4417 887183 : if (swap_stateful)
4418 865709 : neighbor_material_data.swap(*neighbor, neighbor_side);
4419 :
4420 887183 : if (_discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4421 0 : neighbor_material_data.reset(
4422 0 : _discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4423 :
4424 887183 : if (reinit_mats)
4425 21474 : neighbor_material_data.reinit(*reinit_mats);
4426 865709 : else if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4427 865446 : neighbor_material_data.reinit(
4428 865446 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4429 : }
4430 4283297 : }
4431 :
4432 : void
4433 4464197 : FEProblemBase::reinitMaterialsBoundary(const BoundaryID boundary_id,
4434 : const THREAD_ID tid,
4435 : const bool swap_stateful,
4436 : const std::deque<MaterialBase *> * const reinit_mats)
4437 : {
4438 4464197 : if (hasActiveMaterialProperties(tid) && needBoundaryMaterialOnSide(boundary_id, tid))
4439 : {
4440 285958 : auto && elem = _assembly[tid][0]->elem();
4441 285958 : unsigned int side = _assembly[tid][0]->side();
4442 285958 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4443 :
4444 285958 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4445 285958 : bnd_material_data.resize(n_points);
4446 :
4447 285958 : if (swap_stateful && !bnd_material_data.isSwapped())
4448 238273 : bnd_material_data.swap(*elem, side);
4449 :
4450 285958 : if (_discrete_materials.hasActiveBoundaryObjects(boundary_id, tid))
4451 0 : bnd_material_data.reset(_discrete_materials.getActiveBoundaryObjects(boundary_id, tid));
4452 :
4453 285958 : if (reinit_mats)
4454 21474 : bnd_material_data.reinit(*reinit_mats);
4455 264484 : else if (_materials.hasActiveBoundaryObjects(boundary_id, tid))
4456 23154 : bnd_material_data.reinit(_materials.getActiveBoundaryObjects(boundary_id, tid));
4457 : }
4458 4464197 : }
4459 :
4460 : void
4461 46788 : FEProblemBase::reinitMaterialsInterface(BoundaryID boundary_id,
4462 : const THREAD_ID tid,
4463 : bool swap_stateful)
4464 : {
4465 46788 : if (hasActiveMaterialProperties(tid) && needInterfaceMaterialOnSide(boundary_id, tid))
4466 : {
4467 37022 : const Elem * const & elem = _assembly[tid][0]->elem();
4468 37022 : unsigned int side = _assembly[tid][0]->side();
4469 37022 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4470 :
4471 37022 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4472 37022 : bnd_material_data.resize(n_points);
4473 :
4474 37022 : if (swap_stateful && !bnd_material_data.isSwapped())
4475 35527 : bnd_material_data.swap(*elem, side);
4476 :
4477 37022 : if (_interface_materials.hasActiveBoundaryObjects(boundary_id, tid))
4478 1171 : bnd_material_data.reinit(_interface_materials.getActiveBoundaryObjects(boundary_id, tid));
4479 : }
4480 46788 : }
4481 :
4482 : void
4483 362113747 : FEProblemBase::swapBackMaterials(const THREAD_ID tid)
4484 : {
4485 362113747 : auto && elem = _assembly[tid][0]->elem();
4486 362113747 : _material_props.getMaterialData(tid).swapBack(*elem);
4487 362113747 : }
4488 :
4489 : void
4490 8751666 : FEProblemBase::swapBackMaterialsFace(const THREAD_ID tid)
4491 : {
4492 8751666 : auto && elem = _assembly[tid][0]->elem();
4493 8751666 : unsigned int side = _assembly[tid][0]->side();
4494 8751666 : _bnd_material_props.getMaterialData(tid).swapBack(*elem, side);
4495 8751666 : }
4496 :
4497 : void
4498 3630008 : FEProblemBase::swapBackMaterialsNeighbor(const THREAD_ID tid)
4499 : {
4500 : // NOTE: this will not work with h-adaptivity
4501 3630008 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4502 : unsigned int neighbor_side =
4503 3630008 : neighbor ? neighbor->which_neighbor_am_i(_assembly[tid][0]->elem()) : libMesh::invalid_uint;
4504 :
4505 3630008 : if (!neighbor)
4506 : {
4507 0 : if (haveFV())
4508 : {
4509 : // If neighbor is null, then we're on the neighbor side of a mesh boundary, e.g. we're off
4510 : // the mesh in ghost-land. If we're using the finite volume method, then variable values and
4511 : // consequently material properties have well-defined values in this ghost region outside of
4512 : // the mesh and we really do want to reinit our neighbor materials in this case. Since we're
4513 : // off in ghost land it's safe to do swaps with `MaterialPropertyStorage` using the elem and
4514 : // elem_side keys
4515 0 : neighbor = _assembly[tid][0]->elem();
4516 0 : neighbor_side = _assembly[tid][0]->side();
4517 : mooseAssert(neighbor, "We should have an appropriate value for elem coming from Assembly");
4518 : }
4519 : else
4520 0 : mooseError("neighbor is null in Assembly!");
4521 : }
4522 :
4523 3630008 : _neighbor_material_props.getMaterialData(tid).swapBack(*neighbor, neighbor_side);
4524 3630008 : }
4525 :
4526 : void
4527 987951 : FEProblemBase::logAdd(const std::string & system,
4528 : const std::string & name,
4529 : const std::string & type,
4530 : const InputParameters & params) const
4531 : {
4532 987951 : if (_verbose_setup != "false")
4533 102 : _console << "[DBG] Adding " << system << " '" << name << "' of type " << type << std::endl;
4534 987951 : if (_verbose_setup == "extra")
4535 0 : _console << params << std::endl;
4536 987951 : }
4537 :
4538 : void
4539 140069 : FEProblemBase::addObjectParamsHelper(InputParameters & parameters,
4540 : const std::string & object_name,
4541 : const std::string & var_param_name)
4542 : {
4543 : // Due to objects like SolutionUserObject which manipulate libmesh objects
4544 : // and variables directly at the back end, we need a default option here
4545 : // which is going to be the pointer to the first solver system within this
4546 : // problem
4547 140069 : unsigned int sys_num = 0;
4548 140069 : if (parameters.isParamValid(var_param_name))
4549 : {
4550 58583 : const auto variable_name = parameters.varName(var_param_name, object_name);
4551 58583 : if (this->hasVariable(variable_name) || this->hasScalarVariable(variable_name))
4552 51598 : sys_num = getSystem(variable_name).number();
4553 58583 : }
4554 280138 : if (parameters.isParamValid("solver_sys"))
4555 : {
4556 4208 : const auto var_sys_num = sys_num;
4557 4208 : sys_num = getSystemBase(parameters.get<SolverSystemName>("solver_sys")).number();
4558 4208 : if (sys_num != var_sys_num && parameters.isParamValid(var_param_name))
4559 0 : mooseError("We dont support setting 'variable' to a variable that is not set to the same "
4560 : "system as the 'solver_sys' parameter");
4561 : }
4562 :
4563 141233 : if (_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4564 141233 : parameters.get<bool>("use_displaced_mesh"))
4565 : {
4566 1132 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4567 566 : if (sys_num == _aux->number())
4568 1104 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->systemBaseAuxiliary();
4569 : else
4570 594 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(sys_num);
4571 : }
4572 : else
4573 : {
4574 : // The object requested use_displaced_mesh, but it was overridden
4575 : // due to there being no displacements variables in the [Mesh] block.
4576 : // If that happened, update the value of use_displaced_mesh appropriately.
4577 225465 : if (!_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4578 225465 : parameters.get<bool>("use_displaced_mesh"))
4579 144 : parameters.set<bool>("use_displaced_mesh") = false;
4580 :
4581 279006 : parameters.set<SubProblem *>("_subproblem") = this;
4582 :
4583 139503 : if (sys_num == _aux->number())
4584 40062 : parameters.set<SystemBase *>("_sys") = _aux.get();
4585 : else
4586 378447 : parameters.set<SystemBase *>("_sys") = _solver_systems[sys_num].get();
4587 : }
4588 140069 : }
4589 :
4590 : void
4591 63150 : FEProblemBase::checkUserObjectNameCollision(const std::string & name,
4592 : const std::string & type) const
4593 : {
4594 63150 : if (hasUserObject(name))
4595 9 : mooseError("A ",
4596 9 : getUserObjectBase(name).typeAndName(),
4597 : " already exists. You may not add a ",
4598 : type,
4599 : " by the same name.");
4600 :
4601 : #ifdef MOOSE_KOKKOS_ENABLED
4602 47346 : if (hasKokkosUserObject(name))
4603 0 : mooseError("A ",
4604 0 : getKokkosUserObject<UserObjectBase>(name).typeAndName(),
4605 : " already exists. You may not add a ",
4606 : type,
4607 : " by the same name.");
4608 : #endif
4609 63141 : }
4610 :
4611 : void
4612 50222 : FEProblemBase::addPostprocessor(const std::string & pp_name,
4613 : const std::string & name,
4614 : InputParameters & parameters)
4615 : {
4616 50222 : checkUserObjectNameCollision(name, "Postprocessor");
4617 :
4618 50219 : addUserObject(pp_name, name, parameters);
4619 50190 : }
4620 :
4621 : void
4622 5693 : FEProblemBase::addVectorPostprocessor(const std::string & pp_name,
4623 : const std::string & name,
4624 : InputParameters & parameters)
4625 : {
4626 5693 : checkUserObjectNameCollision(name, "VectorPostprocessor");
4627 :
4628 5690 : addUserObject(pp_name, name, parameters);
4629 5660 : }
4630 :
4631 : void
4632 4743 : FEProblemBase::addReporter(const std::string & type,
4633 : const std::string & name,
4634 : InputParameters & parameters)
4635 : {
4636 4743 : checkUserObjectNameCollision(name, "Reporter");
4637 :
4638 4740 : addUserObject(type, name, parameters);
4639 4707 : }
4640 :
4641 : std::vector<std::shared_ptr<UserObject>>
4642 70892 : FEProblemBase::addUserObject(const std::string & user_object_name,
4643 : const std::string & name,
4644 : InputParameters & parameters)
4645 : {
4646 : parallel_object_only();
4647 :
4648 70892 : std::vector<std::shared_ptr<UserObject>> uos;
4649 :
4650 : // Add the _subproblem and _sys parameters depending on use_displaced_mesh
4651 70892 : addObjectParamsHelper(parameters, name);
4652 :
4653 115257 : for (const auto tid : make_range(libMesh::n_threads()))
4654 : {
4655 : // Create the UserObject
4656 : std::shared_ptr<UserObject> user_object =
4657 74713 : _factory.create<UserObject>(user_object_name, name, parameters, tid);
4658 74438 : logAdd("UserObject", name, user_object_name, parameters);
4659 74438 : uos.push_back(user_object);
4660 :
4661 74438 : if (tid != 0)
4662 3821 : user_object->setPrimaryThreadCopy(uos[0].get());
4663 :
4664 74438 : theWarehouse().add(user_object);
4665 :
4666 : // Attempt to create all the possible UserObject types
4667 74432 : auto euo = std::dynamic_pointer_cast<ElementUserObject>(user_object);
4668 74432 : auto suo = std::dynamic_pointer_cast<SideUserObject>(user_object);
4669 74432 : auto isuo = std::dynamic_pointer_cast<InternalSideUserObject>(user_object);
4670 74432 : auto iuo = std::dynamic_pointer_cast<InterfaceUserObjectBase>(user_object);
4671 74432 : auto nuo = std::dynamic_pointer_cast<NodalUserObject>(user_object);
4672 74432 : auto duo = std::dynamic_pointer_cast<DomainUserObject>(user_object);
4673 74432 : auto guo = std::dynamic_pointer_cast<GeneralUserObject>(user_object);
4674 74432 : auto tguo = std::dynamic_pointer_cast<ThreadedGeneralUserObject>(user_object);
4675 74432 : auto muo = std::dynamic_pointer_cast<MortarUserObject>(user_object);
4676 :
4677 : // Account for displaced mesh use
4678 74432 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4679 : {
4680 : // Whether to re-init or not depends on the attributes of the base classes.
4681 : // For example, InterfaceUOBase has "_current_side_elem" and "_neighbor_elem"
4682 : // so it needs to reinit on displaced neighbors and faces
4683 : // _reinit_displaced_elem -> _current_elem will be reinited
4684 : // _reinit_displaced_face -> _current_elem, lowerD if any and _current_side_elem to be
4685 : // reinited _reinit_displaced_neighbor -> _current_elem, lowerD if any and _current_neighbor
4686 : // to be reinited Note that as soon as you use materials on the displaced mesh, all three get
4687 : // turned on.
4688 615 : if (euo || nuo || duo)
4689 555 : _reinit_displaced_elem = true;
4690 615 : if (suo || duo || isuo || iuo)
4691 24 : _reinit_displaced_face = true;
4692 615 : if (iuo || duo || isuo)
4693 0 : _reinit_displaced_neighbor = true;
4694 : }
4695 :
4696 : // These objects only require one thread
4697 74432 : if ((guo && !tguo) || muo)
4698 30067 : break;
4699 345035 : }
4700 :
4701 : // Add as a Functor if it is one. We usually need to add the user object from thread 0 as the
4702 : // registered functor for all threads because when user objects are thread joined, generally only
4703 : // the primary thread copy ends up with all the data
4704 148542 : for (const auto tid : make_range(libMesh::n_threads()))
4705 : {
4706 77931 : const decltype(uos)::size_type uo_index = uos.front()->needThreadedCopy() ? tid : 0;
4707 77931 : if (const auto functor = dynamic_cast<Moose::FunctorBase<Real> *>(uos[uo_index].get()))
4708 : {
4709 60902 : this->addFunctor(name, *functor, tid);
4710 60902 : if (_displaced_problem)
4711 729 : _displaced_problem->addFunctor(name, *functor, tid);
4712 : }
4713 : }
4714 :
4715 70611 : return uos;
4716 2 : }
4717 :
4718 : void
4719 2047 : FEProblemBase::addFVInterpolationMethod(const std::string & method_type,
4720 : const std::string & name,
4721 : InputParameters & parameters)
4722 : {
4723 : parallel_object_only();
4724 :
4725 2047 : addObjectParamsHelper(parameters, name);
4726 :
4727 4094 : for (const auto tid : make_range(libMesh::n_threads()))
4728 : {
4729 2047 : auto method = _factory.create<FVInterpolationMethod>(method_type, name, parameters, tid);
4730 2047 : logAdd("FVInterpolationMethod", name, method_type, parameters);
4731 2047 : theWarehouse().add(method);
4732 2047 : }
4733 2047 : }
4734 :
4735 : const UserObject &
4736 285632 : FEProblemBase::getUserObjectBase(const std::string & name, const THREAD_ID tid /* = 0 */) const
4737 : {
4738 285632 : std::vector<UserObject *> objs;
4739 285632 : theWarehouse()
4740 571264 : .query()
4741 285632 : .condition<AttribSystem>("UserObject")
4742 285632 : .condition<AttribThread>(tid)
4743 285632 : .condition<AttribName>(name)
4744 285632 : .queryInto(objs);
4745 285632 : if (objs.empty())
4746 : {
4747 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_user_object"),
4748 : "A UserObject getter was called before UserObjects have been constructed. The "
4749 : "requested UserObject '" +
4750 : name + "' may exist in the input file, but UserObjects are not available yet.");
4751 :
4752 0 : mooseError("Unable to find user object with name '" + name + "'");
4753 : }
4754 : mooseAssert(objs.size() == 1, "Should only find one UO");
4755 571264 : return *(objs[0]);
4756 285632 : }
4757 :
4758 : const Positions &
4759 1142 : FEProblemBase::getPositionsObject(const std::string & name) const
4760 : {
4761 1142 : std::vector<Positions *> objs;
4762 1142 : theWarehouse()
4763 2284 : .query()
4764 1142 : .condition<AttribSystem>("UserObject")
4765 1142 : .condition<AttribName>(name)
4766 1142 : .queryInto(objs);
4767 1142 : if (objs.empty())
4768 0 : mooseError("Unable to find Positions object with name '" + name + "'");
4769 : mooseAssert(objs.size() == 1, "Should only find one Positions");
4770 2284 : return *(objs[0]);
4771 1142 : }
4772 :
4773 : bool
4774 92212 : FEProblemBase::hasUserObject(const std::string & name) const
4775 : {
4776 92212 : std::vector<UserObject *> objs;
4777 92212 : theWarehouse()
4778 92212 : .query()
4779 92212 : .condition<AttribSystem>("UserObject")
4780 184424 : .condition<AttribThread>(0)
4781 92212 : .condition<AttribName>(name)
4782 92212 : .queryInto(objs);
4783 184424 : return !objs.empty();
4784 92212 : }
4785 :
4786 : const FVInterpolationMethod &
4787 586 : FEProblemBase::getFVInterpolationMethod(const InterpolationMethodName & name,
4788 : const THREAD_ID tid) const
4789 : {
4790 586 : std::vector<FVInterpolationMethod *> methods;
4791 586 : theWarehouse()
4792 1172 : .query()
4793 586 : .condition<AttribSystem>("FVInterpolationMethod")
4794 586 : .condition<AttribThread>(tid)
4795 586 : .condition<AttribName>(name)
4796 586 : .queryInto(methods);
4797 :
4798 586 : if (methods.empty())
4799 : {
4800 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_interpolation_method"),
4801 : "An FVInterpolationMethod getter was called before FVInterpolationMethods have "
4802 : "been constructed. If you are attempting to access this object in the constructor "
4803 : "of another object then make sure that the FVInterpolationMethod is constructed "
4804 : "before the object using it.");
4805 :
4806 0 : mooseError("Unable to find FVInterpolationMethod with name '", name, "'");
4807 : }
4808 :
4809 : mooseAssert(methods.size() == 1, "Expected a single FVInterpolationMethod per thread");
4810 1172 : return *(methods[0]);
4811 586 : }
4812 :
4813 : const FVFaceInterpolationMethod &
4814 15 : FEProblemBase::getFVFaceInterpolationMethod(const InterpolationMethodName & name,
4815 : const THREAD_ID tid) const
4816 : {
4817 15 : const auto & method = getFVInterpolationMethod(name, tid);
4818 15 : const auto * face_method = dynamic_cast<const FVFaceInterpolationMethod *>(&method);
4819 :
4820 15 : if (!face_method)
4821 0 : mooseError("FVInterpolationMethod '",
4822 : name,
4823 : "' (",
4824 0 : method.type(),
4825 : ") is not a scalar face interpolation method.");
4826 :
4827 15 : return *face_method;
4828 : }
4829 :
4830 : const FVAdvectedInterpolationMethod &
4831 571 : FEProblemBase::getFVAdvectedInterpolationMethod(const InterpolationMethodName & name,
4832 : const THREAD_ID tid) const
4833 : {
4834 571 : const auto & method = getFVInterpolationMethod(name, tid);
4835 571 : const auto * advected_method = dynamic_cast<const FVAdvectedInterpolationMethod *>(&method);
4836 :
4837 571 : if (!advected_method)
4838 0 : mooseError("FVInterpolationMethod '",
4839 : name,
4840 : "' (",
4841 0 : method.type(),
4842 : ") is not an advected interpolation method.");
4843 :
4844 571 : return *advected_method;
4845 : }
4846 :
4847 : bool
4848 0 : FEProblemBase::hasFVInterpolationMethod(const InterpolationMethodName & name) const
4849 : {
4850 0 : std::vector<FVInterpolationMethod *> methods;
4851 0 : theWarehouse()
4852 0 : .query()
4853 0 : .condition<AttribSystem>("FVInterpolationMethod")
4854 0 : .condition<AttribThread>(0)
4855 0 : .condition<AttribName>(name)
4856 0 : .queryInto(methods);
4857 0 : return !methods.empty();
4858 0 : }
4859 :
4860 : bool
4861 308 : FEProblemBase::hasPostprocessorValueByName(const PostprocessorName & name) const
4862 : {
4863 308 : return _reporter_data.hasReporterValue<PostprocessorValue>(PostprocessorReporterName(name));
4864 : }
4865 :
4866 : const Postprocessor &
4867 2 : FEProblemBase::getPostprocessorObjectByName(const PostprocessorName & object_name,
4868 : const THREAD_ID tid) const
4869 : {
4870 2 : std::vector<Postprocessor *> objs;
4871 2 : theWarehouse()
4872 2 : .query()
4873 4 : .condition<AttribInterfaces>(Interfaces::Postprocessor)
4874 2 : .condition<AttribThread>(tid)
4875 2 : .condition<AttribName>(object_name)
4876 2 : .queryInto(objs);
4877 :
4878 2 : if (objs.empty())
4879 0 : mooseError("Unable to find Postprocessor with name '", object_name, "'");
4880 : mooseAssert(objs.size() == 1,
4881 : "We shouldn't find more than one postprocessor object for a given name");
4882 4 : return *(objs[0]);
4883 2 : }
4884 :
4885 : const PostprocessorValue &
4886 850134 : FEProblemBase::getPostprocessorValueByName(const PostprocessorName & name,
4887 : std::size_t t_index) const
4888 : {
4889 1700268 : return _reporter_data.getReporterValue<PostprocessorValue>(PostprocessorReporterName(name),
4890 1700268 : t_index);
4891 : }
4892 :
4893 : void
4894 578672 : FEProblemBase::setPostprocessorValueByName(const PostprocessorName & name,
4895 : const PostprocessorValue & value,
4896 : std::size_t t_index)
4897 : {
4898 578672 : _reporter_data.setReporterValue<PostprocessorValue>(
4899 1157344 : PostprocessorReporterName(name), value, t_index);
4900 578672 : }
4901 :
4902 : bool
4903 52 : FEProblemBase::hasPostprocessor(const std::string & name) const
4904 : {
4905 52 : mooseDeprecated("FEProblemBase::hasPostprocssor is being removed; use "
4906 : "hasPostprocessorValueByName instead.");
4907 52 : return hasPostprocessorValueByName(name);
4908 : }
4909 :
4910 : const VectorPostprocessorValue &
4911 50 : FEProblemBase::getVectorPostprocessorValueByName(const std::string & object_name,
4912 : const std::string & vector_name,
4913 : std::size_t t_index) const
4914 : {
4915 50 : return _reporter_data.getReporterValue<VectorPostprocessorValue>(
4916 100 : VectorPostprocessorReporterName(object_name, vector_name), t_index);
4917 : }
4918 :
4919 : void
4920 18 : FEProblemBase::setVectorPostprocessorValueByName(const std::string & object_name,
4921 : const std::string & vector_name,
4922 : const VectorPostprocessorValue & value,
4923 : std::size_t t_index)
4924 : {
4925 18 : _reporter_data.setReporterValue<VectorPostprocessorValue>(
4926 36 : VectorPostprocessorReporterName(object_name, vector_name), value, t_index);
4927 18 : }
4928 :
4929 : const VectorPostprocessor &
4930 9969 : FEProblemBase::getVectorPostprocessorObjectByName(const std::string & object_name,
4931 : const THREAD_ID tid) const
4932 : {
4933 9969 : std::vector<VectorPostprocessor *> objs;
4934 9969 : theWarehouse()
4935 9969 : .query()
4936 19938 : .condition<AttribInterfaces>(Interfaces::VectorPostprocessor)
4937 9969 : .condition<AttribThread>(tid)
4938 9969 : .condition<AttribName>(object_name)
4939 9969 : .queryInto(objs);
4940 :
4941 9969 : if (objs.empty())
4942 : {
4943 : mooseAssert(
4944 : getMooseApp().actionWarehouse().isTaskComplete("add_vector_postprocessor"),
4945 : "A VectorPostprocessor getter was called before VectorPostprocessors have been "
4946 : "constructed. The requested VectorPostprocessor '" +
4947 : object_name +
4948 : "' may exist in the input file, but VectorPostprocessors are not available yet.");
4949 :
4950 0 : mooseError("Unable to find VectorPostprocessor with name '", object_name, "'");
4951 : }
4952 : mooseAssert(objs.size() == 1,
4953 : "We shouldn't find more than one vector postprocessor object for a given name");
4954 19938 : return *(objs[0]);
4955 9969 : }
4956 :
4957 : void
4958 70 : FEProblemBase::parentOutputPositionChanged()
4959 : {
4960 2030 : for (const auto & it : _multi_apps)
4961 : {
4962 1960 : const auto & objects = it.second.getActiveObjects();
4963 1979 : for (const auto & obj : objects)
4964 19 : obj->parentOutputPositionChanged();
4965 : }
4966 70 : }
4967 :
4968 : void
4969 0 : FEProblemBase::computeIndicatorsAndMarkers()
4970 : {
4971 0 : computeIndicators();
4972 0 : computeMarkers();
4973 0 : }
4974 :
4975 : void
4976 226338 : FEProblemBase::computeIndicators()
4977 : {
4978 : // Initialize indicator aux variable fields
4979 226338 : if (_indicators.hasActiveObjects() || _internal_side_indicators.hasActiveObjects())
4980 : {
4981 11875 : TIME_SECTION("computeIndicators", 1, "Computing Indicators");
4982 :
4983 : // Internal side indicators may lead to creating a much larger sparsity pattern than dictated by
4984 : // the actual finite element scheme (e.g. CFEM)
4985 2375 : const auto old_do_derivatives = ADReal::do_derivatives;
4986 2375 : ADReal::do_derivatives = false;
4987 :
4988 2375 : std::vector<std::string> fields;
4989 :
4990 : // Indicator Fields
4991 2375 : const auto & indicators = _indicators.getActiveObjects();
4992 2521 : for (const auto & indicator : indicators)
4993 146 : fields.push_back(indicator->name());
4994 :
4995 : // InternalSideIndicator Fields
4996 2375 : const auto & internal_indicators = _internal_side_indicators.getActiveObjects();
4997 4706 : for (const auto & internal_indicator : internal_indicators)
4998 2331 : fields.push_back(internal_indicator->name());
4999 :
5000 2375 : _aux->zeroVariables(fields);
5001 :
5002 : // compute Indicators
5003 2375 : ComputeIndicatorThread cit(*this);
5004 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cit);
5005 2375 : _aux->solution().close();
5006 2375 : _aux->update();
5007 :
5008 2375 : ComputeIndicatorThread finalize_cit(*this, true);
5009 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), finalize_cit);
5010 2375 : _aux->solution().close();
5011 2375 : _aux->update();
5012 :
5013 2375 : ADReal::do_derivatives = old_do_derivatives;
5014 2375 : }
5015 226338 : }
5016 :
5017 : void
5018 226360 : FEProblemBase::computeMarkers()
5019 : {
5020 226360 : if (_markers.hasActiveObjects())
5021 : {
5022 32580 : TIME_SECTION("computeMarkers", 1, "Computing Markers");
5023 :
5024 6516 : std::vector<std::string> fields;
5025 :
5026 : // Marker Fields
5027 6516 : const auto & markers = _markers.getActiveObjects();
5028 14011 : for (const auto & marker : markers)
5029 7495 : fields.push_back(marker->name());
5030 :
5031 6516 : _aux->zeroVariables(fields);
5032 :
5033 6516 : _adaptivity.updateErrorVectors();
5034 :
5035 13660 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5036 : {
5037 7147 : const auto & markers = _markers.getActiveObjects(tid);
5038 15365 : for (const auto & marker : markers)
5039 8221 : marker->markerSetup();
5040 : }
5041 :
5042 6513 : ComputeMarkerThread cmt(*this);
5043 6513 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cmt);
5044 :
5045 6513 : _aux->solution().close();
5046 6513 : _aux->update();
5047 6513 : }
5048 226357 : }
5049 :
5050 : const ExecFlagType &
5051 6255705 : FEProblemBase::getCurrentExecuteOnFlag() const
5052 : {
5053 6255705 : return _current_execute_on_flag;
5054 : }
5055 :
5056 : void
5057 7044263 : FEProblemBase::setCurrentExecuteOnFlag(const ExecFlagType & flag)
5058 : {
5059 7044263 : _current_execute_on_flag = flag;
5060 7044263 : }
5061 :
5062 : void
5063 72 : FEProblemBase::executeAllObjects(const ExecFlagType & /*exec_type*/)
5064 : {
5065 72 : }
5066 :
5067 : void
5068 1805051 : FEProblemBase::customSetup(const ExecFlagType & exec_type)
5069 : {
5070 1805051 : SubProblem::customSetup(exec_type);
5071 :
5072 1805051 : if (_line_search)
5073 0 : _line_search->customSetup(exec_type);
5074 :
5075 1805051 : unsigned int n_threads = libMesh::n_threads();
5076 3788644 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5077 : {
5078 1983593 : _all_materials.customSetup(exec_type, tid);
5079 1983593 : _functions.customSetup(exec_type, tid);
5080 : }
5081 :
5082 : #ifdef MOOSE_KOKKOS_ENABLED
5083 1320145 : _kokkos_functions.customSetup(exec_type);
5084 : #endif
5085 :
5086 1805051 : _aux->customSetup(exec_type);
5087 3613621 : for (auto & nl : _nl)
5088 1808570 : nl->customSetup(exec_type);
5089 :
5090 1805051 : if (_displaced_problem)
5091 145157 : _displaced_problem->customSetup(exec_type);
5092 :
5093 3788644 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5094 : {
5095 1983593 : _internal_side_indicators.customSetup(exec_type, tid);
5096 1983593 : _indicators.customSetup(exec_type, tid);
5097 1983593 : _markers.customSetup(exec_type, tid);
5098 : }
5099 :
5100 1805051 : std::vector<UserObject *> userobjs;
5101 1805051 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
5102 4460746 : for (auto obj : userobjs)
5103 2655695 : obj->customSetup(exec_type);
5104 :
5105 : #ifdef MOOSE_KOKKOS_ENABLED
5106 : {
5107 1320145 : std::vector<UserObjectBase *> userobjs;
5108 1320145 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
5109 1333661 : for (auto obj : userobjs)
5110 13516 : obj->customSetup(exec_type);
5111 1320145 : }
5112 : #endif
5113 :
5114 1805051 : _app.getOutputWarehouse().customSetup(exec_type);
5115 1805051 : }
5116 :
5117 : void
5118 2180904 : FEProblemBase::execute(const ExecFlagType & exec_type)
5119 : {
5120 : // Set the current flag
5121 2180904 : setCurrentExecuteOnFlag(exec_type);
5122 :
5123 2180904 : if (exec_type != EXEC_INITIAL)
5124 2124273 : executeControls(exec_type);
5125 :
5126 : // intentially call this after executing controls because the setups may rely on the controls
5127 : // FIXME: we skip the following flags because they have dedicated setup functions in
5128 : // SetupInterface and it may not be appropriate to call them here.
5129 3988408 : if (!(exec_type == EXEC_INITIAL || exec_type == EXEC_TIMESTEP_BEGIN ||
5130 1807522 : exec_type == EXEC_SUBDOMAIN || exec_type == EXEC_NONLINEAR || exec_type == EXEC_LINEAR))
5131 1805051 : customSetup(exec_type);
5132 :
5133 2180886 : executeSamplers(exec_type);
5134 :
5135 : // Pre-aux UserObjects
5136 2180862 : computeUserObjects(exec_type, Moose::PRE_AUX);
5137 :
5138 : // Systems (includes system time derivative and aux kernel calculations)
5139 2180862 : computeSystems(exec_type);
5140 : // With the auxiliary system solution computed, sync the displaced problem auxiliary solution
5141 : // before computation of post-aux user objects. The undisplaced auxiliary system current local
5142 : // solution is updated (via System::update) within the AuxiliarySystem class's variable
5143 : // computation methods (e.g. computeElementalVarsHelper, computeNodalVarsHelper), so it is safe to
5144 : // use it here
5145 2180829 : if (_displaced_problem)
5146 179606 : _displaced_problem->syncAuxSolution(*getAuxiliarySystem().currentSolution());
5147 :
5148 : // Post-aux UserObjects
5149 2180829 : computeUserObjects(exec_type, Moose::POST_AUX);
5150 :
5151 : // Return the current flag to None
5152 2180740 : setCurrentExecuteOnFlag(EXEC_NONE);
5153 :
5154 2180740 : if (_uo_aux_state_check && !_checking_uo_aux_state)
5155 : {
5156 : // we will only check aux variables and postprocessors
5157 : // checking more reporter data can be added in the future if needed
5158 559 : std::unique_ptr<NumericVector<Number>> x = _aux->currentSolution()->clone();
5159 559 : DenseVector<Real> pp_values = getReporterData().getAllRealReporterValues();
5160 :
5161 : // call THIS execute one more time for checking the possible states
5162 559 : _checking_uo_aux_state = true;
5163 559 : FEProblemBase::execute(exec_type);
5164 559 : _checking_uo_aux_state = false;
5165 :
5166 559 : const Real check_tol = 1e-8;
5167 :
5168 559 : const Real xnorm = x->l2_norm();
5169 559 : *x -= *_aux->currentSolution();
5170 559 : if (x->l2_norm() > check_tol * xnorm)
5171 : {
5172 3 : const auto & sys = _aux->system();
5173 3 : const unsigned int n_vars = sys.n_vars();
5174 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5175 15 : for (const auto i : make_range(n_vars))
5176 : {
5177 12 : const Real vnorm = sys.calculate_norm(*x, i, DISCRETE_L2);
5178 12 : ordered_map.emplace(vnorm, sys.variable_name(i));
5179 : }
5180 :
5181 3 : std::ostringstream oss;
5182 15 : for (const auto & [error_norm, var_name] : ordered_map)
5183 12 : oss << " {" << var_name << ", " << error_norm << "},\n";
5184 :
5185 3 : mooseError("Aux kernels, user objects appear to have states for aux variables on ",
5186 : exec_type,
5187 : ".\nVariable error norms in descending order:\n",
5188 3 : oss.str());
5189 0 : }
5190 :
5191 556 : const DenseVector<Real> new_pp_values = getReporterData().getAllRealReporterValues();
5192 556 : if (pp_values.size() != new_pp_values.size())
5193 0 : mooseError("Second execution for uo/aux state check should not change the number of "
5194 : "real reporter values");
5195 :
5196 556 : const Real ppnorm = pp_values.l2_norm();
5197 556 : pp_values -= new_pp_values;
5198 556 : if (pp_values.l2_norm() > check_tol * ppnorm)
5199 : {
5200 3 : const auto pp_names = getReporterData().getAllRealReporterFullNames();
5201 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5202 12 : for (const auto i : index_range(pp_names))
5203 9 : ordered_map.emplace(std::abs(pp_values(i)), pp_names[i]);
5204 :
5205 3 : std::ostringstream oss;
5206 12 : for (const auto & [error_norm, pp_name] : ordered_map)
5207 9 : oss << " {" << pp_name << ", " << error_norm << "},\n";
5208 :
5209 3 : mooseError("Aux kernels, user objects appear to have states for real reporter values on ",
5210 : exec_type,
5211 : ".\nErrors of real reporter values in descending order:\n",
5212 3 : oss.str());
5213 0 : }
5214 553 : }
5215 2180734 : }
5216 :
5217 : // Finalize, threadJoin, and update PP values of Elemental/Nodal/Side/InternalSideUserObjects
5218 : void
5219 1346562 : FEProblemBase::joinAndFinalize(TheWarehouse::Query query, bool isgen)
5220 : {
5221 1346562 : std::vector<UserObject *> objs;
5222 1346562 : query.queryInto(objs);
5223 1346562 : if (!isgen)
5224 : {
5225 : // join all threaded user objects (i.e. not regular general user objects) to the primary
5226 : // thread
5227 1426037 : for (auto obj : objs)
5228 372035 : if (obj->primaryThreadCopy())
5229 30799 : obj->primaryThreadCopy()->threadJoin(*obj);
5230 : }
5231 :
5232 1346562 : query.condition<AttribThread>(0).queryInto(objs);
5233 :
5234 : // finalize objects and retrieve/store any postprocessor values
5235 1919785 : for (auto obj : objs)
5236 : {
5237 573303 : if (isgen && dynamic_cast<ThreadedGeneralUserObject *>(obj))
5238 133 : continue;
5239 573170 : if (isgen)
5240 : {
5241 : // general user objects are not run in their own threaded loop object - so run them here
5242 231934 : if (shouldPrintExecution(0))
5243 724 : _console << "[DBG] Initializing, executing & finalizing general UO '" << obj->name()
5244 724 : << "' on " << _current_execute_on_flag.name() << std::endl;
5245 231934 : obj->initialize();
5246 231934 : obj->execute();
5247 : }
5248 :
5249 573120 : obj->finalize();
5250 :
5251 : // These have to be stored piecemeal (with every call to this function) because general
5252 : // postprocessors (which run last after other userobjects have been completed) might depend on
5253 : // them being stored. This wouldn't be a problem if all userobjects satisfied the dependency
5254 : // resolver interface and could be sorted appropriately with the general userobjects, but they
5255 : // don't.
5256 573105 : auto pp = dynamic_cast<const Postprocessor *>(obj);
5257 573105 : if (pp)
5258 : {
5259 500262 : _reporter_data.finalize(obj->name());
5260 500262 : setPostprocessorValueByName(obj->name(), pp->getValue());
5261 : }
5262 :
5263 573096 : auto vpp = dynamic_cast<VectorPostprocessor *>(obj);
5264 573096 : if (vpp)
5265 13523 : _reporter_data.finalize(obj->name());
5266 :
5267 : // Update Reporter data
5268 573096 : auto reporter = dynamic_cast<Reporter *>(obj);
5269 573096 : if (reporter)
5270 5333 : _reporter_data.finalize(obj->name());
5271 : }
5272 1346482 : }
5273 :
5274 : TheWarehouse::Query
5275 20078847 : FEProblemBase::getUOQuery(const std::string & system,
5276 : const ExecFlagType & type,
5277 : const Moose::AuxGroup & group) const
5278 : {
5279 : TheWarehouse::Query query =
5280 20078847 : theWarehouse().query().condition<AttribSystem>(system).condition<AttribExecOns>(type);
5281 :
5282 20078847 : if (group == Moose::PRE_IC)
5283 98707 : query.condition<AttribPreIC>(true);
5284 19980140 : else if (group == Moose::PRE_AUX)
5285 9989691 : query.condition<AttribPreAux>(type);
5286 9990449 : else if (group == Moose::POST_AUX)
5287 9990381 : query.condition<AttribPostAux>(type);
5288 :
5289 20078847 : return query;
5290 0 : }
5291 :
5292 : void
5293 20078847 : FEProblemBase::getUOExecutionGroups(TheWarehouse::Query & query,
5294 : std::set<int> & execution_groups) const
5295 : {
5296 20078847 : std::vector<UserObjectBase *> uos;
5297 20078847 : query.queryIntoUnsorted(uos);
5298 20685390 : for (const auto & uo : uos)
5299 1819629 : execution_groups.insert(uo->getParam<int>("execution_order_group"));
5300 20078847 : }
5301 :
5302 : void
5303 56897 : FEProblemBase::computeUserObjectByName(const ExecFlagType & type,
5304 : const Moose::AuxGroup & group,
5305 : const std::string & name)
5306 : {
5307 56897 : const auto old_exec_flag = _current_execute_on_flag;
5308 56897 : _current_execute_on_flag = type;
5309 :
5310 56897 : std::set<int> execution_groups;
5311 :
5312 : #ifdef MOOSE_KOKKOS_ENABLED
5313 : TheWarehouse::Query kokkos_query =
5314 41026 : getUOQuery("KokkosUserObject", type, group).condition<AttribName>(name);
5315 41026 : getUOExecutionGroups(kokkos_query, execution_groups);
5316 : #endif
5317 :
5318 56897 : TheWarehouse::Query query = getUOQuery("UserObject", type, group).condition<AttribName>(name);
5319 56897 : getUOExecutionGroups(query, execution_groups);
5320 :
5321 82848 : for (const auto execution_group : execution_groups)
5322 : {
5323 : #ifdef MOOSE_KOKKOS_ENABLED
5324 18721 : computeKokkosUserObjectsInternal(
5325 18721 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5326 : #endif
5327 :
5328 25951 : computeUserObjectsInternal(type,
5329 25951 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5330 : }
5331 :
5332 56897 : _current_execute_on_flag = old_exec_flag;
5333 56897 : }
5334 :
5335 : void
5336 11553006 : FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGroup & group)
5337 : {
5338 11553006 : std::set<int> execution_groups;
5339 :
5340 : #ifdef MOOSE_KOKKOS_ENABLED
5341 8427918 : TheWarehouse::Query kokkos_query = getUOQuery("KokkosUserObject", type, group);
5342 8427918 : getUOExecutionGroups(kokkos_query, execution_groups);
5343 : #endif
5344 :
5345 11553006 : TheWarehouse::Query query = getUOQuery("UserObject", type, group);
5346 11553006 : getUOExecutionGroups(query, execution_groups);
5347 :
5348 11821251 : for (const auto execution_group : execution_groups)
5349 : {
5350 : #ifdef MOOSE_KOKKOS_ENABLED
5351 195456 : computeKokkosUserObjectsInternal(
5352 195456 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5353 : #endif
5354 :
5355 268334 : computeUserObjectsInternal(type,
5356 268334 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5357 : }
5358 11552917 : }
5359 :
5360 : void
5361 294285 : FEProblemBase::computeUserObjectsInternal(const ExecFlagType & type, TheWarehouse::Query & query)
5362 : {
5363 : try
5364 : {
5365 1471425 : TIME_SECTION("computeUserObjects", 1, "Computing User Objects");
5366 :
5367 294285 : std::vector<GeneralUserObject *> genobjs;
5368 294285 : query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject).queryInto(genobjs);
5369 :
5370 294285 : std::vector<UserObject *> userobjs;
5371 294285 : query.clone()
5372 588570 : .condition<AttribInterfaces>(Interfaces::ElementUserObject | Interfaces::SideUserObject |
5373 : Interfaces::InternalSideUserObject |
5374 588570 : Interfaces::InterfaceUserObject | Interfaces::DomainUserObject)
5375 294285 : .queryInto(userobjs);
5376 :
5377 294285 : std::vector<UserObject *> tgobjs;
5378 294285 : query.clone()
5379 588570 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5380 294285 : .queryInto(tgobjs);
5381 :
5382 294285 : std::vector<UserObject *> nodal;
5383 294285 : query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).queryInto(nodal);
5384 :
5385 294285 : std::vector<MortarUserObject *> mortar;
5386 294285 : query.clone().condition<AttribInterfaces>(Interfaces::MortarUserObject).queryInto(mortar);
5387 :
5388 294285 : if (userobjs.empty() && genobjs.empty() && tgobjs.empty() && nodal.empty() && mortar.empty())
5389 1710 : return;
5390 :
5391 : // Start the timer here since we have at least one active user object
5392 292575 : std::string compute_uo_tag = "computeUserObjects(" + Moose::stringify(type) + ")";
5393 :
5394 : // Perform Residual/Jacobian setups
5395 292575 : if (type == EXEC_LINEAR)
5396 : {
5397 125261 : for (auto obj : userobjs)
5398 71388 : obj->residualSetup();
5399 58331 : for (auto obj : nodal)
5400 4458 : obj->residualSetup();
5401 53873 : for (auto obj : mortar)
5402 0 : obj->residualSetup();
5403 53882 : for (auto obj : tgobjs)
5404 9 : obj->residualSetup();
5405 65556 : for (auto obj : genobjs)
5406 11683 : obj->residualSetup();
5407 : }
5408 238702 : else if (type == EXEC_NONLINEAR)
5409 : {
5410 14080 : for (auto obj : userobjs)
5411 4420 : obj->jacobianSetup();
5412 10052 : for (auto obj : nodal)
5413 392 : obj->jacobianSetup();
5414 9660 : for (auto obj : mortar)
5415 0 : obj->jacobianSetup();
5416 9663 : for (auto obj : tgobjs)
5417 3 : obj->jacobianSetup();
5418 26470 : for (auto obj : genobjs)
5419 16810 : obj->jacobianSetup();
5420 : }
5421 :
5422 643899 : for (auto obj : userobjs)
5423 351324 : obj->initialize();
5424 :
5425 : // Execute Side/InternalSide/Interface/Elemental/DomainUserObjects
5426 292575 : if (!userobjs.empty())
5427 : {
5428 : // non-nodal user objects have to be run separately before the nodal user objects run
5429 : // because some nodal user objects (NodalNormal related) depend on elemental user objects
5430 : // :-(
5431 207510 : ComputeUserObjectsThread cppt(*this, query);
5432 207510 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cppt);
5433 :
5434 : // There is one instance in rattlesnake where an elemental user object's finalize depends
5435 : // on a side user object having been finalized first :-(
5436 207501 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::SideUserObject));
5437 207498 : joinAndFinalize(
5438 414996 : query.clone().condition<AttribInterfaces>(Interfaces::InternalSideUserObject));
5439 207498 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::InterfaceUserObject));
5440 207498 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::ElementUserObject));
5441 207495 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::DomainUserObject));
5442 207495 : }
5443 :
5444 : // if any elemental user object may have written to variables we need to close the aux solution
5445 643792 : for (const auto & uo : userobjs)
5446 351265 : if (auto euo = dynamic_cast<const ElementUserObject *>(uo);
5447 351265 : euo && euo->hasWritableCoupledVariables())
5448 : {
5449 33 : _aux->solution().close();
5450 33 : _aux->system().update();
5451 33 : break;
5452 : }
5453 :
5454 : // Execute NodalUserObjects
5455 : // BISON has an axial reloc elemental user object that has a finalize func that depends on a
5456 : // nodal user object's prev value. So we can't initialize this until after elemental objects
5457 : // have been finalized :-(
5458 312866 : for (auto obj : nodal)
5459 20306 : obj->initialize();
5460 292560 : if (query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).count() > 0)
5461 : {
5462 16379 : ComputeNodalUserObjectsThread cnppt(*this, query);
5463 16379 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), cnppt);
5464 16379 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject));
5465 16379 : }
5466 :
5467 : // if any nodal user object may have written to variables we need to close the aux solution
5468 312830 : for (const auto & uo : nodal)
5469 20292 : if (auto nuo = dynamic_cast<const NodalUserObject *>(uo);
5470 20292 : nuo && nuo->hasWritableCoupledVariables())
5471 : {
5472 22 : _aux->solution().close();
5473 22 : _aux->system().update();
5474 22 : break;
5475 : }
5476 :
5477 : // Execute MortarUserObjects
5478 : {
5479 292582 : for (auto obj : mortar)
5480 22 : obj->initialize();
5481 292560 : if (!mortar.empty())
5482 : {
5483 33 : auto create_and_run_mortar_functors = [this, type, &mortar](const bool displaced)
5484 : {
5485 : // go over mortar interfaces and construct functors
5486 33 : const auto & mortar_interfaces = getMortarInterfaces(displaced);
5487 55 : for (const auto & [primary_secondary_boundary_pair, interface_config] : mortar_interfaces)
5488 : {
5489 : auto mortar_uos_to_execute =
5490 22 : getMortarUserObjects(primary_secondary_boundary_pair.first,
5491 22 : primary_secondary_boundary_pair.second,
5492 : displaced,
5493 22 : mortar);
5494 :
5495 : auto * const subproblem = displaced
5496 22 : ? static_cast<SubProblem *>(_displaced_problem.get())
5497 22 : : static_cast<SubProblem *>(this);
5498 : MortarUserObjectThread muot(mortar_uos_to_execute,
5499 22 : *interface_config.amg,
5500 : *subproblem,
5501 : *this,
5502 : displaced,
5503 22 : subproblem->assembly(0, 0));
5504 :
5505 22 : muot();
5506 22 : }
5507 55 : };
5508 :
5509 22 : create_and_run_mortar_functors(false);
5510 22 : if (_displaced_problem)
5511 11 : create_and_run_mortar_functors(true);
5512 22 : }
5513 292582 : for (auto obj : mortar)
5514 22 : obj->finalize();
5515 : }
5516 :
5517 : // Execute threaded general user objects
5518 293001 : for (auto obj : tgobjs)
5519 441 : obj->initialize();
5520 292560 : std::vector<GeneralUserObject *> tguos_zero;
5521 292560 : query.clone()
5522 292560 : .condition<AttribThread>(0)
5523 585120 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5524 292560 : .queryInto(tguos_zero);
5525 292693 : for (auto obj : tguos_zero)
5526 : {
5527 133 : std::vector<GeneralUserObject *> tguos;
5528 133 : auto q = query.clone()
5529 133 : .condition<AttribName>(obj->name())
5530 133 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject);
5531 133 : q.queryInto(tguos);
5532 :
5533 133 : ComputeThreadedGeneralUserObjectsThread ctguot(*this);
5534 :
5535 : // Force one thread per ThreadedGeneralUserObject via grainsize
5536 266 : Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(),
5537 133 : tguos.end(),
5538 : /*grainsize=*/1),
5539 : ctguot);
5540 133 : joinAndFinalize(q);
5541 133 : }
5542 :
5543 : // Execute general user objects
5544 292560 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject), true);
5545 302746 : }
5546 0 : catch (...)
5547 : {
5548 0 : handleException("computeUserObjectsInternal");
5549 0 : }
5550 : }
5551 :
5552 : void
5553 5752026 : FEProblemBase::executeControls(const ExecFlagType & exec_type)
5554 : {
5555 5752026 : if (_control_warehouse[exec_type].hasActiveObjects())
5556 : {
5557 40205 : TIME_SECTION("executeControls", 1, "Executing Controls");
5558 :
5559 8041 : DependencyResolver<std::shared_ptr<Control>> resolver;
5560 :
5561 8041 : auto controls_wh = _control_warehouse[exec_type];
5562 : // Add all of the dependencies into the resolver and sort them
5563 20003 : for (const auto & it : controls_wh.getActiveObjects())
5564 : {
5565 : // Make sure an item with no dependencies comes out too!
5566 11965 : resolver.addItem(it);
5567 :
5568 11965 : std::vector<std::string> & dependent_controls = it->getDependencies();
5569 14857 : for (const auto & depend_name : dependent_controls)
5570 : {
5571 2895 : if (controls_wh.hasActiveObject(depend_name))
5572 : {
5573 2892 : auto dep_control = controls_wh.getActiveObject(depend_name);
5574 2892 : resolver.addEdge(dep_control, it);
5575 2892 : }
5576 : else
5577 3 : mooseError("The Control \"",
5578 : depend_name,
5579 : "\" was not created, did you make a "
5580 : "spelling mistake or forget to include it "
5581 : "in your input file?");
5582 : }
5583 : }
5584 :
5585 8038 : const auto & ordered_controls = resolver.getSortedValues();
5586 :
5587 8038 : if (!ordered_controls.empty())
5588 : {
5589 : // already called by initialSetup when exec_type == EXEC_INITIAL
5590 8038 : if (exec_type != EXEC_INITIAL)
5591 7130 : _control_warehouse.setup(exec_type);
5592 :
5593 : // Run the controls in the proper order
5594 19958 : for (const auto & control : ordered_controls)
5595 11962 : control->execute();
5596 : }
5597 7996 : }
5598 5751981 : }
5599 :
5600 : void
5601 2180886 : FEProblemBase::executeSamplers(const ExecFlagType & exec_type)
5602 : {
5603 : // TODO: This should be done in a threaded loop, but this should be super quick so for now
5604 : // do a serial loop.
5605 4577451 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5606 : {
5607 2396589 : std::vector<Sampler *> objects;
5608 2396589 : theWarehouse()
5609 4793178 : .query()
5610 2396589 : .condition<AttribSystem>("Sampler")
5611 2396589 : .condition<AttribThread>(tid)
5612 2396589 : .condition<AttribExecOns>(exec_type)
5613 2396589 : .queryInto(objects);
5614 :
5615 2396589 : if (!objects.empty())
5616 : {
5617 1475 : TIME_SECTION("executeSamplers", 1, "Executing Samplers");
5618 295 : FEProblemBase::objectSetupHelper<Sampler>(objects, exec_type);
5619 295 : FEProblemBase::objectExecuteHelper<Sampler>(objects);
5620 271 : }
5621 2396565 : }
5622 2180862 : }
5623 :
5624 : void
5625 313444 : FEProblemBase::updateActiveObjects()
5626 : {
5627 1567220 : TIME_SECTION("updateActiveObjects", 5, "Updating Active Objects");
5628 :
5629 657911 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5630 : {
5631 691337 : for (auto & nl : _nl)
5632 346870 : nl->updateActive(tid);
5633 344467 : _aux->updateActive(tid);
5634 344467 : _indicators.updateActive(tid);
5635 344467 : _internal_side_indicators.updateActive(tid);
5636 344467 : _markers.updateActive(tid);
5637 344467 : _all_materials.updateActive(tid);
5638 344467 : _materials.updateActive(tid);
5639 344467 : _discrete_materials.updateActive(tid);
5640 : }
5641 :
5642 313444 : _control_warehouse.updateActive();
5643 313444 : _multi_apps.updateActive();
5644 313444 : _transient_multi_apps.updateActive();
5645 313444 : _transfers.updateActive();
5646 313444 : _to_multi_app_transfers.updateActive();
5647 313444 : _from_multi_app_transfers.updateActive();
5648 313444 : _between_multi_app_transfers.updateActive();
5649 :
5650 : #ifdef MOOSE_KOKKOS_ENABLED
5651 228753 : _kokkos_materials.updateActive();
5652 : #endif
5653 313444 : }
5654 :
5655 : void
5656 0 : FEProblemBase::reportMooseObjectDependency(MooseObject * /*a*/, MooseObject * /*b*/)
5657 : {
5658 : //<< "Object " << a->name() << " -> " << b->name() << std::endl;
5659 0 : }
5660 :
5661 : void
5662 68487 : FEProblemBase::reinitBecauseOfGhostingOrNewGeomObjects(const bool mortar_changed)
5663 : {
5664 342435 : TIME_SECTION("reinitBecauseOfGhostingOrNewGeomObjects",
5665 : 3,
5666 : "Reinitializing Because of Geometric Search Objects");
5667 :
5668 : // Need to see if _any_ processor has ghosted elems or geometry objects.
5669 68487 : bool needs_reinit = !_ghosted_elems.empty();
5670 136605 : needs_reinit = needs_reinit || !_geometric_search_data._nearest_node_locators.empty() ||
5671 68118 : (_mortar_data->hasObjects() && mortar_changed);
5672 68487 : needs_reinit =
5673 138167 : needs_reinit || (_displaced_problem &&
5674 4423 : (!_displaced_problem->geomSearchData()._nearest_node_locators.empty() ||
5675 69159 : (_mortar_data->hasDisplacedObjects() && mortar_changed)));
5676 68487 : _communicator.max(needs_reinit);
5677 :
5678 68487 : if (needs_reinit)
5679 : {
5680 : // Call reinit to get the ghosted vectors correct now that some geometric search has been done
5681 2211 : es().reinit();
5682 :
5683 2211 : if (_displaced_mesh)
5684 1607 : _displaced_problem->es().reinit();
5685 : }
5686 68487 : }
5687 :
5688 : void
5689 177 : FEProblemBase::addDamper(const std::string & damper_name,
5690 : const std::string & name,
5691 : InputParameters & parameters)
5692 : {
5693 : parallel_object_only();
5694 :
5695 : const auto nl_sys_num =
5696 177 : parameters.isParamValid("variable")
5697 588 : ? determineSolverSystem(parameters.varName("variable", name), true).second
5698 174 : : (unsigned int)0;
5699 :
5700 174 : if (!isSolverSystemNonlinear(nl_sys_num))
5701 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
5702 : "supported at the moment!");
5703 :
5704 348 : parameters.set<SubProblem *>("_subproblem") = this;
5705 348 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
5706 :
5707 174 : _has_dampers = true;
5708 174 : logAdd("Damper", name, damper_name, parameters);
5709 174 : _nl[nl_sys_num]->addDamper(damper_name, name, parameters);
5710 174 : }
5711 :
5712 : void
5713 162 : FEProblemBase::setupDampers()
5714 : {
5715 324 : for (auto & nl : _nl)
5716 162 : nl->setupDampers();
5717 162 : }
5718 :
5719 : void
5720 659 : FEProblemBase::addIndicator(const std::string & indicator_name,
5721 : const std::string & name,
5722 : InputParameters & parameters)
5723 : {
5724 : parallel_object_only();
5725 :
5726 659 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5727 : {
5728 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5729 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5730 0 : _reinit_displaced_elem = true;
5731 : }
5732 : else
5733 : {
5734 659 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5735 : {
5736 : // We allow Indicators to request that they use_displaced_mesh,
5737 : // but then be overridden when no displacements variables are
5738 : // provided in the Mesh block. If that happened, update the value
5739 : // of use_displaced_mesh appropriately for this Indicator.
5740 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5741 0 : parameters.set<bool>("use_displaced_mesh") = false;
5742 : }
5743 :
5744 1318 : parameters.set<SubProblem *>("_subproblem") = this;
5745 1977 : parameters.set<SystemBase *>("_sys") = _aux.get();
5746 : }
5747 :
5748 1382 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5749 : {
5750 : std::shared_ptr<Indicator> indicator =
5751 723 : _factory.create<Indicator>(indicator_name, name, parameters, tid);
5752 723 : logAdd("Indicator", name, indicator_name, parameters);
5753 : std::shared_ptr<InternalSideIndicatorBase> isi =
5754 723 : std::dynamic_pointer_cast<InternalSideIndicatorBase>(indicator);
5755 723 : if (isi)
5756 622 : _internal_side_indicators.addObject(isi, tid);
5757 : else
5758 101 : _indicators.addObject(indicator, tid);
5759 723 : }
5760 659 : }
5761 :
5762 : void
5763 1966 : FEProblemBase::addMarker(const std::string & marker_name,
5764 : const std::string & name,
5765 : InputParameters & parameters)
5766 : {
5767 : parallel_object_only();
5768 :
5769 1966 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5770 : {
5771 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5772 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5773 0 : _reinit_displaced_elem = true;
5774 : }
5775 : else
5776 : {
5777 1966 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5778 : {
5779 : // We allow Markers to request that they use_displaced_mesh,
5780 : // but then be overridden when no displacements variables are
5781 : // provided in the Mesh block. If that happened, update the value
5782 : // of use_displaced_mesh appropriately for this Marker.
5783 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5784 0 : parameters.set<bool>("use_displaced_mesh") = false;
5785 : }
5786 :
5787 3932 : parameters.set<SubProblem *>("_subproblem") = this;
5788 5898 : parameters.set<SystemBase *>("_sys") = _aux.get();
5789 : }
5790 :
5791 4107 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5792 : {
5793 2144 : std::shared_ptr<Marker> marker = _factory.create<Marker>(marker_name, name, parameters, tid);
5794 2141 : logAdd("Marker", name, marker_name, parameters);
5795 2141 : _markers.addObject(marker, tid);
5796 2141 : }
5797 1963 : }
5798 :
5799 : void
5800 8183 : FEProblemBase::addMultiApp(const std::string & multi_app_name,
5801 : const std::string & name,
5802 : InputParameters & parameters)
5803 : {
5804 : parallel_object_only();
5805 :
5806 16366 : parameters.set<MPI_Comm>("_mpi_comm") = _communicator.get();
5807 :
5808 8183 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5809 : {
5810 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5811 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5812 0 : _reinit_displaced_elem = true;
5813 : }
5814 : else
5815 : {
5816 8183 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5817 : {
5818 : // We allow MultiApps to request that they use_displaced_mesh,
5819 : // but then be overridden when no displacements variables are
5820 : // provided in the Mesh block. If that happened, update the value
5821 : // of use_displaced_mesh appropriately for this MultiApp.
5822 42 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5823 84 : parameters.set<bool>("use_displaced_mesh") = false;
5824 : }
5825 :
5826 16366 : parameters.set<SubProblem *>("_subproblem") = this;
5827 24549 : parameters.set<SystemBase *>("_sys") = _aux.get();
5828 : }
5829 :
5830 8183 : std::shared_ptr<MultiApp> multi_app = _factory.create<MultiApp>(multi_app_name, name, parameters);
5831 8171 : logAdd("MultiApp", name, multi_app_name, parameters);
5832 8171 : multi_app->setupPositions();
5833 :
5834 8084 : _multi_apps.addObject(multi_app);
5835 :
5836 : // Store TransientMultiApp objects in another container, this is needed for calling computeDT
5837 : std::shared_ptr<TransientMultiApp> trans_multi_app =
5838 8084 : std::dynamic_pointer_cast<TransientMultiApp>(multi_app);
5839 8084 : if (trans_multi_app)
5840 5395 : _transient_multi_apps.addObject(trans_multi_app);
5841 8084 : }
5842 :
5843 : bool
5844 112600 : FEProblemBase::hasMultiApps(ExecFlagType type) const
5845 : {
5846 112600 : return _multi_apps[type].hasActiveObjects();
5847 : }
5848 :
5849 : bool
5850 26601 : FEProblemBase::hasMultiApp(const std::string & multi_app_name) const
5851 : {
5852 26601 : return _multi_apps.hasActiveObject(multi_app_name);
5853 : }
5854 :
5855 : std::shared_ptr<MultiApp>
5856 26601 : FEProblemBase::getMultiApp(const std::string & multi_app_name) const
5857 : {
5858 26601 : if (!hasMultiApp(multi_app_name))
5859 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_multi_app"),
5860 : "A MultiApp getter was called before MultiApps have been constructed. "
5861 : "If you are attempting to access this object in the constructor of another object "
5862 : "then make sure that the MultiApp is constructed before the object using it.");
5863 :
5864 26601 : return _multi_apps.getObject(multi_app_name);
5865 : }
5866 :
5867 : void
5868 14524890 : FEProblemBase::execMultiAppTransfers(ExecFlagType type, Transfer::DIRECTION direction)
5869 : {
5870 14524890 : bool to_multiapp = direction == MultiAppTransfer::TO_MULTIAPP;
5871 14524890 : bool from_multiapp = direction == MultiAppTransfer::FROM_MULTIAPP;
5872 14524890 : std::string string_direction;
5873 14524890 : if (to_multiapp)
5874 4841684 : string_direction = " To ";
5875 9683206 : else if (from_multiapp)
5876 4841558 : string_direction = " From ";
5877 : else
5878 4841648 : string_direction = " Between ";
5879 :
5880 14525052 : const MooseObjectWarehouse<Transfer> & wh = to_multiapp ? _to_multi_app_transfers[type]
5881 24208006 : : from_multiapp ? _from_multi_app_transfers[type]
5882 33891374 : : _between_multi_app_transfers[type];
5883 :
5884 14524890 : if (wh.hasActiveObjects())
5885 : {
5886 546105 : TIME_SECTION("execMultiAppTransfers", 1, "Executing Transfers");
5887 :
5888 109221 : const auto & transfers = wh.getActiveObjects();
5889 :
5890 109221 : if (_verbose_multiapps)
5891 : {
5892 1120 : _console << COLOR_CYAN << "\nTransfers on " << Moose::stringify(type) << string_direction
5893 1120 : << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
5894 :
5895 : VariadicTable<std::string, std::string, std::string, std::string> table(
5896 2240 : {"Name", "Type", "From", "To"});
5897 :
5898 : // Build Table of Transfer Info
5899 3060 : for (const auto & transfer : transfers)
5900 : {
5901 1940 : auto multiapp_transfer = dynamic_cast<MultiAppTransfer *>(transfer.get());
5902 :
5903 1940 : table.addRow(multiapp_transfer->name(),
5904 1940 : multiapp_transfer->type(),
5905 3880 : multiapp_transfer->getFromName(),
5906 3880 : multiapp_transfer->getToName());
5907 : }
5908 :
5909 : // Print it
5910 1120 : table.print(_console);
5911 1120 : }
5912 :
5913 223483 : for (const auto & transfer : transfers)
5914 : {
5915 114359 : transfer->setCurrentDirection(direction);
5916 114359 : transfer->execute();
5917 : }
5918 :
5919 109124 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5920 :
5921 109124 : if (_verbose_multiapps)
5922 2228 : _console << COLOR_CYAN << "Transfers on " << Moose::stringify(type) << " Are Finished\n"
5923 1114 : << COLOR_DEFAULT << std::endl;
5924 109124 : }
5925 14415669 : else if (_multi_apps[type].getActiveObjects().size())
5926 : {
5927 110614 : if (_verbose_multiapps)
5928 7076 : _console << COLOR_CYAN << "\nNo Transfers on " << Moose::stringify(type) << string_direction
5929 3538 : << "MultiApps\n"
5930 3538 : << COLOR_DEFAULT << std::endl;
5931 : }
5932 14524793 : }
5933 :
5934 : std::vector<std::shared_ptr<Transfer>>
5935 12 : FEProblemBase::getTransfers(ExecFlagType type, Transfer::DIRECTION direction) const
5936 : {
5937 12 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5938 12 : return _to_multi_app_transfers[type].getActiveObjects();
5939 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5940 0 : return _from_multi_app_transfers[type].getActiveObjects();
5941 : else
5942 0 : return _between_multi_app_transfers[type].getActiveObjects();
5943 : }
5944 :
5945 : std::vector<std::shared_ptr<Transfer>>
5946 0 : FEProblemBase::getTransfers(Transfer::DIRECTION direction) const
5947 : {
5948 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5949 0 : return _to_multi_app_transfers.getActiveObjects();
5950 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5951 0 : return _from_multi_app_transfers.getActiveObjects();
5952 : else
5953 0 : return _between_multi_app_transfers.getActiveObjects();
5954 : }
5955 :
5956 : const ExecuteMooseObjectWarehouse<Transfer> &
5957 0 : FEProblemBase::getMultiAppTransferWarehouse(Transfer::DIRECTION direction) const
5958 : {
5959 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5960 0 : return _to_multi_app_transfers;
5961 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5962 0 : return _from_multi_app_transfers;
5963 : else
5964 0 : return _between_multi_app_transfers;
5965 : }
5966 :
5967 : bool
5968 4841687 : FEProblemBase::execMultiApps(ExecFlagType type, bool auto_advance)
5969 : {
5970 : // Active MultiApps
5971 : const std::vector<MooseSharedPointer<MultiApp>> & multi_apps =
5972 4841687 : _multi_apps[type].getActiveObjects();
5973 :
5974 : // Do anything that needs to be done to Apps before transfers
5975 4906923 : for (const auto & multi_app : multi_apps)
5976 65239 : multi_app->preTransfer(_dt, _time);
5977 :
5978 : // Execute Transfers _to_ MultiApps
5979 4841684 : execMultiAppTransfers(type, MultiAppTransfer::TO_MULTIAPP);
5980 :
5981 : // Execute Transfers _between_ Multiapps
5982 4841648 : execMultiAppTransfers(type, MultiAppTransfer::BETWEEN_MULTIAPP);
5983 :
5984 : // Execute MultiApps
5985 4841648 : if (multi_apps.size())
5986 : {
5987 319895 : TIME_SECTION("execMultiApps", 1, "Executing MultiApps", false);
5988 :
5989 63979 : if (_verbose_multiapps)
5990 2988 : _console << COLOR_CYAN << "\nExecuting MultiApps on " << Moose::stringify(type)
5991 1494 : << COLOR_DEFAULT << std::endl;
5992 :
5993 63979 : bool success = true;
5994 :
5995 129090 : for (const auto & multi_app : multi_apps)
5996 : {
5997 65194 : success = multi_app->solveStep(_dt, _time, auto_advance);
5998 : // no need to finish executing the subapps if one fails
5999 65185 : if (!success)
6000 74 : break;
6001 : }
6002 :
6003 63970 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6004 :
6005 63970 : _communicator.min(success);
6006 :
6007 63970 : if (!success)
6008 81 : return false;
6009 :
6010 63889 : if (_verbose_multiapps)
6011 2988 : _console << COLOR_CYAN << "Finished Executing MultiApps on " << Moose::stringify(type) << "\n"
6012 1494 : << COLOR_DEFAULT << std::endl;
6013 63970 : }
6014 :
6015 : // Execute Transfers _from_ MultiApps
6016 4841558 : execMultiAppTransfers(type, MultiAppTransfer::FROM_MULTIAPP);
6017 :
6018 : // If we made it here then everything passed
6019 4841497 : return true;
6020 : }
6021 :
6022 : void
6023 50055 : FEProblemBase::finalizeMultiApps()
6024 : {
6025 50055 : const auto & multi_apps = _multi_apps.getActiveObjects();
6026 :
6027 57121 : for (const auto & multi_app : multi_apps)
6028 7066 : multi_app->finalize();
6029 50055 : }
6030 :
6031 : void
6032 51620 : FEProblemBase::postExecute()
6033 : {
6034 51620 : const auto & multi_apps = _multi_apps.getActiveObjects();
6035 :
6036 59075 : for (const auto & multi_app : multi_apps)
6037 7455 : multi_app->postExecute();
6038 51620 : }
6039 :
6040 : void
6041 672308 : FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
6042 : {
6043 672308 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6044 :
6045 672308 : if (multi_apps.size())
6046 27809 : for (const auto & multi_app : multi_apps)
6047 14076 : multi_app->incrementTStep(_time);
6048 672308 : }
6049 :
6050 : void
6051 36370 : FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
6052 : {
6053 36370 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6054 :
6055 36370 : if (multi_apps.size())
6056 : {
6057 8584 : if (_verbose_multiapps)
6058 270 : _console << COLOR_CYAN << "\nAdvancing MultiApps on " << type.name() << COLOR_DEFAULT
6059 270 : << std::endl;
6060 :
6061 17174 : for (const auto & multi_app : multi_apps)
6062 8590 : multi_app->finishStep(recurse_through_multiapp_levels);
6063 :
6064 8584 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6065 :
6066 8584 : if (_verbose_multiapps)
6067 270 : _console << COLOR_CYAN << "Finished Advancing MultiApps on " << type.name() << "\n"
6068 270 : << COLOR_DEFAULT << std::endl;
6069 : }
6070 36370 : }
6071 :
6072 : void
6073 1126271 : FEProblemBase::backupMultiApps(ExecFlagType type)
6074 : {
6075 1126271 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6076 :
6077 1126271 : if (multi_apps.size())
6078 : {
6079 105995 : TIME_SECTION("backupMultiApps", 5, "Backing Up MultiApp");
6080 :
6081 21199 : if (_verbose_multiapps)
6082 798 : _console << COLOR_CYAN << "\nBacking Up MultiApps on " << type.name() << COLOR_DEFAULT
6083 798 : << std::endl;
6084 :
6085 43547 : for (const auto & multi_app : multi_apps)
6086 22348 : multi_app->backup();
6087 :
6088 21199 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6089 :
6090 21199 : if (_verbose_multiapps)
6091 798 : _console << COLOR_CYAN << "Finished Backing Up MultiApps on " << type.name() << "\n"
6092 798 : << COLOR_DEFAULT << std::endl;
6093 21199 : }
6094 1126271 : }
6095 :
6096 : void
6097 104740 : FEProblemBase::restoreMultiApps(ExecFlagType type, bool force)
6098 : {
6099 104740 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6100 :
6101 104740 : if (multi_apps.size())
6102 : {
6103 43028 : if (_verbose_multiapps)
6104 : {
6105 528 : if (force)
6106 0 : _console << COLOR_CYAN << "\nRestoring Multiapps on " << type.name()
6107 0 : << " because of solve failure!" << COLOR_DEFAULT << std::endl;
6108 : else
6109 528 : _console << COLOR_CYAN << "\nRestoring MultiApps on " << type.name() << COLOR_DEFAULT
6110 528 : << std::endl;
6111 : }
6112 :
6113 86098 : for (const auto & multi_app : multi_apps)
6114 43073 : multi_app->restore(force);
6115 :
6116 43025 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6117 :
6118 43025 : if (_verbose_multiapps)
6119 528 : _console << COLOR_CYAN << "Finished Restoring MultiApps on " << type.name() << "\n"
6120 528 : << COLOR_DEFAULT << std::endl;
6121 : }
6122 104737 : }
6123 :
6124 : Real
6125 697680 : FEProblemBase::computeMultiAppsDT(ExecFlagType type)
6126 : {
6127 697680 : const auto & multi_apps = _transient_multi_apps[type].getActiveObjects();
6128 :
6129 697680 : Real smallest_dt = std::numeric_limits<Real>::max();
6130 :
6131 715475 : for (const auto & multi_app : multi_apps)
6132 17795 : smallest_dt = std::min(smallest_dt, multi_app->computeDT());
6133 :
6134 697680 : return smallest_dt;
6135 : }
6136 :
6137 : void
6138 4786955 : FEProblemBase::execTransfers(ExecFlagType type)
6139 : {
6140 4786955 : if (_transfers[type].hasActiveObjects())
6141 : {
6142 0 : TIME_SECTION("execTransfers", 3, "Executing Transfers");
6143 :
6144 0 : const auto & transfers = _transfers[type].getActiveObjects();
6145 :
6146 0 : for (const auto & transfer : transfers)
6147 0 : transfer->execute();
6148 0 : }
6149 4786955 : }
6150 :
6151 : void
6152 13221 : FEProblemBase::addTransfer(const std::string & transfer_name,
6153 : const std::string & name,
6154 : InputParameters & parameters)
6155 : {
6156 : parallel_object_only();
6157 :
6158 13221 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
6159 : {
6160 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
6161 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
6162 0 : _reinit_displaced_elem = true;
6163 : }
6164 : else
6165 : {
6166 13221 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
6167 : {
6168 : // We allow Transfers to request that they use_displaced_mesh,
6169 : // but then be overridden when no displacements variables are
6170 : // provided in the Mesh block. If that happened, update the value
6171 : // of use_displaced_mesh appropriately for this Transfer.
6172 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
6173 0 : parameters.set<bool>("use_displaced_mesh") = false;
6174 : }
6175 :
6176 26442 : parameters.set<SubProblem *>("_subproblem") = this;
6177 39663 : parameters.set<SystemBase *>("_sys") = _aux.get();
6178 : }
6179 :
6180 : // Handle the "SAME_AS_MULTIAPP" execute option. The get method is used to test for the
6181 : // flag so the set by user flag is not reset, calling set with the true flag causes the set
6182 : // by user status to be reset, which should only be done if the EXEC_SAME_AS_MULTIAPP is
6183 : // being applied to the object.
6184 13221 : if (parameters.get<ExecFlagEnum>("execute_on").isValueSet(EXEC_SAME_AS_MULTIAPP))
6185 : {
6186 11082 : ExecFlagEnum & exec_enum = parameters.set<ExecFlagEnum>("execute_on", true);
6187 11082 : std::shared_ptr<MultiApp> multiapp;
6188 22164 : if (parameters.isParamValid("multi_app"))
6189 50 : multiapp = getMultiApp(parameters.get<MultiAppName>("multi_app"));
6190 : // This catches the sibling transfer case, where we want to be executing only as often as the
6191 : // receiving application. A transfer 'to' a multiapp is executed before that multiapp
6192 22064 : else if (parameters.isParamValid("to_multi_app"))
6193 5743 : multiapp = getMultiApp(parameters.get<MultiAppName>("to_multi_app"));
6194 10578 : else if (parameters.isParamValid("from_multi_app"))
6195 5283 : multiapp = getMultiApp(parameters.get<MultiAppName>("from_multi_app"));
6196 : // else do nothing because the user has provided invalid input. They should get a nice error
6197 : // about this during transfer construction. This necessitates checking for null in this next
6198 : // line, however
6199 11082 : if (multiapp)
6200 33228 : exec_enum = multiapp->getParam<ExecFlagEnum>("execute_on");
6201 11082 : }
6202 :
6203 : // Create the Transfer objects
6204 13221 : std::shared_ptr<Transfer> transfer = _factory.create<Transfer>(transfer_name, name, parameters);
6205 13164 : logAdd("Transfer", name, transfer_name, parameters);
6206 :
6207 : // Add MultiAppTransfer object
6208 : std::shared_ptr<MultiAppTransfer> multi_app_transfer =
6209 13164 : std::dynamic_pointer_cast<MultiAppTransfer>(transfer);
6210 13164 : if (multi_app_transfer)
6211 : {
6212 13164 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::TO_MULTIAPP))
6213 5274 : _to_multi_app_transfers.addObject(multi_app_transfer);
6214 13164 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::FROM_MULTIAPP))
6215 6420 : _from_multi_app_transfers.addObject(multi_app_transfer);
6216 13164 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::BETWEEN_MULTIAPP))
6217 1470 : _between_multi_app_transfers.addObject(multi_app_transfer);
6218 : }
6219 : else
6220 0 : _transfers.addObject(transfer);
6221 13164 : }
6222 :
6223 : bool
6224 1439843 : FEProblemBase::hasVariable(const std::string & var_name) const
6225 : {
6226 2348579 : for (auto & sys : _solver_systems)
6227 1445744 : if (sys->hasVariable(var_name))
6228 537008 : return true;
6229 902835 : if (_aux->hasVariable(var_name))
6230 832313 : return true;
6231 :
6232 70522 : return false;
6233 : }
6234 :
6235 : bool
6236 65 : FEProblemBase::hasSolverVariable(const std::string & var_name) const
6237 : {
6238 78 : for (auto & sys : _solver_systems)
6239 65 : if (sys->hasVariable(var_name))
6240 52 : return true;
6241 :
6242 13 : return false;
6243 : }
6244 :
6245 : const MooseVariableFieldBase &
6246 4319544 : FEProblemBase::getVariable(const THREAD_ID tid,
6247 : const std::string & var_name,
6248 : Moose::VarKindType expected_var_type,
6249 : Moose::VarFieldType expected_var_field_type) const
6250 : {
6251 12958626 : return getVariableHelper(
6252 4319544 : tid, var_name, expected_var_type, expected_var_field_type, _solver_systems, *_aux);
6253 : }
6254 :
6255 : MooseVariable &
6256 7713 : FEProblemBase::getStandardVariable(const THREAD_ID tid, const std::string & var_name)
6257 : {
6258 11607 : for (auto & sys : _solver_systems)
6259 7713 : if (sys->hasVariable(var_name))
6260 3819 : return sys->getFieldVariable<Real>(tid, var_name);
6261 3894 : if (_aux->hasVariable(var_name))
6262 3891 : return _aux->getFieldVariable<Real>(tid, var_name);
6263 :
6264 3 : mooseError("Unknown variable " + var_name);
6265 : }
6266 :
6267 : MooseVariableFieldBase &
6268 238 : FEProblemBase::getActualFieldVariable(const THREAD_ID tid, const std::string & var_name)
6269 : {
6270 283 : for (auto & sys : _solver_systems)
6271 238 : if (sys->hasVariable(var_name))
6272 193 : return sys->getActualFieldVariable<Real>(tid, var_name);
6273 45 : if (_aux->hasVariable(var_name))
6274 45 : return _aux->getActualFieldVariable<Real>(tid, var_name);
6275 :
6276 0 : mooseError("Unknown variable " + var_name);
6277 : }
6278 :
6279 : VectorMooseVariable &
6280 0 : FEProblemBase::getVectorVariable(const THREAD_ID tid, const std::string & var_name)
6281 : {
6282 0 : for (auto & sys : _solver_systems)
6283 0 : if (sys->hasVariable(var_name))
6284 0 : return sys->getFieldVariable<RealVectorValue>(tid, var_name);
6285 0 : if (_aux->hasVariable(var_name))
6286 0 : return _aux->getFieldVariable<RealVectorValue>(tid, var_name);
6287 :
6288 0 : mooseError("Unknown variable " + var_name);
6289 : }
6290 :
6291 : ArrayMooseVariable &
6292 351 : FEProblemBase::getArrayVariable(const THREAD_ID tid, const std::string & var_name)
6293 : {
6294 611 : for (auto & sys : _solver_systems)
6295 351 : if (sys->hasVariable(var_name))
6296 91 : return sys->getFieldVariable<RealEigenVector>(tid, var_name);
6297 260 : if (_aux->hasVariable(var_name))
6298 260 : return _aux->getFieldVariable<RealEigenVector>(tid, var_name);
6299 :
6300 0 : mooseError("Unknown variable " + var_name);
6301 : }
6302 :
6303 : bool
6304 182279 : FEProblemBase::hasScalarVariable(const std::string & var_name) const
6305 : {
6306 343822 : for (auto & sys : _solver_systems)
6307 183304 : if (sys->hasScalarVariable(var_name))
6308 21761 : return true;
6309 160518 : if (_aux->hasScalarVariable(var_name))
6310 10613 : return true;
6311 :
6312 149905 : return false;
6313 : }
6314 :
6315 : MooseVariableScalar &
6316 44346 : FEProblemBase::getScalarVariable(const THREAD_ID tid, const std::string & var_name)
6317 : {
6318 56695 : for (auto & sys : _solver_systems)
6319 44350 : if (sys->hasScalarVariable(var_name))
6320 32001 : return sys->getScalarVariable(tid, var_name);
6321 12345 : if (_aux->hasScalarVariable(var_name))
6322 12345 : return _aux->getScalarVariable(tid, var_name);
6323 :
6324 0 : mooseError("Unknown variable " + var_name);
6325 : }
6326 :
6327 : System &
6328 59065 : FEProblemBase::getSystem(const std::string & var_name)
6329 : {
6330 59065 : const auto [var_in_sys, sys_num] = determineSolverSystem(var_name);
6331 59065 : if (var_in_sys)
6332 40247 : return _solver_systems[sys_num]->system();
6333 18818 : else if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
6334 18818 : return _aux->system();
6335 : else
6336 0 : mooseError("Unable to find a system containing the variable " + var_name);
6337 : }
6338 :
6339 : const RestartableEquationSystems &
6340 0 : FEProblemBase::getRestartableEquationSystems() const
6341 : {
6342 0 : return _req.get();
6343 : }
6344 :
6345 : void
6346 465552 : FEProblemBase::setActiveFEVariableCoupleableMatrixTags(std::set<TagID> & mtags, const THREAD_ID tid)
6347 : {
6348 465552 : SubProblem::setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6349 :
6350 465552 : if (_displaced_problem)
6351 98260 : _displaced_problem->setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6352 465552 : }
6353 :
6354 : void
6355 6246685 : FEProblemBase::setActiveFEVariableCoupleableVectorTags(std::set<TagID> & vtags, const THREAD_ID tid)
6356 : {
6357 6246685 : SubProblem::setActiveFEVariableCoupleableVectorTags(vtags, tid);
6358 :
6359 6246685 : if (_displaced_problem)
6360 340076 : _displaced_problem->setActiveFEVariableCoupleableVectorTags(vtags, tid);
6361 6246685 : }
6362 :
6363 : void
6364 47814 : FEProblemBase::setActiveScalarVariableCoupleableMatrixTags(std::set<TagID> & mtags,
6365 : const THREAD_ID tid)
6366 : {
6367 47814 : SubProblem::setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6368 :
6369 47814 : if (_displaced_problem)
6370 0 : _displaced_problem->setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6371 47814 : }
6372 :
6373 : void
6374 47814 : FEProblemBase::setActiveScalarVariableCoupleableVectorTags(std::set<TagID> & vtags,
6375 : const THREAD_ID tid)
6376 : {
6377 47814 : SubProblem::setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6378 :
6379 47814 : if (_displaced_problem)
6380 0 : _displaced_problem->setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6381 47814 : }
6382 :
6383 : void
6384 9530882 : FEProblemBase::setActiveElementalMooseVariables(const std::set<MooseVariableFEBase *> & moose_vars,
6385 : const THREAD_ID tid)
6386 : {
6387 9530882 : SubProblem::setActiveElementalMooseVariables(moose_vars, tid);
6388 :
6389 9530882 : if (_displaced_problem)
6390 479387 : _displaced_problem->setActiveElementalMooseVariables(moose_vars, tid);
6391 9530882 : }
6392 :
6393 : void
6394 3840482 : FEProblemBase::clearActiveElementalMooseVariables(const THREAD_ID tid)
6395 : {
6396 3840482 : SubProblem::clearActiveElementalMooseVariables(tid);
6397 :
6398 3840482 : if (_displaced_problem)
6399 170463 : _displaced_problem->clearActiveElementalMooseVariables(tid);
6400 3840482 : }
6401 :
6402 : void
6403 207676 : FEProblemBase::clearActiveFEVariableCoupleableMatrixTags(const THREAD_ID tid)
6404 : {
6405 207676 : SubProblem::clearActiveFEVariableCoupleableMatrixTags(tid);
6406 :
6407 207676 : if (_displaced_problem)
6408 41849 : _displaced_problem->clearActiveFEVariableCoupleableMatrixTags(tid);
6409 207676 : }
6410 :
6411 : void
6412 207676 : FEProblemBase::clearActiveFEVariableCoupleableVectorTags(const THREAD_ID tid)
6413 : {
6414 207676 : SubProblem::clearActiveFEVariableCoupleableVectorTags(tid);
6415 :
6416 207676 : if (_displaced_problem)
6417 41849 : _displaced_problem->clearActiveFEVariableCoupleableVectorTags(tid);
6418 207676 : }
6419 :
6420 : void
6421 47814 : FEProblemBase::clearActiveScalarVariableCoupleableMatrixTags(const THREAD_ID tid)
6422 : {
6423 47814 : SubProblem::clearActiveScalarVariableCoupleableMatrixTags(tid);
6424 :
6425 47814 : if (_displaced_problem)
6426 0 : _displaced_problem->clearActiveScalarVariableCoupleableMatrixTags(tid);
6427 47814 : }
6428 :
6429 : void
6430 47814 : FEProblemBase::clearActiveScalarVariableCoupleableVectorTags(const THREAD_ID tid)
6431 : {
6432 47814 : SubProblem::clearActiveScalarVariableCoupleableVectorTags(tid);
6433 :
6434 47814 : if (_displaced_problem)
6435 0 : _displaced_problem->clearActiveScalarVariableCoupleableVectorTags(tid);
6436 47814 : }
6437 :
6438 : void
6439 4863821 : FEProblemBase::setActiveMaterialProperties(const std::unordered_set<unsigned int> & mat_prop_ids,
6440 : const THREAD_ID tid)
6441 : {
6442 : // mark active properties in every material
6443 5984629 : for (auto & mat : _all_materials.getObjects(tid))
6444 1120808 : mat->setActiveProperties(mat_prop_ids);
6445 5743615 : for (auto & mat : _all_materials[Moose::FACE_MATERIAL_DATA].getObjects(tid))
6446 879794 : mat->setActiveProperties(mat_prop_ids);
6447 5743615 : for (auto & mat : _all_materials[Moose::NEIGHBOR_MATERIAL_DATA].getObjects(tid))
6448 879794 : mat->setActiveProperties(mat_prop_ids);
6449 :
6450 4863821 : _has_active_material_properties[tid] = !mat_prop_ids.empty();
6451 4863821 : }
6452 :
6453 : bool
6454 383622098 : FEProblemBase::hasActiveMaterialProperties(const THREAD_ID tid) const
6455 : {
6456 383622098 : return _has_active_material_properties[tid];
6457 : }
6458 :
6459 : void
6460 3983146 : FEProblemBase::clearActiveMaterialProperties(const THREAD_ID tid)
6461 : {
6462 3983146 : _has_active_material_properties[tid] = 0;
6463 3983146 : }
6464 :
6465 : void
6466 60912 : FEProblemBase::addAnyRedistributers()
6467 : {
6468 : #ifdef LIBMESH_ENABLE_AMR
6469 63181 : if ((_adaptivity.isOn() || _num_grid_steps) &&
6470 2269 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
6471 2206 : _neighbor_material_props.hasStatefulProperties()))
6472 : {
6473 : // Even on a serialized Mesh, we don't keep our material
6474 : // properties serialized, so we'll rely on the callback to
6475 : // redistribute() to redistribute properties at the same time
6476 : // libMesh is redistributing elements.
6477 63 : auto add_redistributer = [this](MooseMesh & mesh,
6478 : const std::string & redistributer_name,
6479 : const bool use_displaced_mesh)
6480 : {
6481 63 : InputParameters redistribute_params = RedistributeProperties::validParams();
6482 63 : redistribute_params.set<MooseApp *>(MooseBase::app_param) = &_app;
6483 126 : redistribute_params.set<std::string>("for_whom") = this->name();
6484 189 : redistribute_params.set<MooseMesh *>("mesh") = &mesh;
6485 63 : redistribute_params.set<Moose::RelationshipManagerType>("rm_type") =
6486 : Moose::RelationshipManagerType::GEOMETRIC;
6487 126 : redistribute_params.set<bool>("use_displaced_mesh") = use_displaced_mesh;
6488 63 : redistribute_params.setHitNode(*parameters().getHitNode(), {});
6489 :
6490 : std::shared_ptr<RedistributeProperties> redistributer =
6491 63 : _factory.create<RedistributeProperties>(
6492 126 : "RedistributeProperties", redistributer_name, redistribute_params);
6493 :
6494 63 : if (_material_props.hasStatefulProperties())
6495 63 : redistributer->addMaterialPropertyStorage(_material_props);
6496 :
6497 63 : if (_bnd_material_props.hasStatefulProperties())
6498 63 : redistributer->addMaterialPropertyStorage(_bnd_material_props);
6499 :
6500 63 : if (_neighbor_material_props.hasStatefulProperties())
6501 63 : redistributer->addMaterialPropertyStorage(_neighbor_material_props);
6502 :
6503 63 : mesh.getMesh().add_ghosting_functor(redistributer);
6504 126 : };
6505 :
6506 63 : add_redistributer(_mesh, "mesh_property_redistributer", false);
6507 63 : if (_displaced_problem)
6508 0 : add_redistributer(_displaced_problem->mesh(), "displaced_mesh_property_redistributer", true);
6509 : }
6510 : #endif // LIBMESH_ENABLE_AMR
6511 60912 : }
6512 :
6513 : void
6514 62843 : FEProblemBase::updateMaxQps()
6515 : {
6516 : // Find the maximum number of quadrature points
6517 : {
6518 62843 : MaxQpsThread mqt(*this);
6519 62843 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mqt);
6520 62843 : _max_qps = mqt.max();
6521 :
6522 : // If we have more shape functions or more quadrature points on
6523 : // another processor, then we may need to handle those elements
6524 : // ourselves later after repartitioning.
6525 62843 : _communicator.max(_max_qps);
6526 : }
6527 :
6528 62843 : unsigned int max_qpts = getMaxQps();
6529 62843 : if (max_qpts > Moose::constMaxQpsPerElem)
6530 0 : mooseError("Max quadrature points per element assumptions made in some code (e.g. Coupleable ",
6531 : "and MaterialPropertyInterface classes) have been violated.\n",
6532 : "Complain to Moose developers to have constMaxQpsPerElem increased from ",
6533 : Moose::constMaxQpsPerElem,
6534 : " to ",
6535 : max_qpts);
6536 132353 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6537 : {
6538 : // the highest available order in libMesh is 43
6539 69510 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
6540 69510 : _zero[tid].resize(max_qpts, 0);
6541 69510 : _ad_zero[tid].resize(max_qpts, 0);
6542 69510 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
6543 69510 : _ad_grad_zero[tid].resize(max_qpts, ADRealGradient(0));
6544 69510 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
6545 69510 : _ad_second_zero[tid].resize(max_qpts, ADRealTensorValue(0));
6546 69510 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
6547 69510 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
6548 : }
6549 62843 : }
6550 :
6551 : void
6552 78 : FEProblemBase::bumpVolumeQRuleOrder(Order order, SubdomainID block)
6553 : {
6554 168 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6555 180 : for (const auto i : index_range(_nl))
6556 90 : _assembly[tid][i]->bumpVolumeQRuleOrder(order, block);
6557 :
6558 78 : if (_displaced_problem)
6559 0 : _displaced_problem->bumpVolumeQRuleOrder(order, block);
6560 :
6561 78 : updateMaxQps();
6562 78 : }
6563 :
6564 : void
6565 13 : FEProblemBase::bumpAllQRuleOrder(Order order, SubdomainID block)
6566 : {
6567 28 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6568 30 : for (const auto i : index_range(_nl))
6569 15 : _assembly[tid][i]->bumpAllQRuleOrder(order, block);
6570 :
6571 13 : if (_displaced_problem)
6572 0 : _displaced_problem->bumpAllQRuleOrder(order, block);
6573 :
6574 13 : updateMaxQps();
6575 13 : }
6576 :
6577 : void
6578 62752 : FEProblemBase::createQRules(QuadratureType type,
6579 : Order order,
6580 : Order volume_order,
6581 : Order face_order,
6582 : SubdomainID block,
6583 : const bool allow_negative_qweights)
6584 : {
6585 62752 : if (order == INVALID_ORDER)
6586 : {
6587 : // automatically determine the integration order
6588 62135 : order = _solver_systems[0]->getMinQuadratureOrder();
6589 62463 : for (const auto i : make_range(std::size_t(1), _solver_systems.size()))
6590 328 : if (order < _solver_systems[i]->getMinQuadratureOrder())
6591 0 : order = _solver_systems[i]->getMinQuadratureOrder();
6592 62135 : if (order < _aux->getMinQuadratureOrder())
6593 6081 : order = _aux->getMinQuadratureOrder();
6594 : }
6595 :
6596 62752 : if (volume_order == INVALID_ORDER)
6597 62564 : volume_order = order;
6598 :
6599 62752 : if (face_order == INVALID_ORDER)
6600 62564 : face_order = order;
6601 :
6602 132157 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6603 139199 : for (const auto i : index_range(_solver_systems))
6604 69794 : _assembly[tid][i]->createQRules(
6605 : type, order, volume_order, face_order, block, allow_negative_qweights);
6606 :
6607 62752 : if (_displaced_problem)
6608 2022 : _displaced_problem->createQRules(
6609 : type, order, volume_order, face_order, block, allow_negative_qweights);
6610 :
6611 62752 : updateMaxQps();
6612 62752 : }
6613 :
6614 : void
6615 20069 : FEProblemBase::setCoupling(Moose::CouplingType type)
6616 : {
6617 20069 : if (_trust_user_coupling_matrix)
6618 : {
6619 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6620 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6621 : "haven't been provided a coupling matrix!");
6622 :
6623 : // We've been told to trust the user coupling matrix, so we're going to leave things alone
6624 3 : return;
6625 : }
6626 :
6627 20066 : _coupling = type;
6628 : }
6629 :
6630 : void
6631 0 : FEProblemBase::setCouplingMatrix(CouplingMatrix * cm, const unsigned int i)
6632 : {
6633 : // TODO: Deprecate method
6634 0 : setCoupling(Moose::COUPLING_CUSTOM);
6635 0 : _cm[i].reset(cm);
6636 0 : }
6637 :
6638 : void
6639 13568 : FEProblemBase::setCouplingMatrix(std::unique_ptr<CouplingMatrix> cm, const unsigned int i)
6640 : {
6641 13568 : setCoupling(Moose::COUPLING_CUSTOM);
6642 13568 : _cm[i] = std::move(cm);
6643 13568 : }
6644 :
6645 : void
6646 3 : FEProblemBase::trustUserCouplingMatrix()
6647 : {
6648 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6649 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6650 : "haven't been provided a coupling matrix!");
6651 :
6652 3 : _trust_user_coupling_matrix = true;
6653 3 : }
6654 :
6655 : void
6656 63 : FEProblemBase::setNonlocalCouplingMatrix()
6657 : {
6658 315 : TIME_SECTION("setNonlocalCouplingMatrix", 5, "Setting Nonlocal Coupling Matrix");
6659 :
6660 63 : if (_nl.size() > 1)
6661 0 : mooseError("Nonlocal kernels are weirdly stored on the FEProblem so we don't currently support "
6662 : "multiple nonlinear systems with nonlocal kernels.");
6663 :
6664 126 : for (const auto nl_sys_num : index_range(_nl))
6665 : {
6666 63 : auto & nl = _nl[nl_sys_num];
6667 63 : auto & nonlocal_cm = _nonlocal_cm[nl_sys_num];
6668 63 : unsigned int n_vars = nl->nVariables();
6669 63 : nonlocal_cm.resize(n_vars);
6670 63 : const auto & vars = nl->getVariables(0);
6671 63 : const auto & nonlocal_kernel = _nonlocal_kernels.getObjects();
6672 63 : const auto & nonlocal_integrated_bc = _nonlocal_integrated_bcs.getObjects();
6673 189 : for (const auto & ivar : vars)
6674 : {
6675 196 : for (const auto & kernel : nonlocal_kernel)
6676 : {
6677 140 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6678 70 : if (i == kernel->variable().number())
6679 105 : for (const auto & jvar : vars)
6680 : {
6681 70 : const auto it = _var_dof_map.find(jvar->name());
6682 70 : if (it != _var_dof_map.end())
6683 : {
6684 62 : unsigned int j = jvar->number();
6685 62 : nonlocal_cm(i, j) = 1;
6686 : }
6687 : }
6688 : }
6689 182 : for (const auto & integrated_bc : nonlocal_integrated_bc)
6690 : {
6691 112 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6692 56 : if (i == integrated_bc->variable().number())
6693 84 : for (const auto & jvar : vars)
6694 : {
6695 56 : const auto it = _var_dof_map.find(jvar->name());
6696 56 : if (it != _var_dof_map.end())
6697 : {
6698 28 : unsigned int j = jvar->number();
6699 28 : nonlocal_cm(i, j) = 1;
6700 : }
6701 : }
6702 : }
6703 : }
6704 : }
6705 63 : }
6706 :
6707 : bool
6708 624 : FEProblemBase::areCoupled(const unsigned int ivar,
6709 : const unsigned int jvar,
6710 : const unsigned int nl_sys) const
6711 : {
6712 624 : return (*_cm[nl_sys])(ivar, jvar);
6713 : }
6714 :
6715 : std::vector<std::pair<MooseVariableFieldBase *, MooseVariableFieldBase *>> &
6716 15071055 : FEProblemBase::couplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6717 : {
6718 15071055 : return _assembly[tid][nl_sys]->couplingEntries();
6719 : }
6720 :
6721 : std::vector<std::pair<MooseVariableFieldBase *, MooseVariableFieldBase *>> &
6722 4162 : FEProblemBase::nonlocalCouplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6723 : {
6724 4162 : return _assembly[tid][nl_sys]->nonlocalCouplingEntries();
6725 : }
6726 :
6727 : void
6728 61406 : FEProblemBase::init()
6729 : {
6730 61406 : if (_initialized)
6731 0 : return;
6732 :
6733 307030 : TIME_SECTION("init", 2, "Initializing");
6734 :
6735 : // call executioner's preProblemInit so that it can do some setups before problem init
6736 61406 : _app.getExecutioner()->preProblemInit();
6737 :
6738 : // If we have AD and we are doing global AD indexing, then we should by default set the matrix
6739 : // coupling to full. If the user has told us to trust their coupling matrix, then this call will
6740 : // not do anything
6741 61406 : if (haveADObjects() && Moose::globalADIndexing())
6742 6408 : setCoupling(Moose::COUPLING_FULL);
6743 :
6744 121906 : for (const auto i : index_range(_nl))
6745 : {
6746 60500 : auto & nl = _nl[i];
6747 60500 : auto & cm = _cm[i];
6748 :
6749 60500 : unsigned int n_vars = nl->nVariables();
6750 : {
6751 302500 : TIME_SECTION("fillCouplingMatrix", 3, "Filling Coupling Matrix");
6752 :
6753 60500 : switch (_coupling)
6754 : {
6755 45186 : case Moose::COUPLING_DIAG:
6756 45186 : cm = std::make_unique<CouplingMatrix>(n_vars);
6757 82186 : for (unsigned int i = 0; i < n_vars; i++)
6758 37000 : (*cm)(i, i) = 1;
6759 45186 : break;
6760 :
6761 : // for full jacobian
6762 6561 : case Moose::COUPLING_FULL:
6763 6561 : cm = std::make_unique<CouplingMatrix>(n_vars);
6764 16802 : for (unsigned int i = 0; i < n_vars; i++)
6765 29656 : for (unsigned int j = 0; j < n_vars; j++)
6766 19415 : (*cm)(i, j) = 1;
6767 6561 : break;
6768 :
6769 8753 : case Moose::COUPLING_CUSTOM:
6770 : // do nothing, _cm was already set through couplingMatrix() call
6771 8753 : break;
6772 : }
6773 60500 : }
6774 :
6775 60500 : nl->dofMap()._dof_coupling = cm.get();
6776 :
6777 : // If there are no variables, make sure to pass a nullptr coupling
6778 : // matrix, to avoid warnings about non-nullptr yet empty
6779 : // CouplingMatrices.
6780 60500 : if (n_vars == 0)
6781 14130 : nl->dofMap()._dof_coupling = nullptr;
6782 :
6783 60500 : nl->dofMap().attach_extra_sparsity_function(&extraSparsity, nl.get());
6784 60500 : nl->dofMap().attach_extra_send_list_function(&extraSendList, nl.get());
6785 60500 : _aux->dofMap().attach_extra_send_list_function(&extraSendList, _aux.get());
6786 :
6787 60500 : if (!_skip_nl_system_check && _solve && n_vars == 0)
6788 0 : mooseError("No variables specified in nonlinear system '", nl->name(), "'.");
6789 : }
6790 :
6791 61406 : ghostGhostedBoundaries(); // We do this again right here in case new boundaries have been added
6792 :
6793 : // We may have added element/nodes to the mesh in ghostGhostedBoundaries so we need to update
6794 : // all of our mesh information. We need to make sure that mesh information is up-to-date before
6795 : // EquationSystems::init because that will call through to updateGeomSearch (for sparsity
6796 : // augmentation) and if we haven't added back boundary node information before that latter call,
6797 : // then we're screwed. We'll get things like "Unable to find closest node!"
6798 61406 : _mesh.meshChanged();
6799 61406 : if (_displaced_problem)
6800 2022 : _displaced_mesh->meshChanged();
6801 :
6802 61406 : if (_mesh.doingPRefinement())
6803 : {
6804 240 : preparePRefinement();
6805 240 : if (_displaced_problem)
6806 0 : _displaced_problem->preparePRefinement();
6807 : }
6808 :
6809 : // do not assemble system matrix for JFNK solve
6810 121906 : for (auto & nl : _nl)
6811 60500 : if (solverParams(nl->number())._type == Moose::ST_JFNK)
6812 120 : nl->turnOffJacobian();
6813 :
6814 123134 : for (auto & sys : _solver_systems)
6815 61728 : sys->preInit();
6816 61406 : _aux->preInit();
6817 :
6818 : // Build the mortar segment meshes, if they haven't been already, for a couple reasons:
6819 : // 1) Get the ghosting correct for both static and dynamic meshes
6820 : // 2) Make sure the mortar mesh is built for mortar constraints that live on the static mesh
6821 : //
6822 : // It is worth-while to note that mortar meshes that live on a dynamic mesh will be built
6823 : // during residual and Jacobian evaluation because when displacements are solution variables
6824 : // the mortar mesh will move and change during the course of a non-linear solve. We DO NOT
6825 : // redo ghosting during non-linear solve, so for purpose 1) the below call has to be made
6826 61406 : if (!_mortar_data->initialized())
6827 45805 : updateMortarMesh();
6828 :
6829 : {
6830 307030 : TIME_SECTION("EquationSystems::Init", 2, "Initializing Equation Systems");
6831 61406 : es().init();
6832 61406 : }
6833 :
6834 123134 : for (auto & sys : _solver_systems)
6835 61728 : sys->postInit();
6836 61406 : _aux->postInit();
6837 :
6838 : // Now that the equation system and the dof distribution is done, we can generate the
6839 : // finite volume-related parts if needed.
6840 61406 : if (haveFV())
6841 4343 : _mesh.setupFiniteVolumeMeshData();
6842 :
6843 123134 : for (auto & sys : _solver_systems)
6844 61728 : sys->update();
6845 61406 : _aux->update();
6846 :
6847 129028 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
6848 134361 : for (const auto i : index_range(_nl))
6849 : {
6850 : mooseAssert(
6851 : _cm[i],
6852 : "Coupling matrix not set for system "
6853 : << i
6854 : << ". This should only happen if a preconditioner was not setup for this system");
6855 66739 : _assembly[tid][i]->init(_cm[i].get());
6856 : }
6857 :
6858 61406 : if (_displaced_problem)
6859 2022 : _displaced_problem->init();
6860 :
6861 : #ifdef MOOSE_KOKKOS_ENABLED
6862 46531 : if (_has_kokkos_objects)
6863 2608 : initKokkos();
6864 : #endif
6865 :
6866 61406 : _initialized = true;
6867 61406 : }
6868 :
6869 : unsigned int
6870 12163 : FEProblemBase::nlSysNum(const NonlinearSystemName & nl_sys_name) const
6871 : {
6872 12163 : std::istringstream ss(nl_sys_name);
6873 : unsigned int nl_sys_num;
6874 12163 : if (!(ss >> nl_sys_num) || !ss.eof())
6875 12163 : nl_sys_num = libmesh_map_find(_nl_sys_name_to_num, nl_sys_name);
6876 :
6877 12163 : return nl_sys_num;
6878 12163 : }
6879 :
6880 : unsigned int
6881 77292 : FEProblemBase::linearSysNum(const LinearSystemName & linear_sys_name) const
6882 : {
6883 77292 : std::istringstream ss(linear_sys_name);
6884 : unsigned int linear_sys_num;
6885 77292 : if (!(ss >> linear_sys_num) || !ss.eof())
6886 77292 : linear_sys_num = libmesh_map_find(_linear_sys_name_to_num, linear_sys_name);
6887 :
6888 77292 : return linear_sys_num;
6889 77292 : }
6890 :
6891 : unsigned int
6892 127605 : FEProblemBase::solverSysNum(const SolverSystemName & solver_sys_name) const
6893 : {
6894 127605 : std::istringstream ss(solver_sys_name);
6895 : unsigned int solver_sys_num;
6896 127605 : if (!(ss >> solver_sys_num) || !ss.eof())
6897 : {
6898 127605 : const auto & search = _solver_sys_name_to_num.find(solver_sys_name);
6899 127605 : if (search == _solver_sys_name_to_num.end())
6900 0 : mooseError("The solver system number was requested for system '" + solver_sys_name,
6901 : "' but this system does not exist in the Problem. Systems can be added to the "
6902 : "problem using the 'nl_sys_names'/'linear_sys_names' parameter.\nSystems in the "
6903 0 : "Problem: " +
6904 0 : Moose::stringify(_solver_sys_names));
6905 127605 : solver_sys_num = search->second;
6906 : }
6907 :
6908 127605 : return solver_sys_num;
6909 127605 : }
6910 :
6911 : unsigned int
6912 1644 : FEProblemBase::systemNumForVariable(const VariableName & variable_name) const
6913 : {
6914 1746 : for (const auto & solver_sys : _solver_systems)
6915 1644 : if (solver_sys->hasVariable(variable_name))
6916 1542 : return solver_sys->number();
6917 : mooseAssert(_aux, "Should have an auxiliary system");
6918 102 : if (_aux->hasVariable(variable_name))
6919 102 : return _aux->number();
6920 :
6921 0 : mooseError("Variable '",
6922 : variable_name,
6923 : "' was not found in any solver (nonlinear/linear) or auxiliary system");
6924 : }
6925 :
6926 : void
6927 323337 : FEProblemBase::solve(const unsigned int nl_sys_num)
6928 : {
6929 1616685 : TIME_SECTION("solve", 1, "Solving", false);
6930 :
6931 323337 : setCurrentNonlinearSystem(nl_sys_num);
6932 :
6933 : // This prevents stale dof indices from lingering around and possibly leading to invalid reads
6934 : // and writes. Dof indices may be made stale through operations like mesh adaptivity
6935 323337 : clearAllDofIndices();
6936 323337 : if (_displaced_problem)
6937 32837 : _displaced_problem->clearAllDofIndices();
6938 :
6939 : // Setup the output system for printing linear/nonlinear iteration information and some solver
6940 : // settings, including setting matrix prefixes. This must occur before petscSetOptions
6941 323337 : initPetscOutputAndSomeSolverSettings();
6942 :
6943 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6944 : Moose::PetscSupport::petscSetOptions(
6945 : _petsc_options, _solver_params); // Make sure the PETSc options are setup for this app
6946 : #else
6947 : // Now this database will be the default
6948 : // Each app should have only one database
6949 323337 : if (!_app.isUltimateMaster())
6950 85527 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6951 : // We did not add PETSc options to database yet
6952 323337 : if (!_is_petsc_options_inserted)
6953 : {
6954 : // Insert options for all systems all at once
6955 39557 : Moose::PetscSupport::petscSetOptions(_petsc_options, _solver_params, this);
6956 39554 : _is_petsc_options_inserted = true;
6957 : }
6958 : #endif
6959 :
6960 : // set up DM which is required if use a field split preconditioner
6961 : // We need to setup DM every "solve()" because libMesh destroy SNES after solve()
6962 : // Do not worry, DM setup is very cheap
6963 323334 : _current_nl_sys->setupDM();
6964 :
6965 323334 : possiblyRebuildGeomSearchPatches();
6966 :
6967 : // reset flag so that residual evaluation does not get skipped
6968 : // and the next non-linear iteration does not automatically fail with
6969 : // "DIVERGED_NANORINF", when we throw an exception and stop solve
6970 323334 : _fail_next_system_convergence_check = false;
6971 :
6972 323334 : if (_solve)
6973 : {
6974 289563 : _current_nl_sys->solve();
6975 289508 : _current_nl_sys->update();
6976 : }
6977 :
6978 : // sync solutions in displaced problem
6979 323279 : if (_displaced_problem)
6980 32834 : _displaced_problem->syncSolutions();
6981 :
6982 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6983 323279 : if (!_app.isUltimateMaster())
6984 85527 : LibmeshPetscCall(PetscOptionsPop());
6985 : #endif
6986 323279 : }
6987 :
6988 : void
6989 199 : FEProblemBase::setException(const std::string & message)
6990 : {
6991 199 : _has_exception = true;
6992 199 : _exception_message = message;
6993 199 : }
6994 :
6995 : void
6996 19982943 : FEProblemBase::checkExceptionAndStopSolve(bool print_message)
6997 : {
6998 19982943 : if (_skip_exception_check)
6999 216 : return;
7000 :
7001 59948181 : TIME_SECTION("checkExceptionAndStopSolve", 5);
7002 :
7003 : // See if any processor had an exception. If it did, get back the
7004 : // processor that the exception occurred on.
7005 : unsigned int processor_id;
7006 :
7007 19982727 : _communicator.maxloc(_has_exception, processor_id);
7008 :
7009 19982727 : if (_has_exception)
7010 : {
7011 302 : _communicator.broadcast(_exception_message, processor_id);
7012 :
7013 432 : if (_current_execute_on_flag == EXEC_LINEAR || _current_execute_on_flag == EXEC_NONLINEAR ||
7014 130 : _current_execute_on_flag == EXEC_POSTCHECK)
7015 : {
7016 : // Print the message
7017 302 : if (_communicator.rank() == 0 && print_message)
7018 : {
7019 184 : _console << "\n" << _exception_message << "\n";
7020 184 : if (isTransient())
7021 : _console
7022 : << "To recover, the solution will fail and then be re-attempted with a reduced time "
7023 172 : "step.\n"
7024 172 : << std::endl;
7025 : }
7026 :
7027 : // Stop the solve -- this entails setting
7028 : // SNESSetFunctionDomainError() or directly inserting NaNs in the
7029 : // residual vector to let PETSc >= 3.6 return DIVERGED_NANORINF.
7030 302 : if (_current_nl_sys)
7031 302 : _current_nl_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7032 :
7033 302 : if (_current_linear_sys)
7034 0 : _current_linear_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7035 :
7036 : // and close Aux system (we MUST do this here; see #11525)
7037 302 : _aux->solution().close();
7038 :
7039 : // We've handled this exception, so we no longer have one.
7040 302 : _has_exception = false;
7041 :
7042 : // Force the next non-linear convergence check to fail (and all further residual evaluation
7043 : // to be skipped).
7044 302 : _fail_next_system_convergence_check = true;
7045 :
7046 : // Repropagate the exception, so it can be caught at a higher level, typically
7047 : // this is NonlinearSystem::computeResidual().
7048 302 : throw MooseException(_exception_message);
7049 : }
7050 : else
7051 0 : mooseError("The following parallel-communicated exception was detected during " +
7052 0 : Moose::stringify(_current_execute_on_flag) + " evaluation:\n" +
7053 0 : _exception_message +
7054 : "\nBecause this did not occur during residual evaluation, there"
7055 : " is no way to handle this, so the solution is aborting.\n");
7056 : }
7057 19982727 : }
7058 :
7059 : void
7060 3547928 : FEProblemBase::resetState()
7061 : {
7062 : // Our default state is to allow computing derivatives
7063 3547928 : ADReal::do_derivatives = true;
7064 3547928 : _current_execute_on_flag = EXEC_NONE;
7065 :
7066 : // Clear the VectorTags and MatrixTags
7067 3547928 : clearCurrentResidualVectorTags();
7068 3547928 : clearCurrentJacobianMatrixTags();
7069 :
7070 3547928 : _safe_access_tagged_vectors = true;
7071 3547928 : _safe_access_tagged_matrices = true;
7072 :
7073 3547928 : setCurrentlyComputingResidual(false);
7074 3547928 : setCurrentlyComputingJacobian(false);
7075 3547928 : setCurrentlyComputingResidualAndJacobian(false);
7076 3547928 : if (_displaced_problem)
7077 : {
7078 145291 : _displaced_problem->setCurrentlyComputingResidual(false);
7079 145291 : _displaced_problem->setCurrentlyComputingJacobian(false);
7080 145291 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(false);
7081 : }
7082 3547928 : }
7083 :
7084 : void
7085 25718 : FEProblemBase::solveLinearSystem(const unsigned int linear_sys_num,
7086 : const Moose::PetscSupport::PetscOptions * po)
7087 : {
7088 128590 : TIME_SECTION("solve", 1, "Solving", false);
7089 :
7090 25718 : setCurrentLinearSystem(linear_sys_num);
7091 :
7092 25718 : const Moose::PetscSupport::PetscOptions & options = po ? *po : _petsc_options;
7093 25718 : auto & solver_params = _solver_params[numNonlinearSystems() + linear_sys_num];
7094 :
7095 : // Set custom convergence criteria
7096 25718 : Moose::PetscSupport::petscSetDefaults(*this);
7097 :
7098 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
7099 : LibmeshPetscCall(Moose::PetscSupport::petscSetOptions(
7100 : options, solver_params)); // Make sure the PETSc options are setup for this app
7101 : #else
7102 : // Now this database will be the default
7103 : // Each app should have only one database
7104 25718 : if (!_app.isUltimateMaster())
7105 195 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
7106 :
7107 : // We did not add PETSc options to database yet
7108 25718 : if (!_is_petsc_options_inserted)
7109 : {
7110 1119 : Moose::PetscSupport::petscSetOptions(options, solver_params, this);
7111 1119 : _is_petsc_options_inserted = true;
7112 : }
7113 : #endif
7114 :
7115 25718 : if (_solve)
7116 25708 : _current_linear_sys->solve();
7117 :
7118 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
7119 25718 : if (!_app.isUltimateMaster())
7120 195 : LibmeshPetscCall(PetscOptionsPop());
7121 : #endif
7122 25718 : }
7123 :
7124 : bool
7125 336401 : FEProblemBase::solverSystemConverged(const unsigned int sys_num)
7126 : {
7127 336401 : if (_solve)
7128 336379 : return _solver_systems[sys_num]->converged();
7129 : else
7130 22 : return true;
7131 : }
7132 :
7133 : unsigned int
7134 4901 : FEProblemBase::nNonlinearIterations(const unsigned int nl_sys_num) const
7135 : {
7136 4901 : return _nl[nl_sys_num]->nNonlinearIterations();
7137 : }
7138 :
7139 : unsigned int
7140 4051 : FEProblemBase::nLinearIterations(const unsigned int nl_sys_num) const
7141 : {
7142 4051 : return _nl[nl_sys_num]->nLinearIterations();
7143 : }
7144 :
7145 : Real
7146 242 : FEProblemBase::finalNonlinearResidual(const unsigned int nl_sys_num) const
7147 : {
7148 242 : return _nl[nl_sys_num]->finalNonlinearResidual();
7149 : }
7150 :
7151 : bool
7152 760320 : FEProblemBase::computingPreSMOResidual(const unsigned int nl_sys_num) const
7153 : {
7154 760320 : return _nl[nl_sys_num]->computingPreSMOResidual();
7155 : }
7156 :
7157 : void
7158 56124 : FEProblemBase::copySolutionsBackwards()
7159 : {
7160 280620 : TIME_SECTION("copySolutionsBackwards", 3, "Copying Solutions Backward");
7161 :
7162 112549 : for (auto & sys : _solver_systems)
7163 56425 : sys->copySolutionsBackwards();
7164 56124 : _aux->copySolutionsBackwards();
7165 56124 : }
7166 :
7167 : void
7168 152 : FEProblemBase::skipNextForwardSolutionCopyToOld()
7169 : {
7170 324 : for (auto & sys : _solver_systems)
7171 172 : sys->skipNextSolutionToOldCopy();
7172 152 : _aux->skipNextSolutionToOldCopy();
7173 152 : }
7174 :
7175 : void
7176 234989 : FEProblemBase::advanceState()
7177 : {
7178 1174945 : TIME_SECTION("advanceState", 5, "Advancing State");
7179 :
7180 473385 : for (auto & sys : _solver_systems)
7181 238396 : sys->copyOldSolutions();
7182 234989 : _aux->copyOldSolutions();
7183 :
7184 234989 : if (_displaced_problem)
7185 : {
7186 61858 : for (const auto i : index_range(_solver_systems))
7187 30929 : _displaced_problem->solverSys(i).copyOldSolutions();
7188 30929 : _displaced_problem->auxSys().copyOldSolutions();
7189 : }
7190 :
7191 234989 : _reporter_data.copyValuesBack();
7192 :
7193 234989 : getMooseApp().getChainControlDataSystem().copyValuesBack();
7194 :
7195 234989 : if (_material_props.hasStatefulProperties())
7196 1760 : _material_props.shift();
7197 :
7198 234989 : if (_bnd_material_props.hasStatefulProperties())
7199 1612 : _bnd_material_props.shift();
7200 :
7201 234989 : if (_neighbor_material_props.hasStatefulProperties())
7202 1496 : _neighbor_material_props.shift();
7203 :
7204 : #ifdef MOOSE_KOKKOS_ENABLED
7205 172786 : if (_kokkos_material_props.hasStatefulProperties())
7206 566 : _kokkos_material_props.shift();
7207 :
7208 172786 : if (_kokkos_bnd_material_props.hasStatefulProperties())
7209 641 : _kokkos_bnd_material_props.shift();
7210 :
7211 172786 : if (_kokkos_neighbor_material_props.hasStatefulProperties())
7212 566 : _kokkos_neighbor_material_props.shift();
7213 : #endif
7214 234989 : }
7215 :
7216 : void
7217 3360 : FEProblemBase::restoreSolutions()
7218 : {
7219 16800 : TIME_SECTION("restoreSolutions", 5, "Restoring Solutions");
7220 :
7221 3360 : if (!_not_zeroed_tagged_vectors.empty())
7222 0 : paramError("not_zeroed_tag_vectors",
7223 : "There is currently no way to restore not-zeroed vectors.");
7224 :
7225 6722 : for (auto & sys : _solver_systems)
7226 : {
7227 3362 : if (_verbose_restore)
7228 18 : _console << "Restoring solutions on system " << sys->name() << "..." << std::endl;
7229 3362 : sys->restoreSolutions();
7230 : }
7231 :
7232 3360 : if (_verbose_restore)
7233 18 : _console << "Restoring solutions on Auxiliary system..." << std::endl;
7234 3360 : _aux->restoreSolutions();
7235 :
7236 3360 : if (_verbose_restore)
7237 18 : _console << "Restoring postprocessor, vector-postprocessor, and reporter data..." << std::endl;
7238 3360 : _reporter_data.restoreState(_verbose_restore);
7239 :
7240 3360 : if (_displaced_problem)
7241 132 : _displaced_problem->updateMesh();
7242 3360 : }
7243 :
7244 : void
7245 100 : FEProblemBase::saveOldSolutions()
7246 : {
7247 500 : TIME_SECTION("saveOldSolutions", 5, "Saving Old Solutions");
7248 :
7249 200 : for (auto & sys : _solver_systems)
7250 100 : sys->saveOldSolutions();
7251 100 : _aux->saveOldSolutions();
7252 100 : }
7253 :
7254 : void
7255 100 : FEProblemBase::restoreOldSolutions()
7256 : {
7257 500 : TIME_SECTION("restoreOldSolutions", 5, "Restoring Old Solutions");
7258 :
7259 200 : for (auto & sys : _solver_systems)
7260 100 : sys->restoreOldSolutions();
7261 100 : _aux->restoreOldSolutions();
7262 100 : }
7263 :
7264 : void
7265 1246570 : FEProblemBase::outputStep(ExecFlagType type)
7266 : {
7267 6232850 : TIME_SECTION("outputStep", 1, "Outputting");
7268 :
7269 1246570 : setCurrentExecuteOnFlag(type);
7270 :
7271 2507645 : for (auto & sys : _solver_systems)
7272 1261075 : sys->update();
7273 1246570 : _aux->update();
7274 :
7275 1246570 : if (_displaced_problem)
7276 126253 : _displaced_problem->syncSolutions();
7277 1246570 : _app.getOutputWarehouse().outputStep(type);
7278 :
7279 1246561 : setCurrentExecuteOnFlag(EXEC_NONE);
7280 1246561 : }
7281 :
7282 : void
7283 83891 : FEProblemBase::allowOutput(bool state)
7284 : {
7285 83891 : _app.getOutputWarehouse().allowOutput(state);
7286 83891 : }
7287 :
7288 : void
7289 23 : FEProblemBase::forceOutput()
7290 : {
7291 23 : _app.getOutputWarehouse().forceOutput();
7292 23 : }
7293 :
7294 : void
7295 342362 : FEProblemBase::initPetscOutputAndSomeSolverSettings()
7296 : {
7297 342362 : _app.getOutputWarehouse().solveSetup();
7298 342362 : Moose::PetscSupport::petscSetDefaults(*this);
7299 342362 : }
7300 :
7301 : void
7302 240775 : FEProblemBase::onTimestepBegin()
7303 : {
7304 722325 : TIME_SECTION("onTimestepBegin", 2);
7305 :
7306 484618 : for (auto & nl : _nl)
7307 243843 : nl->onTimestepBegin();
7308 240775 : }
7309 :
7310 : void
7311 311572 : FEProblemBase::onTimestepEnd()
7312 : {
7313 311572 : }
7314 :
7315 : Real
7316 6839503 : FEProblemBase::getTimeFromStateArg(const Moose::StateArg & state) const
7317 : {
7318 6839503 : if (state.iteration_type != Moose::SolutionIterationType::Time)
7319 : // If we are any iteration type other than time (e.g. nonlinear), then temporally we are still
7320 : // in the present time
7321 0 : return time();
7322 :
7323 6839503 : switch (state.state)
7324 : {
7325 6839503 : case 0:
7326 6839503 : return time();
7327 :
7328 0 : case 1:
7329 0 : return timeOld();
7330 :
7331 0 : default:
7332 0 : mooseError("Unhandled state ", state.state, " in FEProblemBase::getTimeFromStateArg");
7333 : }
7334 : }
7335 :
7336 : void
7337 30650 : FEProblemBase::addTimeIntegrator(const std::string & type,
7338 : const std::string & name,
7339 : InputParameters & parameters)
7340 : {
7341 : parallel_object_only();
7342 :
7343 91950 : parameters.set<SubProblem *>("_subproblem") = this;
7344 30650 : logAdd("TimeIntegrator", name, type, parameters);
7345 30650 : _aux->addTimeIntegrator(type, name + ":aux", parameters);
7346 61354 : for (auto & sys : _solver_systems)
7347 30704 : sys->addTimeIntegrator(type, name + ":" + sys->name(), parameters);
7348 30650 : _has_time_integrator = true;
7349 :
7350 : // add vectors to store u_dot, u_dotdot, udot_old, u_dotdot_old and
7351 : // solution vectors older than 2 time steps, if requested by the time
7352 : // integrator
7353 30650 : _aux->addDotVectors();
7354 61323 : for (auto & nl : _nl)
7355 : {
7356 30673 : nl->addDotVectors();
7357 :
7358 30673 : auto tag_udot = nl->getTimeIntegrators()[0]->uDotFactorTag();
7359 30673 : if (!nl->hasVector(tag_udot))
7360 30637 : nl->associateVectorToTag(*nl->solutionUDot(), tag_udot);
7361 30673 : auto tag_udotdot = nl->getTimeIntegrators()[0]->uDotDotFactorTag();
7362 30673 : if (!nl->hasVector(tag_udotdot) && uDotDotRequested())
7363 150 : nl->associateVectorToTag(*nl->solutionUDotDot(), tag_udotdot);
7364 : }
7365 :
7366 30650 : if (_displaced_problem)
7367 : // Time integrator does not exist when displaced problem is created.
7368 1728 : _displaced_problem->addTimeIntegrator();
7369 30650 : }
7370 :
7371 : void
7372 22 : FEProblemBase::addPredictor(const std::string & type,
7373 : const std::string & name,
7374 : InputParameters & parameters)
7375 : {
7376 : parallel_object_only();
7377 :
7378 22 : if (!numNonlinearSystems() && numLinearSystems())
7379 0 : mooseError("Vector bounds cannot be used with LinearSystems!");
7380 :
7381 44 : parameters.set<SubProblem *>("_subproblem") = this;
7382 22 : std::shared_ptr<Predictor> predictor = _factory.create<Predictor>(type, name, parameters);
7383 22 : logAdd("Predictor", name, type, parameters);
7384 :
7385 44 : for (auto & nl : _nl)
7386 22 : nl->setPredictor(predictor);
7387 22 : }
7388 :
7389 : Real
7390 62919 : FEProblemBase::computeResidualL2Norm(NonlinearSystemBase & sys)
7391 : {
7392 62919 : _current_nl_sys = &sys;
7393 62919 : computeResidual(*sys.currentSolution(), sys.RHS(), sys.number());
7394 62919 : return sys.RHS().l2_norm();
7395 : }
7396 :
7397 : Real
7398 84 : FEProblemBase::computeResidualL2Norm(LinearSystem & sys)
7399 : {
7400 84 : _current_linear_sys = &sys;
7401 :
7402 : // We assemble the current system to check the current residual
7403 84 : computeLinearSystemSys(sys.linearImplicitSystem(),
7404 84 : *sys.linearImplicitSystem().matrix,
7405 84 : *sys.linearImplicitSystem().rhs,
7406 : /*compute fresh gradients*/ true);
7407 :
7408 : // Unfortunate, but we have to allocate a new vector for the residual
7409 84 : auto residual = sys.linearImplicitSystem().rhs->clone();
7410 84 : residual->scale(-1.0);
7411 84 : residual->add_vector(*sys.currentSolution(), *sys.linearImplicitSystem().matrix);
7412 168 : return residual->l2_norm();
7413 84 : }
7414 :
7415 : Real
7416 62825 : FEProblemBase::computeResidualL2Norm()
7417 : {
7418 314125 : TIME_SECTION("computeResidualL2Norm", 2, "Computing L2 Norm of Residual");
7419 :
7420 : // We use sum the squared norms of the individual systems and then take the square root of it
7421 62825 : Real l2_norm = 0.0;
7422 125744 : for (auto sys : _nl)
7423 : {
7424 62919 : const auto norm = computeResidualL2Norm(*sys);
7425 62919 : l2_norm += norm * norm;
7426 62919 : }
7427 :
7428 62909 : for (auto sys : _linear_systems)
7429 : {
7430 84 : const auto norm = computeResidualL2Norm(*sys);
7431 84 : l2_norm += norm * norm;
7432 84 : }
7433 :
7434 125650 : return std::sqrt(l2_norm);
7435 62825 : }
7436 :
7437 : void
7438 2956608 : FEProblemBase::computeResidualSys(NonlinearImplicitSystem & sys,
7439 : const NumericVector<Number> & soln,
7440 : NumericVector<Number> & residual)
7441 : {
7442 : parallel_object_only();
7443 :
7444 8869824 : TIME_SECTION("computeResidualSys", 5);
7445 : // Reset before residual setup, calculation & execution
7446 2956608 : _app.solutionInvalidity().resetIterationOccurences();
7447 :
7448 2956608 : computeResidual(soln, residual, sys.number());
7449 2956578 : }
7450 :
7451 : void
7452 0 : FEProblemBase::computeResidual(NonlinearImplicitSystem & sys,
7453 : const NumericVector<Number> & soln,
7454 : NumericVector<Number> & residual)
7455 : {
7456 0 : mooseDeprecated("Please use computeResidualSys");
7457 :
7458 0 : computeResidualSys(sys, soln, residual);
7459 0 : }
7460 :
7461 : void
7462 3016349 : FEProblemBase::computeResidual(const NumericVector<Number> & soln,
7463 : NumericVector<Number> & residual,
7464 : const unsigned int nl_sys_num)
7465 : {
7466 3016349 : setCurrentNonlinearSystem(nl_sys_num);
7467 :
7468 : // We associate the residual tag with the given residual vector to make sure we
7469 : // don't filter it out below
7470 3016349 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7471 3016349 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7472 :
7473 : mooseAssert(_fe_vector_tags.empty(), "This should be empty indicating a clean starting state");
7474 : // We filter out tags which do not have associated vectors in the current nonlinear
7475 : // system. This is essential to be able to use system-dependent residual tags.
7476 3016349 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7477 :
7478 3016349 : computeResidualInternal(soln, residual, _fe_vector_tags);
7479 3016319 : _fe_vector_tags.clear();
7480 3016319 : }
7481 :
7482 : void
7483 9899 : FEProblemBase::computeResidualAndJacobian(const NumericVector<Number> & soln,
7484 : NumericVector<Number> & residual,
7485 : SparseMatrix<Number> & jacobian)
7486 : {
7487 : try
7488 : {
7489 : try
7490 : {
7491 : // vector tags
7492 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7493 9899 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7494 :
7495 : mooseAssert(_fe_vector_tags.empty(),
7496 : "This should be empty indicating a clean starting state");
7497 : // We filter out tags which do not have associated vectors in the current nonlinear
7498 : // system. This is essential to be able to use system-dependent residual tags.
7499 9899 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7500 :
7501 9899 : setCurrentResidualVectorTags(_fe_vector_tags);
7502 :
7503 : // matrix tags
7504 : {
7505 9899 : _fe_matrix_tags.clear();
7506 :
7507 9899 : auto & tags = getMatrixTags();
7508 29697 : for (auto & tag : tags)
7509 19798 : _fe_matrix_tags.insert(tag.second);
7510 : }
7511 :
7512 9899 : _current_nl_sys->setSolution(soln);
7513 :
7514 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7515 9899 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7516 :
7517 29697 : for (const auto tag : _fe_matrix_tags)
7518 19798 : if (_current_nl_sys->hasMatrix(tag))
7519 : {
7520 9899 : auto & matrix = _current_nl_sys->getMatrix(tag);
7521 9899 : matrix.zero();
7522 9899 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7523 : // PETSc algorithms require diagonal allocations regardless of whether there is non-zero
7524 : // diagonal dependence. With global AD indexing we only add non-zero
7525 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7526 123018 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7527 120004 : matrix.add(index, index, 0);
7528 : }
7529 :
7530 9899 : _aux->zeroVariablesForResidual();
7531 :
7532 9899 : unsigned int n_threads = libMesh::n_threads();
7533 :
7534 9899 : _current_execute_on_flag = EXEC_LINEAR;
7535 :
7536 : // Random interface objects
7537 9899 : for (const auto & it : _random_data_objects)
7538 0 : it.second->updateSeeds(EXEC_LINEAR);
7539 :
7540 9899 : setCurrentlyComputingResidual(true);
7541 9899 : setCurrentlyComputingJacobian(true);
7542 9899 : setCurrentlyComputingResidualAndJacobian(true);
7543 9899 : if (_displaced_problem)
7544 : {
7545 1057 : _displaced_problem->setCurrentlyComputingResidual(true);
7546 1057 : _displaced_problem->setCurrentlyComputingJacobian(true);
7547 1057 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(true);
7548 : }
7549 :
7550 9899 : execTransfers(EXEC_LINEAR);
7551 :
7552 9899 : execMultiApps(EXEC_LINEAR);
7553 :
7554 21234 : for (unsigned int tid = 0; tid < n_threads; tid++)
7555 11335 : reinitScalars(tid);
7556 :
7557 9899 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7558 :
7559 9899 : _aux->residualSetup();
7560 :
7561 9899 : if (_displaced_problem)
7562 : {
7563 1057 : computeSystems(EXEC_PRE_DISPLACE);
7564 1057 : _displaced_problem->updateMesh();
7565 1057 : if (_mortar_data->hasDisplacedObjects())
7566 1057 : updateMortarMesh();
7567 : }
7568 :
7569 21234 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7570 : {
7571 11335 : _all_materials.residualSetup(tid);
7572 11335 : _functions.residualSetup(tid);
7573 : }
7574 :
7575 : #ifdef MOOSE_KOKKOS_ENABLED
7576 8839 : _kokkos_functions.residualSetup();
7577 : #endif
7578 :
7579 9899 : computeSystems(EXEC_LINEAR);
7580 :
7581 9899 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7582 :
7583 9899 : executeControls(EXEC_LINEAR);
7584 :
7585 9899 : _app.getOutputWarehouse().residualSetup();
7586 :
7587 9899 : _safe_access_tagged_vectors = false;
7588 9899 : _safe_access_tagged_matrices = false;
7589 :
7590 9899 : _current_nl_sys->computeResidualAndJacobianTags(_fe_vector_tags, _fe_matrix_tags);
7591 :
7592 9899 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7593 9899 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7594 : }
7595 0 : catch (...)
7596 : {
7597 0 : handleException("computeResidualAndJacobian");
7598 0 : }
7599 : }
7600 0 : catch (const MooseException &)
7601 : {
7602 : // The buck stops here, we have already handled the exception by
7603 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7604 : // "diverged" reason during the next solve.
7605 0 : }
7606 0 : catch (...)
7607 : {
7608 0 : mooseError("Unexpected exception type");
7609 0 : }
7610 :
7611 9899 : resetState();
7612 9899 : _fe_vector_tags.clear();
7613 9899 : _fe_matrix_tags.clear();
7614 9899 : }
7615 :
7616 : void
7617 0 : FEProblemBase::computeResidualTag(const NumericVector<Number> & soln,
7618 : NumericVector<Number> & residual,
7619 : TagID tag)
7620 : {
7621 : try
7622 : {
7623 0 : _current_nl_sys->setSolution(soln);
7624 :
7625 0 : _current_nl_sys->associateVectorToTag(residual, tag);
7626 :
7627 0 : computeResidualTags({tag});
7628 :
7629 0 : _current_nl_sys->disassociateVectorFromTag(residual, tag);
7630 : }
7631 0 : catch (MooseException & e)
7632 : {
7633 : // If a MooseException propagates all the way to here, it means
7634 : // that it was thrown from a MOOSE system where we do not
7635 : // (currently) properly support the throwing of exceptions, and
7636 : // therefore we have no choice but to error out. It may be
7637 : // *possible* to handle exceptions from other systems, but in the
7638 : // meantime, we don't want to silently swallow any unhandled
7639 : // exceptions here.
7640 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7641 : "contact the MOOSE team for assistance.");
7642 0 : }
7643 0 : }
7644 :
7645 : void
7646 3019933 : FEProblemBase::computeResidualInternal(const NumericVector<Number> & soln,
7647 : NumericVector<Number> & residual,
7648 : const std::set<TagID> & tags)
7649 : {
7650 : parallel_object_only();
7651 :
7652 9059799 : TIME_SECTION("computeResidualInternal", 1);
7653 :
7654 : try
7655 : {
7656 3019933 : _current_nl_sys->setSolution(soln);
7657 :
7658 3019933 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7659 :
7660 3019933 : computeResidualTags(tags);
7661 :
7662 3019903 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7663 : }
7664 0 : catch (MooseException & e)
7665 : {
7666 : // If a MooseException propagates all the way to here, it means
7667 : // that it was thrown from a MOOSE system where we do not
7668 : // (currently) properly support the throwing of exceptions, and
7669 : // therefore we have no choice but to error out. It may be
7670 : // *possible* to handle exceptions from other systems, but in the
7671 : // meantime, we don't want to silently swallow any unhandled
7672 : // exceptions here.
7673 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7674 : "contact the MOOSE team for assistance.");
7675 0 : }
7676 3019903 : }
7677 :
7678 : void
7679 0 : FEProblemBase::computeResidualType(const NumericVector<Number> & soln,
7680 : NumericVector<Number> & residual,
7681 : TagID tag)
7682 : {
7683 0 : TIME_SECTION("computeResidualType", 5);
7684 :
7685 : try
7686 : {
7687 0 : _current_nl_sys->setSolution(soln);
7688 :
7689 0 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7690 :
7691 0 : computeResidualTags({tag, _current_nl_sys->residualVectorTag()});
7692 :
7693 0 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7694 : }
7695 0 : catch (MooseException & e)
7696 : {
7697 : // If a MooseException propagates all the way to here, it means
7698 : // that it was thrown from a MOOSE system where we do not
7699 : // (currently) properly support the throwing of exceptions, and
7700 : // therefore we have no choice but to error out. It may be
7701 : // *possible* to handle exceptions from other systems, but in the
7702 : // meantime, we don't want to silently swallow any unhandled
7703 : // exceptions here.
7704 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7705 : "contact the MOOSE team for assistance.");
7706 0 : }
7707 0 : }
7708 :
7709 : void
7710 3 : FEProblemBase::handleException(const std::string & calling_method)
7711 : {
7712 : auto create_exception_message =
7713 3 : [&calling_method](const std::string & exception_type, const auto & exception)
7714 : {
7715 : return std::string("A " + exception_type + " was raised during FEProblemBase::" +
7716 6 : calling_method + "\n" + std::string(exception.what()));
7717 3 : };
7718 :
7719 : try
7720 : {
7721 3 : throw;
7722 : }
7723 3 : catch (const MooseException & e)
7724 : {
7725 0 : setException(create_exception_message("MooseException", e));
7726 0 : }
7727 0 : catch (const MetaPhysicL::LogicError & e)
7728 : {
7729 0 : moose::translateMetaPhysicLError(e);
7730 0 : }
7731 3 : catch (const libMesh::PetscSolverException & e)
7732 : {
7733 : // One PETSc solver exception that we cannot currently recover from are new nonzero errors. In
7734 : // particular I have observed the following scenario in a parallel test:
7735 : // - Both processes throw because of a new nonzero during MOOSE's computeJacobianTags
7736 : // - We potentially handle the exceptions nicely here
7737 : // - When the matrix is closed in libMesh's libmesh_petsc_snes_solver, there is a new nonzero
7738 : // throw which we do not catch here in MOOSE and the simulation terminates. This only appears
7739 : // in parallel (and not all the time; a test I was examining threw with distributed mesh, but
7740 : // not with replicated). In serial there are no new throws from libmesh_petsc_snes_solver.
7741 : // So for uniformity of behavior across serial/parallel, we will choose to abort here and always
7742 : // produce a non-zero exit code
7743 6 : mooseError(create_exception_message("libMesh::PetscSolverException", e));
7744 0 : }
7745 0 : catch (const std::exception & e)
7746 : {
7747 : // This might be libMesh detecting a degenerate Jacobian or matrix
7748 0 : if (strstr(e.what(), "Jacobian") || strstr(e.what(), "singular") ||
7749 0 : strstr(e.what(), "det != 0"))
7750 0 : setException(create_exception_message("libMesh DegenerateMap", e));
7751 : else
7752 : {
7753 0 : const auto message = create_exception_message("std::exception", e);
7754 0 : if (_regard_general_exceptions_as_errors)
7755 0 : mooseError(message);
7756 : else
7757 0 : setException(message);
7758 0 : }
7759 0 : }
7760 :
7761 0 : checkExceptionAndStopSolve();
7762 0 : }
7763 :
7764 : void
7765 3055336 : FEProblemBase::computeResidualTags(const std::set<TagID> & tags)
7766 : {
7767 : parallel_object_only();
7768 :
7769 : try
7770 : {
7771 : try
7772 : {
7773 15276680 : TIME_SECTION("computeResidualTags", 5, "Computing Residual");
7774 :
7775 3055336 : ADReal::do_derivatives = false;
7776 :
7777 3055336 : setCurrentResidualVectorTags(tags);
7778 :
7779 3055336 : _aux->zeroVariablesForResidual();
7780 :
7781 3055336 : unsigned int n_threads = libMesh::n_threads();
7782 :
7783 3055336 : _current_execute_on_flag = EXEC_LINEAR;
7784 :
7785 : // Random interface objects
7786 3068218 : for (const auto & it : _random_data_objects)
7787 12882 : it.second->updateSeeds(EXEC_LINEAR);
7788 :
7789 3055336 : execTransfers(EXEC_LINEAR);
7790 :
7791 3055336 : execMultiApps(EXEC_LINEAR);
7792 :
7793 6419947 : for (unsigned int tid = 0; tid < n_threads; tid++)
7794 3364611 : reinitScalars(tid);
7795 :
7796 3055336 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7797 :
7798 3055336 : _aux->residualSetup();
7799 :
7800 3055336 : if (_displaced_problem)
7801 : {
7802 123090 : computeSystems(EXEC_PRE_DISPLACE);
7803 123090 : _displaced_problem->updateMesh();
7804 123090 : if (_mortar_data->hasDisplacedObjects())
7805 2211 : updateMortarMesh();
7806 : }
7807 :
7808 6419947 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7809 : {
7810 3364611 : _all_materials.residualSetup(tid);
7811 3364611 : _functions.residualSetup(tid);
7812 : }
7813 :
7814 : #ifdef MOOSE_KOKKOS_ENABLED
7815 2222826 : _kokkos_functions.residualSetup();
7816 : #endif
7817 :
7818 3055336 : computeSystems(EXEC_LINEAR);
7819 :
7820 3055336 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7821 :
7822 3055336 : executeControls(EXEC_LINEAR);
7823 :
7824 3055336 : _app.getOutputWarehouse().residualSetup();
7825 :
7826 3055336 : _safe_access_tagged_vectors = false;
7827 3055336 : _current_nl_sys->computeResidualTags(tags);
7828 3055306 : }
7829 0 : catch (...)
7830 : {
7831 0 : handleException("computeResidualTags");
7832 0 : }
7833 : }
7834 0 : catch (const MooseException &)
7835 : {
7836 : // The buck stops here, we have already handled the exception by
7837 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7838 : // "diverged" reason during the next solve.
7839 0 : }
7840 0 : catch (...)
7841 : {
7842 0 : mooseError("Unexpected exception type");
7843 0 : }
7844 :
7845 3055306 : resetState();
7846 3055306 : }
7847 :
7848 : void
7849 474332 : FEProblemBase::computeJacobianSys(NonlinearImplicitSystem & sys,
7850 : const NumericVector<Number> & soln,
7851 : SparseMatrix<Number> & jacobian)
7852 : {
7853 : // Reset before Jacobian setup, calculation & execution
7854 474332 : _app.solutionInvalidity().resetIterationOccurences();
7855 474332 : computeJacobian(soln, jacobian, sys.number());
7856 474319 : }
7857 :
7858 : void
7859 4467 : FEProblemBase::computeJacobianTag(const NumericVector<Number> & soln,
7860 : SparseMatrix<Number> & jacobian,
7861 : TagID tag)
7862 : {
7863 4467 : _current_nl_sys->setSolution(soln);
7864 :
7865 4467 : _current_nl_sys->associateMatrixToTag(jacobian, tag);
7866 :
7867 8934 : computeJacobianTags({tag});
7868 :
7869 4467 : _current_nl_sys->disassociateMatrixFromTag(jacobian, tag);
7870 4467 : }
7871 :
7872 : void
7873 473488 : FEProblemBase::computeJacobian(const NumericVector<Number> & soln,
7874 : SparseMatrix<Number> & jacobian,
7875 : const unsigned int nl_sys_num)
7876 : {
7877 473488 : setCurrentNonlinearSystem(nl_sys_num);
7878 :
7879 473488 : _fe_matrix_tags.clear();
7880 :
7881 473488 : auto & tags = getMatrixTags();
7882 1420738 : for (auto & tag : tags)
7883 947250 : _fe_matrix_tags.insert(tag.second);
7884 :
7885 473488 : computeJacobianInternal(soln, jacobian, _fe_matrix_tags);
7886 473475 : }
7887 :
7888 : void
7889 473488 : FEProblemBase::computeJacobianInternal(const NumericVector<Number> & soln,
7890 : SparseMatrix<Number> & jacobian,
7891 : const std::set<TagID> & tags)
7892 : {
7893 1420464 : TIME_SECTION("computeJacobianInternal", 1);
7894 :
7895 473488 : _current_nl_sys->setSolution(soln);
7896 :
7897 473488 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7898 :
7899 473488 : computeJacobianTags(tags);
7900 :
7901 473475 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7902 473475 : }
7903 :
7904 : void
7905 482736 : FEProblemBase::computeJacobianTags(const std::set<TagID> & tags)
7906 : {
7907 : try
7908 : {
7909 : try
7910 : {
7911 482736 : if (!_has_jacobian || !_const_jacobian)
7912 : {
7913 2381405 : TIME_SECTION("computeJacobianTags", 5, "Computing Jacobian");
7914 :
7915 1421525 : for (auto tag : tags)
7916 945244 : if (_current_nl_sys->hasMatrix(tag))
7917 : {
7918 478211 : auto & matrix = _current_nl_sys->getMatrix(tag);
7919 478211 : if (_restore_original_nonzero_pattern)
7920 7033 : matrix.restore_original_nonzero_pattern();
7921 : else
7922 471178 : matrix.zero();
7923 478211 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7924 : // PETSc algorithms require diagonal allocations regardless of whether there is
7925 : // non-zero diagonal dependence. With global AD indexing we only add non-zero
7926 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7927 4863057 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7928 4819252 : matrix.add(index, index, 0);
7929 : }
7930 :
7931 476281 : _aux->zeroVariablesForJacobian();
7932 :
7933 476281 : unsigned int n_threads = libMesh::n_threads();
7934 :
7935 : // Random interface objects
7936 478303 : for (const auto & it : _random_data_objects)
7937 2022 : it.second->updateSeeds(EXEC_NONLINEAR);
7938 :
7939 476281 : _current_execute_on_flag = EXEC_NONLINEAR;
7940 476281 : _currently_computing_jacobian = true;
7941 476281 : if (_displaced_problem)
7942 21147 : _displaced_problem->setCurrentlyComputingJacobian(true);
7943 :
7944 476281 : execTransfers(EXEC_NONLINEAR);
7945 476281 : execMultiApps(EXEC_NONLINEAR);
7946 :
7947 1002923 : for (unsigned int tid = 0; tid < n_threads; tid++)
7948 526642 : reinitScalars(tid);
7949 :
7950 476281 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7951 :
7952 476281 : _aux->jacobianSetup();
7953 :
7954 476281 : if (_displaced_problem)
7955 : {
7956 21147 : computeSystems(EXEC_PRE_DISPLACE);
7957 21147 : _displaced_problem->updateMesh();
7958 : }
7959 :
7960 1002916 : for (unsigned int tid = 0; tid < n_threads; tid++)
7961 : {
7962 526638 : _all_materials.jacobianSetup(tid);
7963 526638 : _functions.jacobianSetup(tid);
7964 : }
7965 :
7966 : #ifdef MOOSE_KOKKOS_ENABLED
7967 347491 : _kokkos_functions.jacobianSetup();
7968 : #endif
7969 :
7970 476278 : computeSystems(EXEC_NONLINEAR);
7971 :
7972 476278 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7973 :
7974 476278 : executeControls(EXEC_NONLINEAR);
7975 :
7976 476278 : _app.getOutputWarehouse().jacobianSetup();
7977 :
7978 476278 : _safe_access_tagged_matrices = false;
7979 :
7980 476278 : _current_nl_sys->computeJacobianTags(tags);
7981 :
7982 : // For explicit Euler calculations for example we often compute the Jacobian one time and
7983 : // then re-use it over and over. If we're performing automatic scaling, we don't want to
7984 : // use that kernel, diagonal-block only Jacobian for our actual matrix when performing
7985 : // solves!
7986 476268 : if (!_current_nl_sys->computingScalingJacobian())
7987 475703 : _has_jacobian = true;
7988 476271 : }
7989 : }
7990 3 : catch (...)
7991 : {
7992 3 : handleException("computeJacobianTags");
7993 0 : }
7994 : }
7995 0 : catch (const MooseException &)
7996 : {
7997 : // The buck stops here, we have already handled the exception by
7998 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7999 : // "diverged" reason during the next solve.
8000 0 : }
8001 0 : catch (...)
8002 : {
8003 0 : mooseError("Unexpected exception type");
8004 0 : }
8005 :
8006 482723 : resetState();
8007 482723 : }
8008 :
8009 : void
8010 263 : FEProblemBase::computeJacobianBlocks(std::vector<JacobianBlock *> & blocks,
8011 : const unsigned int nl_sys_num)
8012 : {
8013 789 : TIME_SECTION("computeTransientImplicitJacobian", 2);
8014 263 : setCurrentNonlinearSystem(nl_sys_num);
8015 :
8016 263 : if (_displaced_problem)
8017 : {
8018 0 : computeSystems(EXEC_PRE_DISPLACE);
8019 0 : _displaced_problem->updateMesh();
8020 : }
8021 :
8022 263 : computeSystems(EXEC_NONLINEAR);
8023 :
8024 263 : _currently_computing_jacobian = true;
8025 263 : _current_nl_sys->computeJacobianBlocks(blocks);
8026 263 : _currently_computing_jacobian = false;
8027 263 : }
8028 :
8029 : void
8030 0 : FEProblemBase::computeJacobianBlock(SparseMatrix<Number> & jacobian,
8031 : libMesh::System & precond_system,
8032 : unsigned int ivar,
8033 : unsigned int jvar)
8034 : {
8035 0 : JacobianBlock jac_block(precond_system, jacobian, ivar, jvar);
8036 0 : std::vector<JacobianBlock *> blocks = {&jac_block};
8037 : mooseAssert(_current_nl_sys, "This should be non-null");
8038 0 : computeJacobianBlocks(blocks, _current_nl_sys->number());
8039 0 : }
8040 :
8041 : void
8042 714 : FEProblemBase::computeBounds(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8043 : NumericVector<Number> & lower,
8044 : NumericVector<Number> & upper)
8045 : {
8046 : try
8047 : {
8048 : try
8049 : {
8050 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8051 : "I expect these system numbers to be the same");
8052 :
8053 3570 : if (!_current_nl_sys->hasVector("lower_bound") || !_current_nl_sys->hasVector("upper_bound"))
8054 0 : return;
8055 :
8056 3570 : TIME_SECTION("computeBounds", 1, "Computing Bounds");
8057 :
8058 1428 : NumericVector<Number> & _lower = _current_nl_sys->getVector("lower_bound");
8059 1428 : NumericVector<Number> & _upper = _current_nl_sys->getVector("upper_bound");
8060 714 : _lower.swap(lower);
8061 714 : _upper.swap(upper);
8062 1492 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
8063 778 : _all_materials.residualSetup(tid);
8064 :
8065 714 : _aux->residualSetup();
8066 714 : computeSystems(EXEC_LINEAR);
8067 714 : _lower.swap(lower);
8068 714 : _upper.swap(upper);
8069 714 : }
8070 0 : catch (...)
8071 : {
8072 0 : handleException("computeBounds");
8073 0 : }
8074 : }
8075 0 : catch (MooseException & e)
8076 : {
8077 0 : mooseError("Irrecoverable exception: " + std::string(e.what()));
8078 0 : }
8079 0 : catch (...)
8080 : {
8081 0 : mooseError("Unexpected exception type");
8082 0 : }
8083 : }
8084 :
8085 : void
8086 25792 : FEProblemBase::computeLinearSystemSys(LinearImplicitSystem & sys,
8087 : SparseMatrix<Number> & system_matrix,
8088 : NumericVector<Number> & rhs,
8089 : const bool compute_gradients)
8090 : {
8091 77376 : TIME_SECTION("computeLinearSystemSys", 5);
8092 :
8093 25792 : setCurrentLinearSystem(linearSysNum(sys.name()));
8094 :
8095 25792 : _current_linear_sys->associateVectorToTag(rhs, _current_linear_sys->rightHandSideVectorTag());
8096 25792 : _current_linear_sys->associateMatrixToTag(system_matrix, _current_linear_sys->systemMatrixTag());
8097 :
8098 : // We are using the residual tag system for right hand sides so we fetch everything
8099 25792 : const auto & vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
8100 :
8101 : // We filter out tags which do not have associated vectors in the current
8102 : // system. This is essential to be able to use system-dependent vector tags.
8103 25792 : selectVectorTagsFromSystem(*_current_linear_sys, vector_tags, _linear_vector_tags);
8104 25792 : selectMatrixTagsFromSystem(*_current_linear_sys, getMatrixTags(), _linear_matrix_tags);
8105 :
8106 25792 : computeLinearSystemTags(*(_current_linear_sys->currentSolution()),
8107 25792 : _linear_vector_tags,
8108 25792 : _linear_matrix_tags,
8109 : compute_gradients);
8110 :
8111 25792 : _current_linear_sys->disassociateMatrixFromTag(system_matrix,
8112 25792 : _current_linear_sys->systemMatrixTag());
8113 25792 : _current_linear_sys->disassociateVectorFromTag(rhs,
8114 25792 : _current_linear_sys->rightHandSideVectorTag());
8115 : // We reset the tags to the default containers for further operations
8116 25792 : _current_linear_sys->associateVectorToTag(_current_linear_sys->getRightHandSideVector(),
8117 25792 : _current_linear_sys->rightHandSideVectorTag());
8118 25792 : _current_linear_sys->associateMatrixToTag(_current_linear_sys->getSystemMatrix(),
8119 25792 : _current_linear_sys->systemMatrixTag());
8120 25792 : }
8121 :
8122 : void
8123 25792 : FEProblemBase::computeLinearSystemTags(const NumericVector<Number> & soln,
8124 : const std::set<TagID> & vector_tags,
8125 : const std::set<TagID> & matrix_tags,
8126 : const bool compute_gradients)
8127 : {
8128 128960 : TIME_SECTION("computeLinearSystemTags", 5, "Computing Linear System");
8129 :
8130 25792 : _current_linear_sys->setSolution(soln);
8131 :
8132 51604 : for (auto tag : matrix_tags)
8133 : {
8134 25812 : auto & matrix = _current_linear_sys->getMatrix(tag);
8135 25812 : matrix.zero();
8136 : }
8137 :
8138 25792 : unsigned int n_threads = libMesh::n_threads();
8139 :
8140 25792 : _current_execute_on_flag = EXEC_NONLINEAR;
8141 :
8142 : // Random interface objects
8143 25792 : for (const auto & it : _random_data_objects)
8144 0 : it.second->updateSeeds(EXEC_NONLINEAR);
8145 :
8146 25792 : execTransfers(EXEC_NONLINEAR);
8147 25792 : execMultiApps(EXEC_NONLINEAR);
8148 :
8149 25792 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
8150 :
8151 25792 : _aux->jacobianSetup();
8152 :
8153 51584 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
8154 : {
8155 25792 : _functions.jacobianSetup(tid);
8156 : }
8157 :
8158 : #ifdef MOOSE_KOKKOS_ENABLED
8159 17842 : _kokkos_functions.jacobianSetup();
8160 : #endif
8161 :
8162 : try
8163 : {
8164 25792 : computeSystems(EXEC_NONLINEAR);
8165 : }
8166 0 : catch (MooseException & e)
8167 : {
8168 0 : _console << "\nA MooseException was raised during Auxiliary variable computation.\n"
8169 0 : << "The next solve will fail, the timestep will be reduced, and we will try again.\n"
8170 0 : << std::endl;
8171 :
8172 : // We know the next solve is going to fail, so there's no point in
8173 : // computing anything else after this. Plus, using incompletely
8174 : // computed AuxVariables in subsequent calculations could lead to
8175 : // other errors or unhandled exceptions being thrown.
8176 0 : return;
8177 0 : }
8178 :
8179 25792 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
8180 25792 : executeControls(EXEC_NONLINEAR);
8181 :
8182 25792 : _app.getOutputWarehouse().jacobianSetup();
8183 :
8184 25792 : _current_linear_sys->computeLinearSystemTags(vector_tags, matrix_tags, compute_gradients);
8185 :
8186 : // Reset execution flag as after this point we are no longer on LINEAR
8187 25792 : _current_execute_on_flag = EXEC_NONE;
8188 :
8189 : // These are the relevant parts of resetState()
8190 25792 : _safe_access_tagged_vectors = true;
8191 25792 : _safe_access_tagged_matrices = true;
8192 25792 : }
8193 :
8194 : void
8195 296692 : FEProblemBase::computeNearNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8196 : std::vector<NumericVector<Number> *> & sp)
8197 : {
8198 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8199 : "I expect these system numbers to be the same");
8200 :
8201 296692 : sp.clear();
8202 890076 : for (unsigned int i = 0; i < subspaceDim("NearNullSpace"); ++i)
8203 : {
8204 0 : std::stringstream postfix;
8205 0 : postfix << "_" << i;
8206 0 : std::string modename = "NearNullSpace" + postfix.str();
8207 0 : sp.push_back(&_current_nl_sys->getVector(modename));
8208 0 : }
8209 296692 : }
8210 :
8211 : void
8212 296692 : FEProblemBase::computeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8213 : std::vector<NumericVector<Number> *> & sp)
8214 : {
8215 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8216 : "I expect these system numbers to be the same");
8217 296692 : sp.clear();
8218 890142 : for (unsigned int i = 0; i < subspaceDim("NullSpace"); ++i)
8219 : {
8220 22 : std::stringstream postfix;
8221 22 : postfix << "_" << i;
8222 22 : sp.push_back(&_current_nl_sys->getVector("NullSpace" + postfix.str()));
8223 22 : }
8224 296692 : }
8225 :
8226 : void
8227 296692 : FEProblemBase::computeTransposeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8228 : std::vector<NumericVector<Number> *> & sp)
8229 : {
8230 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8231 : "I expect these system numbers to be the same");
8232 296692 : sp.clear();
8233 890109 : for (unsigned int i = 0; i < subspaceDim("TransposeNullSpace"); ++i)
8234 : {
8235 11 : std::stringstream postfix;
8236 11 : postfix << "_" << i;
8237 11 : sp.push_back(&_current_nl_sys->getVector("TransposeNullSpace" + postfix.str()));
8238 11 : }
8239 296692 : }
8240 :
8241 : void
8242 2128 : FEProblemBase::computePostCheck(NonlinearImplicitSystem & sys,
8243 : const NumericVector<Number> & old_soln,
8244 : NumericVector<Number> & search_direction,
8245 : NumericVector<Number> & new_soln,
8246 : bool & changed_search_direction,
8247 : bool & changed_new_soln)
8248 : {
8249 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8250 : "I expect these system numbers to be the same");
8251 :
8252 : // This function replaces the old PetscSupport::dampedCheck() function.
8253 : //
8254 : // 1.) Recreate code in PetscSupport::dampedCheck() for constructing
8255 : // ghosted "soln" and "update" vectors.
8256 : // 2.) Call FEProblemBase::computeDamping() with these ghost vectors.
8257 : // 3.) Recreate the code in PetscSupport::dampedCheck() to actually update
8258 : // the solution vector based on the damping, and set the "changed" flags
8259 : // appropriately.
8260 :
8261 10640 : TIME_SECTION("computePostCheck", 2, "Computing Post Check");
8262 :
8263 2128 : _current_execute_on_flag = EXEC_POSTCHECK;
8264 :
8265 : // MOOSE's FEProblemBase doesn't update the solution during the
8266 : // postcheck, but FEProblemBase-derived classes might.
8267 2128 : if (_has_dampers || shouldUpdateSolution())
8268 : {
8269 : // We need ghosted versions of new_soln and search_direction (the
8270 : // ones we get from libmesh/PETSc are PARALLEL vectors. To make
8271 : // our lives simpler, we use the same ghosting pattern as the
8272 : // system's current_local_solution to create new ghosted vectors.
8273 :
8274 : // Construct zeroed-out clones with the same ghosted dofs as the
8275 : // System's current_local_solution.
8276 : std::unique_ptr<NumericVector<Number>> ghosted_solution =
8277 1625 : sys.current_local_solution->zero_clone(),
8278 : ghosted_search_direction =
8279 1625 : sys.current_local_solution->zero_clone();
8280 :
8281 : // Copy values from input vectors into clones with ghosted values.
8282 1625 : *ghosted_solution = new_soln;
8283 1625 : *ghosted_search_direction = search_direction;
8284 :
8285 1625 : if (_has_dampers)
8286 : {
8287 : // Compute the damping coefficient using the ghosted vectors
8288 1625 : Real damping = computeDamping(*ghosted_solution, *ghosted_search_direction);
8289 :
8290 : // If some non-trivial damping was computed, update the new_soln
8291 : // vector accordingly.
8292 1625 : if (damping < 1.0)
8293 : {
8294 1221 : new_soln = old_soln;
8295 1221 : new_soln.add(-damping, search_direction);
8296 1221 : changed_new_soln = true;
8297 : }
8298 : }
8299 :
8300 1625 : if (shouldUpdateSolution())
8301 : {
8302 : // Update the ghosted copy of the new solution, if necessary.
8303 0 : if (changed_new_soln)
8304 0 : *ghosted_solution = new_soln;
8305 :
8306 0 : bool updated_solution = updateSolution(new_soln, *ghosted_solution);
8307 0 : if (updated_solution)
8308 0 : changed_new_soln = true;
8309 : }
8310 1625 : }
8311 :
8312 2128 : if (vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
8313 : {
8314 503 : _current_nl_sys->setPreviousNewtonSolution(old_soln);
8315 503 : _aux->copyCurrentIntoPreviousNL();
8316 : }
8317 :
8318 : // MOOSE doesn't change the search_direction
8319 2128 : changed_search_direction = false;
8320 :
8321 2128 : _current_execute_on_flag = EXEC_NONE;
8322 2128 : }
8323 :
8324 : Real
8325 1625 : FEProblemBase::computeDamping(const NumericVector<Number> & soln,
8326 : const NumericVector<Number> & update)
8327 : {
8328 : // Default to no damping
8329 1625 : Real damping = 1.0;
8330 :
8331 1625 : if (_has_dampers)
8332 : {
8333 8125 : TIME_SECTION("computeDamping", 1, "Computing Damping");
8334 :
8335 : // Save pointer to the current solution
8336 1625 : const NumericVector<Number> * _saved_current_solution = _current_nl_sys->currentSolution();
8337 :
8338 1625 : _current_nl_sys->setSolution(soln);
8339 : // For now, do not re-compute auxiliary variables. Doing so allows a wild solution increment
8340 : // to get to the material models, which may not be able to cope with drastically different
8341 : // values. Once more complete dependency checking is in place, auxiliary variables (and
8342 : // material properties) will be computed as needed by dampers.
8343 : // _aux.compute();
8344 1625 : damping = _current_nl_sys->computeDamping(soln, update);
8345 :
8346 : // restore saved solution
8347 1625 : _current_nl_sys->setSolution(*_saved_current_solution);
8348 1625 : }
8349 :
8350 1625 : return damping;
8351 : }
8352 :
8353 : bool
8354 291288 : FEProblemBase::shouldUpdateSolution()
8355 : {
8356 291288 : return false;
8357 : }
8358 :
8359 : bool
8360 0 : FEProblemBase::updateSolution(NumericVector<Number> & /*vec_solution*/,
8361 : NumericVector<Number> & /*ghosted_solution*/)
8362 : {
8363 0 : return false;
8364 : }
8365 :
8366 : void
8367 203 : FEProblemBase::predictorCleanup(NumericVector<Number> & /*ghosted_solution*/)
8368 : {
8369 203 : }
8370 :
8371 : void
8372 2022 : FEProblemBase::addDisplacedProblem(std::shared_ptr<DisplacedProblem> displaced_problem)
8373 : {
8374 : parallel_object_only();
8375 :
8376 2022 : _displaced_mesh = &displaced_problem->mesh();
8377 2022 : _displaced_problem = displaced_problem;
8378 2022 : }
8379 :
8380 : void
8381 122704 : FEProblemBase::updateGeomSearch(GeometricSearchData::GeometricSearchType type)
8382 : {
8383 613520 : TIME_SECTION("updateGeometricSearch", 3, "Updating Geometric Search");
8384 :
8385 122704 : _geometric_search_data.update(type);
8386 :
8387 122704 : if (_displaced_problem)
8388 4209 : _displaced_problem->updateGeomSearch(type);
8389 122704 : }
8390 :
8391 : void
8392 64794 : FEProblemBase::updateMortarMesh()
8393 : {
8394 323970 : TIME_SECTION("updateMortarMesh", 5, "Updating Mortar Mesh");
8395 :
8396 64794 : FloatingPointExceptionGuard fpe_guard(_app);
8397 :
8398 64794 : _mortar_data->update();
8399 64788 : }
8400 :
8401 : void
8402 1481 : FEProblemBase::createMortarInterface(
8403 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8404 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8405 : bool on_displaced,
8406 : bool periodic,
8407 : const bool debug,
8408 : const bool correct_edge_dropping,
8409 : const Real minimum_projection_angle,
8410 : const Mortar3DSubpatchPlane mortar_3d_subpatch_plane,
8411 : const MooseEnum & triangulation,
8412 : const bool triangulate_triangles,
8413 : const Mortar3DQuadraturePointMapping mortar_3d_qp_mapping)
8414 : {
8415 1481 : _has_mortar = true;
8416 :
8417 1481 : if (on_displaced)
8418 254 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8419 : primary_secondary_subdomain_pair,
8420 127 : *_displaced_problem,
8421 : on_displaced,
8422 : periodic,
8423 : debug,
8424 : correct_edge_dropping,
8425 : minimum_projection_angle,
8426 : mortar_3d_subpatch_plane,
8427 : triangulation,
8428 : triangulate_triangles,
8429 127 : mortar_3d_qp_mapping);
8430 : else
8431 1354 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8432 : primary_secondary_subdomain_pair,
8433 : *this,
8434 : on_displaced,
8435 : periodic,
8436 : debug,
8437 : correct_edge_dropping,
8438 : minimum_projection_angle,
8439 : mortar_3d_subpatch_plane,
8440 : triangulation,
8441 : triangulate_triangles,
8442 1348 : mortar_3d_qp_mapping);
8443 : }
8444 :
8445 : const AutomaticMortarGeneration &
8446 0 : FEProblemBase::getMortarInterface(
8447 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8448 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8449 : bool on_displaced) const
8450 : {
8451 0 : return _mortar_data->getMortarInterface(
8452 0 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8453 : }
8454 :
8455 : AutomaticMortarGeneration &
8456 256642 : FEProblemBase::getMortarInterface(
8457 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8458 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8459 : bool on_displaced)
8460 : {
8461 256642 : return _mortar_data->getMortarInterface(
8462 256642 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8463 : }
8464 :
8465 : void
8466 323334 : FEProblemBase::possiblyRebuildGeomSearchPatches()
8467 : {
8468 323334 : if (_displaced_problem) // Only need to do this if things are moving...
8469 : {
8470 164185 : TIME_SECTION("possiblyRebuildGeomSearchPatches", 5, "Rebuilding Geometric Search Patches");
8471 :
8472 32837 : switch (_mesh.getPatchUpdateStrategy())
8473 : {
8474 31742 : case Moose::Never:
8475 31742 : break;
8476 365 : case Moose::Iteration:
8477 : // Update the list of ghosted elements at the start of the time step
8478 365 : _geometric_search_data.updateGhostedElems();
8479 365 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8480 :
8481 365 : _displaced_problem->geomSearchData().updateGhostedElems();
8482 365 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8483 :
8484 : // The commands below ensure that the sparsity of the Jacobian matrix is
8485 : // augmented at the start of the time step using neighbor nodes from the end
8486 : // of the previous time step.
8487 :
8488 365 : reinitBecauseOfGhostingOrNewGeomObjects();
8489 :
8490 : // This is needed to reinitialize PETSc output
8491 365 : initPetscOutputAndSomeSolverSettings();
8492 :
8493 365 : break;
8494 :
8495 331 : case Moose::Auto:
8496 : {
8497 331 : Real max = _displaced_problem->geomSearchData().maxPatchPercentage();
8498 331 : _communicator.max(max);
8499 :
8500 : // If we haven't moved very far through the patch
8501 331 : if (max < 0.4)
8502 298 : break;
8503 : }
8504 : libmesh_fallthrough();
8505 :
8506 : // Let this fall through if things do need to be updated...
8507 : case Moose::Always:
8508 : // Flush output here to see the message before the reinitialization, which could take a
8509 : // while
8510 432 : _console << "\n\nUpdating geometric search patches\n" << std::endl;
8511 :
8512 432 : _geometric_search_data.clearNearestNodeLocators();
8513 432 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8514 :
8515 432 : _displaced_problem->geomSearchData().clearNearestNodeLocators();
8516 432 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8517 :
8518 432 : reinitBecauseOfGhostingOrNewGeomObjects();
8519 :
8520 : // This is needed to reinitialize PETSc output
8521 432 : initPetscOutputAndSomeSolverSettings();
8522 : }
8523 32837 : }
8524 323334 : }
8525 :
8526 : #ifdef LIBMESH_ENABLE_AMR
8527 : void
8528 56628 : FEProblemBase::initialAdaptMesh()
8529 : {
8530 56628 : unsigned int n = adaptivity().getInitialSteps();
8531 56628 : _cycles_completed = 0;
8532 56628 : if (n)
8533 : {
8534 631 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8535 3 : mooseError("HFEM does not support mesh adaptivity currently.");
8536 :
8537 3140 : TIME_SECTION("initialAdaptMesh", 2, "Performing Initial Adaptivity");
8538 :
8539 1510 : for (unsigned int i = 0; i < n; i++)
8540 : {
8541 1057 : computeIndicators();
8542 1057 : computeMarkers();
8543 :
8544 1057 : if (_adaptivity.initialAdaptMesh())
8545 : {
8546 882 : meshChanged(
8547 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8548 :
8549 : // reproject the initial condition
8550 882 : projectSolution();
8551 :
8552 882 : _cycles_completed++;
8553 : }
8554 : else
8555 : {
8556 175 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8557 175 : return;
8558 : }
8559 : }
8560 628 : }
8561 : }
8562 :
8563 : bool
8564 168993 : FEProblemBase::adaptMesh()
8565 : {
8566 : // reset cycle counter
8567 168993 : _cycles_completed = 0;
8568 :
8569 168993 : if (!_adaptivity.isAdaptivityDue())
8570 164416 : return false;
8571 :
8572 22885 : TIME_SECTION("adaptMesh", 3, "Adapting Mesh");
8573 :
8574 4577 : unsigned int cycles_per_step = _adaptivity.getCyclesPerStep();
8575 :
8576 4577 : bool mesh_changed = false;
8577 :
8578 8056 : for (unsigned int i = 0; i < cycles_per_step; ++i)
8579 : {
8580 4729 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8581 0 : mooseError("HFEM does not support mesh adaptivity currently.");
8582 :
8583 : // Markers were already computed once by Executioner
8584 4729 : if (_adaptivity.getRecomputeMarkersFlag() && i > 0)
8585 22 : computeMarkers();
8586 :
8587 : bool mesh_changed_this_step;
8588 4729 : mesh_changed_this_step = _adaptivity.adaptMesh();
8589 :
8590 4729 : if (mesh_changed_this_step)
8591 : {
8592 3479 : mesh_changed = true;
8593 :
8594 3479 : meshChanged(
8595 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8596 3479 : _cycles_completed++;
8597 : }
8598 : else
8599 : {
8600 : // If the mesh didn't change, we still need to update the displaced mesh
8601 : // to undo the undisplacement performed in Adaptivity::adaptMesh
8602 1250 : if (_displaced_problem)
8603 44 : _displaced_problem->updateMesh();
8604 :
8605 1250 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8606 1250 : break;
8607 : }
8608 :
8609 : // Show adaptivity progress
8610 3479 : _console << std::flush;
8611 : }
8612 :
8613 : // We're done with all intermediate changes; now get systems ready
8614 : // for real if necessary.
8615 4577 : if (mesh_changed)
8616 3338 : es().reinit_systems();
8617 :
8618 : // Execute multi-apps that need to run after adaptivity, but before the next timestep.
8619 4577 : execMultiApps(EXEC_POST_ADAPTIVITY);
8620 :
8621 4577 : return mesh_changed;
8622 4577 : }
8623 : #endif // LIBMESH_ENABLE_AMR
8624 :
8625 : void
8626 0 : FEProblemBase::initXFEM(std::shared_ptr<XFEMInterface> xfem)
8627 : {
8628 0 : _xfem = xfem;
8629 0 : _xfem->setMesh(&_mesh);
8630 0 : if (_displaced_mesh)
8631 0 : _xfem->setDisplacedMesh(_displaced_mesh);
8632 :
8633 0 : auto fill_data = [](auto & storage)
8634 : {
8635 0 : std::vector<MaterialData *> data(libMesh::n_threads());
8636 0 : for (const auto tid : make_range(libMesh::n_threads()))
8637 0 : data[tid] = &storage.getMaterialData(tid);
8638 0 : return data;
8639 0 : };
8640 0 : _xfem->setMaterialData(fill_data(_material_props));
8641 0 : _xfem->setBoundaryMaterialData(fill_data(_bnd_material_props));
8642 :
8643 0 : unsigned int n_threads = libMesh::n_threads();
8644 0 : for (unsigned int i = 0; i < n_threads; ++i)
8645 0 : for (const auto nl_sys_num : index_range(_nl))
8646 : {
8647 0 : _assembly[i][nl_sys_num]->setXFEM(_xfem);
8648 0 : if (_displaced_problem)
8649 0 : _displaced_problem->assembly(i, nl_sys_num).setXFEM(_xfem);
8650 : }
8651 0 : }
8652 :
8653 : bool
8654 0 : FEProblemBase::updateMeshXFEM()
8655 : {
8656 0 : TIME_SECTION("updateMeshXFEM", 5, "Updating XFEM");
8657 :
8658 0 : bool updated = false;
8659 0 : if (haveXFEM())
8660 : {
8661 0 : if (_xfem->updateHeal())
8662 : // XFEM exodiff tests rely on a given numbering because they cannot use map = true due to
8663 : // having coincident elements. While conceptually speaking we do not need to contract the
8664 : // mesh, we need its call to renumber_nodes_and_elements in order to preserve these tests
8665 0 : meshChanged(
8666 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8667 :
8668 0 : updated = _xfem->update(_time, _nl, *_aux);
8669 0 : if (updated)
8670 : {
8671 0 : meshChanged(
8672 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8673 0 : _xfem->initSolution(_nl, *_aux);
8674 0 : restoreSolutions();
8675 0 : _console << "\nXFEM update complete: Mesh modified" << std::endl;
8676 : }
8677 : else
8678 0 : _console << "\nXFEM update complete: Mesh not modified" << std::endl;
8679 : }
8680 0 : return updated;
8681 0 : }
8682 :
8683 : void
8684 7026 : FEProblemBase::meshChanged(const bool intermediate_change,
8685 : const bool contract_mesh,
8686 : const bool clean_refinement_flags)
8687 : {
8688 35130 : TIME_SECTION("meshChanged", 3, "Handling Mesh Changes");
8689 :
8690 7026 : _app.markMeshChangedForBackup();
8691 :
8692 13844 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8693 6818 : _neighbor_material_props.hasStatefulProperties())
8694 208 : _mesh.cacheChangedLists(); // Currently only used with adaptivity and stateful material
8695 : // properties
8696 :
8697 : // Clear these out because they corresponded to the old mesh
8698 7026 : _ghosted_elems.clear();
8699 7026 : ghostGhostedBoundaries();
8700 :
8701 : // The mesh changed. We notify the MooseMesh first, because
8702 : // callbacks (e.g. for sparsity calculations) triggered by the
8703 : // EquationSystems reinit may require up-to-date MooseMesh caches.
8704 7026 : _mesh.meshChanged();
8705 :
8706 : // If we're just going to alter the mesh again, all we need to
8707 : // handle here is AMR and projections, not full system reinit
8708 7026 : if (intermediate_change)
8709 3510 : es().reinit_solutions();
8710 : else
8711 3516 : es().reinit();
8712 :
8713 7026 : if (contract_mesh)
8714 : // Once vectors are restricted, we can delete children of coarsened elements
8715 4452 : _mesh.getMesh().contract();
8716 7026 : if (clean_refinement_flags)
8717 : {
8718 : // Finally clear refinement flags so that if someone tries to project vectors again without
8719 : // an intervening mesh refinement to clear flags they won't run into trouble
8720 4452 : MeshRefinement refinement(_mesh.getMesh());
8721 4452 : refinement.clean_refinement_flags();
8722 4452 : }
8723 :
8724 7026 : if (!intermediate_change)
8725 : {
8726 : // Since the mesh has changed, we need to make sure that we update any of our
8727 : // MOOSE-system specific data.
8728 7032 : for (auto & sys : _solver_systems)
8729 3516 : sys->reinit();
8730 3516 : _aux->reinit();
8731 : }
8732 :
8733 : // Updating MooseMesh first breaks other adaptivity code, unless we
8734 : // then *again* update the MooseMesh caches. E.g. the definition of
8735 : // "active" and "local" may have been *changed* by refinement and
8736 : // repartitioning done in EquationSystems::reinit().
8737 7026 : _mesh.meshChanged();
8738 :
8739 : // If we have finite volume variables, we will need to recompute additional elemental/face
8740 : // quantities
8741 7026 : if (haveFV() && _mesh.isFiniteVolumeInfoDirty())
8742 351 : _mesh.setupFiniteVolumeMeshData();
8743 :
8744 : // Let the meshChangedInterface notify the mesh changed event before we update the active
8745 : // semilocal nodes, because the set of ghosted elements may potentially be updated during a mesh
8746 : // changed event.
8747 125503 : for (const auto & mci : _notify_when_mesh_changes)
8748 118477 : mci->meshChanged();
8749 :
8750 : // Since the Mesh changed, update the PointLocator object used by DiracKernels.
8751 7026 : _dirac_kernel_info.updatePointLocator(_mesh);
8752 :
8753 : // Need to redo ghosting
8754 7026 : _geometric_search_data.reinit();
8755 :
8756 7026 : if (_displaced_problem)
8757 : {
8758 569 : _displaced_problem->meshChanged(contract_mesh, clean_refinement_flags);
8759 569 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8760 : }
8761 :
8762 7026 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8763 :
8764 7026 : _evaluable_local_elem_range.reset();
8765 7026 : _nl_evaluable_local_elem_range.reset();
8766 :
8767 : // Just like we reinitialized our geometric search objects, we also need to reinitialize our
8768 : // mortar meshes. Note that this needs to happen after DisplacedProblem::meshChanged because the
8769 : // mortar mesh discretization will depend necessarily on the displaced mesh being re-displaced
8770 7026 : _mortar_data->meshChanged();
8771 :
8772 : // Nonlinear systems hold the mortar mesh functors. The domains of definition of the mortar
8773 : // functors might have changed when the mesh changed.
8774 14042 : for (auto & nl_sys : _nl)
8775 7016 : nl_sys->reinitMortarFunctors();
8776 :
8777 7026 : reinitBecauseOfGhostingOrNewGeomObjects(/*mortar_changed=*/true);
8778 :
8779 : // We need to create new storage for newly active elements, and copy
8780 : // stateful properties from the old elements.
8781 7234 : if (_has_initialized_stateful &&
8782 208 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties()))
8783 : {
8784 208 : if (havePRefinement())
8785 66 : _mesh.buildPRefinementAndCoarseningMaps(_assembly[0][0].get());
8786 :
8787 : // Prolong properties onto newly refined elements' children
8788 : {
8789 : ProjectMaterialProperties pmp(
8790 208 : /* refine = */ true, *this, _material_props, _bnd_material_props, _assembly);
8791 208 : const auto & range = *_mesh.refinedElementRange();
8792 208 : Threads::parallel_reduce(range, pmp);
8793 :
8794 : // Concurrent erasure from the shared hash map is not safe while we are reading from it in
8795 : // ProjectMaterialProperties, so we handle erasure here. Moreover, erasure based on key is
8796 : // not thread safe in and of itself because it is a read-write operation. Note that we do not
8797 : // do the erasure for p-refinement because the coarse level element is the same as our active
8798 : // refined level element
8799 208 : if (!doingPRefinement())
8800 3272 : for (const auto & elem : range)
8801 : {
8802 3130 : _material_props.eraseProperty(elem);
8803 3130 : _bnd_material_props.eraseProperty(elem);
8804 3130 : _neighbor_material_props.eraseProperty(elem);
8805 : }
8806 208 : }
8807 :
8808 : // Restrict properties onto newly coarsened elements
8809 : {
8810 : ProjectMaterialProperties pmp(
8811 208 : /* refine = */ false, *this, _material_props, _bnd_material_props, _assembly);
8812 208 : const auto & range = *_mesh.coarsenedElementRange();
8813 208 : Threads::parallel_reduce(range, pmp);
8814 : // Note that we do not do the erasure for p-refinement because the coarse level element is the
8815 : // same as our active refined level element
8816 208 : if (!doingPRefinement())
8817 1322 : for (const auto & elem : range)
8818 : {
8819 1180 : auto && coarsened_children = _mesh.coarsenedElementChildren(elem);
8820 7732 : for (auto && child : coarsened_children)
8821 : {
8822 6552 : _material_props.eraseProperty(child);
8823 6552 : _bnd_material_props.eraseProperty(child);
8824 6552 : _neighbor_material_props.eraseProperty(child);
8825 : }
8826 : }
8827 208 : }
8828 : }
8829 :
8830 7026 : if (_calculate_jacobian_in_uo)
8831 0 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
8832 :
8833 7026 : _has_jacobian = false; // we have to recompute jacobian when mesh changed
8834 :
8835 : // Now for backwards compatibility with user code that overrode the old no-arg meshChanged we must
8836 : // call it here
8837 7026 : meshChanged();
8838 7026 : }
8839 :
8840 : void
8841 931571 : FEProblemBase::notifyWhenMeshChanges(MeshChangedInterface * mci)
8842 : {
8843 931571 : _notify_when_mesh_changes.push_back(mci);
8844 931571 : }
8845 :
8846 : void
8847 77456 : FEProblemBase::notifyWhenMeshDisplaces(MeshDisplacedInterface * mdi)
8848 : {
8849 77456 : _notify_when_mesh_displaces.push_back(mdi);
8850 77456 : }
8851 :
8852 : void
8853 63048 : FEProblemBase::meshDisplaced()
8854 : {
8855 91725 : for (const auto & mdi : _notify_when_mesh_displaces)
8856 28677 : mdi->meshDisplaced();
8857 63048 : }
8858 :
8859 : void
8860 9805 : FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const bool threaded)
8861 : {
8862 : ComputeMaterialsObjectThread cmt(
8863 9805 : *this, _material_props, _bnd_material_props, _neighbor_material_props, _assembly);
8864 9805 : if (threaded)
8865 9805 : Threads::parallel_reduce(elem_range, cmt);
8866 : else
8867 0 : cmt(elem_range, true);
8868 :
8869 : #ifdef MOOSE_KOKKOS_ENABLED
8870 7383 : if (_has_kokkos_objects)
8871 781 : initKokkosStatefulProps();
8872 : #endif
8873 9805 : }
8874 :
8875 : void
8876 61354 : FEProblemBase::checkProblemIntegrity()
8877 : {
8878 184062 : TIME_SECTION("checkProblemIntegrity", 5);
8879 :
8880 : // Subdomains specified by the "Problem/block" parameter
8881 122708 : const auto & subdomain_names = getParam<std::vector<SubdomainName>>("block");
8882 61354 : auto mesh_subdomains_vec = MooseMeshUtils::getSubdomainIDs(_mesh, subdomain_names);
8883 61354 : std::set<SubdomainID> mesh_subdomains(mesh_subdomains_vec.begin(), mesh_subdomains_vec.end());
8884 :
8885 : // Check kernel coverage of subdomains (blocks) in the mesh
8886 61354 : if (!_skip_nl_system_check && _solve && _kernel_coverage_check != CoverageCheckMode::FALSE &&
8887 41966 : _kernel_coverage_check != CoverageCheckMode::OFF)
8888 : {
8889 41954 : std::set<SubdomainID> blocks;
8890 41954 : if (_kernel_coverage_check == CoverageCheckMode::TRUE ||
8891 231 : _kernel_coverage_check == CoverageCheckMode::ON)
8892 41723 : blocks = mesh_subdomains;
8893 231 : else if (_kernel_coverage_check == CoverageCheckMode::SKIP_LIST)
8894 : {
8895 12 : blocks = mesh_subdomains;
8896 24 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8897 : {
8898 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8899 12 : if (id == Moose::INVALID_BLOCK_ID)
8900 0 : paramError("kernel_coverage_block_list",
8901 : "Subdomain \"",
8902 : subdomain_name,
8903 : "\" not found in mesh.");
8904 12 : blocks.erase(id);
8905 : }
8906 : }
8907 219 : else if (_kernel_coverage_check == CoverageCheckMode::ONLY_LIST)
8908 438 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8909 : {
8910 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8911 219 : if (id == Moose::INVALID_BLOCK_ID)
8912 0 : paramError("kernel_coverage_block_list",
8913 : "Subdomain \"",
8914 : subdomain_name,
8915 : "\" not found in mesh.");
8916 219 : blocks.insert(id);
8917 : }
8918 41954 : if (!blocks.empty())
8919 82977 : for (auto & nl : _nl)
8920 41035 : nl->checkKernelCoverage(blocks);
8921 41942 : }
8922 :
8923 : // Check materials
8924 : {
8925 : #ifdef LIBMESH_ENABLE_AMR
8926 63611 : if ((_adaptivity.isOn() || _num_grid_steps) &&
8927 2269 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8928 2206 : _neighbor_material_props.hasStatefulProperties()))
8929 : {
8930 63 : _console << "Using EXPERIMENTAL Stateful Material Property projection with Adaptivity!\n"
8931 63 : << std::flush;
8932 : }
8933 : #endif
8934 :
8935 61342 : std::set<SubdomainID> local_mesh_subs(mesh_subdomains);
8936 :
8937 61342 : if (_material_coverage_check != CoverageCheckMode::FALSE &&
8938 61258 : _material_coverage_check != CoverageCheckMode::OFF)
8939 : {
8940 : /**
8941 : * If a material is specified for any block in the simulation, then all blocks must
8942 : * have a material specified.
8943 : */
8944 61258 : bool check_material_coverage = false;
8945 61258 : std::set<SubdomainID> ids = _all_materials.getActiveBlocks();
8946 73933 : for (const auto & id : ids)
8947 : {
8948 12675 : local_mesh_subs.erase(id);
8949 12675 : check_material_coverage = true;
8950 : }
8951 :
8952 : // did the user limit the subdomains to be checked?
8953 61258 : if (_material_coverage_check == CoverageCheckMode::SKIP_LIST)
8954 : {
8955 24 : for (const auto & subdomain_name : _material_coverage_blocks)
8956 : {
8957 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8958 12 : if (id == Moose::INVALID_BLOCK_ID)
8959 0 : paramError("material_coverage_block_list",
8960 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8961 12 : local_mesh_subs.erase(id);
8962 : }
8963 : }
8964 61246 : else if (_material_coverage_check == CoverageCheckMode::ONLY_LIST)
8965 : {
8966 219 : std::set<SubdomainID> blocks(local_mesh_subs);
8967 438 : for (const auto & subdomain_name : _material_coverage_blocks)
8968 : {
8969 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8970 219 : if (id == Moose::INVALID_BLOCK_ID)
8971 0 : paramError("material_coverage_block_list",
8972 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8973 219 : blocks.erase(id);
8974 : }
8975 231 : for (const auto id : blocks)
8976 12 : local_mesh_subs.erase(id);
8977 219 : }
8978 :
8979 : // also exclude mortar spaces from the material check
8980 61258 : auto && mortar_subdomain_ids = _mortar_data->getMortarSubdomainIDs();
8981 63386 : for (auto subdomain_id : mortar_subdomain_ids)
8982 2128 : local_mesh_subs.erase(subdomain_id);
8983 :
8984 : // Check Material Coverage
8985 61258 : if (check_material_coverage && !local_mesh_subs.empty())
8986 : {
8987 6 : std::stringstream extra_subdomain_ids;
8988 : /// unsigned int is necessary to print SubdomainIDs in the statement below
8989 6 : std::copy(local_mesh_subs.begin(),
8990 : local_mesh_subs.end(),
8991 12 : std::ostream_iterator<unsigned int>(extra_subdomain_ids, " "));
8992 : /// vector is necessary to get the subdomain names
8993 : std::vector<SubdomainID> local_mesh_subs_vec(local_mesh_subs.begin(),
8994 6 : local_mesh_subs.end());
8995 :
8996 18 : mooseError("The following blocks from your input mesh do not contain an active material: " +
8997 12 : extra_subdomain_ids.str() +
8998 18 : "(names: " + Moose::stringify(_mesh.getSubdomainNames(local_mesh_subs_vec)) +
8999 : ")\nWhen ANY mesh block contains a Material object, "
9000 : "all blocks must contain a Material object.\n");
9001 0 : }
9002 61252 : }
9003 :
9004 : // Check material properties on blocks and boundaries
9005 61336 : checkBlockMatProps();
9006 61304 : checkBoundaryMatProps();
9007 :
9008 : // Check that material properties exist when requested by other properties on a given block
9009 61295 : const auto & materials = _all_materials.getActiveObjects();
9010 75496 : for (const auto & material : materials)
9011 14201 : material->checkStatefulSanity();
9012 :
9013 : // auto mats_to_check = _materials.getActiveBlockObjects();
9014 : // const auto & discrete_materials = _discrete_materials.getActiveBlockObjects();
9015 : // for (const auto & map_it : discrete_materials)
9016 : // for (const auto & container_element : map_it.second)
9017 : // mats_to_check[map_it.first].push_back(container_element);
9018 61295 : if (_material_dependency_check)
9019 61271 : checkDependMaterialsHelper(_all_materials.getActiveBlockObjects());
9020 61282 : }
9021 :
9022 61282 : checkUserObjects();
9023 :
9024 : // Verify that we don't have any Element type/Coordinate Type conflicts
9025 61282 : checkCoordinateSystems();
9026 :
9027 : // Coordinate transforms are only intended for use with MultiApps at this time. If you are not
9028 : // using multiapps but still require these, contact a moose developer
9029 61419 : if (_mesh.coordTransform().hasScalingOrRotationTransformation() && _app.isUltimateMaster() &&
9030 140 : !hasMultiApps())
9031 3 : mooseError("Coordinate transformation parameters, listed below, are only to be used in the "
9032 : "context of application to application field transfers at this time. The mesh is "
9033 : "not modified by these parameters within an application.\n"
9034 : "You should likely use a 'TransformGenerator' in the [Mesh] block to achieve the "
9035 : "desired mesh modification.\n\n",
9036 3 : Moose::stringify(MooseAppCoordTransform::validParams()));
9037 :
9038 : // If using displacements, verify that the order of the displacement
9039 : // variables matches the order of the elements in the displaced
9040 : // mesh.
9041 61276 : checkDisplacementOrders();
9042 :
9043 : // Check for postprocessor names with same name as a scalar variable
9044 61273 : checkDuplicatePostprocessorVariableNames();
9045 61273 : }
9046 :
9047 : void
9048 61276 : FEProblemBase::checkDisplacementOrders()
9049 : {
9050 61276 : if (_displaced_problem)
9051 : {
9052 2022 : bool mesh_has_second_order_elements = false;
9053 4044 : for (const auto & elem : as_range(_displaced_mesh->activeLocalElementsBegin(),
9054 464948 : _displaced_mesh->activeLocalElementsEnd()))
9055 : {
9056 229785 : if (elem->default_order() == SECOND)
9057 : {
9058 344 : mesh_has_second_order_elements = true;
9059 344 : break;
9060 : }
9061 2022 : }
9062 :
9063 : // We checked our local elements, so take the max over all processors.
9064 2022 : _displaced_mesh->comm().max(mesh_has_second_order_elements);
9065 :
9066 : // If the Mesh has second order elements, make sure the
9067 : // displacement variables are second-order.
9068 2022 : if (mesh_has_second_order_elements)
9069 : {
9070 : const std::vector<std::string> & displacement_variables =
9071 344 : _displaced_problem->getDisplacementVarNames();
9072 :
9073 1133 : for (const auto & var_name : displacement_variables)
9074 : {
9075 : MooseVariableFEBase & mv =
9076 792 : _displaced_problem->getVariable(/*tid=*/0,
9077 : var_name,
9078 : Moose::VarKindType::VAR_ANY,
9079 : Moose::VarFieldType::VAR_FIELD_STANDARD);
9080 792 : if (mv.order() != SECOND)
9081 3 : mooseError("Error: mesh has SECOND order elements, so all displacement variables must be "
9082 : "SECOND order.");
9083 : }
9084 : }
9085 : }
9086 61273 : }
9087 :
9088 : void
9089 61282 : FEProblemBase::checkUserObjects()
9090 : {
9091 : // Check user_objects block coverage
9092 61282 : std::set<SubdomainID> mesh_subdomains = _mesh.meshSubdomains();
9093 61282 : std::set<SubdomainID> user_objects_blocks;
9094 :
9095 : // gather names of all user_objects that were defined in the input file
9096 : // and the blocks that they are defined on
9097 61282 : std::set<std::string> names;
9098 :
9099 61282 : std::vector<UserObjectBase *> objects;
9100 61282 : theWarehouse().query().condition<AttribInterfaces>(Interfaces::UserObject).queryInto(objects);
9101 :
9102 136441 : for (const auto & obj : objects)
9103 75159 : names.insert(obj->name());
9104 :
9105 : // See if all referenced blocks are covered
9106 61282 : std::set<SubdomainID> difference;
9107 61282 : std::set_difference(user_objects_blocks.begin(),
9108 : user_objects_blocks.end(),
9109 : mesh_subdomains.begin(),
9110 : mesh_subdomains.end(),
9111 : std::inserter(difference, difference.end()));
9112 :
9113 61282 : if (!difference.empty())
9114 : {
9115 0 : std::ostringstream oss;
9116 0 : oss << "One or more UserObjects is referencing a nonexistent block:\n";
9117 0 : for (const auto & id : difference)
9118 0 : oss << id << "\n";
9119 0 : mooseError(oss.str());
9120 0 : }
9121 61282 : }
9122 :
9123 : void
9124 61271 : FEProblemBase::checkDependMaterialsHelper(
9125 : const std::map<SubdomainID, std::vector<std::shared_ptr<MaterialBase>>> & materials_map)
9126 : {
9127 73888 : for (const auto & it : materials_map)
9128 : {
9129 : /// These two sets are used to make sure that all dependent props on a block are actually supplied
9130 12625 : std::set<std::string> block_depend_props, block_supplied_props;
9131 :
9132 31390 : for (const auto & mat1 : it.second)
9133 : {
9134 18765 : auto & alldeps = mat1->getMatPropDependencies(); // includes requested stateful props
9135 21084 : for (auto & dep : alldeps)
9136 2319 : block_depend_props.insert(_material_prop_registry.getName(dep));
9137 :
9138 : // See if any of the active materials supply this property
9139 57970 : for (const auto & mat2 : it.second)
9140 : {
9141 39205 : const std::set<std::string> & supplied_props = mat2->MaterialBase::getSuppliedItems();
9142 39205 : block_supplied_props.insert(supplied_props.begin(), supplied_props.end());
9143 : }
9144 : }
9145 :
9146 : // Add zero material properties specific to this block and unrestricted
9147 12625 : block_supplied_props.insert(_zero_block_material_props[it.first].begin(),
9148 12625 : _zero_block_material_props[it.first].end());
9149 :
9150 : // Error check to make sure all properties consumed by materials are supplied on this block
9151 12625 : std::set<std::string> difference;
9152 12625 : std::set_difference(block_depend_props.begin(),
9153 : block_depend_props.end(),
9154 : block_supplied_props.begin(),
9155 : block_supplied_props.end(),
9156 : std::inserter(difference, difference.end()));
9157 :
9158 12625 : if (!difference.empty())
9159 : {
9160 8 : std::ostringstream oss;
9161 8 : oss << "One or more Material Properties were not supplied on block ";
9162 8 : const std::string & subdomain_name = _mesh.getSubdomainName(it.first);
9163 8 : if (subdomain_name.length() > 0)
9164 0 : oss << subdomain_name << " (" << it.first << ")";
9165 : else
9166 8 : oss << it.first;
9167 8 : oss << ":\n";
9168 16 : for (const auto & name : difference)
9169 8 : oss << name << "\n";
9170 8 : mooseError(oss.str());
9171 0 : }
9172 12617 : }
9173 :
9174 : // This loop checks that materials are not supplied by multiple Material objects
9175 73875 : for (const auto & it : materials_map)
9176 : {
9177 12617 : const auto & materials = it.second;
9178 12617 : std::set<std::string> inner_supplied, outer_supplied;
9179 :
9180 31362 : for (const auto & outer_mat : materials)
9181 : {
9182 : // Storage for properties for this material (outer) and all other materials (inner)
9183 18750 : outer_supplied = outer_mat->getSuppliedItems();
9184 18750 : inner_supplied.clear();
9185 :
9186 : // Property to material map for error reporting
9187 18750 : std::map<std::string, std::set<std::string>> prop_to_mat;
9188 39632 : for (const auto & name : outer_supplied)
9189 20882 : prop_to_mat[name].insert(outer_mat->name());
9190 :
9191 57931 : for (const auto & inner_mat : materials)
9192 : {
9193 39181 : if (outer_mat == inner_mat)
9194 18750 : continue;
9195 :
9196 : // Check whether these materials are an AD pair
9197 20431 : auto outer_mat_type = outer_mat->type();
9198 20431 : auto inner_mat_type = inner_mat->type();
9199 40862 : removeSubstring(outer_mat_type, "<RESIDUAL>");
9200 40862 : removeSubstring(outer_mat_type, "<JACOBIAN>");
9201 40862 : removeSubstring(inner_mat_type, "<RESIDUAL>");
9202 20431 : removeSubstring(inner_mat_type, "<JACOBIAN>");
9203 20431 : if (outer_mat_type == inner_mat_type && outer_mat_type != outer_mat->type() &&
9204 0 : inner_mat_type != inner_mat->type())
9205 0 : continue;
9206 :
9207 20431 : inner_supplied.insert(inner_mat->getSuppliedItems().begin(),
9208 20431 : inner_mat->getSuppliedItems().end());
9209 :
9210 115501 : for (const auto & inner_supplied_name : inner_supplied)
9211 95070 : prop_to_mat[inner_supplied_name].insert(inner_mat->name());
9212 20431 : }
9213 :
9214 : // Test that a property isn't supplied on multiple blocks
9215 18750 : std::set<std::string> intersection;
9216 18750 : std::set_intersection(outer_supplied.begin(),
9217 : outer_supplied.end(),
9218 : inner_supplied.begin(),
9219 : inner_supplied.end(),
9220 : std::inserter(intersection, intersection.end()));
9221 :
9222 18750 : if (!intersection.empty())
9223 : {
9224 5 : std::ostringstream oss;
9225 5 : oss << "The following material properties are declared on block " << it.first
9226 5 : << " by multiple materials:\n";
9227 10 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << "Material Property"
9228 5 : << "Material Objects\n";
9229 20 : for (const auto & outer_name : intersection)
9230 : {
9231 15 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << outer_name;
9232 45 : for (const auto & inner_name : prop_to_mat[outer_name])
9233 30 : oss << inner_name << " ";
9234 15 : oss << '\n';
9235 : }
9236 :
9237 5 : mooseError(oss.str());
9238 : break;
9239 0 : }
9240 18745 : }
9241 12612 : }
9242 61258 : }
9243 :
9244 : void
9245 61282 : FEProblemBase::checkCoordinateSystems()
9246 : {
9247 61282 : _mesh.checkCoordinateSystems();
9248 61279 : }
9249 :
9250 : void
9251 473 : FEProblemBase::setRestartFile(const std::string & file_name)
9252 : {
9253 473 : if (_app.isRecovering())
9254 : {
9255 23 : mooseInfo("Restart file ", file_name, " is NOT being used since we are performing recovery.");
9256 : }
9257 : else
9258 : {
9259 450 : _app.setRestart(true);
9260 450 : _app.setRestartRecoverFileBase(file_name);
9261 450 : mooseInfo("Using ", file_name, " for restart.");
9262 : }
9263 473 : }
9264 :
9265 : std::vector<VariableName>
9266 361075 : FEProblemBase::getVariableNames()
9267 : {
9268 361075 : std::vector<VariableName> names;
9269 :
9270 726541 : for (auto & sys : _solver_systems)
9271 : {
9272 365466 : const std::vector<VariableName> & var_names = sys->getVariableNames();
9273 365466 : names.insert(names.end(), var_names.begin(), var_names.end());
9274 : }
9275 :
9276 361075 : const std::vector<VariableName> & aux_var_names = _aux->getVariableNames();
9277 361075 : names.insert(names.end(), aux_var_names.begin(), aux_var_names.end());
9278 :
9279 361075 : return names;
9280 0 : }
9281 :
9282 : SolverParams &
9283 1636902 : FEProblemBase::solverParams(const unsigned int solver_sys_num)
9284 : {
9285 : mooseAssert(solver_sys_num < numSolverSystems(),
9286 : "Solver system number '" << solver_sys_num << "' is out of bounds. We have '"
9287 : << numSolverSystems() << "' solver systems");
9288 1636902 : return _solver_params[solver_sys_num];
9289 : }
9290 :
9291 : const SolverParams &
9292 14652 : FEProblemBase::solverParams(const unsigned int solver_sys_num) const
9293 : {
9294 14652 : return const_cast<FEProblemBase *>(this)->solverParams(solver_sys_num);
9295 : }
9296 :
9297 : void
9298 371 : FEProblemBase::registerRandomInterface(RandomInterface & random_interface, const std::string & name)
9299 : {
9300 371 : auto insert_pair = moose_try_emplace(
9301 371 : _random_data_objects, name, std::make_unique<RandomData>(*this, random_interface));
9302 :
9303 371 : auto random_data_ptr = insert_pair.first->second.get();
9304 371 : random_interface.setRandomDataPointer(random_data_ptr);
9305 371 : }
9306 :
9307 : bool
9308 1603787 : FEProblemBase::needBoundaryMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9309 : {
9310 1603787 : if (_bnd_mat_side_cache[tid].find(bnd_id) == _bnd_mat_side_cache[tid].end())
9311 : {
9312 28384 : auto & bnd_mat_side_cache = _bnd_mat_side_cache[tid][bnd_id];
9313 28384 : bnd_mat_side_cache = false;
9314 :
9315 : // Check systems
9316 28384 : if (_aux->needMaterialOnSide(bnd_id))
9317 : {
9318 506 : bnd_mat_side_cache = true;
9319 506 : return true;
9320 : }
9321 53484 : for (auto & nl : _nl)
9322 27690 : if (nl->needBoundaryMaterialOnSide(bnd_id, tid))
9323 : {
9324 2084 : bnd_mat_side_cache = true;
9325 2084 : return true;
9326 : }
9327 :
9328 : // TODO: these objects should be checked for whether they actually consume materials
9329 : // NOTE: InterfaceUO can use use boundary properties too
9330 25794 : if (theWarehouse()
9331 51588 : .query()
9332 25794 : .condition<AttribThread>(tid)
9333 25794 : .condition<AttribInterfaces>(Interfaces::SideUserObject | Interfaces::DomainUserObject |
9334 : Interfaces::InterfaceUserObject)
9335 25794 : .condition<AttribBoundaries>(bnd_id)
9336 25794 : .count() > 0)
9337 : {
9338 560 : bnd_mat_side_cache = true;
9339 560 : return true;
9340 : }
9341 : }
9342 :
9343 1600637 : return _bnd_mat_side_cache[tid][bnd_id];
9344 : }
9345 :
9346 : bool
9347 386695 : FEProblemBase::needInterfaceMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9348 : {
9349 386695 : if (_interface_mat_side_cache[tid].find(bnd_id) == _interface_mat_side_cache[tid].end())
9350 : {
9351 2700 : auto & interface_mat_side_cache = _interface_mat_side_cache[tid][bnd_id];
9352 2700 : interface_mat_side_cache = false;
9353 :
9354 : // Aux-system has not needed interface materials so far
9355 5184 : for (auto & nl : _nl)
9356 2700 : if (nl->needInterfaceMaterialOnSide(bnd_id, tid))
9357 : {
9358 216 : interface_mat_side_cache = true;
9359 216 : return true;
9360 : }
9361 :
9362 : // TODO: these objects should be checked for whether they actually consume materials
9363 2484 : if (theWarehouse()
9364 4968 : .query()
9365 2484 : .condition<AttribThread>(tid)
9366 2484 : .condition<AttribInterfaces>(Interfaces::InterfaceUserObject |
9367 : Interfaces::DomainUserObject)
9368 2484 : .condition<AttribBoundaries>(bnd_id)
9369 2484 : .count() > 0)
9370 : {
9371 79 : interface_mat_side_cache = true;
9372 79 : return true;
9373 : }
9374 2405 : else if (_interface_materials.hasActiveBoundaryObjects(bnd_id, tid))
9375 : {
9376 9 : interface_mat_side_cache = true;
9377 9 : return true;
9378 : }
9379 : }
9380 386391 : return _interface_mat_side_cache[tid][bnd_id];
9381 : }
9382 :
9383 : bool
9384 422937 : FEProblemBase::needInternalNeighborSideMaterial(SubdomainID subdomain_id, const THREAD_ID tid)
9385 : {
9386 422937 : if (_block_mat_side_cache[tid].find(subdomain_id) == _block_mat_side_cache[tid].end())
9387 : {
9388 12258 : _block_mat_side_cache[tid][subdomain_id] = false;
9389 :
9390 24071 : for (auto & nl : _nl)
9391 12223 : if (nl->needInternalNeighborSideMaterial(subdomain_id, tid))
9392 : {
9393 410 : _block_mat_side_cache[tid][subdomain_id] = true;
9394 410 : return true;
9395 : }
9396 :
9397 : // TODO: these objects should be checked for whether they actually consume materials
9398 11848 : if (theWarehouse()
9399 23696 : .query()
9400 11848 : .condition<AttribThread>(tid)
9401 11848 : .condition<AttribInterfaces>(Interfaces::InternalSideUserObject |
9402 : Interfaces::DomainUserObject)
9403 11848 : .condition<AttribSubdomains>(subdomain_id)
9404 11848 : .count() > 0)
9405 : {
9406 33 : _block_mat_side_cache[tid][subdomain_id] = true;
9407 33 : return true;
9408 : }
9409 : }
9410 :
9411 422494 : return _block_mat_side_cache[tid][subdomain_id];
9412 : }
9413 :
9414 : bool
9415 289160 : FEProblemBase::needsPreviousNewtonIteration() const
9416 : {
9417 289160 : return vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG);
9418 : }
9419 :
9420 : void
9421 76 : FEProblemBase::needsPreviousNewtonIteration(bool state)
9422 : {
9423 76 : if (state && !vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
9424 0 : mooseError("Previous nonlinear solution is required but not added through "
9425 : "Problem/previous_nl_solution_required=true");
9426 76 : }
9427 :
9428 : void
9429 52 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(bool needed,
9430 : const unsigned int solver_sys_num)
9431 : {
9432 52 : _previous_multiapp_fp_nl_solution_required[solver_sys_num] = needed;
9433 52 : }
9434 :
9435 : bool
9436 59123 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(
9437 : const unsigned int solver_sys_num) const
9438 : {
9439 59123 : return _previous_multiapp_fp_nl_solution_required[solver_sys_num];
9440 : }
9441 :
9442 : void
9443 13 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary(bool state)
9444 : {
9445 13 : _previous_multiapp_fp_aux_solution_required = state;
9446 13 : }
9447 :
9448 : bool
9449 58924 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary() const
9450 : {
9451 58924 : return _previous_multiapp_fp_aux_solution_required;
9452 : }
9453 :
9454 : bool
9455 7561278 : FEProblemBase::hasJacobian() const
9456 : {
9457 7561278 : return _has_jacobian;
9458 : }
9459 :
9460 : bool
9461 7076750 : FEProblemBase::constJacobian() const
9462 : {
9463 7076750 : return _const_jacobian;
9464 : }
9465 :
9466 : void
9467 292174 : FEProblemBase::addOutput(const std::string & object_type,
9468 : const std::string & object_name,
9469 : InputParameters & parameters)
9470 : {
9471 : parallel_object_only();
9472 :
9473 : // Get a reference to the OutputWarehouse
9474 292174 : OutputWarehouse & output_warehouse = _app.getOutputWarehouse();
9475 :
9476 : // Reject the reserved names for objects not built by MOOSE
9477 292174 : if (!parameters.get<bool>("_built_by_moose") && output_warehouse.isReservedName(object_name))
9478 6 : mooseError("The name '", object_name, "' is a reserved name for output objects");
9479 :
9480 : // Check that an object by the same name does not already exist; this must be done before the
9481 : // object is created to avoid getting misleading errors from the Parser
9482 292168 : if (output_warehouse.hasOutput(object_name))
9483 3 : mooseError("An output object named '", object_name, "' already exists");
9484 :
9485 : // Add a pointer to the FEProblemBase class
9486 584330 : parameters.addPrivateParam<FEProblemBase *>("_fe_problem_base", this);
9487 :
9488 : // --show-input should enable the display of the input file on the screen
9489 708121 : if (object_type == "Console" && _app.getParam<bool>("show_input") &&
9490 292192 : parameters.get<bool>("output_screen"))
9491 54 : parameters.set<ExecFlagEnum>("execute_input_on") = EXEC_INITIAL;
9492 :
9493 : // Apply only user-set parameters from the common [Outputs] block so that
9494 : // each output type's own defaults are not overridden by common defaults.
9495 292165 : const InputParameters * common = output_warehouse.getCommonParameters();
9496 292165 : if (common)
9497 292165 : parameters.applyCommonUserSetParameters(*common);
9498 :
9499 : // Set the correct value for the binary flag for XDA/XDR output
9500 292165 : if (object_type == "XDR")
9501 120 : parameters.set<bool>("_binary") = true;
9502 292105 : else if (object_type == "XDA")
9503 244 : parameters.set<bool>("_binary") = false;
9504 :
9505 : // Adjust the checkpoint suffix if auto recovery was enabled
9506 292165 : if (object_name == "auto_recovery_checkpoint")
9507 0 : parameters.set<std::string>("suffix") = "auto_recovery";
9508 :
9509 : // Create the object and add it to the warehouse
9510 292165 : std::shared_ptr<Output> output = _factory.create<Output>(object_type, object_name, parameters);
9511 292153 : logAdd("Output", object_name, object_type, parameters);
9512 292153 : output_warehouse.addOutput(output);
9513 292153 : }
9514 :
9515 : void
9516 23078 : FEProblemBase::haveADObjects(const bool have_ad_objects)
9517 : {
9518 23078 : _have_ad_objects = have_ad_objects;
9519 23078 : if (_displaced_problem)
9520 227 : _displaced_problem->SubProblem::haveADObjects(have_ad_objects);
9521 23078 : }
9522 :
9523 : const SystemBase &
9524 0 : FEProblemBase::getSystemBase(const unsigned int sys_num) const
9525 : {
9526 0 : if (sys_num < _solver_systems.size())
9527 0 : return *_solver_systems[sys_num];
9528 :
9529 0 : return *_aux;
9530 : }
9531 :
9532 : SystemBase &
9533 4208 : FEProblemBase::getSystemBase(const std::string & sys_name)
9534 : {
9535 4208 : if (std::find(_solver_sys_names.begin(), _solver_sys_names.end(), sys_name) !=
9536 8416 : _solver_sys_names.end())
9537 4208 : return getSystemBase(solverSysNum(sys_name));
9538 0 : else if (sys_name == "aux0")
9539 0 : return *_aux;
9540 : else
9541 0 : mooseError("System '" + sys_name + "' was requested from problem but does not exist.");
9542 : }
9543 :
9544 : SystemBase &
9545 5852 : FEProblemBase::getSystemBase(const unsigned int sys_num)
9546 : {
9547 5852 : if (sys_num < _solver_systems.size())
9548 5750 : return *_solver_systems[sys_num];
9549 :
9550 102 : return *_aux;
9551 : }
9552 :
9553 : const SystemBase &
9554 12804 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num) const
9555 : {
9556 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9557 12804 : return *_nl[sys_num];
9558 : }
9559 :
9560 : SystemBase &
9561 2194077 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num)
9562 : {
9563 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9564 2194077 : return *_nl[sys_num];
9565 : }
9566 :
9567 : const SystemBase &
9568 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num) const
9569 : {
9570 : mooseAssert(sys_num < _linear_systems.size(),
9571 : "System number greater than the number of linear systems");
9572 0 : return *_linear_systems[sys_num];
9573 : }
9574 :
9575 : SystemBase &
9576 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num)
9577 : {
9578 : mooseAssert(sys_num < _linear_systems.size(),
9579 : "System number greater than the number of linear systems");
9580 0 : return *_linear_systems[sys_num];
9581 : }
9582 :
9583 : const SystemBase &
9584 0 : FEProblemBase::systemBaseSolver(const unsigned int sys_num) const
9585 : {
9586 : mooseAssert(sys_num < _solver_systems.size(),
9587 : "System number greater than the number of solver systems");
9588 0 : return *_solver_systems[sys_num];
9589 : }
9590 :
9591 : SystemBase &
9592 6366816 : FEProblemBase::systemBaseSolver(const unsigned int sys_num)
9593 : {
9594 : mooseAssert(sys_num < _solver_systems.size(),
9595 : "System number greater than the number of solver systems");
9596 6366816 : return *_solver_systems[sys_num];
9597 : }
9598 :
9599 : const SystemBase &
9600 417 : FEProblemBase::systemBaseAuxiliary() const
9601 : {
9602 417 : return *_aux;
9603 : }
9604 :
9605 : SystemBase &
9606 10030786 : FEProblemBase::systemBaseAuxiliary()
9607 : {
9608 10030786 : return *_aux;
9609 : }
9610 :
9611 : void
9612 3913795 : FEProblemBase::computingNonlinearResid(bool computing_nonlinear_residual)
9613 : {
9614 : parallel_object_only();
9615 :
9616 3913795 : if (_displaced_problem)
9617 192292 : _displaced_problem->computingNonlinearResid(computing_nonlinear_residual);
9618 3913795 : _computing_nonlinear_residual = computing_nonlinear_residual;
9619 3913795 : }
9620 :
9621 : void
9622 9668616 : FEProblemBase::setCurrentlyComputingResidual(bool currently_computing_residual)
9623 : {
9624 9668616 : if (_displaced_problem)
9625 392528 : _displaced_problem->setCurrentlyComputingResidual(currently_computing_residual);
9626 9668616 : _currently_computing_residual = currently_computing_residual;
9627 9668616 : }
9628 :
9629 : void
9630 50 : FEProblemBase::uniformRefine()
9631 : {
9632 : // ResetDisplacedMeshThread::onNode looks up the reference mesh by ID, so we need to make sure
9633 : // we undisplace before adapting the reference mesh
9634 50 : if (_displaced_problem)
9635 34 : _displaced_problem->undisplaceMesh();
9636 :
9637 50 : Adaptivity::uniformRefine(&_mesh, 1);
9638 50 : if (_displaced_problem)
9639 34 : Adaptivity::uniformRefine(&_displaced_problem->mesh(), 1);
9640 :
9641 50 : meshChanged(
9642 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
9643 50 : }
9644 :
9645 : void
9646 60959 : FEProblemBase::automaticScaling(bool automatic_scaling)
9647 : {
9648 60959 : if (_displaced_problem)
9649 2022 : _displaced_problem->automaticScaling(automatic_scaling);
9650 :
9651 60959 : SubProblem::automaticScaling(automatic_scaling);
9652 60959 : }
9653 :
9654 : void
9655 564004 : FEProblemBase::reinitElemFaceRef(const Elem * elem,
9656 : unsigned int side,
9657 : Real tolerance,
9658 : const std::vector<Point> * const pts,
9659 : const std::vector<Real> * const weights,
9660 : const THREAD_ID tid)
9661 : {
9662 564004 : SubProblem::reinitElemFaceRef(elem, side, tolerance, pts, weights, tid);
9663 :
9664 564004 : if (_displaced_problem)
9665 20096 : _displaced_problem->reinitElemFaceRef(
9666 20096 : _displaced_mesh->elemPtr(elem->id()), side, tolerance, pts, weights, tid);
9667 564004 : }
9668 :
9669 : void
9670 564004 : FEProblemBase::reinitNeighborFaceRef(const Elem * neighbor_elem,
9671 : unsigned int neighbor_side,
9672 : Real tolerance,
9673 : const std::vector<Point> * const pts,
9674 : const std::vector<Real> * const weights,
9675 : const THREAD_ID tid)
9676 : {
9677 564004 : SubProblem::reinitNeighborFaceRef(neighbor_elem, neighbor_side, tolerance, pts, weights, tid);
9678 :
9679 564004 : if (_displaced_problem)
9680 20096 : _displaced_problem->reinitNeighborFaceRef(
9681 20096 : _displaced_mesh->elemPtr(neighbor_elem->id()), neighbor_side, tolerance, pts, weights, tid);
9682 564004 : }
9683 :
9684 : void
9685 3036116 : FEProblemBase::getFVMatsAndDependencies(
9686 : const SubdomainID blk_id,
9687 : std::vector<std::shared_ptr<MaterialBase>> & face_materials,
9688 : std::vector<std::shared_ptr<MaterialBase>> & neighbor_materials,
9689 : std::set<MooseVariableFieldBase *> & variables,
9690 : const THREAD_ID tid)
9691 : {
9692 3036116 : if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9693 : {
9694 : auto & this_face_mats =
9695 3544 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9696 7268 : for (std::shared_ptr<MaterialBase> face_mat : this_face_mats)
9697 3724 : if (face_mat->ghostable())
9698 : {
9699 3724 : face_materials.push_back(face_mat);
9700 3724 : auto & var_deps = face_mat->getMooseVariableDependencies();
9701 4084 : for (auto * var : var_deps)
9702 : {
9703 360 : if (!var->isFV())
9704 0 : mooseError(
9705 : "Ghostable materials should only have finite volume variables coupled into them.");
9706 360 : else if (face_mat->hasStatefulProperties())
9707 0 : mooseError("Finite volume materials do not currently support stateful properties.");
9708 360 : variables.insert(var);
9709 : }
9710 3724 : }
9711 : }
9712 :
9713 3036116 : if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9714 : {
9715 : auto & this_neighbor_mats =
9716 3544 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9717 7268 : for (std::shared_ptr<MaterialBase> neighbor_mat : this_neighbor_mats)
9718 3724 : if (neighbor_mat->ghostable())
9719 : {
9720 3724 : neighbor_materials.push_back(neighbor_mat);
9721 : #ifndef NDEBUG
9722 : auto & var_deps = neighbor_mat->getMooseVariableDependencies();
9723 : for (auto * var : var_deps)
9724 : {
9725 : if (!var->isFV())
9726 : mooseError(
9727 : "Ghostable materials should only have finite volume variables coupled into them.");
9728 : else if (neighbor_mat->hasStatefulProperties())
9729 : mooseError("Finite volume materials do not currently support stateful properties.");
9730 : auto pr = variables.insert(var);
9731 : mooseAssert(!pr.second,
9732 : "We should not have inserted any new variables dependencies from our "
9733 : "neighbor materials that didn't exist for our face materials");
9734 : }
9735 : #endif
9736 3724 : }
9737 : }
9738 3036116 : }
9739 :
9740 : void
9741 31843481 : FEProblemBase::resizeMaterialData(const Moose::MaterialDataType data_type,
9742 : const unsigned int nqp,
9743 : const THREAD_ID tid)
9744 : {
9745 31843481 : getMaterialData(data_type, tid).resize(nqp);
9746 31843481 : }
9747 :
9748 : void
9749 60899 : FEProblemBase::setNonlinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9750 : {
9751 60899 : if (convergence_names.size() != numNonlinearSystems())
9752 0 : paramError("nonlinear_convergence",
9753 : "There must be one convergence object per nonlinear system");
9754 60899 : _nonlinear_convergence_names = convergence_names;
9755 60899 : }
9756 :
9757 : void
9758 62537 : FEProblemBase::setMultiAppFixedPointConvergenceName(const ConvergenceName & convergence_name)
9759 : {
9760 62537 : _multiapp_fixed_point_convergence_name = convergence_name;
9761 62537 : }
9762 :
9763 : void
9764 30584 : FEProblemBase::setSteadyStateConvergenceName(const ConvergenceName & convergence_name)
9765 : {
9766 30584 : _steady_state_convergence_name = convergence_name;
9767 30584 : }
9768 :
9769 : const std::vector<ConvergenceName> &
9770 991555 : FEProblemBase::getNonlinearConvergenceNames() const
9771 : {
9772 991555 : if (_nonlinear_convergence_names)
9773 991555 : return *_nonlinear_convergence_names;
9774 0 : mooseError("The nonlinear system convergence name(s) have not been set.");
9775 : }
9776 :
9777 : bool
9778 26206 : FEProblemBase::hasLinearConvergenceObjects() const
9779 : {
9780 : // If false,this means we have not set one, not that we are querying this too early
9781 : // TODO: once there is a default linear CV object, error on the 'not set' case
9782 26206 : return _linear_convergence_names.has_value();
9783 : }
9784 :
9785 : void
9786 134 : FEProblemBase::setLinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9787 : {
9788 134 : if (convergence_names.size() != numLinearSystems())
9789 0 : paramError("linear_convergence", "There must be one convergence object per linear system");
9790 134 : _linear_convergence_names = convergence_names;
9791 134 : }
9792 :
9793 : const std::vector<ConvergenceName> &
9794 4281 : FEProblemBase::getLinearConvergenceNames() const
9795 : {
9796 4281 : if (_linear_convergence_names)
9797 4281 : return *_linear_convergence_names;
9798 0 : mooseError("The linear convergence name(s) have not been set.");
9799 : }
9800 :
9801 : const ConvergenceName &
9802 252172 : FEProblemBase::getMultiAppFixedPointConvergenceName() const
9803 : {
9804 252172 : if (_multiapp_fixed_point_convergence_name)
9805 252172 : return _multiapp_fixed_point_convergence_name.value();
9806 : else
9807 0 : mooseError("The fixed point convergence name has not been set.");
9808 : }
9809 :
9810 : const ConvergenceName &
9811 103850 : FEProblemBase::getSteadyStateConvergenceName() const
9812 : {
9813 103850 : if (_steady_state_convergence_name)
9814 103850 : return _steady_state_convergence_name.value();
9815 : else
9816 0 : mooseError("The steady convergence name has not been set.");
9817 : }
9818 :
9819 : void
9820 3065331 : FEProblemBase::residualSetup()
9821 : {
9822 3065331 : SubProblem::residualSetup();
9823 : // We need to setup all the nonlinear systems other than our current one which actually called
9824 : // this method (so we have to make sure we don't go in a circle)
9825 6220464 : for (const auto i : make_range(numNonlinearSystems()))
9826 3155133 : if (i != currentNlSysNum())
9827 89802 : _nl[i]->residualSetup();
9828 : // We don't setup the aux sys because that's been done elsewhere
9829 3065331 : if (_displaced_problem)
9830 124147 : _displaced_problem->residualSetup();
9831 3065331 : }
9832 :
9833 : void
9834 476278 : FEProblemBase::jacobianSetup()
9835 : {
9836 476278 : SubProblem::jacobianSetup();
9837 : // We need to setup all the nonlinear systems other than our current one which actually called
9838 : // this method (so we have to make sure we don't go in a circle)
9839 968089 : for (const auto i : make_range(numNonlinearSystems()))
9840 491811 : if (i != currentNlSysNum())
9841 15533 : _nl[i]->jacobianSetup();
9842 : // We don't setup the aux sys because that's been done elsewhere
9843 476278 : if (_displaced_problem)
9844 21144 : _displaced_problem->jacobianSetup();
9845 476278 : }
9846 :
9847 : MooseAppCoordTransform &
9848 96032 : FEProblemBase::coordTransform()
9849 : {
9850 96032 : return mesh().coordTransform();
9851 : }
9852 :
9853 : unsigned int
9854 478103642 : FEProblemBase::currentNlSysNum() const
9855 : {
9856 : // If we don't have nonlinear systems this should be an invalid number
9857 478103642 : unsigned int current_nl_sys_num = libMesh::invalid_uint;
9858 478103642 : if (_nl.size())
9859 478101530 : current_nl_sys_num = currentNonlinearSystem().number();
9860 :
9861 478103642 : return current_nl_sys_num;
9862 : }
9863 :
9864 : unsigned int
9865 0 : FEProblemBase::currentLinearSysNum() const
9866 : {
9867 : // If we don't have linear systems this should be an invalid number
9868 0 : unsigned int current_linear_sys_num = libMesh::invalid_uint;
9869 0 : if (_linear_systems.size())
9870 0 : current_linear_sys_num = currentLinearSystem().number();
9871 :
9872 0 : return current_linear_sys_num;
9873 : }
9874 :
9875 : bool
9876 123388502 : FEProblemBase::shouldPrintExecution(const THREAD_ID tid) const
9877 : {
9878 : // For now, only support printing from thread 0
9879 123388502 : if (tid != 0)
9880 516904 : return false;
9881 :
9882 245505814 : if (_print_execution_on.isValueSet(_current_execute_on_flag) ||
9883 122634216 : _print_execution_on.isValueSet(EXEC_ALWAYS))
9884 337814 : return true;
9885 : else
9886 122533784 : return false;
9887 : }
9888 :
9889 : std::vector<MortarUserObject *>
9890 562830 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9891 : const BoundaryID secondary_boundary_id,
9892 : const bool displaced,
9893 : const std::vector<MortarUserObject *> & mortar_uo_superset)
9894 : {
9895 562830 : std::vector<MortarUserObject *> mortar_uos;
9896 562830 : auto * const subproblem = displaced ? static_cast<SubProblem *>(_displaced_problem.get())
9897 562830 : : static_cast<SubProblem *>(this);
9898 562852 : for (auto * const obj : mortar_uo_superset)
9899 44 : if (obj->onInterface(primary_boundary_id, secondary_boundary_id) &&
9900 22 : (&obj->getSubProblem() == subproblem))
9901 22 : mortar_uos.push_back(obj);
9902 :
9903 562830 : return mortar_uos;
9904 0 : }
9905 :
9906 : std::vector<MortarUserObject *>
9907 562808 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9908 : const BoundaryID secondary_boundary_id,
9909 : const bool displaced)
9910 : {
9911 562808 : std::vector<MortarUserObject *> mortar_uos;
9912 562808 : theWarehouse()
9913 562808 : .query()
9914 1125616 : .condition<AttribInterfaces>(Interfaces::MortarUserObject)
9915 562808 : .queryInto(mortar_uos);
9916 1125616 : return getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced, mortar_uos);
9917 562808 : }
9918 :
9919 : void
9920 562808 : FEProblemBase::reinitMortarUserObjects(const BoundaryID primary_boundary_id,
9921 : const BoundaryID secondary_boundary_id,
9922 : const bool displaced)
9923 : {
9924 : const auto mortar_uos =
9925 562808 : getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced);
9926 562808 : for (auto * const mortar_uo : mortar_uos)
9927 : {
9928 0 : mortar_uo->setNormals();
9929 0 : mortar_uo->reinit();
9930 : }
9931 562808 : }
9932 :
9933 : void
9934 0 : FEProblemBase::setVerboseProblem(bool verbose)
9935 : {
9936 0 : _verbose_setup = verbose ? "true" : "false";
9937 0 : _verbose_multiapps = verbose;
9938 0 : _verbose_restore = verbose;
9939 0 : }
9940 :
9941 : void
9942 111832 : FEProblemBase::setCurrentLowerDElem(const Elem * const lower_d_elem, const THREAD_ID tid)
9943 : {
9944 111832 : SubProblem::setCurrentLowerDElem(lower_d_elem, tid);
9945 111832 : if (_displaced_problem)
9946 27619 : _displaced_problem->setCurrentLowerDElem(
9947 0 : lower_d_elem ? _displaced_mesh->elemPtr(lower_d_elem->id()) : nullptr, tid);
9948 111832 : }
9949 :
9950 : void
9951 121869873 : FEProblemBase::setCurrentBoundaryID(BoundaryID bid, const THREAD_ID tid)
9952 : {
9953 121869873 : SubProblem::setCurrentBoundaryID(bid, tid);
9954 121869873 : if (_displaced_problem)
9955 8992635 : _displaced_problem->setCurrentBoundaryID(bid, tid);
9956 121869873 : }
9957 :
9958 : void
9959 7442730 : FEProblemBase::setCurrentNonlinearSystem(const unsigned int nl_sys_num)
9960 : {
9961 : mooseAssert(nl_sys_num < _nl.size(),
9962 : "System number greater than the number of nonlinear systems");
9963 7442730 : _current_nl_sys = _nl[nl_sys_num].get();
9964 7442730 : _current_solver_sys = _current_nl_sys;
9965 7442730 : }
9966 :
9967 : void
9968 77302 : FEProblemBase::setCurrentLinearSystem(const unsigned int sys_num)
9969 : {
9970 : mooseAssert(sys_num < _linear_systems.size(),
9971 : "System number greater than the number of linear systems");
9972 77302 : _current_linear_sys = _linear_systems[sys_num].get();
9973 77302 : _current_solver_sys = _current_linear_sys;
9974 77302 : }
9975 :
9976 : void
9977 5894762 : FEProblemBase::computeSystems(const ExecFlagType & type)
9978 : {
9979 : // When performing an adjoint solve in the optimization module, the current solver system is the
9980 : // adjoint. However, the adjoint solve requires having accurate time derivative calculations for
9981 : // the forward system. The cleanest way to handle such uses is just to compute the time
9982 : // derivatives for all solver systems instead of trying to guess which ones we need and don't need
9983 11941338 : for (auto & solver_sys : _solver_systems)
9984 6046576 : solver_sys->compute(type);
9985 :
9986 5894762 : _aux->compute(type);
9987 5894729 : }
9988 :
9989 : const ConstElemRange &
9990 3952898 : FEProblemBase::getCurrentAlgebraicElementRange()
9991 : {
9992 3952898 : if (!_current_algebraic_elem_range)
9993 3952898 : return *_mesh.getActiveLocalElementRange();
9994 :
9995 0 : return *_current_algebraic_elem_range;
9996 : }
9997 : const ConstNodeRange &
9998 97040 : FEProblemBase::getCurrentAlgebraicNodeRange()
9999 : {
10000 97040 : if (!_current_algebraic_node_range)
10001 97040 : return *_mesh.getLocalNodeRange();
10002 :
10003 0 : return *_current_algebraic_node_range;
10004 : }
10005 : const ConstBndNodeRange &
10006 3492536 : FEProblemBase::getCurrentAlgebraicBndNodeRange()
10007 : {
10008 3492536 : if (!_current_algebraic_bnd_node_range)
10009 3492536 : return *_mesh.getBoundaryNodeRange();
10010 :
10011 0 : return *_current_algebraic_bnd_node_range;
10012 : }
10013 :
10014 : void
10015 0 : FEProblemBase::setCurrentAlgebraicElementRange(ConstElemRange * range)
10016 : {
10017 0 : if (!range)
10018 : {
10019 0 : _current_algebraic_elem_range = nullptr;
10020 0 : return;
10021 : }
10022 :
10023 0 : _current_algebraic_elem_range = std::make_unique<ConstElemRange>(*range);
10024 : }
10025 : void
10026 0 : FEProblemBase::setCurrentAlgebraicNodeRange(ConstNodeRange * range)
10027 : {
10028 0 : if (!range)
10029 : {
10030 0 : _current_algebraic_node_range = nullptr;
10031 0 : return;
10032 : }
10033 :
10034 0 : _current_algebraic_node_range = std::make_unique<ConstNodeRange>(*range);
10035 : }
10036 : void
10037 0 : FEProblemBase::setCurrentAlgebraicBndNodeRange(ConstBndNodeRange * range)
10038 : {
10039 0 : if (!range)
10040 : {
10041 0 : _current_algebraic_bnd_node_range = nullptr;
10042 0 : return;
10043 : }
10044 :
10045 0 : _current_algebraic_bnd_node_range = std::make_unique<ConstBndNodeRange>(*range);
10046 : }
10047 :
10048 : unsigned short
10049 59677 : FEProblemBase::getCurrentICState()
10050 : {
10051 59677 : return _current_ic_state;
10052 : }
10053 :
10054 : std::string
10055 54300 : FEProblemBase::solverTypeString(const unsigned int solver_sys_num)
10056 : {
10057 54300 : return Moose::stringify(solverParams(solver_sys_num)._type);
10058 : }
10059 :
10060 : SolverParams
10061 1264 : FEProblemBase::makeLinearSolverParams()
10062 : {
10063 1264 : SolverParams solver_params;
10064 1264 : solver_params._type = Moose::SolveType::ST_LINEAR;
10065 1264 : solver_params._line_search = Moose::LineSearchType::LS_NONE;
10066 1264 : return solver_params;
10067 : }
10068 :
10069 : const libMesh::CouplingMatrix &
10070 72852 : FEProblemBase::nonlocalCouplingMatrix(const unsigned i) const
10071 : {
10072 72852 : return _nonlocal_cm[i];
10073 : }
10074 :
10075 : bool
10076 90523819 : FEProblemBase::checkNonlocalCouplingRequirement() const
10077 : {
10078 90523819 : return _requires_nonlocal_coupling;
10079 : }
10080 :
10081 : const std::unordered_map<std::pair<BoundaryID, BoundaryID>, MortarInterfaceConfig> &
10082 119607 : FEProblemBase::getMortarInterfaces(bool on_displaced) const
10083 : {
10084 119607 : return _mortar_data->getMortarInterfaces(on_displaced);
10085 : }
|
