Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #ifdef MOOSE_KOKKOS_ENABLED
11 : #include "KokkosMaterialPropertyStorage.h"
12 : #endif
13 :
14 : #include "FEProblemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialPropertyStorage.h"
17 : #include "MooseEnum.h"
18 : #include "Factory.h"
19 : #include "MooseUtils.h"
20 : #include "DisplacedProblem.h"
21 : #include "SystemBase.h"
22 : #include "MaterialData.h"
23 : #include "ComputeUserObjectsThread.h"
24 : #include "ComputeNodalUserObjectsThread.h"
25 : #include "ComputeThreadedGeneralUserObjectsThread.h"
26 : #include "ComputeMaterialsObjectThread.h"
27 : #include "ProjectMaterialProperties.h"
28 : #include "ComputeIndicatorThread.h"
29 : #include "ComputeMarkerThread.h"
30 : #include "ComputeInitialConditionThread.h"
31 : #include "ComputeFVInitialConditionThread.h"
32 : #include "ComputeBoundaryInitialConditionThread.h"
33 : #include "MaxQpsThread.h"
34 : #include "ActionWarehouse.h"
35 : #include "Conversion.h"
36 : #include "Material.h"
37 : #include "FunctorMaterial.h"
38 : #include "ConstantIC.h"
39 : #include "Parser.h"
40 : #include "ElementH1Error.h"
41 : #include "Function.h"
42 : #include "Convergence.h"
43 : #include "NonlinearSystem.h"
44 : #include "LinearSystem.h"
45 : #include "SolverSystem.h"
46 : #include "Distribution.h"
47 : #include "Sampler.h"
48 : #include "FVAdvectedInterpolationMethod.h"
49 : #include "FVFaceInterpolationMethod.h"
50 : #include "FVInterpolationMethod.h"
51 : #include "PetscSupport.h"
52 : #include "RandomInterface.h"
53 : #include "RandomData.h"
54 : #include "MooseEigenSystem.h"
55 : #include "MooseParsedFunction.h"
56 : #include "MeshChangedInterface.h"
57 : #include "MeshDisplacedInterface.h"
58 : #include "ComputeJacobianBlocksThread.h"
59 : #include "ScalarInitialCondition.h"
60 : #include "FVInitialConditionTempl.h"
61 : #include "ElementPostprocessor.h"
62 : #include "NodalPostprocessor.h"
63 : #include "SidePostprocessor.h"
64 : #include "InternalSidePostprocessor.h"
65 : #include "InterfacePostprocessor.h"
66 : #include "GeneralPostprocessor.h"
67 : #include "ElementVectorPostprocessor.h"
68 : #include "NodalVectorPostprocessor.h"
69 : #include "SideVectorPostprocessor.h"
70 : #include "InternalSideVectorPostprocessor.h"
71 : #include "GeneralVectorPostprocessor.h"
72 : #include "Positions.h"
73 : #include "Indicator.h"
74 : #include "Marker.h"
75 : #include "MultiApp.h"
76 : #include "MultiAppTransfer.h"
77 : #include "TransientMultiApp.h"
78 : #include "ElementUserObject.h"
79 : #include "DomainUserObject.h"
80 : #include "NodalUserObject.h"
81 : #include "SideUserObject.h"
82 : #include "InternalSideUserObject.h"
83 : #include "InterfaceUserObject.h"
84 : #include "GeneralUserObject.h"
85 : #include "ThreadedGeneralUserObject.h"
86 : #include "InternalSideIndicatorBase.h"
87 : #include "Transfer.h"
88 : #include "MultiAppTransfer.h"
89 : #include "MultiMooseEnum.h"
90 : #include "Predictor.h"
91 : #include "Assembly.h"
92 : #include "Control.h"
93 : #include "XFEMInterface.h"
94 : #include "ConsoleUtils.h"
95 : #include "NonlocalKernel.h"
96 : #include "NonlocalIntegratedBC.h"
97 : #include "ShapeElementUserObject.h"
98 : #include "ShapeSideUserObject.h"
99 : #include "MooseVariableFE.h"
100 : #include "MooseVariableScalar.h"
101 : #include "InputParameterWarehouse.h"
102 : #include "TimeIntegrator.h"
103 : #include "LineSearch.h"
104 : #include "FloatingPointExceptionGuard.h"
105 : #include "MaxVarNDofsPerElem.h"
106 : #include "MaxVarNDofsPerNode.h"
107 : #include "FVKernel.h"
108 : #include "LinearFVKernel.h"
109 : #include "FVTimeKernel.h"
110 : #include "MooseVariableFV.h"
111 : #include "MooseLinearVariableFV.h"
112 : #include "FVBoundaryCondition.h"
113 : #include "LinearFVBoundaryCondition.h"
114 : #include "FVInterfaceKernel.h"
115 : #include "Reporter.h"
116 : #include "ADUtils.h"
117 : #include "Executioner.h"
118 : #include "VariadicTable.h"
119 : #include "BoundaryNodeIntegrityCheckThread.h"
120 : #include "BoundaryElemIntegrityCheckThread.h"
121 : #include "NodalBCBase.h"
122 : #include "MortarUserObject.h"
123 : #include "MortarUserObjectThread.h"
124 : #include "RedistributeProperties.h"
125 : #include "Checkpoint.h"
126 : #include "MortarInterfaceWarehouse.h"
127 : #include "AutomaticMortarGeneration.h"
128 :
129 : #include "libmesh/exodusII_io.h"
130 : #include "libmesh/quadrature.h"
131 : #include "libmesh/coupling_matrix.h"
132 : #include "libmesh/nonlinear_solver.h"
133 : #include "libmesh/sparse_matrix.h"
134 : #include "libmesh/string_to_enum.h"
135 : #include "libmesh/fe_interface.h"
136 : #include "libmesh/enum_norm_type.h"
137 : #include "libmesh/petsc_solver_exception.h"
138 :
139 : #include "metaphysicl/dualnumber.h"
140 :
141 : // C++
142 : #include <cstring> // for "Jacobian" exception test
143 :
144 : using namespace libMesh;
145 :
146 : // Anonymous namespace for helper function
147 : namespace
148 : {
149 : /**
150 : * Method for sorting the MooseVariableFEBases based on variable numbers
151 : */
152 : bool
153 58 : sortMooseVariables(const MooseVariableFEBase * a, const MooseVariableFEBase * b)
154 : {
155 58 : return a->number() < b->number();
156 : }
157 : } // namespace
158 :
159 : Threads::spin_mutex get_function_mutex;
160 :
161 : InputParameters
162 191541 : FEProblemBase::validParams()
163 : {
164 191541 : InputParameters params = SubProblem::validParams();
165 766164 : params.addParam<unsigned int>("null_space_dimension", 0, "The dimension of the nullspace");
166 574623 : params.addParam<unsigned int>(
167 383082 : "transpose_null_space_dimension", 0, "The dimension of the transpose nullspace");
168 574623 : params.addParam<unsigned int>(
169 383082 : "near_null_space_dimension", 0, "The dimension of the near nullspace");
170 574623 : params.addParam<bool>("solve",
171 383082 : true,
172 : "Whether or not to actually solve the Nonlinear system. "
173 : "This is handy in the case that all you want to do is "
174 : "execute AuxKernels, Transfers, etc. without actually "
175 : "solving anything");
176 574623 : params.addParam<bool>("use_nonlinear",
177 383082 : true,
178 : "Determines whether to use a Nonlinear vs a "
179 : "Eigenvalue system (Automatically determined based "
180 : "on executioner)");
181 766164 : params.addParam<bool>("error_on_jacobian_nonzero_reallocation",
182 : "This causes PETSc to error if it had to reallocate memory in the Jacobian "
183 : "matrix due to not having enough nonzeros");
184 574623 : params.addParam<bool>("ignore_zeros_in_jacobian",
185 383082 : false,
186 : "Do not explicitly store zero values in "
187 : "the Jacobian matrix if true");
188 574623 : params.addParam<bool>("force_restart",
189 383082 : false,
190 : "EXPERIMENTAL: If true, a sub_app may use a "
191 : "restart file instead of using of using the master "
192 : "backup file");
193 957705 : params.addDeprecatedParam<bool>("skip_additional_restart_data",
194 383082 : false,
195 : "True to skip additional data in equation system for restart.",
196 : "This parameter is no longer used, as we do not load additional "
197 : "vectors by default with restart");
198 574623 : params.addParam<bool>("skip_nl_system_check",
199 383082 : false,
200 : "True to skip the NonlinearSystem check for work to do (e.g. Make sure "
201 : "that there are variables to solve for).");
202 574623 : params.addParam<bool>("allow_initial_conditions_with_restart",
203 383082 : false,
204 : "True to allow the user to specify initial conditions when restarting. "
205 : "Initial conditions can override any restarted field");
206 :
207 383082 : auto coverage_check_description = [](std::string scope, std::string list_param_name)
208 : {
209 766164 : return "Controls, if and how a " + scope +
210 : " subdomain coverage check is performed. "
211 : "With 'TRUE' or 'ON' all subdomains are checked (the default). Setting 'FALSE' or 'OFF' "
212 : "will disable the check for all subdomains. "
213 : "To exclude a predefined set of subdomains 'SKIP_LIST' is to "
214 766164 : "be used, while the subdomains to skip are to be defined in the parameter '" +
215 766164 : list_param_name +
216 : "'. To limit the check to a list of subdomains, 'ONLY_LIST' is to "
217 1149246 : "be used (again, using the parameter '" +
218 766164 : list_param_name + "').";
219 : };
220 :
221 957705 : params.addParam<std::vector<SubdomainName>>(
222 : "block",
223 : {"ANY_BLOCK_ID"},
224 : "List of subdomains for kernel coverage and material coverage checks. Setting this parameter "
225 : "is equivalent to setting 'kernel_coverage_block_list' and 'material_coverage_block_list' as "
226 : "well as using 'ONLY_LIST' as the coverage check mode.");
227 :
228 766164 : MooseEnum kernel_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
229 191541 : params.addParam<MooseEnum>("kernel_coverage_check",
230 : kernel_coverage_check_modes,
231 957705 : coverage_check_description("kernel", "kernel_coverage_block_list"));
232 766164 : params.addParam<std::vector<SubdomainName>>(
233 : "kernel_coverage_block_list",
234 : {},
235 : "List of subdomains for kernel coverage check. The meaning of this list is controlled by the "
236 : "parameter 'kernel_coverage_check' (whether this is the list of subdomains to be checked, "
237 : "not to be checked or not taken into account).");
238 574623 : params.addParam<bool>(
239 : "boundary_restricted_node_integrity_check",
240 383082 : true,
241 : "Set to false to disable checking of boundary restricted nodal object variable dependencies, "
242 : "e.g. are the variable dependencies defined on the selected boundaries?");
243 574623 : params.addParam<bool>("boundary_restricted_elem_integrity_check",
244 383082 : true,
245 : "Set to false to disable checking of boundary restricted elemental object "
246 : "variable dependencies, e.g. are the variable dependencies defined on the "
247 : "selected boundaries?");
248 766164 : MooseEnum material_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
249 191541 : params.addParam<MooseEnum>(
250 : "material_coverage_check",
251 : material_coverage_check_modes,
252 957705 : coverage_check_description("material", "material_coverage_block_list"));
253 766164 : params.addParam<std::vector<SubdomainName>>(
254 : "material_coverage_block_list",
255 : {},
256 : "List of subdomains for material coverage check. The meaning of this list is controlled by "
257 : "the parameter 'material_coverage_check' (whether this is the list of subdomains to be "
258 : "checked, not to be checked or not taken into account).");
259 :
260 574623 : params.addParam<bool>("fv_bcs_integrity_check",
261 383082 : true,
262 : "Set to false to disable checking of overlapping Dirichlet and Flux BCs "
263 : "and/or multiple DirichletBCs per sideset");
264 :
265 574623 : params.addParam<bool>(
266 383082 : "material_dependency_check", true, "Set to false to disable material dependency check");
267 574623 : params.addParam<bool>("parallel_barrier_messaging",
268 383082 : false,
269 : "Displays messaging from parallel "
270 : "barrier notifications when executing "
271 : "or transferring to/from Multiapps "
272 : "(default: false)");
273 :
274 766164 : MooseEnum verbosity("false true extra", "false");
275 766164 : params.addParam<MooseEnum>("verbose_setup",
276 : verbosity,
277 : "Set to 'true' to have the problem report on any object created. Set "
278 : "to 'extra' to also display all parameters.");
279 574623 : params.addParam<bool>("verbose_multiapps",
280 383082 : false,
281 : "Set to True to enable verbose screen printing related to MultiApps");
282 574623 : params.addParam<bool>(
283 : "verbose_restore",
284 383082 : false,
285 : "Set to True to enable verbose screen printing related to solution restoration");
286 :
287 766164 : params.addParam<FileNameNoExtension>("restart_file_base",
288 : "File base name used for restart (e.g. "
289 : "<path>/<filebase> or <path>/LATEST to "
290 : "grab the latest file available)");
291 :
292 766164 : params.addParam<std::vector<std::vector<TagName>>>(
293 : "extra_tag_vectors",
294 : {},
295 : "Extra vectors to add to the system that can be filled by objects which compute residuals "
296 : "and Jacobians (Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
297 : "nonlinear system the extra tag vectors should be added for");
298 :
299 766164 : params.addParam<std::vector<std::vector<TagName>>>(
300 : "not_zeroed_tag_vectors",
301 : {},
302 : "Extra vector tags which the sytem will not zero when other vector tags are zeroed. "
303 : "The outer index is for which nonlinear system the extra tag vectors should be added for");
304 :
305 766164 : params.addParam<std::vector<std::vector<TagName>>>(
306 : "extra_tag_matrices",
307 : {},
308 : "Extra matrices to add to the system that can be filled "
309 : "by objects which compute residuals and Jacobians "
310 : "(Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
311 : "nonlinear system the extra tag vectors should be added for");
312 :
313 766164 : params.addParam<std::vector<TagName>>(
314 : "extra_tag_solutions",
315 : {},
316 : "Extra solution vectors to add to the system that can be used by "
317 : "objects for coupling variable values stored in them.");
318 :
319 574623 : params.addParam<bool>("previous_nl_solution_required",
320 383082 : false,
321 : "True to indicate that this calculation requires a solution vector for "
322 : "storing the previous nonlinear iteration.");
323 :
324 574623 : params.addParam<std::vector<NonlinearSystemName>>(
325 766164 : "nl_sys_names", std::vector<NonlinearSystemName>{"nl0"}, "The nonlinear system names");
326 :
327 766164 : params.addParam<std::vector<LinearSystemName>>("linear_sys_names", {}, "The linear system names");
328 :
329 574623 : params.addParam<bool>("check_uo_aux_state",
330 383082 : false,
331 : "True to turn on a check that no state presents during the evaluation of "
332 : "user objects and aux kernels");
333 :
334 191541 : params.addPrivateParam<MooseMesh *>("mesh");
335 :
336 574623 : params.declareControllable("solve");
337 :
338 574623 : params.addParam<bool>(
339 : "allow_invalid_solution",
340 383082 : false,
341 : "Set to true to allow convergence even though the solution has been marked as 'invalid'");
342 574623 : params.addParam<bool>("show_invalid_solution_console",
343 383082 : true,
344 : "Set to true to show the invalid solution occurrence summary in console");
345 574623 : params.addParam<bool>("immediately_print_invalid_solution",
346 383082 : false,
347 : "Whether or not to report invalid solution warnings at the time the "
348 : "warning is produced instead of after the calculation");
349 :
350 574623 : params.addParam<bool>(
351 : "identify_variable_groups_in_nl",
352 383082 : true,
353 : "Whether to identify variable groups in nonlinear systems. This affects dof ordering");
354 :
355 574623 : params.addParam<bool>(
356 : "regard_general_exceptions_as_errors",
357 383082 : false,
358 : "If we catch an exception during residual/Jacobian evaluaton for which we don't have "
359 : "specific handling, immediately error instead of allowing the time step to be cut");
360 :
361 574623 : params.addParam<bool>("use_hash_table_matrix_assembly",
362 383082 : false,
363 : "Whether to assemble matrices using hash tables instead of preallocating "
364 : "matrix memory. This can be a good option if the sparsity pattern changes "
365 : "throughout the course of the simulation.");
366 766164 : params.addParam<bool>(
367 : "restore_original_nonzero_pattern",
368 : "Whether we should reset matrix memory for every Jacobian evaluation. This option is useful "
369 : "if the sparsity pattern is constantly changing and you are using hash table assembly or if "
370 : "you wish to continually restore the matrix to the originally preallocated sparsity pattern "
371 : "computed by relationship managers.");
372 :
373 766164 : params.addParamNamesToGroup(
374 : "skip_nl_system_check kernel_coverage_check kernel_coverage_block_list "
375 : "boundary_restricted_node_integrity_check "
376 : "boundary_restricted_elem_integrity_check material_coverage_check "
377 : "material_coverage_block_list fv_bcs_integrity_check "
378 : "material_dependency_check check_uo_aux_state error_on_jacobian_nonzero_reallocation",
379 : "Simulation checks");
380 766164 : params.addParamNamesToGroup("use_nonlinear previous_nl_solution_required nl_sys_names "
381 : "ignore_zeros_in_jacobian identify_variable_groups_in_nl "
382 : "use_hash_table_matrix_assembly restore_original_nonzero_pattern",
383 : "Nonlinear system(s)");
384 766164 : params.addParamNamesToGroup(
385 : "restart_file_base force_restart allow_initial_conditions_with_restart", "Restart");
386 766164 : params.addParamNamesToGroup(
387 : "verbose_setup verbose_multiapps verbose_restore parallel_barrier_messaging", "Verbosity");
388 766164 : params.addParamNamesToGroup(
389 : "null_space_dimension transpose_null_space_dimension near_null_space_dimension",
390 : "Null space removal");
391 766164 : params.addParamNamesToGroup(
392 : "extra_tag_vectors extra_tag_matrices extra_tag_solutions not_zeroed_tag_vectors",
393 : "Contribution to tagged field data");
394 574623 : params.addParamNamesToGroup(
395 : "allow_invalid_solution show_invalid_solution_console immediately_print_invalid_solution",
396 : "Solution validity control");
397 :
398 383082 : return params;
399 574623 : }
400 :
401 62202 : FEProblemBase::FEProblemBase(const InputParameters & parameters)
402 : : SubProblem(parameters),
403 : Restartable(this, "FEProblemBase"),
404 248808 : _mesh(*getCheckedPointerParam<MooseMesh *>("mesh")),
405 124404 : _req(declareManagedRestartableDataWithContext<RestartableEquationSystems>(
406 : "equation_systems", nullptr, _mesh)),
407 62202 : _initialized(false),
408 124404 : _solve(getParam<bool>("solve")),
409 62202 : _transient(false),
410 124404 : _time(declareRestartableData<Real>("time")),
411 124404 : _time_old(declareRestartableData<Real>("time_old")),
412 124404 : _t_step(declareRecoverableData<int>("t_step")),
413 124404 : _dt(declareRestartableData<Real>("dt")),
414 124404 : _dt_old(declareRestartableData<Real>("dt_old")),
415 62202 : _need_to_add_default_nonlinear_convergence(false),
416 62202 : _need_to_add_default_multiapp_fixed_point_convergence(false),
417 62202 : _need_to_add_default_steady_state_convergence(false),
418 124404 : _linear_sys_names(getParam<std::vector<LinearSystemName>>("linear_sys_names")),
419 62202 : _num_linear_sys(_linear_sys_names.size()),
420 124404 : _linear_systems(_num_linear_sys, nullptr),
421 62202 : _current_linear_sys(nullptr),
422 124404 : _using_default_nl(!isParamSetByUser("nl_sys_names")),
423 184646 : _nl_sys_names(!_using_default_nl || (_using_default_nl && !_linear_sys_names.size())
424 62202 : ? getParam<std::vector<NonlinearSystemName>>("nl_sys_names")
425 : : std::vector<NonlinearSystemName>()),
426 62202 : _num_nl_sys(_nl_sys_names.size()),
427 124404 : _nl(_num_nl_sys, nullptr),
428 62202 : _current_nl_sys(nullptr),
429 124404 : _solver_systems(_num_nl_sys + _num_linear_sys, nullptr),
430 62202 : _aux(nullptr),
431 62202 : _coupling(Moose::COUPLING_DIAG),
432 : #ifdef MOOSE_KOKKOS_ENABLED
433 47018 : _kokkos_assembly(*this),
434 : #endif
435 62202 : _mesh_divisions(/*threaded=*/true),
436 62202 : _material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
437 62202 : "material_props", &_mesh, _material_prop_registry, *this)),
438 62202 : _bnd_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
439 62202 : "bnd_material_props", &_mesh, _material_prop_registry, *this)),
440 62202 : _neighbor_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
441 62202 : "neighbor_material_props", &_mesh, _material_prop_registry, *this)),
442 : #ifdef MOOSE_KOKKOS_ENABLED
443 47018 : _kokkos_material_props(
444 47018 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
445 47018 : "kokkos_material_props", &_mesh, _material_prop_registry, *this)),
446 47018 : _kokkos_bnd_material_props(
447 47018 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
448 47018 : "kokkos_bnd_material_props", &_mesh, _material_prop_registry, *this)),
449 47018 : _kokkos_neighbor_material_props(
450 47018 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
451 47018 : "kokkos_neighbor_material_props", &_mesh, _material_prop_registry, *this)),
452 : #endif
453 62202 : _reporter_data(_app),
454 62202 : _multi_apps(_app.getExecuteOnEnum()),
455 62202 : _transient_multi_apps(_app.getExecuteOnEnum()),
456 62202 : _transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
457 62202 : _to_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
458 62202 : _from_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
459 62202 : _between_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
460 : #ifdef LIBMESH_ENABLE_AMR
461 62202 : _adaptivity(*this),
462 62202 : _cycles_completed(0),
463 : #endif
464 62202 : _displaced_mesh(nullptr),
465 62202 : _geometric_search_data(*this, _mesh),
466 62202 : _mortar_data(std::make_unique<MortarInterfaceWarehouse>(*this)),
467 62202 : _reinit_displaced_elem(false),
468 62202 : _reinit_displaced_face(false),
469 62202 : _reinit_displaced_neighbor(false),
470 62202 : _input_file_saved(false),
471 62202 : _has_dampers(false),
472 62202 : _has_constraints(false),
473 62202 : _snesmf_reuse_base(true),
474 62202 : _skip_exception_check(false),
475 62202 : _snesmf_reuse_base_set_by_user(false),
476 62202 : _has_initialized_stateful(false),
477 62202 : _const_jacobian(false),
478 62202 : _has_jacobian(false),
479 62202 : _needs_old_newton_iter(false),
480 124404 : _previous_nl_solution_required(getParam<bool>("previous_nl_solution_required")),
481 124404 : _previous_multiapp_fp_nl_solution_required(_num_nl_sys + _num_linear_sys, false),
482 62202 : _previous_multiapp_fp_aux_solution_required(false),
483 62202 : _has_nonlocal_coupling(false),
484 62202 : _calculate_jacobian_in_uo(false),
485 62202 : _kernel_coverage_check(
486 124404 : getParam<MooseEnum>("kernel_coverage_check").getEnum<CoverageCheckMode>()),
487 124404 : _kernel_coverage_blocks(getParam<std::vector<SubdomainName>>("kernel_coverage_block_list")),
488 62202 : _boundary_restricted_node_integrity_check(
489 124404 : getParam<bool>("boundary_restricted_node_integrity_check")),
490 62202 : _boundary_restricted_elem_integrity_check(
491 124404 : getParam<bool>("boundary_restricted_elem_integrity_check")),
492 62202 : _material_coverage_check(
493 124404 : getParam<MooseEnum>("material_coverage_check").getEnum<CoverageCheckMode>()),
494 124404 : _material_coverage_blocks(getParam<std::vector<SubdomainName>>("material_coverage_block_list")),
495 124404 : _fv_bcs_integrity_check(getParam<bool>("fv_bcs_integrity_check")),
496 124404 : _material_dependency_check(getParam<bool>("material_dependency_check")),
497 124404 : _uo_aux_state_check(getParam<bool>("check_uo_aux_state")),
498 : #ifndef NDEBUG
499 : _check_residual_for_nans(false),
500 : #endif
501 62202 : _max_qps(std::numeric_limits<unsigned int>::max()),
502 62202 : _max_scalar_order(INVALID_ORDER),
503 62202 : _has_time_integrator(false),
504 62202 : _has_exception(false),
505 124404 : _parallel_barrier_messaging(getParam<bool>("parallel_barrier_messaging")),
506 124404 : _verbose_setup(getParam<MooseEnum>("verbose_setup")),
507 124404 : _verbose_multiapps(getParam<bool>("verbose_multiapps")),
508 124404 : _verbose_restore(getParam<bool>("verbose_restore")),
509 62202 : _current_execute_on_flag(EXEC_NONE),
510 62202 : _control_warehouse(_app.getExecuteOnEnum(), /*threaded=*/false),
511 62202 : _is_petsc_options_inserted(false),
512 62202 : _line_search(nullptr),
513 62202 : _using_ad_mat_props(false),
514 62202 : _current_ic_state(0),
515 124404 : _use_hash_table_matrix_assembly(getParam<bool>("use_hash_table_matrix_assembly")),
516 62202 : _error_on_jacobian_nonzero_reallocation(
517 124404 : isParamValid("error_on_jacobian_nonzero_reallocation")
518 124931 : ? getParam<bool>("error_on_jacobian_nonzero_reallocation")
519 61675 : : _app.errorOnJacobianNonzeroReallocation()),
520 124404 : _restore_original_nonzero_pattern(isParamValid("restore_original_nonzero_pattern")
521 124404 : ? getParam<bool>("restore_original_nonzero_pattern")
522 62202 : : _use_hash_table_matrix_assembly),
523 124404 : _ignore_zeros_in_jacobian(getParam<bool>("ignore_zeros_in_jacobian")),
524 62202 : _preserve_matrix_sparsity_pattern(true),
525 124404 : _force_restart(getParam<bool>("force_restart")),
526 124404 : _allow_ics_during_restart(getParam<bool>("allow_initial_conditions_with_restart")),
527 124404 : _skip_nl_system_check(getParam<bool>("skip_nl_system_check")),
528 62202 : _fail_next_system_convergence_check(false),
529 124404 : _allow_invalid_solution(getParam<bool>("allow_invalid_solution")),
530 124404 : _show_invalid_solution_console(getParam<bool>("show_invalid_solution_console")),
531 124404 : _immediately_print_invalid_solution(getParam<bool>("immediately_print_invalid_solution")),
532 62202 : _started_initial_setup(false),
533 62202 : _has_internal_edge_residual_objects(false),
534 62202 : _u_dot_requested(false),
535 62202 : _u_dotdot_requested(false),
536 62202 : _u_dot_old_requested(false),
537 62202 : _u_dotdot_old_requested(false),
538 62202 : _has_mortar(false),
539 62202 : _num_grid_steps(0),
540 62202 : _print_execution_on(),
541 124404 : _identify_variable_groups_in_nl(getParam<bool>("identify_variable_groups_in_nl")),
542 62202 : _regard_general_exceptions_as_errors(getParam<bool>("regard_general_exceptions_as_errors")),
543 1181838 : _requires_nonlocal_coupling(false)
544 : {
545 : auto checkCoverageCheckConflict =
546 124404 : [this](const std::string & coverage_check,
547 : const CoverageCheckMode & coverage_check_mode,
548 : const std::vector<SubdomainName> & coverage_blocks) -> void
549 : {
550 124404 : if (coverage_check_mode != CoverageCheckMode::FALSE &&
551 120001 : coverage_check_mode != CoverageCheckMode::OFF)
552 119983 : if (coverage_blocks.size() > 1)
553 0 : if (std::find(coverage_blocks.begin(), coverage_blocks.end(), "ANY_BLOCK_ID") !=
554 0 : coverage_blocks.end())
555 0 : paramError(coverage_check,
556 : "The list of blocks used for ",
557 : coverage_check,
558 : " cannot contain 'ANY_BLOCK_ID' along with other blocks. ");
559 124404 : };
560 :
561 62202 : checkCoverageCheckConflict(
562 62202 : "kernel_coverage_check", _kernel_coverage_check, _kernel_coverage_blocks);
563 62202 : checkCoverageCheckConflict(
564 62202 : "material_coverage_check", _material_coverage_check, _material_coverage_blocks);
565 :
566 : // Initialize static do_derivatives member. We initialize this to true so that all the
567 : // default AD things that we setup early in the simulation actually get their derivative
568 : // vectors initalized. We will toggle this to false when doing residual evaluations
569 62202 : ADReal::do_derivatives = true;
570 :
571 : // Disable refinement/coarsening in EquationSystems::reinit because we already do this ourselves
572 62202 : es().disable_refine_in_reinit();
573 :
574 62202 : _solver_params.reserve(_num_nl_sys + _num_linear_sys);
575 : // Default constructor fine for nonlinear because it will be populated later by framework
576 : // executioner/solve object parameters
577 62202 : _solver_params.resize(_num_nl_sys);
578 123655 : for (const auto i : index_range(_nl_sys_names))
579 : {
580 61453 : const auto & name = _nl_sys_names[i];
581 61453 : _nl_sys_name_to_num[name] = i;
582 61453 : _solver_sys_name_to_num[name] = i;
583 61453 : _solver_sys_names.push_back(name);
584 : }
585 :
586 63234 : for (const auto i : index_range(_linear_sys_names))
587 : {
588 1032 : const auto & name = _linear_sys_names[i];
589 1032 : _linear_sys_name_to_num[name] = i;
590 1032 : _solver_sys_name_to_num[name] = i + _num_nl_sys;
591 1032 : _solver_sys_names.push_back(name);
592 : // Unlike for nonlinear these are basically dummy parameters
593 1032 : _solver_params.push_back(makeLinearSolverParams());
594 : }
595 :
596 62202 : _nonlocal_cm.resize(numSolverSystems());
597 62202 : _cm.resize(numSolverSystems());
598 :
599 62202 : _time = 0.0;
600 62202 : _time_old = 0.0;
601 62202 : _t_step = 0;
602 62202 : _dt = 0;
603 62202 : _dt_old = _dt;
604 :
605 62202 : unsigned int n_threads = libMesh::n_threads();
606 :
607 62202 : _real_zero.resize(n_threads, 0.);
608 62202 : _scalar_zero.resize(n_threads);
609 62202 : _zero.resize(n_threads);
610 62202 : _phi_zero.resize(n_threads);
611 62202 : _ad_zero.resize(n_threads);
612 62202 : _grad_zero.resize(n_threads);
613 62202 : _ad_grad_zero.resize(n_threads);
614 62202 : _grad_phi_zero.resize(n_threads);
615 62202 : _second_zero.resize(n_threads);
616 62202 : _ad_second_zero.resize(n_threads);
617 62202 : _second_phi_zero.resize(n_threads);
618 62202 : _point_zero.resize(n_threads);
619 62202 : _vector_zero.resize(n_threads);
620 62202 : _vector_curl_zero.resize(n_threads);
621 62202 : _uo_jacobian_moose_vars.resize(n_threads);
622 :
623 62202 : _has_active_material_properties.resize(n_threads, 0);
624 :
625 62202 : _block_mat_side_cache.resize(n_threads);
626 62202 : _bnd_mat_side_cache.resize(n_threads);
627 62202 : _interface_mat_side_cache.resize(n_threads);
628 :
629 124404 : es().parameters.set<FEProblemBase *>("_fe_problem_base") = this;
630 :
631 186606 : if (isParamValid("restart_file_base"))
632 : {
633 934 : std::string restart_file_base = getParam<FileNameNoExtension>("restart_file_base");
634 :
635 : // This check reverts to old behavior of providing "restart_file_base=" to mean
636 : // don't restart... BISON currently relies on this. It could probably be removed.
637 : // The new MooseUtils::convertLatestCheckpoint will error out if a checkpoint file
638 : // is not found, which I think makes sense. Which means, without this, if you
639 : // set "restart_file_base=", you'll get a "No checkpoint file found" error
640 467 : if (restart_file_base.size())
641 : {
642 467 : restart_file_base = MooseUtils::convertLatestCheckpoint(restart_file_base);
643 467 : setRestartFile(restart_file_base);
644 : }
645 467 : }
646 :
647 : // // Generally speaking, the mesh is prepared for use, and consequently remote elements are deleted
648 : // // well before our Problem(s) are constructed. Historically, in MooseMesh we have a bunch of
649 : // // needs_prepare type flags that make it so we never call prepare_for_use (and consequently
650 : // // delete_remote_elements) again. So the below line, historically, has had no impact. HOWEVER:
651 : // // I've added some code in SetupMeshCompleteAction for deleting remote elements post
652 : // // EquationSystems::init. If I execute that code without default ghosting, then I get > 40 MOOSE
653 : // // test failures, so we clearly have some simulations that are not yet covered properly by
654 : // // relationship managers. Until that is resolved, I am going to retain default geometric ghosting
655 : // if (!_default_ghosting)
656 : // _mesh.getMesh().remove_ghosting_functor(_mesh.getMesh().default_ghosting());
657 :
658 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
659 : // Main app should hold the default database to handle system petsc options
660 62202 : if (!_app.isUltimateMaster())
661 12213 : LibmeshPetscCall(PetscOptionsCreate(&_petsc_option_data_base));
662 : #endif
663 :
664 62202 : if (!_solve)
665 : {
666 : // If we are not solving, we do not care about seeing unused petsc options
667 49929 : Moose::PetscSupport::setSinglePetscOption("-options_left", "0");
668 : // We don't want petscSetOptions being called in solve and clearing the option that was just set
669 16643 : _is_petsc_options_inserted = true;
670 : }
671 62202 : }
672 :
673 : const MooseMesh &
674 0 : FEProblemBase::mesh(bool use_displaced) const
675 : {
676 0 : if (use_displaced && !_displaced_problem)
677 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
678 : "Regular mesh will be returned");
679 0 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
680 : }
681 :
682 : MooseMesh &
683 359630 : FEProblemBase::mesh(bool use_displaced)
684 : {
685 359630 : if (use_displaced && !_displaced_problem)
686 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
687 : "Regular mesh will be returned");
688 359630 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
689 : }
690 :
691 : void
692 62202 : FEProblemBase::createTagVectors()
693 : {
694 : // add vectors and their tags to system
695 124404 : auto & vectors = getParam<std::vector<std::vector<TagName>>>("extra_tag_vectors");
696 62948 : for (const auto sys_num : index_range(vectors))
697 1800 : for (auto & vector : vectors[sys_num])
698 : {
699 1054 : auto tag = addVectorTag(vector);
700 1054 : _solver_systems[sys_num]->addVector(tag, false, libMesh::GHOSTED);
701 : }
702 :
703 124404 : auto & not_zeroed_vectors = getParam<std::vector<std::vector<TagName>>>("not_zeroed_tag_vectors");
704 62213 : for (const auto sys_num : index_range(not_zeroed_vectors))
705 22 : for (auto & vector : not_zeroed_vectors[sys_num])
706 : {
707 11 : auto tag = addVectorTag(vector);
708 11 : _solver_systems[sys_num]->addVector(tag, false, GHOSTED);
709 11 : addNotZeroedVectorTag(tag);
710 : }
711 62202 : }
712 :
713 : void
714 61306 : FEProblemBase::createTagMatrices(CreateTaggedMatrixKey)
715 : {
716 122612 : auto & matrices = getParam<std::vector<std::vector<TagName>>>("extra_tag_matrices");
717 61552 : for (const auto sys_num : index_range(matrices))
718 692 : for (auto & matrix : matrices[sys_num])
719 : {
720 446 : auto tag = addMatrixTag(matrix);
721 446 : _solver_systems[sys_num]->addMatrix(tag);
722 : }
723 :
724 122871 : for (auto & sys : _solver_systems)
725 61565 : sys->sizeVariableMatrixData();
726 61306 : _aux->sizeVariableMatrixData();
727 61306 : }
728 :
729 : void
730 62202 : FEProblemBase::createTagSolutions()
731 : {
732 186631 : for (auto & vector : getParam<std::vector<TagName>>("extra_tag_solutions"))
733 : {
734 25 : auto tag = addVectorTag(vector, Moose::VECTOR_TAG_SOLUTION);
735 50 : for (auto & sys : _solver_systems)
736 25 : sys->addVector(tag, false, libMesh::GHOSTED);
737 25 : _aux->addVector(tag, false, libMesh::GHOSTED);
738 : }
739 :
740 62202 : if (_previous_nl_solution_required)
741 : {
742 : // We'll populate the zeroth state of the nonlinear iterations with the current solution for
743 : // ease of use in doing things like copying solutions backwards. We're just storing pointers in
744 : // the solution states containers so populating the zeroth state does not cost us the memory of
745 : // a new vector
746 83 : needSolutionState(1, Moose::SolutionIterationType::Nonlinear);
747 : }
748 :
749 62202 : auto tag = addVectorTag(Moose::SOLUTION_TAG, Moose::VECTOR_TAG_SOLUTION);
750 124687 : for (auto & sys : _solver_systems)
751 62485 : sys->associateVectorToTag(*sys->system().current_local_solution.get(), tag);
752 62202 : _aux->associateVectorToTag(*_aux->system().current_local_solution.get(), tag);
753 62202 : }
754 :
755 : void
756 135 : FEProblemBase::needSolutionState(unsigned int state, Moose::SolutionIterationType iteration_type)
757 : {
758 270 : for (auto & sys : _solver_systems)
759 135 : sys->needSolutionState(state, iteration_type);
760 135 : _aux->needSolutionState(state, iteration_type);
761 135 : }
762 :
763 : bool
764 48 : FEProblemBase::hasSolutionState(unsigned int state,
765 : Moose::SolutionIterationType iteration_type) const
766 : {
767 48 : bool has_solution_state = false;
768 107 : for (auto & sys : _solver_systems)
769 59 : has_solution_state |= sys->hasSolutionState(state, iteration_type);
770 48 : has_solution_state |= _aux->hasSolutionState(state, iteration_type);
771 48 : return has_solution_state;
772 : }
773 :
774 : void
775 62202 : FEProblemBase::newAssemblyArray(std::vector<std::shared_ptr<SolverSystem>> & solver_systems)
776 : {
777 62202 : unsigned int n_threads = libMesh::n_threads();
778 :
779 62202 : _assembly.resize(n_threads);
780 131272 : for (const auto i : make_range(n_threads))
781 : {
782 69070 : _assembly[i].resize(solver_systems.size());
783 138449 : for (const auto j : index_range(solver_systems))
784 69379 : _assembly[i][j] = std::make_unique<Assembly>(*solver_systems[j], i);
785 : }
786 62202 : }
787 :
788 : void
789 59300 : FEProblemBase::initNullSpaceVectors(const InputParameters & parameters,
790 : std::vector<std::shared_ptr<NonlinearSystemBase>> & nls)
791 : {
792 296500 : TIME_SECTION("initNullSpaceVectors", 5, "Initializing Null Space Vectors");
793 :
794 59300 : unsigned int dimNullSpace = parameters.get<unsigned int>("null_space_dimension");
795 : unsigned int dimTransposeNullSpace =
796 59300 : parameters.get<unsigned int>("transpose_null_space_dimension");
797 59300 : unsigned int dimNearNullSpace = parameters.get<unsigned int>("near_null_space_dimension");
798 59324 : for (unsigned int i = 0; i < dimNullSpace; ++i)
799 : {
800 24 : std::ostringstream oss;
801 24 : oss << "_" << i;
802 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
803 : // builder might march over all nodes
804 48 : for (auto & nl : nls)
805 24 : nl->addVector("NullSpace" + oss.str(), false, libMesh::GHOSTED);
806 24 : }
807 118600 : _subspace_dim["NullSpace"] = dimNullSpace;
808 59312 : for (unsigned int i = 0; i < dimTransposeNullSpace; ++i)
809 : {
810 12 : std::ostringstream oss;
811 12 : oss << "_" << i;
812 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
813 : // builder might march over all nodes
814 24 : for (auto & nl : nls)
815 12 : nl->addVector("TransposeNullSpace" + oss.str(), false, libMesh::GHOSTED);
816 12 : }
817 118600 : _subspace_dim["TransposeNullSpace"] = dimTransposeNullSpace;
818 59300 : for (unsigned int i = 0; i < dimNearNullSpace; ++i)
819 : {
820 0 : std::ostringstream oss;
821 0 : oss << "_" << i;
822 : // do not project, since this will be recomputed, but make it ghosted, since the near-nullspace
823 : // builder might march over all semilocal nodes
824 0 : for (auto & nl : nls)
825 0 : nl->addVector("NearNullSpace" + oss.str(), false, libMesh::GHOSTED);
826 0 : }
827 118600 : _subspace_dim["NearNullSpace"] = dimNearNullSpace;
828 59300 : }
829 :
830 177351 : FEProblemBase::~FEProblemBase()
831 : {
832 : // Flush the Console stream, the underlying call to Console::mooseConsole
833 : // relies on a call to Output::checkInterval that has references to
834 : // _time, etc. If it is not flushed here memory problems arise if you have
835 : // an unflushed stream and start destructing things.
836 59117 : _console << std::flush;
837 :
838 59117 : unsigned int n_threads = libMesh::n_threads();
839 123809 : for (unsigned int i = 0; i < n_threads; i++)
840 : {
841 64692 : _zero[i].release();
842 64692 : _phi_zero[i].release();
843 64692 : _scalar_zero[i].release();
844 64692 : _grad_zero[i].release();
845 64692 : _grad_phi_zero[i].release();
846 64692 : _second_zero[i].release();
847 64692 : _second_phi_zero[i].release();
848 64692 : _vector_zero[i].release();
849 64692 : _vector_curl_zero[i].release();
850 64692 : _ad_zero[i].release();
851 64692 : _ad_grad_zero[i].release();
852 64692 : _ad_second_zero[i].release();
853 : }
854 :
855 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
856 59117 : if (!_app.isUltimateMaster())
857 : {
858 11449 : auto ierr = PetscOptionsDestroy(&_petsc_option_data_base);
859 : // Don't throw on destruction
860 11449 : CHKERRABORT(this->comm().get(), ierr);
861 : }
862 : #endif
863 59117 : }
864 :
865 : void
866 0 : FEProblemBase::setCoordSystem(const std::vector<SubdomainName> & blocks,
867 : const MultiMooseEnum & coord_sys)
868 : {
869 0 : TIME_SECTION("setCoordSystem", 5, "Setting Coordinate System");
870 0 : _mesh.setCoordSystem(blocks, coord_sys);
871 0 : }
872 :
873 : void
874 0 : FEProblemBase::setAxisymmetricCoordAxis(const MooseEnum & rz_coord_axis)
875 : {
876 0 : _mesh.setAxisymmetricCoordAxis(rz_coord_axis);
877 0 : }
878 :
879 : const ConstElemRange &
880 1383 : FEProblemBase::getEvaluableElementRange()
881 : {
882 1383 : if (!_evaluable_local_elem_range)
883 : {
884 677 : std::vector<const DofMap *> dof_maps(es().n_systems());
885 2031 : for (const auto i : make_range(es().n_systems()))
886 : {
887 1354 : const auto & sys = es().get_system(i);
888 1354 : dof_maps[i] = &sys.get_dof_map();
889 : }
890 : _evaluable_local_elem_range =
891 1354 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
892 2031 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
893 677 : }
894 1383 : return *_evaluable_local_elem_range;
895 : }
896 :
897 : const ConstElemRange &
898 208 : FEProblemBase::getNonlinearEvaluableElementRange()
899 : {
900 208 : if (!_nl_evaluable_local_elem_range)
901 : {
902 208 : std::vector<const DofMap *> dof_maps(_nl.size());
903 416 : for (const auto i : index_range(dof_maps))
904 208 : dof_maps[i] = &_nl[i]->dofMap();
905 : _nl_evaluable_local_elem_range =
906 416 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
907 624 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
908 208 : }
909 :
910 208 : return *_nl_evaluable_local_elem_range;
911 : }
912 :
913 : void
914 59930 : FEProblemBase::initialSetup()
915 : {
916 299650 : TIME_SECTION("initialSetup", 2, "Performing Initial Setup");
917 :
918 59930 : SubProblem::initialSetup();
919 :
920 59930 : if (_app.isRecovering() + _app.isRestarting() + bool(_app.getExReaderForRestart()) > 1)
921 0 : mooseError("Checkpoint recovery and restart and exodus restart are all mutually exclusive.");
922 :
923 59930 : if (_skip_exception_check)
924 9 : mooseWarning("MOOSE may fail to catch an exception when the \"skip_exception_check\" parameter "
925 : "is used. If you receive a terse MPI error during execution, remove this "
926 : "parameter and rerun your simulation");
927 :
928 : // set state flag indicating that we are in or beyond initialSetup.
929 : // This can be used to throw errors in methods that _must_ be called at construction time.
930 59930 : _started_initial_setup = true;
931 59930 : setCurrentExecuteOnFlag(EXEC_INITIAL);
932 :
933 : // Setup the solution states (current, old, etc) in each system based on
934 : // its default and the states requested of each of its variables
935 120119 : for (const auto i : index_range(_solver_systems))
936 : {
937 60189 : _solver_systems[i]->initSolutionState();
938 60189 : if (getDisplacedProblem())
939 2013 : getDisplacedProblem()->solverSys(i).initSolutionState();
940 : }
941 59930 : _aux->initSolutionState();
942 59930 : if (getDisplacedProblem())
943 2013 : getDisplacedProblem()->auxSys().initSolutionState();
944 :
945 : // always execute to get the max number of DoF per element and node needed to initialize phi_zero
946 : // variables
947 59930 : dof_id_type global_max_var_n_dofs_per_elem = 0;
948 120119 : for (const auto i : index_range(_solver_systems))
949 : {
950 60189 : auto & sys = *_solver_systems[i];
951 : dof_id_type max_var_n_dofs_per_elem;
952 : dof_id_type max_var_n_dofs_per_node;
953 : {
954 300945 : TIME_SECTION("computingMaxDofs", 3, "Computing Max Dofs Per Element");
955 :
956 60189 : MaxVarNDofsPerElem mvndpe(*this, sys);
957 60189 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mvndpe);
958 60189 : max_var_n_dofs_per_elem = mvndpe.max();
959 60189 : _communicator.max(max_var_n_dofs_per_elem);
960 :
961 60189 : MaxVarNDofsPerNode mvndpn(*this, sys);
962 60189 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), mvndpn);
963 60189 : max_var_n_dofs_per_node = mvndpn.max();
964 60189 : _communicator.max(max_var_n_dofs_per_node);
965 60189 : global_max_var_n_dofs_per_elem =
966 60189 : std::max(global_max_var_n_dofs_per_elem, max_var_n_dofs_per_elem);
967 60189 : }
968 :
969 : {
970 300945 : TIME_SECTION("assignMaxDofs", 5, "Assigning Maximum Dofs Per Elem");
971 :
972 60189 : sys.assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
973 60189 : auto displaced_problem = getDisplacedProblem();
974 60189 : if (displaced_problem)
975 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
976 :
977 60189 : sys.assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
978 60189 : if (displaced_problem)
979 2013 : displaced_problem->solverSys(i).assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
980 60189 : }
981 : }
982 :
983 : {
984 299650 : TIME_SECTION("resizingVarValues", 5, "Resizing Variable Values");
985 :
986 125868 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
987 : {
988 131876 : _phi_zero[tid].resize(global_max_var_n_dofs_per_elem, std::vector<Real>(getMaxQps(), 0.));
989 131876 : _grad_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
990 131876 : std::vector<RealGradient>(getMaxQps(), RealGradient(0.)));
991 131876 : _second_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
992 131876 : std::vector<RealTensor>(getMaxQps(), RealTensor(0.)));
993 : }
994 59930 : }
995 :
996 : // Set up stateful material property redistribution, if we suspect
997 : // it may be necessary later.
998 59930 : addAnyRedistributers();
999 :
1000 59930 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1001 : {
1002 : // Only load all of the vectors if we're recovering
1003 4524 : _req.set().setLoadAllVectors(_app.isRecovering());
1004 :
1005 : // This forces stateful material property loading to be an exact one-to-one match
1006 4524 : if (_app.isRecovering())
1007 : {
1008 15984 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1009 11988 : props->setRecovering();
1010 :
1011 : #ifdef MOOSE_KOKKOS_ENABLED
1012 15844 : for (auto props :
1013 19805 : {&_kokkos_material_props, &_kokkos_bnd_material_props, &_kokkos_neighbor_material_props})
1014 11883 : props->setRecovering();
1015 : #endif
1016 : }
1017 :
1018 22620 : TIME_SECTION("restore", 3, "Restoring from backup");
1019 :
1020 : // We could have a cached backup when this app is a sub-app and has been given a Backup
1021 4524 : if (!_app.hasInitialBackup())
1022 3745 : _app.restore(_app.restartFolderBase(_app.getRestartRecoverFileBase()), _app.isRestarting());
1023 : else
1024 779 : _app.restoreFromInitialBackup(_app.isRestarting());
1025 :
1026 : /**
1027 : * If this is a restart run, the user may want to override the start time, which we already set
1028 : * in the constructor. "_time" however will have been "restored" from the restart file. We need
1029 : * to honor the original request of the developer now that the restore has been completed.
1030 : */
1031 4491 : if (_app.isRestarting())
1032 : {
1033 495 : if (_app.hasStartTime())
1034 173 : _time = _time_old = _app.getStartTime();
1035 : else
1036 322 : _time_old = _time;
1037 : }
1038 4491 : }
1039 : else
1040 : {
1041 55406 : libMesh::ExodusII_IO * reader = _app.getExReaderForRestart();
1042 :
1043 55406 : if (reader)
1044 : {
1045 1890 : TIME_SECTION("copyingFromExodus", 3, "Copying Variables From Exodus");
1046 :
1047 764 : for (auto & sys : _solver_systems)
1048 389 : sys->copyVars(*reader);
1049 375 : _aux->copyVars(*reader);
1050 375 : }
1051 : else
1052 : {
1053 55028 : if (_solver_systems[0]->hasVarCopy() || _aux->hasVarCopy())
1054 0 : mooseError("Need Exodus reader to restart variables but the reader is not available\n"
1055 : "Use either FileMesh with an Exodus mesh file or FileMeshGenerator with an "
1056 : "Exodus mesh file and with use_for_exodus_restart equal to true");
1057 : }
1058 : }
1059 :
1060 : // Perform output related setups
1061 59894 : _app.getOutputWarehouse().initialSetup();
1062 :
1063 : // Flush all output to _console that occur during construction and initialization of objects
1064 59848 : _app.getOutputWarehouse().mooseConsole();
1065 :
1066 : // Build Refinement and Coarsening maps for stateful material projections if necessary
1067 62113 : if ((_adaptivity.isOn() || _num_grid_steps) &&
1068 2265 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1069 2202 : _neighbor_material_props.hasStatefulProperties()))
1070 : {
1071 63 : if (_has_internal_edge_residual_objects)
1072 6 : mooseError("Stateful neighbor material properties do not work with mesh adaptivity");
1073 :
1074 57 : _mesh.buildRefinementAndCoarseningMaps(_assembly[0][0].get());
1075 : }
1076 :
1077 59842 : if (!_app.isRecovering())
1078 : {
1079 : /**
1080 : * If we are not recovering but we are doing restart (_app.getExodusFileRestart() == true) with
1081 : * additional uniform refinements. We have to delay the refinement until this point
1082 : * in time so that the equation systems are initialized and projections can be performed.
1083 : */
1084 55846 : if (_mesh.uniformRefineLevel() > 0 && _app.getExodusFileRestart())
1085 : {
1086 10 : if (!_app.isUltimateMaster())
1087 0 : mooseError(
1088 : "Doing extra refinements when restarting is NOT supported for sub-apps of a MultiApp");
1089 :
1090 10 : adaptivity().uniformRefineWithProjection();
1091 : }
1092 : }
1093 :
1094 59842 : unsigned int n_threads = libMesh::n_threads();
1095 :
1096 : // Convergence initial setup
1097 : {
1098 299210 : TIME_SECTION("convergenceInitialSetup", 5, "Initializing Convergence objects");
1099 :
1100 125641 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1101 65814 : _convergences.initialSetup(tid);
1102 59827 : }
1103 :
1104 : // UserObject initialSetup
1105 59827 : std::set<std::string> depend_objects_ic = _ics.getDependObjects();
1106 59827 : std::set<std::string> depend_objects_aux = _aux->getDependObjects();
1107 :
1108 59827 : std::map<int, std::vector<UserObjectBase *>> group_userobjs;
1109 :
1110 : // This replaces all prior updateDependObjects calls on the old user object warehouses.
1111 59827 : TheWarehouse::Query uo_query = theWarehouse().query().condition<AttribSystem>("UserObject");
1112 59827 : std::vector<UserObjectBase *> userobjs;
1113 59827 : uo_query.queryInto(userobjs);
1114 59827 : groupUserObjects(
1115 59827 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1116 :
1117 131849 : for (auto obj : userobjs)
1118 216066 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1119 :
1120 : #ifdef MOOSE_KOKKOS_ENABLED
1121 : {
1122 : TheWarehouse::Query uo_query =
1123 45270 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject");
1124 45270 : std::vector<UserObjectBase *> userobjs;
1125 45270 : uo_query.queryInto(userobjs);
1126 45270 : groupUserObjects(
1127 45270 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1128 :
1129 46495 : for (auto obj : userobjs)
1130 3675 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1131 45270 : }
1132 : #endif
1133 :
1134 90196 : for (auto & [group, objs] : group_userobjs)
1135 103563 : for (auto obj : objs)
1136 73194 : obj->initialSetup();
1137 :
1138 : // check if jacobian calculation is done in userobject
1139 125440 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1140 65693 : checkUserObjectJacobianRequirement(tid);
1141 :
1142 : // Check whether nonlocal coupling is required or not
1143 59747 : checkNonlocalCoupling();
1144 59747 : if (_requires_nonlocal_coupling)
1145 63 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
1146 :
1147 : {
1148 298735 : TIME_SECTION("initializingFunctions", 5, "Initializing Functions");
1149 :
1150 : // Call the initialSetup methods for functions
1151 125408 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1152 : {
1153 65685 : reinitScalars(tid); // initialize scalars so they are properly sized for use as input into
1154 : // ParsedFunctions
1155 65685 : _functions.initialSetup(tid);
1156 : }
1157 :
1158 : #ifdef MOOSE_KOKKOS_ENABLED
1159 45201 : _kokkos_functions.initialSetup();
1160 : #endif
1161 59723 : }
1162 :
1163 : {
1164 298615 : TIME_SECTION("initializingRandomObjects", 5, "Initializing Random Objects");
1165 :
1166 : // Random interface objects
1167 60047 : for (const auto & it : _random_data_objects)
1168 324 : it.second->updateSeeds(EXEC_INITIAL);
1169 59723 : }
1170 :
1171 59723 : if (!_app.isRecovering())
1172 : {
1173 55727 : computeUserObjects(EXEC_INITIAL, Moose::PRE_IC);
1174 :
1175 : {
1176 278635 : TIME_SECTION("ICinitialSetup", 5, "Setting Up Initial Conditions");
1177 :
1178 117339 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1179 61618 : _ics.initialSetup(tid);
1180 :
1181 55721 : _scalar_ics.initialSetup();
1182 55721 : }
1183 :
1184 55721 : projectSolution();
1185 : }
1186 :
1187 : // Materials
1188 59711 : if (_all_materials.hasActiveObjects(0))
1189 : {
1190 41905 : TIME_SECTION("materialInitialSetup", 3, "Setting Up Materials");
1191 :
1192 17489 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1193 : {
1194 : // Sort the Material objects, these will be actually computed by MOOSE in reinit methods.
1195 9126 : _materials.sort(tid);
1196 9123 : _interface_materials.sort(tid);
1197 :
1198 : // Call initialSetup on all material objects
1199 9123 : _all_materials.initialSetup(tid);
1200 :
1201 : // Discrete materials may insert additional dependencies on materials during the initial
1202 : // setup. Therefore we resolve the dependencies once more, now with the additional
1203 : // dependencies due to discrete materials.
1204 9108 : if (_discrete_materials.hasActiveObjects())
1205 : {
1206 56 : _materials.sort(tid);
1207 56 : _interface_materials.sort(tid);
1208 : }
1209 : }
1210 :
1211 : #ifdef MOOSE_KOKKOS_ENABLED
1212 6291 : _kokkos_materials.sort(0, true);
1213 : #endif
1214 :
1215 : {
1216 41805 : TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");
1217 :
1218 8361 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1219 :
1220 16047 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1221 7686 : _neighbor_material_props.hasStatefulProperties())
1222 675 : _has_initialized_stateful = true;
1223 : #ifdef MOOSE_KOKKOS_ENABLED
1224 6289 : if (_kokkos_material_props.hasStatefulProperties() ||
1225 12361 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1226 6072 : _kokkos_neighbor_material_props.hasStatefulProperties())
1227 217 : _has_initialized_stateful = true;
1228 : #endif
1229 8361 : }
1230 8361 : }
1231 :
1232 : // setRestartInPlace() is set because the property maps have now been setup and we can
1233 : // dataLoad() them directly in place
1234 : // setRecovering() is set because from now on we require a one-to-one mapping of
1235 : // stateful properties because we shouldn't be declaring any more
1236 238764 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1237 : {
1238 179073 : props->setRestartInPlace();
1239 179073 : props->setRecovering();
1240 : }
1241 :
1242 125309 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1243 : {
1244 65618 : _internal_side_indicators.initialSetup(tid);
1245 65618 : _indicators.initialSetup(tid);
1246 65618 : _markers.sort(tid);
1247 65618 : _markers.initialSetup(tid);
1248 : }
1249 :
1250 : #ifdef LIBMESH_ENABLE_AMR
1251 :
1252 59691 : if (!_app.isRecovering())
1253 : {
1254 55695 : unsigned int n = adaptivity().getInitialSteps();
1255 55695 : if (n && !_app.isUltimateMaster() && _app.isRestarting())
1256 0 : mooseError("Cannot perform initial adaptivity during restart on sub-apps of a MultiApp!");
1257 :
1258 55695 : initialAdaptMesh();
1259 : }
1260 :
1261 : #endif // LIBMESH_ENABLE_AMR
1262 :
1263 59688 : if (!_app.isRecovering() && !_app.isRestarting())
1264 : {
1265 : // During initial setup the solution is copied to the older solution states (old, older, etc)
1266 55197 : copySolutionsBackwards();
1267 :
1268 : // Check if there are old state initial conditions
1269 55197 : auto ics = _ics.getActiveObjects();
1270 55197 : auto fv_ics = _fv_ics.getActiveObjects();
1271 55197 : auto scalar_ics = _scalar_ics.getActiveObjects();
1272 55197 : unsigned short ic_state_max = 0;
1273 :
1274 165591 : auto findMax = [&ic_state_max](const auto & obj_list)
1275 : {
1276 196312 : for (auto ic : obj_list.getActiveObjects())
1277 30721 : ic_state_max = std::max(ic_state_max, ic->getState());
1278 220788 : };
1279 55197 : findMax(_ics);
1280 55197 : findMax(_fv_ics);
1281 55197 : findMax(_scalar_ics);
1282 :
1283 : // if there are old state ICs, compute them and write to old states accordingly
1284 55197 : if (ic_state_max > 0)
1285 : {
1286 : // state 0 copy (we'll overwrite current state when evaluating ICs and need to restore it once
1287 : // we're done with the old/older state ICs)
1288 0 : std::vector<std::unique_ptr<NumericVector<Real>>> state0_sys_buffers(_solver_systems.size());
1289 0 : std::unique_ptr<NumericVector<Real>> state0_aux_buffer;
1290 :
1291 : // save state 0
1292 0 : for (const auto i : index_range(_solver_systems))
1293 0 : state0_sys_buffers[i] = _solver_systems[i]->solutionState(0).clone();
1294 :
1295 0 : state0_aux_buffer = _aux->solutionState(0).clone();
1296 :
1297 : // compute old state ICs
1298 0 : for (_current_ic_state = 1; _current_ic_state <= ic_state_max; _current_ic_state++)
1299 : {
1300 0 : projectSolution();
1301 :
1302 0 : for (auto & sys : _solver_systems)
1303 0 : sys->solutionState(_current_ic_state) = sys->solutionState(0);
1304 :
1305 0 : _aux->solutionState(_current_ic_state) = _aux->solutionState(0);
1306 : }
1307 0 : _current_ic_state = 0;
1308 :
1309 : // recover state 0
1310 0 : for (const auto i : index_range(_solver_systems))
1311 : {
1312 0 : _solver_systems[i]->solutionState(0) = *state0_sys_buffers[i];
1313 0 : _solver_systems[i]->solutionState(0).close();
1314 0 : _solver_systems[i]->update();
1315 : }
1316 0 : _aux->solutionState(0) = *state0_aux_buffer;
1317 0 : _aux->solutionState(0).close();
1318 0 : _aux->update();
1319 0 : }
1320 55197 : }
1321 :
1322 59688 : if (!_app.isRecovering())
1323 : {
1324 55692 : if (haveXFEM())
1325 0 : updateMeshXFEM();
1326 : }
1327 :
1328 : // Call initialSetup on the solver systems
1329 119635 : for (auto & sys : _solver_systems)
1330 59947 : sys->initialSetup();
1331 :
1332 : // Auxilary variable initialSetup calls
1333 59688 : _aux->initialSetup();
1334 :
1335 59682 : if (_displaced_problem)
1336 : // initialSetup for displaced systems
1337 2013 : _displaced_problem->initialSetup();
1338 :
1339 119623 : for (auto & sys : _solver_systems)
1340 59941 : sys->setSolution(*(sys->system().current_local_solution.get()));
1341 :
1342 : // Update the nearest node searches (has to be called after the problem is all set up)
1343 : // We do this here because this sets up the Element's DoFs to ghost
1344 59682 : updateGeomSearch(GeometricSearchData::NEAREST_NODE);
1345 :
1346 59682 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
1347 59682 : if (_displaced_mesh)
1348 2013 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
1349 :
1350 : // We need to move the mesh in order to build a map between mortar secondary and primary
1351 : // interfaces. This map will then be used by the AgumentSparsityOnInterface ghosting functor to
1352 : // know which dofs we need ghosted when we call EquationSystems::reinit
1353 59682 : if (_displaced_problem && _mortar_data->hasDisplacedObjects())
1354 : {
1355 114 : _displaced_problem->updateMesh();
1356 : // if displacements were applied to the mesh, the mortar mesh should be updated too
1357 114 : updateMortarMesh();
1358 : }
1359 :
1360 : // Possibly reinit one more time to get ghosting correct
1361 59682 : reinitBecauseOfGhostingOrNewGeomObjects();
1362 :
1363 59682 : if (_displaced_mesh)
1364 2013 : _displaced_problem->updateMesh();
1365 :
1366 59682 : updateGeomSearch(); // Call all of the rest of the geometric searches
1367 :
1368 119620 : for (auto & sys : _solver_systems)
1369 : {
1370 59941 : const auto & tis = sys->getTimeIntegrators();
1371 :
1372 : {
1373 299705 : TIME_SECTION("timeIntegratorInitialSetup", 5, "Initializing Time Integrator");
1374 89683 : for (auto & ti : tis)
1375 29745 : ti->initialSetup();
1376 59938 : }
1377 : }
1378 :
1379 : // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ main-app restartable data
1380 : // has been restored
1381 :
1382 : // Call initialSetup on the MultiApps
1383 59679 : if (_multi_apps.hasObjects())
1384 : {
1385 35150 : TIME_SECTION("initialSetupMultiApps", 2, "Initializing MultiApps", false);
1386 7030 : _multi_apps.initialSetup();
1387 7024 : }
1388 :
1389 : // Call initialSetup on the transfers
1390 : {
1391 298365 : TIME_SECTION("initialSetupTransfers", 2, "Initializing Transfers");
1392 :
1393 59673 : _transfers.initialSetup();
1394 :
1395 : // Call initialSetup on the MultiAppTransfers to be executed on TO_MULTIAPP
1396 59673 : const auto & to_multi_app_objects = _to_multi_app_transfers.getActiveObjects();
1397 64797 : for (const auto & transfer : to_multi_app_objects)
1398 : {
1399 5172 : transfer->setCurrentDirection(Transfer::DIRECTION::TO_MULTIAPP);
1400 5172 : transfer->initialSetup();
1401 : }
1402 :
1403 : // Call initialSetup on the MultiAppTransfers to be executed on FROM_MULTIAPP
1404 59625 : const auto & from_multi_app_objects = _from_multi_app_transfers.getActiveObjects();
1405 65861 : for (const auto & transfer : from_multi_app_objects)
1406 : {
1407 6278 : transfer->setCurrentDirection(Transfer::DIRECTION::FROM_MULTIAPP);
1408 6278 : transfer->initialSetup();
1409 : }
1410 :
1411 : // Call initialSetup on the MultiAppTransfers to be executed on BETWEEN_MULTIAPP
1412 59583 : const auto & between_multi_app_objects = _between_multi_app_transfers.getActiveObjects();
1413 61053 : for (const auto & transfer : between_multi_app_objects)
1414 : {
1415 1470 : transfer->setCurrentDirection(Transfer::DIRECTION::BETWEEN_MULTIAPP);
1416 1470 : transfer->initialSetup();
1417 : }
1418 59583 : }
1419 :
1420 59583 : if (_boundary_restricted_node_integrity_check)
1421 : {
1422 178713 : TIME_SECTION("BoundaryRestrictedNodeIntegrityCheck", 5);
1423 :
1424 : // check that variables are defined along boundaries of boundary restricted nodal objects
1425 59571 : const auto & bnd_nodes = getCurrentAlgebraicBndNodeRange();
1426 59571 : BoundaryNodeIntegrityCheckThread bnict(*this, uo_query);
1427 59571 : Threads::parallel_reduce(bnd_nodes, bnict);
1428 :
1429 : // Nodal bcs aren't threaded
1430 118344 : for (auto & nl : _nl)
1431 : {
1432 58791 : const auto & nodal_bcs = nl->getNodalBCWarehouse();
1433 58791 : if (!nodal_bcs.hasBoundaryObjects())
1434 27466 : continue;
1435 :
1436 2469825 : for (const auto & bnode : bnd_nodes)
1437 : {
1438 2438509 : const auto boundary_id = bnode->_bnd_id;
1439 2438509 : const Node * const node = bnode->_node;
1440 :
1441 2438509 : if (node->processor_id() != this->processor_id())
1442 555094 : continue;
1443 :
1444 1883415 : const auto & bnd_name = _mesh.getBoundaryName(boundary_id);
1445 :
1446 : // Avoid assertion in getBoundaryObjects that we have boundary objects for this boundary ID
1447 1883415 : if (!nodal_bcs.hasBoundaryObjects(boundary_id))
1448 965683 : continue;
1449 :
1450 917732 : const auto & bnd_objects = nodal_bcs.getBoundaryObjects(boundary_id);
1451 1926892 : for (const auto & bnd_object : bnd_objects)
1452 : {
1453 1009169 : const auto & bnd_variable = bnd_object->variable();
1454 : // Skip if this object uses geometric search because coupled variables may be defined on
1455 : // paired boundaries instead of the boundary this node is on. Also skip if this boundary
1456 : // condition isn't applicable to the current node, e.g. if the node doesn't have any
1457 : // degrees of freedom for the boundary condition's variable
1458 1009169 : if (!bnd_object->requiresGeometricSearch() &&
1459 2018338 : bnd_object->checkVariableBoundaryIntegrity() &&
1460 1009169 : node->n_dofs(nl->number(), bnd_variable.number()))
1461 : {
1462 : std::set<MooseVariableFieldBase *> vars_to_omit = {
1463 : &static_cast<MooseVariableFieldBase &>(
1464 1926870 : const_cast<MooseVariableBase &>(bnd_variable))};
1465 :
1466 1926870 : boundaryIntegrityCheckError(
1467 1926861 : *bnd_object, bnd_object->checkAllVariables(*node, vars_to_omit), bnd_name);
1468 963426 : }
1469 : }
1470 : }
1471 : }
1472 59553 : }
1473 :
1474 59565 : if (_boundary_restricted_elem_integrity_check)
1475 : {
1476 178605 : TIME_SECTION("BoundaryRestrictedElemIntegrityCheck", 5);
1477 :
1478 : // check that variables are defined along boundaries of boundary restricted elemental objects
1479 59535 : ConstBndElemRange & bnd_elems = *mesh().getBoundaryElementRange();
1480 59535 : BoundaryElemIntegrityCheckThread beict(*this, uo_query);
1481 59535 : Threads::parallel_reduce(bnd_elems, beict);
1482 59523 : }
1483 :
1484 59553 : if (!_app.isRecovering())
1485 : {
1486 55557 : execTransfers(EXEC_INITIAL);
1487 :
1488 55557 : bool converged = execMultiApps(EXEC_INITIAL);
1489 55551 : if (!converged)
1490 3 : mooseError("failed to converge initial MultiApp");
1491 :
1492 : // We'll backup the Multiapp here
1493 55548 : backupMultiApps(EXEC_INITIAL);
1494 :
1495 116929 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1496 61381 : reinitScalars(tid);
1497 :
1498 55548 : execute(EXEC_INITIAL);
1499 :
1500 : // The FEProblemBase::execute method doesn't call all the systems on EXEC_INITIAL, but it does
1501 : // set/unset the current flag. Therefore, this resets the current flag to EXEC_INITIAL so that
1502 : // subsequent calls (e.g., executeControls) have the proper flag.
1503 55473 : setCurrentExecuteOnFlag(EXEC_INITIAL);
1504 : }
1505 :
1506 : // Here we will initialize the stateful properties once more since they may have been updated
1507 : // during initialSetup by calls to computeProperties.
1508 : //
1509 : // It's really bad that we don't allow this during restart. It means that we can't add new
1510 : // stateful materials
1511 : // during restart. This is only happening because this _has_ to be below initial userobject
1512 : // execution.
1513 : // Otherwise this could be done up above... _before_ restoring restartable data... which would
1514 : // allow you to have
1515 : // this happen during restart. I honestly have no idea why this has to happen after initial user
1516 : // object computation.
1517 : // THAT is something we should fix... so I've opened this ticket: #5804
1518 59469 : if (!_app.isRecovering() && !_app.isRestarting())
1519 : {
1520 109384 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1521 54406 : _neighbor_material_props.hasStatefulProperties())
1522 : {
1523 2860 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1524 :
1525 572 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1526 572 : }
1527 : #ifdef MOOSE_KOKKOS_ENABLED
1528 40704 : if (_kokkos_material_props.hasStatefulProperties() ||
1529 81243 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1530 40539 : _kokkos_neighbor_material_props.hasStatefulProperties())
1531 : {
1532 825 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1533 :
1534 165 : initElementStatefulProps(getCurrentAlgebraicElementRange(), true);
1535 165 : }
1536 : #endif
1537 : }
1538 :
1539 : // Control Logic
1540 59469 : _control_warehouse.initialSetup();
1541 59469 : executeControls(EXEC_INITIAL);
1542 :
1543 : // Scalar variables need to reinited for the initial conditions to be available for output
1544 124729 : for (unsigned int tid = 0; tid < n_threads; tid++)
1545 65287 : reinitScalars(tid);
1546 :
1547 59442 : if (_displaced_mesh)
1548 2013 : _displaced_problem->syncSolutions();
1549 :
1550 : // Writes all calls to _console from initialSetup() methods
1551 59442 : _app.getOutputWarehouse().mooseConsole();
1552 :
1553 59442 : if (_requires_nonlocal_coupling)
1554 : {
1555 63 : setNonlocalCouplingMatrix();
1556 126 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1557 126 : for (auto & assembly : _assembly[tid])
1558 63 : assembly->initNonlocalCoupling();
1559 : }
1560 :
1561 : {
1562 297210 : TIME_SECTION("lineSearchInitialSetup", 5, "Initializing Line Search");
1563 :
1564 59442 : if (_line_search)
1565 0 : _line_search->initialSetup();
1566 59442 : }
1567 :
1568 : // Perform Reporter get/declare check
1569 59442 : _reporter_data.check();
1570 :
1571 : // We do this late to allow objects to get late restartable data
1572 59442 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1573 4491 : _app.finalizeRestore();
1574 :
1575 59442 : setCurrentExecuteOnFlag(EXEC_NONE);
1576 59442 : }
1577 :
1578 : void
1579 60291 : FEProblemBase::checkDuplicatePostprocessorVariableNames()
1580 : {
1581 111288 : for (const auto & pp : _reporter_data.getPostprocessorNames())
1582 50997 : if (hasScalarVariable(pp))
1583 0 : mooseError("Postprocessor \"" + pp +
1584 60291 : "\" has the same name as a scalar variable in the system.");
1585 60291 : }
1586 :
1587 : void
1588 269054 : FEProblemBase::timestepSetup()
1589 : {
1590 269054 : SubProblem::timestepSetup();
1591 :
1592 269054 : if (_t_step > 1 && _num_grid_steps)
1593 : {
1594 31 : libMesh::MeshRefinement mesh_refinement(_mesh);
1595 31 : std::unique_ptr<libMesh::MeshRefinement> displaced_mesh_refinement(nullptr);
1596 31 : if (_displaced_mesh)
1597 23 : displaced_mesh_refinement = std::make_unique<libMesh::MeshRefinement>(*_displaced_mesh);
1598 :
1599 62 : for (MooseIndex(_num_grid_steps) i = 0; i < _num_grid_steps; ++i)
1600 : {
1601 31 : if (_displaced_problem)
1602 : // If the DisplacedProblem is active, undisplace the DisplacedMesh in preparation for
1603 : // refinement. We can't safely refine the DisplacedMesh directly, since the Hilbert keys
1604 : // computed on the inconsistenly-displaced Mesh are different on different processors,
1605 : // leading to inconsistent Hilbert keys. We must do this before the undisplaced Mesh is
1606 : // coarsensed, so that the element and node numbering is still consistent. We also have to
1607 : // make sure this is done during every step of coarsening otherwise different partitions
1608 : // will be generated for the reference and displaced meshes (even for replicated)
1609 23 : _displaced_problem->undisplaceMesh();
1610 :
1611 31 : mesh_refinement.uniformly_coarsen();
1612 31 : if (_displaced_mesh)
1613 23 : displaced_mesh_refinement->uniformly_coarsen();
1614 :
1615 : // Mark this as an intermediate change because we do not yet want to reinit_systems. E.g. we
1616 : // need things to happen in the following order for the undisplaced problem:
1617 : // u1) EquationSystems::reinit_solutions. This will restrict the solution vectors and then
1618 : // contract the mesh
1619 : // u2) MooseMesh::meshChanged. This will update the node/side lists and other
1620 : // things which needs to happen after the contraction
1621 : // u3) GeometricSearchData::reinit. Once the node/side lists are updated we can perform our
1622 : // geometric searches which will aid in determining sparsity patterns
1623 : //
1624 : // We do these things for the displaced problem (if it exists)
1625 : // d1) EquationSystems::reinit. Restrict the displaced problem vector copies and then contract
1626 : // the mesh. It's safe to do a full reinit with the displaced because there are no
1627 : // matrices that sparsity pattern calculations will be conducted for
1628 : // d2) MooseMesh::meshChanged. This will update the node/side lists and other
1629 : // things which needs to happen after the contraction
1630 : // d3) UpdateDisplacedMeshThread::operator(). Re-displace the mesh using the *displaced*
1631 : // solution vector copy because we don't know the state of the reference solution vector.
1632 : // It's safe to use the displaced copy because we are outside of a non-linear solve,
1633 : // and there is no concern about differences between solution and current_local_solution
1634 : // d4) GeometricSearchData::reinit. With the node/side lists updated and the mesh
1635 : // re-displaced, we can perform our geometric searches, which will aid in determining the
1636 : // sparsity pattern of the matrix held by the libMesh::ImplicitSystem held by the
1637 : // NonlinearSystem held by this
1638 31 : meshChanged(
1639 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
1640 : }
1641 :
1642 : // u4) Now that all the geometric searches have been done (both undisplaced and displaced),
1643 : // we're ready to update the sparsity pattern
1644 31 : es().reinit_systems();
1645 31 : }
1646 :
1647 269054 : _control_warehouse.timestepSetup();
1648 269054 : if (_line_search)
1649 0 : _line_search->timestepSetup();
1650 :
1651 : // Random interface objects
1652 270434 : for (const auto & it : _random_data_objects)
1653 1380 : it.second->updateSeeds(EXEC_TIMESTEP_BEGIN);
1654 :
1655 269054 : unsigned int n_threads = libMesh::n_threads();
1656 564790 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1657 : {
1658 295736 : _all_materials.timestepSetup(tid);
1659 295736 : _functions.timestepSetup(tid);
1660 : }
1661 :
1662 : #ifdef MOOSE_KOKKOS_ENABLED
1663 197269 : _kokkos_functions.timestepSetup();
1664 : #endif
1665 :
1666 269054 : _aux->timestepSetup();
1667 541572 : for (auto & sys : _solver_systems)
1668 272518 : sys->timestepSetup();
1669 :
1670 269054 : if (_displaced_problem)
1671 : // timestepSetup for displaced systems
1672 30674 : _displaced_problem->timestepSetup();
1673 :
1674 564790 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1675 : {
1676 295736 : _internal_side_indicators.timestepSetup(tid);
1677 295736 : _indicators.timestepSetup(tid);
1678 295736 : _markers.timestepSetup(tid);
1679 : }
1680 :
1681 269054 : std::vector<UserObject *> userobjs;
1682 269054 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
1683 603073 : for (auto obj : userobjs)
1684 334019 : obj->timestepSetup();
1685 :
1686 : #ifdef MOOSE_KOKKOS_ENABLED
1687 : {
1688 197269 : std::vector<UserObjectBase *> userobjs;
1689 197269 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
1690 199448 : for (auto obj : userobjs)
1691 2179 : obj->timestepSetup();
1692 197269 : }
1693 : #endif
1694 :
1695 : // Timestep setup of output objects
1696 269054 : _app.getOutputWarehouse().timestepSetup();
1697 :
1698 269054 : if (_requires_nonlocal_coupling)
1699 97 : if (_nonlocal_kernels.hasActiveObjects() || _nonlocal_integrated_bcs.hasActiveObjects())
1700 97 : _has_nonlocal_coupling = true;
1701 269054 : }
1702 :
1703 : unsigned int
1704 755876 : FEProblemBase::getMaxQps() const
1705 : {
1706 755876 : if (_max_qps == std::numeric_limits<unsigned int>::max())
1707 0 : mooseError("Max QPS uninitialized");
1708 755876 : return _max_qps;
1709 : }
1710 :
1711 : Order
1712 52 : FEProblemBase::getMaxScalarOrder() const
1713 : {
1714 52 : return _max_scalar_order;
1715 : }
1716 :
1717 : void
1718 59747 : FEProblemBase::checkNonlocalCoupling()
1719 : {
1720 298735 : TIME_SECTION("checkNonlocalCoupling", 5, "Checking Nonlocal Coupling");
1721 :
1722 125440 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
1723 130633 : for (auto & nl : _nl)
1724 : {
1725 64940 : const auto & all_kernels = nl->getKernelWarehouse();
1726 64940 : const auto & kernels = all_kernels.getObjects(tid);
1727 148927 : for (const auto & kernel : kernels)
1728 : {
1729 : std::shared_ptr<NonlocalKernel> nonlocal_kernel =
1730 83987 : std::dynamic_pointer_cast<NonlocalKernel>(kernel);
1731 83987 : if (nonlocal_kernel)
1732 : {
1733 35 : if (_calculate_jacobian_in_uo)
1734 35 : _requires_nonlocal_coupling = true;
1735 35 : _nonlocal_kernels.addObject(kernel, tid);
1736 : }
1737 83987 : }
1738 : const MooseObjectWarehouse<IntegratedBCBase> & all_integrated_bcs =
1739 64940 : nl->getIntegratedBCWarehouse();
1740 64940 : const auto & integrated_bcs = all_integrated_bcs.getObjects(tid);
1741 73758 : for (const auto & integrated_bc : integrated_bcs)
1742 : {
1743 : std::shared_ptr<NonlocalIntegratedBC> nonlocal_integrated_bc =
1744 8818 : std::dynamic_pointer_cast<NonlocalIntegratedBC>(integrated_bc);
1745 8818 : if (nonlocal_integrated_bc)
1746 : {
1747 28 : if (_calculate_jacobian_in_uo)
1748 28 : _requires_nonlocal_coupling = true;
1749 28 : _nonlocal_integrated_bcs.addObject(integrated_bc, tid);
1750 : }
1751 8818 : }
1752 : }
1753 59747 : }
1754 :
1755 : void
1756 65693 : FEProblemBase::checkUserObjectJacobianRequirement(THREAD_ID tid)
1757 : {
1758 65693 : std::set<const MooseVariableFEBase *> uo_jacobian_moose_vars;
1759 : {
1760 65693 : std::vector<ShapeElementUserObject *> objs;
1761 65693 : theWarehouse()
1762 65693 : .query()
1763 131386 : .condition<AttribInterfaces>(Interfaces::ShapeElementUserObject)
1764 65693 : .condition<AttribThread>(tid)
1765 65693 : .queryInto(objs);
1766 :
1767 65737 : for (const auto & uo : objs)
1768 : {
1769 44 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1770 44 : const auto & mv_deps = uo->jacobianMooseVariables();
1771 44 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1772 : }
1773 65693 : }
1774 : {
1775 65693 : std::vector<ShapeSideUserObject *> objs;
1776 65693 : theWarehouse()
1777 65693 : .query()
1778 131386 : .condition<AttribInterfaces>(Interfaces::ShapeSideUserObject)
1779 65693 : .condition<AttribThread>(tid)
1780 65693 : .queryInto(objs);
1781 65749 : for (const auto & uo : objs)
1782 : {
1783 56 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1784 56 : const auto & mv_deps = uo->jacobianMooseVariables();
1785 56 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1786 : }
1787 65693 : }
1788 :
1789 65693 : _uo_jacobian_moose_vars[tid].assign(uo_jacobian_moose_vars.begin(), uo_jacobian_moose_vars.end());
1790 131386 : std::sort(
1791 131386 : _uo_jacobian_moose_vars[tid].begin(), _uo_jacobian_moose_vars[tid].end(), sortMooseVariables);
1792 65693 : }
1793 :
1794 : void
1795 63 : FEProblemBase::setVariableAllDoFMap(const std::vector<const MooseVariableFEBase *> & moose_vars)
1796 : {
1797 153 : for (unsigned int i = 0; i < moose_vars.size(); ++i)
1798 : {
1799 90 : VariableName var_name = moose_vars[i]->name();
1800 90 : auto & sys = _solver_systems[moose_vars[i]->sys().number()];
1801 90 : sys->setVariableGlobalDoFs(var_name);
1802 90 : _var_dof_map[var_name] = sys->getVariableGlobalDoFs();
1803 90 : }
1804 63 : }
1805 :
1806 : void
1807 374515929 : FEProblemBase::prepare(const Elem * elem, const THREAD_ID tid)
1808 : {
1809 751537401 : for (const auto i : index_range(_solver_systems))
1810 : {
1811 377021472 : _assembly[tid][i]->reinit(elem);
1812 377021472 : _solver_systems[i]->prepare(tid);
1813 :
1814 377021472 : if (i < _num_nl_sys)
1815 : {
1816 : // This method is called outside of residual/Jacobian callbacks during initial condition
1817 : // evaluation
1818 376453424 : if ((!_has_jacobian || !_const_jacobian) && currentlyComputingJacobian())
1819 47281348 : _assembly[tid][i]->prepareJacobianBlock();
1820 376453424 : _assembly[tid][i]->prepareResidual();
1821 376453424 : if (_has_nonlocal_coupling && currentlyComputingJacobian())
1822 8824 : _assembly[tid][i]->prepareNonlocal();
1823 : }
1824 : }
1825 374515929 : _aux->prepare(tid);
1826 :
1827 386160424 : if (_displaced_problem &&
1828 : // _reinit_displaced_neighbor applies to interface type objects which will do computations
1829 : // based on both elem and neighbor. Consequently, despite what you might think by its name, we
1830 : // must make sure we prepare the displaced elem
1831 386160424 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1832 : {
1833 7561302 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), tid);
1834 7561278 : if (_has_nonlocal_coupling)
1835 0 : _displaced_problem->prepareNonlocal(tid);
1836 : }
1837 374515905 : }
1838 :
1839 : void
1840 25659 : FEProblemBase::prepareFace(const Elem * elem, const THREAD_ID tid)
1841 : {
1842 51318 : for (auto & nl : _nl)
1843 25659 : nl->prepareFace(tid, true);
1844 25659 : _aux->prepareFace(tid, false);
1845 :
1846 25659 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1847 0 : _displaced_problem->prepareFace(_displaced_mesh->elemPtr(elem->id()), tid);
1848 25659 : }
1849 :
1850 : void
1851 0 : FEProblemBase::prepare(const Elem * elem,
1852 : unsigned int ivar,
1853 : unsigned int jvar,
1854 : const std::vector<dof_id_type> & dof_indices,
1855 : const THREAD_ID tid)
1856 : {
1857 0 : for (const auto i : index_range(_nl))
1858 : {
1859 0 : _assembly[tid][i]->reinit(elem);
1860 0 : _nl[i]->prepare(tid);
1861 : }
1862 :
1863 0 : _aux->prepare(tid);
1864 0 : const auto current_nl_sys_num = _current_nl_sys->number();
1865 0 : _assembly[tid][current_nl_sys_num]->prepareBlock(ivar, jvar, dof_indices);
1866 0 : if (_has_nonlocal_coupling)
1867 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1868 : {
1869 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1870 0 : _assembly[tid][current_nl_sys_num]->prepareBlockNonlocal(
1871 : ivar, jvar, dof_indices, jv.allDofIndices());
1872 : }
1873 :
1874 0 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1875 : {
1876 0 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), ivar, jvar, dof_indices, tid);
1877 0 : if (_has_nonlocal_coupling)
1878 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1879 : {
1880 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1881 0 : _displaced_problem->prepareBlockNonlocal(ivar, jvar, dof_indices, jv.allDofIndices(), tid);
1882 : }
1883 : }
1884 0 : }
1885 :
1886 : void
1887 405800714 : FEProblemBase::setCurrentSubdomainID(const Elem * elem, const THREAD_ID tid)
1888 : {
1889 405800714 : SubdomainID did = elem->subdomain_id();
1890 815115081 : for (const auto i : index_range(_solver_systems))
1891 : {
1892 409314367 : _assembly[tid][i]->setCurrentSubdomainID(did);
1893 421775903 : if (_displaced_problem &&
1894 421775903 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1895 7814055 : _displaced_problem->assembly(tid, i).setCurrentSubdomainID(did);
1896 : }
1897 405800714 : }
1898 :
1899 : void
1900 1455924424 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, unsigned int side, const THREAD_ID tid)
1901 : {
1902 1455924424 : SubdomainID did = elem->neighbor_ptr(side)->subdomain_id();
1903 2916541359 : for (const auto i : index_range(_nl))
1904 : {
1905 1460616935 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1906 1504193393 : if (_displaced_problem &&
1907 1504193393 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1908 25839139 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1909 : }
1910 1455924424 : }
1911 :
1912 : void
1913 15257576 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, const THREAD_ID tid)
1914 : {
1915 15257576 : SubdomainID did = elem->subdomain_id();
1916 31466464 : for (const auto i : index_range(_nl))
1917 : {
1918 16208888 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1919 16260184 : if (_displaced_problem &&
1920 16260184 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1921 51296 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1922 : }
1923 15257576 : }
1924 :
1925 : void
1926 136662 : FEProblemBase::prepareAssembly(const THREAD_ID tid)
1927 : {
1928 136662 : _assembly[tid][_current_nl_sys->number()]->prepare();
1929 136662 : if (_has_nonlocal_coupling)
1930 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
1931 :
1932 136662 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1933 : {
1934 51296 : _displaced_problem->prepareAssembly(tid);
1935 51296 : if (_has_nonlocal_coupling)
1936 0 : _displaced_problem->prepareNonlocal(tid);
1937 : }
1938 136662 : }
1939 :
1940 : void
1941 266613 : FEProblemBase::addResidual(const THREAD_ID tid)
1942 : {
1943 533226 : _assembly[tid][_current_nl_sys->number()]->addResidual(Assembly::GlobalDataKey{},
1944 266613 : currentResidualVectorTags());
1945 :
1946 266613 : if (_displaced_problem)
1947 4776 : _displaced_problem->addResidual(tid);
1948 266613 : }
1949 :
1950 : void
1951 1985081 : FEProblemBase::addResidualNeighbor(const THREAD_ID tid)
1952 : {
1953 3970162 : _assembly[tid][_current_nl_sys->number()]->addResidualNeighbor(Assembly::GlobalDataKey{},
1954 1985081 : currentResidualVectorTags());
1955 :
1956 1985081 : if (_displaced_problem)
1957 61744 : _displaced_problem->addResidualNeighbor(tid);
1958 1985081 : }
1959 :
1960 : void
1961 1967412 : FEProblemBase::addResidualLower(const THREAD_ID tid)
1962 : {
1963 3934824 : _assembly[tid][_current_nl_sys->number()]->addResidualLower(Assembly::GlobalDataKey{},
1964 1967412 : currentResidualVectorTags());
1965 :
1966 1967412 : if (_displaced_problem)
1967 61956 : _displaced_problem->addResidualLower(tid);
1968 1967412 : }
1969 :
1970 : void
1971 48426 : FEProblemBase::addResidualScalar(const THREAD_ID tid /* = 0*/)
1972 : {
1973 96852 : _assembly[tid][_current_nl_sys->number()]->addResidualScalar(Assembly::GlobalDataKey{},
1974 48426 : currentResidualVectorTags());
1975 48426 : }
1976 :
1977 : void
1978 288191240 : FEProblemBase::cacheResidual(const THREAD_ID tid)
1979 : {
1980 288191240 : SubProblem::cacheResidual(tid);
1981 288191240 : if (_displaced_problem)
1982 7590898 : _displaced_problem->cacheResidual(tid);
1983 288191240 : }
1984 :
1985 : void
1986 50077 : FEProblemBase::cacheResidualNeighbor(const THREAD_ID tid)
1987 : {
1988 50077 : SubProblem::cacheResidualNeighbor(tid);
1989 50077 : if (_displaced_problem)
1990 32 : _displaced_problem->cacheResidualNeighbor(tid);
1991 50077 : }
1992 :
1993 : void
1994 16991304 : FEProblemBase::addCachedResidual(const THREAD_ID tid)
1995 : {
1996 16991304 : SubProblem::addCachedResidual(tid);
1997 16991304 : if (_displaced_problem)
1998 495407 : _displaced_problem->addCachedResidual(tid);
1999 16991304 : }
2000 :
2001 : void
2002 11332 : FEProblemBase::addCachedResidualDirectly(NumericVector<Number> & residual, const THREAD_ID tid)
2003 : {
2004 11332 : if (_current_nl_sys->hasVector(_current_nl_sys->timeVectorTag()))
2005 30252 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2006 30252 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->timeVectorTag()));
2007 :
2008 11332 : if (_current_nl_sys->hasVector(_current_nl_sys->nonTimeVectorTag()))
2009 33996 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
2010 33996 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->nonTimeVectorTag()));
2011 :
2012 11332 : std::vector<VectorTag> extra_residual_vector_tags;
2013 11332 : extra_residual_vector_tags.reserve(currentResidualVectorTags().size());
2014 11332 : const auto time_tag = _current_nl_sys->timeVectorTag();
2015 11332 : const auto non_time_tag = _current_nl_sys->nonTimeVectorTag();
2016 44152 : for (const auto & vector_tag : currentResidualVectorTags())
2017 32820 : if (vector_tag._id != time_tag && vector_tag._id != non_time_tag)
2018 11404 : extra_residual_vector_tags.push_back(vector_tag);
2019 :
2020 : // Flush extra vector tag caches (e.g. from extra_vector_tags on NodalConstraints)
2021 : // to their respective system vectors after the standard TIME/NONTIME caches above.
2022 : // Without this, NodalConstraint contributions to extra vector tags are silently
2023 : // discarded by the blanket clearCachedResiduals.
2024 11332 : _assembly[tid][_current_nl_sys->number()]->addCachedResiduals(Assembly::GlobalDataKey{},
2025 : extra_residual_vector_tags);
2026 :
2027 : // We do this because by adding the cached residual directly, we cannot ensure that all of the
2028 : // cached residuals are emptied after only the two add calls above
2029 11332 : _assembly[tid][_current_nl_sys->number()]->clearCachedResiduals(Assembly::GlobalDataKey{});
2030 :
2031 11332 : if (_displaced_problem)
2032 35 : _displaced_problem->addCachedResidualDirectly(residual, tid);
2033 11332 : }
2034 :
2035 : void
2036 0 : FEProblemBase::setResidual(NumericVector<Number> & residual, const THREAD_ID tid)
2037 : {
2038 0 : _assembly[tid][_current_nl_sys->number()]->setResidual(
2039 : residual,
2040 0 : Assembly::GlobalDataKey{},
2041 0 : getVectorTag(_nl[_current_nl_sys->number()]->residualVectorTag()));
2042 0 : if (_displaced_problem)
2043 0 : _displaced_problem->setResidual(residual, tid);
2044 0 : }
2045 :
2046 : void
2047 0 : FEProblemBase::setResidualNeighbor(NumericVector<Number> & residual, const THREAD_ID tid)
2048 : {
2049 0 : _assembly[tid][_current_nl_sys->number()]->setResidualNeighbor(
2050 0 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->residualVectorTag()));
2051 0 : if (_displaced_problem)
2052 0 : _displaced_problem->setResidualNeighbor(residual, tid);
2053 0 : }
2054 :
2055 : void
2056 37415 : FEProblemBase::addJacobian(const THREAD_ID tid)
2057 : {
2058 37415 : _assembly[tid][_current_nl_sys->number()]->addJacobian(Assembly::GlobalDataKey{});
2059 37415 : if (_has_nonlocal_coupling)
2060 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianNonlocal(Assembly::GlobalDataKey{});
2061 37415 : if (_displaced_problem)
2062 : {
2063 200 : _displaced_problem->addJacobian(tid);
2064 200 : if (_has_nonlocal_coupling)
2065 0 : _displaced_problem->addJacobianNonlocal(tid);
2066 : }
2067 37415 : }
2068 :
2069 : void
2070 7852 : FEProblemBase::addJacobianNeighbor(const THREAD_ID tid)
2071 : {
2072 7852 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighbor(Assembly::GlobalDataKey{});
2073 7852 : if (_displaced_problem)
2074 44 : _displaced_problem->addJacobianNeighbor(tid);
2075 7852 : }
2076 :
2077 : void
2078 109320 : FEProblemBase::addJacobianNeighborLowerD(const THREAD_ID tid)
2079 : {
2080 109320 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborLowerD(Assembly::GlobalDataKey{});
2081 109320 : if (_displaced_problem)
2082 3072 : _displaced_problem->addJacobianNeighborLowerD(tid);
2083 109320 : }
2084 :
2085 : void
2086 4696 : FEProblemBase::addJacobianLowerD(const THREAD_ID tid)
2087 : {
2088 4696 : _assembly[tid][_current_nl_sys->number()]->addJacobianLowerD(Assembly::GlobalDataKey{});
2089 4696 : if (_displaced_problem)
2090 192 : _displaced_problem->addJacobianLowerD(tid);
2091 4696 : }
2092 :
2093 : void
2094 11603 : FEProblemBase::addJacobianScalar(const THREAD_ID tid /* = 0*/)
2095 : {
2096 11603 : _assembly[tid][_current_nl_sys->number()]->addJacobianScalar(Assembly::GlobalDataKey{});
2097 11603 : }
2098 :
2099 : void
2100 30002 : FEProblemBase::addJacobianOffDiagScalar(unsigned int ivar, const THREAD_ID tid /* = 0*/)
2101 : {
2102 60004 : _assembly[tid][_current_nl_sys->number()]->addJacobianOffDiagScalar(ivar,
2103 30002 : Assembly::GlobalDataKey{});
2104 30002 : }
2105 :
2106 : void
2107 46713220 : FEProblemBase::cacheJacobian(const THREAD_ID tid)
2108 : {
2109 46713220 : SubProblem::cacheJacobian(tid);
2110 46713220 : if (_displaced_problem)
2111 1544138 : _displaced_problem->cacheJacobian(tid);
2112 46713220 : }
2113 :
2114 : void
2115 9409 : FEProblemBase::cacheJacobianNeighbor(const THREAD_ID tid)
2116 : {
2117 9409 : SubProblem::cacheJacobianNeighbor(tid);
2118 9409 : if (_displaced_problem)
2119 0 : _displaced_problem->cacheJacobianNeighbor(tid);
2120 9409 : }
2121 :
2122 : void
2123 2888441 : FEProblemBase::addCachedJacobian(const THREAD_ID tid)
2124 : {
2125 2888441 : SubProblem::addCachedJacobian(tid);
2126 2888438 : if (_displaced_problem)
2127 96451 : _displaced_problem->addCachedJacobian(tid);
2128 2888438 : }
2129 :
2130 : void
2131 70832 : FEProblemBase::addJacobianBlockTags(SparseMatrix<Number> & jacobian,
2132 : unsigned int ivar,
2133 : unsigned int jvar,
2134 : const DofMap & dof_map,
2135 : std::vector<dof_id_type> & dof_indices,
2136 : const std::set<TagID> & tags,
2137 : const THREAD_ID tid)
2138 : {
2139 141664 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockTags(
2140 70832 : jacobian, ivar, jvar, dof_map, dof_indices, Assembly::GlobalDataKey{}, tags);
2141 :
2142 70832 : if (_has_nonlocal_coupling)
2143 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2144 : {
2145 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2146 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockNonlocalTags(
2147 : jacobian,
2148 : ivar,
2149 : jvar,
2150 : dof_map,
2151 : dof_indices,
2152 : jv.allDofIndices(),
2153 0 : Assembly::GlobalDataKey{},
2154 : tags);
2155 : }
2156 :
2157 70832 : if (_displaced_problem)
2158 : {
2159 0 : _displaced_problem->addJacobianBlockTags(jacobian, ivar, jvar, dof_map, dof_indices, tags, tid);
2160 0 : if (_has_nonlocal_coupling)
2161 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2162 : {
2163 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2164 0 : _displaced_problem->addJacobianBlockNonlocal(
2165 : jacobian, ivar, jvar, dof_map, dof_indices, jv.allDofIndices(), tags, tid);
2166 : }
2167 : }
2168 70832 : }
2169 :
2170 : void
2171 768 : FEProblemBase::addJacobianNeighbor(SparseMatrix<Number> & jacobian,
2172 : unsigned int ivar,
2173 : unsigned int jvar,
2174 : const DofMap & dof_map,
2175 : std::vector<dof_id_type> & dof_indices,
2176 : std::vector<dof_id_type> & neighbor_dof_indices,
2177 : const std::set<TagID> & tags,
2178 : const THREAD_ID tid)
2179 : {
2180 1536 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborTags(jacobian,
2181 : ivar,
2182 : jvar,
2183 : dof_map,
2184 : dof_indices,
2185 : neighbor_dof_indices,
2186 768 : Assembly::GlobalDataKey{},
2187 : tags);
2188 768 : if (_displaced_problem)
2189 0 : _displaced_problem->addJacobianNeighbor(
2190 : jacobian, ivar, jvar, dof_map, dof_indices, neighbor_dof_indices, tags, tid);
2191 768 : }
2192 :
2193 : void
2194 125748635 : FEProblemBase::prepareShapes(unsigned int var, const THREAD_ID tid)
2195 : {
2196 125748635 : _assembly[tid][_current_nl_sys->number()]->copyShapes(var);
2197 125748635 : }
2198 :
2199 : void
2200 574535 : FEProblemBase::prepareFaceShapes(unsigned int var, const THREAD_ID tid)
2201 : {
2202 574535 : _assembly[tid][_current_nl_sys->number()]->copyFaceShapes(var);
2203 574535 : }
2204 :
2205 : void
2206 185398 : FEProblemBase::prepareNeighborShapes(unsigned int var, const THREAD_ID tid)
2207 : {
2208 185398 : _assembly[tid][_current_nl_sys->number()]->copyNeighborShapes(var);
2209 185398 : }
2210 :
2211 : void
2212 874909 : FEProblemBase::addGhostedElem(dof_id_type elem_id)
2213 : {
2214 874909 : if (_mesh.elemPtr(elem_id)->processor_id() != processor_id())
2215 205736 : _ghosted_elems.insert(elem_id);
2216 874909 : }
2217 :
2218 : void
2219 29442 : FEProblemBase::addGhostedBoundary(BoundaryID boundary_id)
2220 : {
2221 29442 : _mesh.addGhostedBoundary(boundary_id);
2222 29442 : if (_displaced_problem)
2223 26702 : _displaced_mesh->addGhostedBoundary(boundary_id);
2224 29442 : }
2225 :
2226 : void
2227 67446 : FEProblemBase::ghostGhostedBoundaries()
2228 : {
2229 337230 : TIME_SECTION("ghostGhostedBoundaries", 3, "Ghosting Ghosted Boundaries");
2230 :
2231 67446 : _mesh.ghostGhostedBoundaries();
2232 :
2233 67446 : if (_displaced_problem)
2234 2591 : _displaced_mesh->ghostGhostedBoundaries();
2235 67446 : }
2236 :
2237 : void
2238 0 : FEProblemBase::sizeZeroes(unsigned int /*size*/, const THREAD_ID /*tid*/)
2239 : {
2240 0 : mooseDoOnce(mooseWarning(
2241 : "This function is deprecated and no longer performs any function. Please do not call it."));
2242 0 : }
2243 :
2244 : bool
2245 304028 : FEProblemBase::reinitDirac(const Elem * elem, const THREAD_ID tid)
2246 : {
2247 304028 : std::vector<Point> & points = _dirac_kernel_info.getPoints()[elem].first;
2248 :
2249 304028 : unsigned int n_points = points.size();
2250 :
2251 304028 : if (n_points)
2252 : {
2253 299052 : if (n_points > _max_qps)
2254 : {
2255 0 : _max_qps = n_points;
2256 :
2257 : /**
2258 : * The maximum number of qps can rise if several Dirac points are added to a single element.
2259 : * In that case we need to resize the zeros to compensate.
2260 : */
2261 0 : unsigned int max_qpts = getMaxQps();
2262 0 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
2263 : {
2264 : // the highest available order in libMesh is 43
2265 0 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
2266 0 : _zero[tid].resize(max_qpts, 0);
2267 0 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
2268 0 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
2269 0 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
2270 0 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
2271 : }
2272 : }
2273 :
2274 598104 : for (const auto i : index_range(_nl))
2275 : {
2276 299052 : _assembly[tid][i]->reinitAtPhysical(elem, points);
2277 299052 : _nl[i]->prepare(tid);
2278 : }
2279 299052 : _aux->prepare(tid);
2280 :
2281 299052 : reinitElem(elem, tid);
2282 : }
2283 :
2284 304028 : _assembly[tid][_current_nl_sys->number()]->prepare();
2285 304028 : if (_has_nonlocal_coupling)
2286 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
2287 :
2288 304028 : bool have_points = n_points > 0;
2289 304028 : if (_displaced_problem && (_reinit_displaced_elem))
2290 : {
2291 4976 : have_points |= _displaced_problem->reinitDirac(_displaced_mesh->elemPtr(elem->id()), tid);
2292 4976 : if (_has_nonlocal_coupling)
2293 0 : _displaced_problem->prepareNonlocal(tid);
2294 : }
2295 :
2296 304028 : return have_points;
2297 : }
2298 :
2299 : void
2300 374762065 : FEProblemBase::reinitElem(const Elem * elem, const THREAD_ID tid)
2301 : {
2302 752029673 : for (auto & sys : _solver_systems)
2303 377267608 : sys->reinitElem(elem, tid);
2304 374762065 : _aux->reinitElem(elem, tid);
2305 :
2306 374762065 : if (_displaced_problem && _reinit_displaced_elem)
2307 4423675 : _displaced_problem->reinitElem(_displaced_mesh->elemPtr(elem->id()), tid);
2308 374762065 : }
2309 :
2310 : void
2311 58460 : FEProblemBase::reinitElemPhys(const Elem * const elem,
2312 : const std::vector<Point> & phys_points_in_elem,
2313 : const THREAD_ID tid)
2314 : {
2315 : mooseAssert(_mesh.queryElemPtr(elem->id()) == elem,
2316 : "Are you calling this method with a displaced mesh element?");
2317 :
2318 116920 : for (const auto i : index_range(_solver_systems))
2319 : {
2320 58460 : _assembly[tid][i]->reinitAtPhysical(elem, phys_points_in_elem);
2321 58460 : _solver_systems[i]->prepare(tid);
2322 58460 : _assembly[tid][i]->prepare();
2323 58460 : if (_has_nonlocal_coupling)
2324 0 : _assembly[tid][i]->prepareNonlocal();
2325 : }
2326 58460 : _aux->prepare(tid);
2327 :
2328 58460 : reinitElem(elem, tid);
2329 58460 : }
2330 :
2331 : void
2332 0 : FEProblemBase::reinitElemFace(const Elem * const elem,
2333 : const unsigned int side,
2334 : const BoundaryID,
2335 : const THREAD_ID tid)
2336 : {
2337 0 : mooseDeprecated(
2338 : "reinitElemFace with a BoundaryID argument is deprecated because the boundary id was never "
2339 : "used. Please call reinitElemFace without the BoundaryID argument instead");
2340 :
2341 0 : reinitElemFace(elem, side, tid);
2342 0 : }
2343 :
2344 : void
2345 5124456 : FEProblemBase::reinitElemFace(const Elem * const elem, const unsigned int side, const THREAD_ID tid)
2346 : {
2347 10249260 : for (const auto i : index_range(_solver_systems))
2348 : {
2349 5124804 : _assembly[tid][i]->reinit(elem, side);
2350 5124804 : _solver_systems[i]->reinitElemFace(elem, side, tid);
2351 : }
2352 5124456 : _aux->reinitElemFace(elem, side, tid);
2353 :
2354 5124456 : if (_displaced_problem && _reinit_displaced_face)
2355 82248 : _displaced_problem->reinitElemFace(_displaced_mesh->elemPtr(elem->id()), side, tid);
2356 5124456 : }
2357 :
2358 : void
2359 268555 : FEProblemBase::reinitLowerDElem(const Elem * lower_d_elem,
2360 : const THREAD_ID tid,
2361 : const std::vector<Point> * const pts,
2362 : const std::vector<Real> * const weights)
2363 : {
2364 268555 : SubProblem::reinitLowerDElem(lower_d_elem, tid, pts, weights);
2365 :
2366 268555 : if (_displaced_problem && _displaced_mesh)
2367 960 : _displaced_problem->reinitLowerDElem(
2368 960 : _displaced_mesh->elemPtr(lower_d_elem->id()), tid, pts, weights);
2369 268555 : }
2370 :
2371 : void
2372 25953223 : FEProblemBase::reinitNode(const Node * node, const THREAD_ID tid)
2373 : {
2374 25953223 : if (_displaced_problem && _reinit_displaced_elem)
2375 1001711 : _displaced_problem->reinitNode(&_displaced_mesh->nodeRef(node->id()), tid);
2376 :
2377 51909362 : for (const auto i : index_range(_nl))
2378 : {
2379 25956139 : _assembly[tid][i]->reinit(node);
2380 25956139 : _nl[i]->reinitNode(node, tid);
2381 : }
2382 25953223 : _aux->reinitNode(node, tid);
2383 25953223 : }
2384 :
2385 : void
2386 65199147 : FEProblemBase::reinitNodeFace(const Node * node, BoundaryID bnd_id, const THREAD_ID tid)
2387 : {
2388 65199147 : if (_displaced_problem && _reinit_displaced_face)
2389 3388880 : _displaced_problem->reinitNodeFace(&_displaced_mesh->nodeRef(node->id()), bnd_id, tid);
2390 :
2391 131187658 : for (const auto i : index_range(_nl))
2392 : {
2393 65988511 : _assembly[tid][i]->reinit(node);
2394 65988511 : _nl[i]->reinitNodeFace(node, bnd_id, tid);
2395 : }
2396 65199147 : _aux->reinitNodeFace(node, bnd_id, tid);
2397 65199147 : }
2398 :
2399 : void
2400 5097 : FEProblemBase::reinitNodes(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2401 : {
2402 5097 : if (_displaced_problem && _reinit_displaced_elem)
2403 0 : _displaced_problem->reinitNodes(nodes, tid);
2404 :
2405 10194 : for (auto & nl : _nl)
2406 5097 : nl->reinitNodes(nodes, tid);
2407 5097 : _aux->reinitNodes(nodes, tid);
2408 5097 : }
2409 :
2410 : void
2411 1003 : FEProblemBase::reinitNodesNeighbor(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2412 : {
2413 1003 : if (_displaced_problem && _reinit_displaced_elem)
2414 0 : _displaced_problem->reinitNodesNeighbor(nodes, tid);
2415 :
2416 2006 : for (auto & nl : _nl)
2417 1003 : nl->reinitNodesNeighbor(nodes, tid);
2418 1003 : _aux->reinitNodesNeighbor(nodes, tid);
2419 1003 : }
2420 :
2421 : void
2422 8429445 : FEProblemBase::reinitScalars(const THREAD_ID tid, bool reinit_for_derivative_reordering /*=false*/)
2423 : {
2424 42147225 : TIME_SECTION("reinitScalars", 3, "Reinitializing Scalar Variables");
2425 :
2426 8429445 : if (_displaced_problem && _reinit_displaced_elem)
2427 113549 : _displaced_problem->reinitScalars(tid, reinit_for_derivative_reordering);
2428 :
2429 17087784 : for (auto & nl : _nl)
2430 8658339 : nl->reinitScalars(tid, reinit_for_derivative_reordering);
2431 8429445 : _aux->reinitScalars(tid, reinit_for_derivative_reordering);
2432 :
2433 : // This is called outside of residual/Jacobian call-backs
2434 17093108 : for (auto & assembly : _assembly[tid])
2435 8663663 : assembly->prepareScalar();
2436 8429445 : }
2437 :
2438 : void
2439 185551 : FEProblemBase::reinitOffDiagScalars(const THREAD_ID tid)
2440 : {
2441 185551 : _assembly[tid][_current_nl_sys->number()]->prepareOffDiagScalar();
2442 185551 : if (_displaced_problem)
2443 60 : _displaced_problem->reinitOffDiagScalars(tid);
2444 185551 : }
2445 :
2446 : void
2447 3643016 : FEProblemBase::reinitNeighbor(const Elem * elem, unsigned int side, const THREAD_ID tid)
2448 : {
2449 3643016 : setNeighborSubdomainID(elem, side, tid);
2450 :
2451 3643016 : const Elem * neighbor = elem->neighbor_ptr(side);
2452 3643016 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(elem);
2453 :
2454 7286059 : for (const auto i : index_range(_nl))
2455 : {
2456 3643043 : _assembly[tid][i]->reinitElemAndNeighbor(elem, side, neighbor, neighbor_side);
2457 3643043 : _nl[i]->prepareNeighbor(tid);
2458 : // Called during stateful material property evaluation outside of solve
2459 3643043 : _assembly[tid][i]->prepareNeighbor();
2460 : }
2461 3643016 : _aux->prepareNeighbor(tid);
2462 :
2463 7286059 : for (auto & nl : _nl)
2464 : {
2465 3643043 : nl->reinitElemFace(elem, side, tid);
2466 3643043 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2467 : }
2468 3643016 : _aux->reinitElemFace(elem, side, tid);
2469 3643016 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2470 :
2471 3643016 : if (_displaced_problem && _reinit_displaced_neighbor)
2472 : {
2473 : // There are cases like for cohesive zone modeling without significant sliding where we cannot
2474 : // use FEInterface::inverse_map in Assembly::reinitElemAndNeighbor in the displaced problem
2475 : // because the physical points coming from the element don't actually lie on the neighbor.
2476 : // Moreover, what's the point of doing another physical point inversion in other cases? We only
2477 : // care about the reference points which we can just take from the undisplaced computation
2478 64800 : const auto & displaced_ref_pts = _assembly[tid][0]->qRuleNeighbor()->get_points();
2479 :
2480 64800 : _displaced_problem->reinitNeighbor(
2481 64800 : _displaced_mesh->elemPtr(elem->id()), side, tid, &displaced_ref_pts);
2482 : }
2483 3643016 : }
2484 :
2485 : void
2486 2069965 : FEProblemBase::reinitElemNeighborAndLowerD(const Elem * elem,
2487 : unsigned int side,
2488 : const THREAD_ID tid)
2489 : {
2490 2069965 : reinitNeighbor(elem, side, tid);
2491 :
2492 2069965 : const Elem * lower_d_elem = _mesh.getLowerDElem(elem, side);
2493 2069965 : if (lower_d_elem && _mesh.interiorLowerDBlocks().count(lower_d_elem->subdomain_id()) > 0)
2494 10332 : reinitLowerDElem(lower_d_elem, tid);
2495 : else
2496 : {
2497 : // with mesh refinement, lower-dimensional element might be defined on neighbor side
2498 2059633 : auto & neighbor = _assembly[tid][0]->neighbor();
2499 2059633 : auto & neighbor_side = _assembly[tid][0]->neighborSide();
2500 2059633 : const Elem * lower_d_elem_neighbor = _mesh.getLowerDElem(neighbor, neighbor_side);
2501 2059633 : if (lower_d_elem_neighbor &&
2502 2059633 : _mesh.interiorLowerDBlocks().count(lower_d_elem_neighbor->subdomain_id()) > 0)
2503 : {
2504 0 : auto qps = _assembly[tid][0]->qPointsFaceNeighbor().stdVector();
2505 0 : std::vector<Point> reference_points;
2506 0 : FEMap::inverse_map(
2507 0 : lower_d_elem_neighbor->dim(), lower_d_elem_neighbor, qps, reference_points);
2508 0 : reinitLowerDElem(lower_d_elem_neighbor, tid, &reference_points);
2509 0 : }
2510 : }
2511 :
2512 2069965 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
2513 64740 : _displaced_problem->reinitElemNeighborAndLowerD(
2514 64740 : _displaced_mesh->elemPtr(elem->id()), side, tid);
2515 2069965 : }
2516 :
2517 : void
2518 97230 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2519 : unsigned int neighbor_side,
2520 : const std::vector<Point> & physical_points,
2521 : const THREAD_ID tid)
2522 : {
2523 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2524 : "Are you calling this method with a displaced mesh element?");
2525 :
2526 194460 : for (const auto i : index_range(_nl))
2527 : {
2528 : // Reinits shape the functions at the physical points
2529 97230 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, neighbor_side, physical_points);
2530 :
2531 : // Sets the neighbor dof indices
2532 97230 : _nl[i]->prepareNeighbor(tid);
2533 : }
2534 97230 : _aux->prepareNeighbor(tid);
2535 :
2536 : // Resizes Re and Ke
2537 97230 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2538 :
2539 : // Compute the values of each variable at the points
2540 194460 : for (auto & nl : _nl)
2541 97230 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2542 97230 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2543 97230 : }
2544 :
2545 : void
2546 19880 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2547 : const std::vector<Point> & physical_points,
2548 : const THREAD_ID tid)
2549 : {
2550 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2551 : "Are you calling this method with a displaced mesh element?");
2552 :
2553 39760 : for (const auto i : index_range(_nl))
2554 : {
2555 : // Reinits shape the functions at the physical points
2556 19880 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, physical_points);
2557 :
2558 : // Sets the neighbor dof indices
2559 19880 : _nl[i]->prepareNeighbor(tid);
2560 : }
2561 19880 : _aux->prepareNeighbor(tid);
2562 :
2563 : // Resizes Re and Ke
2564 19880 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2565 :
2566 : // Compute the values of each variable at the points
2567 39760 : for (auto & nl : _nl)
2568 19880 : nl->reinitNeighbor(neighbor, tid);
2569 19880 : _aux->reinitNeighbor(neighbor, tid);
2570 19880 : }
2571 :
2572 : void
2573 35518 : FEProblemBase::getDiracElements(std::set<const Elem *> & elems)
2574 : {
2575 : // First add in the undisplaced elements
2576 35518 : elems = _dirac_kernel_info.getElements();
2577 :
2578 35518 : if (_displaced_problem)
2579 : {
2580 2359 : std::set<const Elem *> displaced_elements;
2581 2359 : _displaced_problem->getDiracElements(displaced_elements);
2582 :
2583 : { // Use the ids from the displaced elements to get the undisplaced elements
2584 : // and add them to the list
2585 7335 : for (const auto & elem : displaced_elements)
2586 4976 : elems.insert(_mesh.elemPtr(elem->id()));
2587 : }
2588 2359 : }
2589 35518 : }
2590 :
2591 : void
2592 3516908 : FEProblemBase::clearDiracInfo()
2593 : {
2594 3516908 : _dirac_kernel_info.clearPoints();
2595 :
2596 3516908 : if (_displaced_problem)
2597 144138 : _displaced_problem->clearDiracInfo();
2598 3516908 : }
2599 :
2600 : void
2601 5091350 : FEProblemBase::subdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2602 : {
2603 5091350 : _all_materials.subdomainSetup(subdomain, tid);
2604 : // Call the subdomain methods of the output system, these are not threaded so only call it once
2605 5091350 : if (tid == 0)
2606 5078598 : _app.getOutputWarehouse().subdomainSetup();
2607 :
2608 10313296 : for (auto & nl : _nl)
2609 5221946 : nl->subdomainSetup(subdomain, tid);
2610 :
2611 : // FIXME: call displaced_problem->subdomainSetup() ?
2612 : // When adding possibility with materials being evaluated on displaced mesh
2613 5091350 : }
2614 :
2615 : void
2616 16548225 : FEProblemBase::neighborSubdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2617 : {
2618 16548225 : _all_materials.neighborSubdomainSetup(subdomain, tid);
2619 16548225 : }
2620 :
2621 : void
2622 49142 : FEProblemBase::addFunction(const std::string & type,
2623 : const std::string & name,
2624 : InputParameters & parameters)
2625 : {
2626 : parallel_object_only();
2627 :
2628 98284 : parameters.set<SubProblem *>("_subproblem") = this;
2629 :
2630 102585 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2631 : {
2632 53549 : std::shared_ptr<Function> func = _factory.create<Function>(type, name, parameters, tid);
2633 53443 : logAdd("Function", name, type, parameters);
2634 53443 : _functions.addObject(func, tid);
2635 :
2636 53443 : if (auto * const functor = dynamic_cast<Moose::FunctorBase<Real> *>(func.get()))
2637 : {
2638 53443 : this->addFunctor(name, *functor, tid);
2639 53443 : if (_displaced_problem)
2640 1780 : _displaced_problem->addFunctor(name, *functor, tid);
2641 : }
2642 : else
2643 0 : mooseError("Unrecognized function functor type");
2644 53443 : }
2645 49036 : }
2646 :
2647 : void
2648 152249 : FEProblemBase::addConvergence(const std::string & type,
2649 : const std::string & name,
2650 : InputParameters & parameters)
2651 : {
2652 : parallel_object_only();
2653 :
2654 320872 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2655 : {
2656 168650 : std::shared_ptr<Convergence> conv = _factory.create<Convergence>(type, name, parameters, tid);
2657 168623 : _convergences.addObject(conv, tid);
2658 168623 : }
2659 152222 : }
2660 :
2661 : void
2662 59548 : FEProblemBase::addDefaultNonlinearConvergence(const InputParameters & params_to_apply)
2663 : {
2664 59548 : const std::string class_name = "DefaultNonlinearConvergence";
2665 59548 : InputParameters params = _factory.getValidParams(class_name);
2666 59548 : params.applyParameters(params_to_apply);
2667 59548 : params.applyParameters(parameters());
2668 59548 : params.set<bool>("added_as_default") = true;
2669 118329 : for (const auto & conv_name : getNonlinearConvergenceNames())
2670 58781 : addConvergence(class_name, conv_name, params);
2671 59548 : }
2672 :
2673 : void
2674 61455 : FEProblemBase::addDefaultMultiAppFixedPointConvergence(const InputParameters & params_to_apply)
2675 : {
2676 61455 : const std::string class_name = "DefaultMultiAppFixedPointConvergence";
2677 61455 : InputParameters params = _factory.getValidParams(class_name);
2678 61455 : params.applyParameters(params_to_apply);
2679 61455 : params.applyParameters(parameters());
2680 61455 : params.set<bool>("added_as_default") = true;
2681 61455 : addConvergence(class_name, getMultiAppFixedPointConvergenceName(), params);
2682 61446 : }
2683 :
2684 : void
2685 30295 : FEProblemBase::addDefaultSteadyStateConvergence(const InputParameters & params_to_apply)
2686 : {
2687 30295 : const std::string class_name = "DefaultSteadyStateConvergence";
2688 30295 : InputParameters params = _factory.getValidParams(class_name);
2689 30295 : params.applyParameters(params_to_apply);
2690 30295 : params.applyParameters(parameters());
2691 30295 : params.set<bool>("added_as_default") = true;
2692 30295 : addConvergence(class_name, getSteadyStateConvergenceName(), params);
2693 30295 : }
2694 :
2695 : bool
2696 86359 : FEProblemBase::hasFunction(const std::string & name, const THREAD_ID tid)
2697 : {
2698 86359 : return _functions.hasActiveObject(name, tid);
2699 : }
2700 :
2701 : Function &
2702 62643 : FEProblemBase::getFunction(const std::string & name, const THREAD_ID tid)
2703 : {
2704 : // This thread lock is necessary since this method will create functions
2705 : // for all threads if one is missing.
2706 62643 : Threads::spin_mutex::scoped_lock lock(get_function_mutex);
2707 :
2708 62643 : if (!hasFunction(name, tid))
2709 : {
2710 : // If we didn't find a function, it might be a default function, attempt to construct one now
2711 20426 : std::istringstream ss(name);
2712 : Real real_value;
2713 :
2714 : // First see if it's just a constant. If it is, build a ConstantFunction
2715 20426 : if (ss >> real_value && ss.eof())
2716 : {
2717 12740 : InputParameters params = _factory.getValidParams("ConstantFunction");
2718 12740 : params.set<Real>("value") = real_value;
2719 19110 : addFunction("ConstantFunction", ss.str(), params);
2720 6370 : }
2721 : else
2722 : {
2723 14056 : FunctionParserBase<Real> fp;
2724 14056 : std::string vars = "x,y,z,t,NaN,pi,e";
2725 14056 : if (fp.Parse(name, vars) == -1) // -1 for success
2726 : {
2727 : // It parsed ok, so build a MooseParsedFunction
2728 42153 : InputParameters params = _factory.getValidParams("ParsedFunction");
2729 14051 : params.set<std::string>("expression") = name;
2730 28102 : addFunction("ParsedFunction", name, params);
2731 14051 : }
2732 14056 : }
2733 :
2734 : // Try once more
2735 20426 : if (!hasFunction(name, tid))
2736 : {
2737 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_function"),
2738 : "getFunction() was called before Functions have been constructed. The requested "
2739 : "Function '" +
2740 : name + "' may exist in the input file, but Functions are not available yet.");
2741 :
2742 7 : mooseError("Unable to find function " + name);
2743 : }
2744 20423 : }
2745 :
2746 62638 : auto * const ret = dynamic_cast<Function *>(_functions.getActiveObject(name, tid).get());
2747 62638 : if (!ret)
2748 0 : mooseError("No function named ", name, " of appropriate type");
2749 :
2750 62638 : return *ret;
2751 62640 : }
2752 :
2753 : bool
2754 210864 : FEProblemBase::hasConvergence(const std::string & name, const THREAD_ID tid) const
2755 : {
2756 210864 : return _convergences.hasActiveObject(name, tid);
2757 : }
2758 :
2759 : Convergence &
2760 1131874 : FEProblemBase::getConvergence(const std::string & name, const THREAD_ID tid) const
2761 : {
2762 1131874 : auto * const ret = dynamic_cast<Convergence *>(_convergences.getActiveObject(name, tid).get());
2763 1131874 : if (!ret)
2764 0 : mooseError("The Convergence object '", name, "' does not exist.");
2765 :
2766 1131874 : return *ret;
2767 : }
2768 :
2769 : const std::vector<std::shared_ptr<Convergence>> &
2770 192942 : FEProblemBase::getConvergenceObjects(const THREAD_ID tid) const
2771 : {
2772 192942 : return _convergences.getActiveObjects(tid);
2773 : }
2774 :
2775 : void
2776 687 : FEProblemBase::addMeshDivision(const std::string & type,
2777 : const std::string & name,
2778 : InputParameters & parameters)
2779 : {
2780 : parallel_object_only();
2781 1374 : parameters.set<FEProblemBase *>("_fe_problem_base") = this;
2782 1374 : parameters.set<SubProblem *>("_subproblem") = this;
2783 1462 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2784 : {
2785 784 : std::shared_ptr<MeshDivision> func = _factory.create<MeshDivision>(type, name, parameters, tid);
2786 775 : _mesh_divisions.addObject(func, tid);
2787 775 : }
2788 678 : }
2789 :
2790 : MeshDivision &
2791 1567 : FEProblemBase::getMeshDivision(const std::string & name, const THREAD_ID tid) const
2792 : {
2793 1567 : auto * const ret = dynamic_cast<MeshDivision *>(_mesh_divisions.getActiveObject(name, tid).get());
2794 1567 : if (!ret)
2795 0 : mooseError("No MeshDivision object named ", name, " of appropriate type");
2796 1567 : return *ret;
2797 : }
2798 :
2799 : void
2800 0 : FEProblemBase::lineSearch()
2801 : {
2802 0 : _line_search->lineSearch();
2803 0 : }
2804 :
2805 : NonlinearSystem &
2806 0 : FEProblemBase::getNonlinearSystem(const unsigned int sys_num)
2807 : {
2808 0 : mooseDeprecated("FEProblemBase::getNonlinearSystem() is deprecated, please use "
2809 : "FEProblemBase::getNonlinearSystemBase() \n");
2810 :
2811 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
2812 0 : auto nl_sys = std::dynamic_pointer_cast<NonlinearSystem>(_nl[sys_num]);
2813 :
2814 0 : if (!nl_sys)
2815 0 : mooseError("This is not a NonlinearSystem");
2816 :
2817 0 : return *nl_sys;
2818 0 : }
2819 :
2820 : void
2821 2 : FEProblemBase::addDistribution(const std::string & type,
2822 : const std::string & name,
2823 : InputParameters & parameters)
2824 : {
2825 4 : parameters.set<std::string>("type") = type;
2826 2 : addObject<Distribution>(type, name, parameters, /* threaded = */ false);
2827 2 : }
2828 :
2829 : bool
2830 4 : FEProblemBase::hasDistribution(const std::string & name) const
2831 : {
2832 4 : std::vector<Distribution *> objs;
2833 4 : theWarehouse()
2834 8 : .query()
2835 4 : .condition<AttribSystem>("Distribution")
2836 4 : .condition<AttribName>(name)
2837 4 : .queryInto(objs);
2838 8 : return !objs.empty();
2839 4 : }
2840 :
2841 : Distribution &
2842 4 : FEProblemBase::getDistribution(const std::string & name)
2843 : {
2844 4 : std::vector<Distribution *> objs;
2845 4 : theWarehouse()
2846 8 : .query()
2847 4 : .condition<AttribSystem>("Distribution")
2848 4 : .condition<AttribName>(name)
2849 4 : .queryInto(objs);
2850 4 : if (objs.empty())
2851 : {
2852 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_distribution"),
2853 : "A Distribution getter was called before Distributions have been constructed. "
2854 : "If you are attempting to access this object in the constructor of another object "
2855 : "then make sure that the Distribution is constructed before the object using it.");
2856 0 : mooseError("Unable to find Distribution with name '" + name + "'");
2857 : }
2858 8 : return *(objs[0]);
2859 4 : }
2860 :
2861 : void
2862 275 : FEProblemBase::addSampler(const std::string & type,
2863 : const std::string & name,
2864 : InputParameters & parameters)
2865 : {
2866 275 : const auto samplers = addObject<Sampler>(type, name, parameters);
2867 569 : for (auto & sampler : samplers)
2868 303 : sampler->init();
2869 266 : }
2870 :
2871 : Sampler &
2872 266 : FEProblemBase::getSampler(const std::string & name, const THREAD_ID tid)
2873 : {
2874 266 : std::vector<Sampler *> objs;
2875 266 : theWarehouse()
2876 532 : .query()
2877 266 : .condition<AttribSystem>("Sampler")
2878 266 : .condition<AttribThread>(tid)
2879 266 : .condition<AttribName>(name)
2880 266 : .queryInto(objs);
2881 266 : if (objs.empty())
2882 : {
2883 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_sampler"),
2884 : "A Sampler getter was called before Samplers have been constructed. "
2885 : "If you are attempting to access this object in the constructor of another object "
2886 : "then make sure that the Sampler is constructed before the object using it.");
2887 :
2888 0 : mooseError(
2889 0 : "Unable to find Sampler with name '" + name +
2890 : "', if you are attempting to access this object in the constructor of another object then "
2891 : "make sure that the Sampler is constructed before the object using it.");
2892 : }
2893 532 : return *(objs[0]);
2894 266 : }
2895 :
2896 : bool
2897 153783 : FEProblemBase::duplicateVariableCheck(const std::string & var_name,
2898 : const FEType & type,
2899 : bool is_aux,
2900 : const std::set<SubdomainID> * const active_subdomains)
2901 : {
2902 153783 : std::set<SubdomainID> subdomainIDs;
2903 153783 : if (active_subdomains->size() == 0)
2904 : {
2905 145680 : const auto subdomains = _mesh.meshSubdomains();
2906 145680 : subdomainIDs.insert(subdomains.begin(), subdomains.end());
2907 145680 : }
2908 : else
2909 8103 : subdomainIDs.insert(active_subdomains->begin(), active_subdomains->end());
2910 :
2911 308258 : for (auto & sys : _solver_systems)
2912 : {
2913 154487 : SystemBase * curr_sys_ptr = sys.get();
2914 154487 : SystemBase * other_sys_ptr = _aux.get();
2915 154487 : std::string error_prefix = "";
2916 154487 : if (is_aux)
2917 : {
2918 94121 : curr_sys_ptr = _aux.get();
2919 94121 : other_sys_ptr = sys.get();
2920 94121 : error_prefix = "aux";
2921 : }
2922 :
2923 154487 : if (other_sys_ptr->hasVariable(var_name))
2924 3 : mooseError("Cannot have an auxiliary variable and a solver variable with the same name: ",
2925 : var_name);
2926 :
2927 154484 : if (curr_sys_ptr->hasVariable(var_name))
2928 : {
2929 : const Variable & var =
2930 9 : curr_sys_ptr->system().variable(curr_sys_ptr->system().variable_number(var_name));
2931 :
2932 : // variable type
2933 9 : if (var.type() != type)
2934 : {
2935 12 : const auto stringifyType = [](FEType t)
2936 12 : { return Moose::stringify(t.family) + " of order " + Moose::stringify(t.order); };
2937 :
2938 6 : mooseError("Mismatching types are specified for ",
2939 : error_prefix,
2940 : "variable with name '",
2941 : var_name,
2942 : "': '",
2943 6 : stringifyType(var.type()),
2944 : "' and '",
2945 6 : stringifyType(type),
2946 : "'");
2947 : }
2948 :
2949 : // block-restriction
2950 3 : if (!(active_subdomains->size() == 0 && var.active_subdomains().size() == 0))
2951 : {
2952 3 : const auto varActiveSubdomains = var.active_subdomains();
2953 3 : std::set<SubdomainID> varSubdomainIDs;
2954 3 : if (varActiveSubdomains.size() == 0)
2955 : {
2956 0 : const auto subdomains = _mesh.meshSubdomains();
2957 0 : varSubdomainIDs.insert(subdomains.begin(), subdomains.end());
2958 0 : }
2959 : else
2960 3 : varSubdomainIDs.insert(varActiveSubdomains.begin(), varActiveSubdomains.end());
2961 :
2962 : // Is subdomainIDs a subset of varSubdomainIDs? With this we allow the case that the newly
2963 : // requested block restriction is only a subset of the existing one.
2964 3 : const auto isSubset = std::includes(varSubdomainIDs.begin(),
2965 : varSubdomainIDs.end(),
2966 : subdomainIDs.begin(),
2967 : subdomainIDs.end());
2968 :
2969 3 : if (!isSubset)
2970 : {
2971 : // helper function: make a string from a set of subdomain ids
2972 6 : const auto stringifySubdomains = [this](std::set<SubdomainID> subdomainIDs)
2973 : {
2974 6 : std::stringstream s;
2975 15 : for (auto const i : subdomainIDs)
2976 : {
2977 : // do we need to insert a comma?
2978 9 : if (s.tellp() != 0)
2979 3 : s << ", ";
2980 :
2981 : // insert subdomain name and id -or- only the id (if no name is given)
2982 9 : const auto subdomainName = _mesh.getSubdomainName(i);
2983 9 : if (subdomainName.empty())
2984 9 : s << i;
2985 : else
2986 0 : s << subdomainName << " (" << i << ")";
2987 9 : }
2988 12 : return s.str();
2989 6 : };
2990 :
2991 6 : const std::string msg = "Mismatching block-restrictions are specified for " +
2992 6 : error_prefix + "variable with name '" + var_name + "': {" +
2993 12 : stringifySubdomains(varSubdomainIDs) + "} and {" +
2994 9 : stringifySubdomains(subdomainIDs) + "}";
2995 :
2996 3 : mooseError(msg);
2997 0 : }
2998 0 : }
2999 :
3000 0 : return true;
3001 : }
3002 154475 : }
3003 :
3004 153771 : return false;
3005 153771 : }
3006 :
3007 : void
3008 59794 : FEProblemBase::addVariable(const std::string & var_type,
3009 : const std::string & var_name,
3010 : InputParameters & params)
3011 : {
3012 : parallel_object_only();
3013 :
3014 59794 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3015 59794 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3016 59794 : const auto fe_type = FEType(order, family);
3017 :
3018 : const auto active_subdomains_vector =
3019 59794 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3020 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3021 59794 : active_subdomains_vector.end());
3022 :
3023 59794 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ false, &active_subdomains))
3024 0 : return;
3025 :
3026 179373 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3027 59791 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_SOLVER;
3028 59791 : SolverSystemName sys_name = params.get<SolverSystemName>("solver_sys");
3029 :
3030 59791 : const auto solver_system_number = solverSysNum(sys_name);
3031 59791 : logAdd("Variable", var_name, var_type, params);
3032 59791 : _solver_systems[solver_system_number]->addVariable(var_type, var_name, params);
3033 59779 : if (_displaced_problem)
3034 : // MooseObjects need to be unique so change the name here
3035 3453 : _displaced_problem->addVariable(var_type, var_name, params, solver_system_number);
3036 :
3037 59779 : _solver_var_to_sys_num[var_name] = solver_system_number;
3038 :
3039 59779 : markFamilyPRefinement(params);
3040 59779 : if (_displaced_problem)
3041 3453 : _displaced_problem->markFamilyPRefinement(params);
3042 59779 : }
3043 :
3044 : std::pair<bool, unsigned int>
3045 4548614 : FEProblemBase::determineSolverSystem(const std::string & var_name,
3046 : const bool error_if_not_found) const
3047 : {
3048 4548614 : auto map_it = _solver_var_to_sys_num.find(var_name);
3049 4548614 : const bool var_in_sys = map_it != _solver_var_to_sys_num.end();
3050 4548614 : if (var_in_sys)
3051 : mooseAssert(_solver_systems[map_it->second]->hasVariable(var_name) ||
3052 : _solver_systems[map_it->second]->hasScalarVariable(var_name),
3053 : "If the variable is in our FEProblem solver system map, then it must be in the "
3054 : "solver system we expect");
3055 3101538 : else if (error_if_not_found)
3056 : {
3057 32 : if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
3058 21 : mooseError("No solver variable named ",
3059 : var_name,
3060 : " found. Did you specify an auxiliary variable when you meant to specify a "
3061 : "solver variable?");
3062 : else
3063 11 : mooseError("Unknown variable '",
3064 : var_name,
3065 : "'. It does not exist in the solver system(s) or auxiliary system");
3066 : }
3067 :
3068 9097164 : return std::make_pair(var_in_sys, var_in_sys ? map_it->second : libMesh::invalid_uint);
3069 : }
3070 :
3071 : void
3072 159605 : FEProblemBase::setResidualObjectParamsAndLog(const std::string & ro_name,
3073 : const std::string & name,
3074 : InputParameters & parameters,
3075 : const unsigned int nl_sys_num,
3076 : const std::string & base_name,
3077 : bool & reinit_displaced)
3078 : {
3079 159605 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3080 : {
3081 1920 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3082 1920 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3083 960 : reinit_displaced = true;
3084 : }
3085 : else
3086 : {
3087 158645 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3088 : {
3089 : // We allow Kernels to request that they use_displaced_mesh,
3090 : // but then be overridden when no displacements variables are
3091 : // provided in the Mesh block. If that happened, update the value
3092 : // of use_displaced_mesh appropriately for this Kernel.
3093 105 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3094 210 : parameters.set<bool>("use_displaced_mesh") = false;
3095 : }
3096 :
3097 317290 : parameters.set<SubProblem *>("_subproblem") = this;
3098 475935 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3099 : }
3100 :
3101 159605 : logAdd(base_name, name, ro_name, parameters);
3102 159605 : }
3103 :
3104 : void
3105 65099 : FEProblemBase::setAuxKernelParamsAndLog(const std::string & ak_name,
3106 : const std::string & name,
3107 : InputParameters & parameters,
3108 : const std::string & base_name)
3109 : {
3110 65099 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3111 : {
3112 22872 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3113 22872 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3114 22872 : parameters.set<SystemBase *>("_nl_sys") = &_displaced_problem->solverSys(0);
3115 11436 : if (!parameters.get<std::vector<BoundaryName>>("boundary").empty())
3116 10990 : _reinit_displaced_face = true;
3117 : else
3118 446 : _reinit_displaced_elem = true;
3119 : }
3120 : else
3121 : {
3122 53663 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3123 : {
3124 : // We allow AuxKernels to request that they use_displaced_mesh,
3125 : // but then be overridden when no displacements variables are
3126 : // provided in the Mesh block. If that happened, update the value
3127 : // of use_displaced_mesh appropriately for this AuxKernel.
3128 795 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3129 1590 : parameters.set<bool>("use_displaced_mesh") = false;
3130 : }
3131 :
3132 107326 : parameters.set<SubProblem *>("_subproblem") = this;
3133 107326 : parameters.set<SystemBase *>("_sys") = _aux.get();
3134 160989 : parameters.set<SystemBase *>("_nl_sys") = _solver_systems[0].get();
3135 : }
3136 :
3137 65099 : logAdd(base_name, name, ak_name, parameters);
3138 65099 : }
3139 :
3140 : void
3141 76701 : FEProblemBase::addKernel(const std::string & kernel_name,
3142 : const std::string & name,
3143 : InputParameters & parameters)
3144 : {
3145 : parallel_object_only();
3146 153402 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3147 76689 : if (!isSolverSystemNonlinear(nl_sys_num))
3148 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
3149 : "supported at the moment!");
3150 76689 : setResidualObjectParamsAndLog(
3151 76689 : kernel_name, name, parameters, nl_sys_num, "Kernel", _reinit_displaced_elem);
3152 :
3153 76689 : _nl[nl_sys_num]->addKernel(kernel_name, name, parameters);
3154 76536 : }
3155 :
3156 : void
3157 431 : FEProblemBase::addHDGKernel(const std::string & kernel_name,
3158 : const std::string & name,
3159 : InputParameters & parameters)
3160 : {
3161 : parallel_object_only();
3162 862 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3163 431 : if (!isSolverSystemNonlinear(nl_sys_num))
3164 0 : mooseError("You are trying to add a HDGKernel to a linear variable/system, which is not "
3165 : "supported at the moment!");
3166 431 : setResidualObjectParamsAndLog(
3167 431 : kernel_name, name, parameters, nl_sys_num, "HDGKernel", _reinit_displaced_elem);
3168 :
3169 431 : _nl[nl_sys_num]->addHDGKernel(kernel_name, name, parameters);
3170 431 : }
3171 :
3172 : void
3173 602 : FEProblemBase::addNodalKernel(const std::string & kernel_name,
3174 : const std::string & name,
3175 : InputParameters & parameters)
3176 : {
3177 : parallel_object_only();
3178 :
3179 1204 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3180 599 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3181 : {
3182 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3183 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3184 0 : _reinit_displaced_elem = true;
3185 : }
3186 : else
3187 : {
3188 599 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3189 : {
3190 : // We allow NodalKernels to request that they use_displaced_mesh,
3191 : // but then be overridden when no displacements variables are
3192 : // provided in the Mesh block. If that happened, update the value
3193 : // of use_displaced_mesh appropriately for this NodalKernel.
3194 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3195 0 : parameters.set<bool>("use_displaced_mesh") = false;
3196 : }
3197 :
3198 1198 : parameters.set<SubProblem *>("_subproblem") = this;
3199 1797 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3200 : }
3201 599 : logAdd("NodalKernel", name, kernel_name, parameters);
3202 599 : _nl[nl_sys_num]->addNodalKernel(kernel_name, name, parameters);
3203 599 : }
3204 :
3205 : void
3206 1319 : FEProblemBase::addScalarKernel(const std::string & kernel_name,
3207 : const std::string & name,
3208 : InputParameters & parameters)
3209 : {
3210 : parallel_object_only();
3211 :
3212 2638 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3213 1319 : if (!isSolverSystemNonlinear(nl_sys_num))
3214 0 : mooseError("You are trying to add a ScalarKernel to a linear variable/system, which is not "
3215 : "supported at the moment!");
3216 :
3217 1319 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3218 : {
3219 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3220 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3221 : }
3222 : else
3223 : {
3224 1319 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3225 : {
3226 : // We allow ScalarKernels to request that they use_displaced_mesh,
3227 : // but then be overridden when no displacements variables are
3228 : // provided in the Mesh block. If that happened, update the value
3229 : // of use_displaced_mesh appropriately for this ScalarKernel.
3230 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3231 0 : parameters.set<bool>("use_displaced_mesh") = false;
3232 : }
3233 :
3234 2638 : parameters.set<SubProblem *>("_subproblem") = this;
3235 3957 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3236 : }
3237 :
3238 1319 : logAdd("ScalarKernel", name, kernel_name, parameters);
3239 1319 : _nl[nl_sys_num]->addScalarKernel(kernel_name, name, parameters);
3240 1313 : }
3241 :
3242 : void
3243 74985 : FEProblemBase::addBoundaryCondition(const std::string & bc_name,
3244 : const std::string & name,
3245 : InputParameters & parameters)
3246 : {
3247 : parallel_object_only();
3248 :
3249 149974 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3250 74977 : if (!isSolverSystemNonlinear(nl_sys_num))
3251 0 : mooseError(
3252 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
3253 : "supported at the moment!");
3254 :
3255 74977 : setResidualObjectParamsAndLog(
3256 74977 : bc_name, name, parameters, nl_sys_num, "BoundaryCondition", _reinit_displaced_face);
3257 74977 : _nl[nl_sys_num]->addBoundaryCondition(bc_name, name, parameters);
3258 74933 : }
3259 :
3260 : void
3261 1628 : FEProblemBase::addConstraint(const std::string & c_name,
3262 : const std::string & name,
3263 : InputParameters & parameters)
3264 : {
3265 : parallel_object_only();
3266 :
3267 1628 : _has_constraints = true;
3268 :
3269 1628 : auto determine_var_param_name = [¶meters, this]()
3270 : {
3271 4884 : if (parameters.isParamValid("variable"))
3272 1067 : return "variable";
3273 : else
3274 : {
3275 : // must be a mortar constraint
3276 1122 : const bool has_secondary_var = parameters.isParamValid("secondary_variable");
3277 1122 : const bool has_primary_var = parameters.isParamValid("primary_variable");
3278 561 : if (!has_secondary_var && !has_primary_var)
3279 0 : mooseError(
3280 : "Either a 'secondary_variable' or 'primary_variable' parameter must be supplied for '",
3281 0 : parameters.getObjectName(),
3282 : "'");
3283 561 : return has_secondary_var ? "secondary_variable" : "primary_variable";
3284 : }
3285 1628 : };
3286 :
3287 : const auto nl_sys_num =
3288 3256 : determineSolverSystem(parameters.varName(determine_var_param_name(), name), true).second;
3289 1625 : if (!isSolverSystemNonlinear(nl_sys_num))
3290 0 : mooseError("You are trying to add a Constraint to a linear variable/system, which is not "
3291 : "supported at the moment!");
3292 :
3293 1625 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3294 : {
3295 266 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3296 266 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3297 133 : _reinit_displaced_face = true;
3298 : }
3299 : else
3300 : {
3301 : // It might _want_ to use a displaced mesh... but we're not so set it to false
3302 1492 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3303 2984 : parameters.set<bool>("use_displaced_mesh") = false;
3304 :
3305 2984 : parameters.set<SubProblem *>("_subproblem") = this;
3306 4476 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3307 : }
3308 :
3309 1625 : logAdd("Constraint", name, c_name, parameters);
3310 1625 : _nl[nl_sys_num]->addConstraint(c_name, name, parameters);
3311 1610 : }
3312 :
3313 : void
3314 93989 : FEProblemBase::addAuxVariable(const std::string & var_type,
3315 : const std::string & var_name,
3316 : InputParameters & params)
3317 : {
3318 : parallel_object_only();
3319 :
3320 93989 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3321 93989 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3322 93989 : const auto fe_type = FEType(order, family);
3323 :
3324 : const auto active_subdomains_vector =
3325 93989 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3326 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3327 93989 : active_subdomains_vector.end());
3328 :
3329 93989 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ true, &active_subdomains))
3330 0 : return;
3331 :
3332 281940 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3333 187960 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3334 :
3335 93980 : logAdd("AuxVariable", var_name, var_type, params);
3336 93980 : _aux->addVariable(var_type, var_name, params);
3337 93980 : if (_displaced_problem)
3338 : // MooseObjects need to be unique so change the name here
3339 10166 : _displaced_problem->addAuxVariable(var_type, var_name, params);
3340 :
3341 93980 : markFamilyPRefinement(params);
3342 93980 : if (_displaced_problem)
3343 10166 : _displaced_problem->markFamilyPRefinement(params);
3344 93980 : }
3345 :
3346 : void
3347 2624 : FEProblemBase::addElementalFieldVariable(const std::string & var_type,
3348 : const std::string & var_name,
3349 : InputParameters & params)
3350 : {
3351 2624 : addAuxVariable(var_type, var_name, params);
3352 2624 : }
3353 :
3354 : void
3355 0 : FEProblemBase::addAuxVariable(const std::string & var_name,
3356 : const FEType & type,
3357 : const std::set<SubdomainID> * const active_subdomains)
3358 : {
3359 : parallel_object_only();
3360 :
3361 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3362 :
3363 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3364 0 : return;
3365 :
3366 0 : std::string var_type;
3367 0 : if (type == FEType(0, MONOMIAL))
3368 0 : var_type = "MooseVariableConstMonomial";
3369 0 : else if (type.family == SCALAR)
3370 0 : var_type = "MooseVariableScalar";
3371 0 : else if (FEInterface::field_type(type) == TYPE_VECTOR)
3372 0 : var_type = "VectorMooseVariable";
3373 : else
3374 0 : var_type = "MooseVariable";
3375 :
3376 0 : InputParameters params = _factory.getValidParams(var_type);
3377 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3378 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3379 0 : params.set<MooseEnum>("order") = type.order.get_order();
3380 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3381 :
3382 0 : if (active_subdomains)
3383 0 : for (const SubdomainID & id : *active_subdomains)
3384 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3385 :
3386 0 : logAdd("AuxVariable", var_name, var_type, params);
3387 0 : _aux->addVariable(var_type, var_name, params);
3388 0 : if (_displaced_problem)
3389 0 : _displaced_problem->addAuxVariable("MooseVariable", var_name, params);
3390 :
3391 0 : markFamilyPRefinement(params);
3392 0 : if (_displaced_problem)
3393 0 : _displaced_problem->markFamilyPRefinement(params);
3394 0 : }
3395 :
3396 : void
3397 0 : FEProblemBase::addAuxArrayVariable(const std::string & var_name,
3398 : const FEType & type,
3399 : unsigned int components,
3400 : const std::set<SubdomainID> * const active_subdomains)
3401 : {
3402 : parallel_object_only();
3403 :
3404 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3405 :
3406 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3407 0 : return;
3408 :
3409 0 : InputParameters params = _factory.getValidParams("ArrayMooseVariable");
3410 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3411 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3412 0 : params.set<MooseEnum>("order") = type.order.get_order();
3413 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3414 0 : params.set<unsigned int>("components") = components;
3415 :
3416 0 : if (active_subdomains)
3417 0 : for (const SubdomainID & id : *active_subdomains)
3418 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3419 :
3420 0 : logAdd("Variable", var_name, "ArrayMooseVariable", params);
3421 0 : _aux->addVariable("ArrayMooseVariable", var_name, params);
3422 0 : if (_displaced_problem)
3423 0 : _displaced_problem->addAuxVariable("ArrayMooseVariable", var_name, params);
3424 :
3425 0 : markFamilyPRefinement(params);
3426 0 : if (_displaced_problem)
3427 0 : _displaced_problem->markFamilyPRefinement(params);
3428 0 : }
3429 :
3430 : void
3431 0 : FEProblemBase::addAuxScalarVariable(const std::string & var_name,
3432 : Order order,
3433 : Real /*scale_factor*/,
3434 : const std::set<SubdomainID> * const active_subdomains)
3435 : {
3436 : parallel_object_only();
3437 :
3438 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3439 :
3440 0 : if (order > _max_scalar_order)
3441 0 : _max_scalar_order = order;
3442 :
3443 0 : FEType type(order, SCALAR);
3444 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3445 0 : return;
3446 :
3447 0 : InputParameters params = _factory.getValidParams("MooseVariableScalar");
3448 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3449 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3450 :
3451 0 : params.set<MooseEnum>("order") = type.order.get_order();
3452 0 : params.set<MooseEnum>("family") = "SCALAR";
3453 0 : params.set<std::vector<Real>>("scaling") = std::vector<Real>{1};
3454 0 : if (active_subdomains)
3455 0 : for (const SubdomainID & id : *active_subdomains)
3456 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3457 :
3458 0 : logAdd("ScalarVariable", var_name, "MooseVariableScalar", params);
3459 0 : _aux->addVariable("MooseVariableScalar", var_name, params);
3460 0 : if (_displaced_problem)
3461 0 : _displaced_problem->addAuxVariable("MooseVariableScalar", var_name, params);
3462 0 : }
3463 :
3464 : void
3465 64478 : FEProblemBase::addAuxKernel(const std::string & kernel_name,
3466 : const std::string & name,
3467 : InputParameters & parameters)
3468 : {
3469 : parallel_object_only();
3470 :
3471 64478 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "AuxKernel");
3472 :
3473 64478 : _aux->addKernel(kernel_name, name, parameters);
3474 64367 : }
3475 :
3476 : void
3477 475 : FEProblemBase::addAuxScalarKernel(const std::string & kernel_name,
3478 : const std::string & name,
3479 : InputParameters & parameters)
3480 : {
3481 : parallel_object_only();
3482 :
3483 475 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3484 : {
3485 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3486 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3487 : }
3488 : else
3489 : {
3490 475 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3491 : {
3492 : // We allow AuxScalarKernels to request that they use_displaced_mesh,
3493 : // but then be overridden when no displacements variables are
3494 : // provided in the Mesh block. If that happened, update the value
3495 : // of use_displaced_mesh appropriately for this AuxScalarKernel.
3496 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3497 0 : parameters.set<bool>("use_displaced_mesh") = false;
3498 : }
3499 :
3500 950 : parameters.set<SubProblem *>("_subproblem") = this;
3501 1425 : parameters.set<SystemBase *>("_sys") = _aux.get();
3502 : }
3503 :
3504 475 : logAdd("AuxScalarKernel", name, kernel_name, parameters);
3505 475 : _aux->addScalarKernel(kernel_name, name, parameters);
3506 472 : }
3507 :
3508 : void
3509 871 : FEProblemBase::addDiracKernel(const std::string & kernel_name,
3510 : const std::string & name,
3511 : InputParameters & parameters)
3512 : {
3513 : parallel_object_only();
3514 :
3515 1742 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3516 868 : if (!isSolverSystemNonlinear(nl_sys_num))
3517 0 : mooseError("You are trying to add a DiracKernel to a linear variable/system, which is not "
3518 : "supported at the moment!");
3519 :
3520 868 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3521 : {
3522 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3523 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3524 12 : _reinit_displaced_elem = true;
3525 : }
3526 : else
3527 : {
3528 856 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3529 : {
3530 : // We allow DiracKernels to request that they use_displaced_mesh,
3531 : // but then be overridden when no displacements variables are
3532 : // provided in the Mesh block. If that happened, update the value
3533 : // of use_displaced_mesh appropriately for this DiracKernel.
3534 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3535 0 : parameters.set<bool>("use_displaced_mesh") = false;
3536 : }
3537 :
3538 1712 : parameters.set<SubProblem *>("_subproblem") = this;
3539 2568 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3540 : }
3541 :
3542 868 : logAdd("DiracKernel", name, kernel_name, parameters);
3543 868 : _nl[nl_sys_num]->addDiracKernel(kernel_name, name, parameters);
3544 862 : }
3545 :
3546 : // DGKernels ////
3547 :
3548 : void
3549 1298 : FEProblemBase::addDGKernel(const std::string & dg_kernel_name,
3550 : const std::string & name,
3551 : InputParameters & parameters)
3552 : {
3553 : parallel_object_only();
3554 :
3555 2596 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3556 1295 : if (!isSolverSystemNonlinear(nl_sys_num))
3557 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
3558 : "supported at the moment!");
3559 :
3560 1295 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3561 : {
3562 48 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3563 48 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3564 24 : _reinit_displaced_neighbor = true;
3565 : }
3566 : else
3567 : {
3568 1271 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3569 : {
3570 : // We allow DGKernels to request that they use_displaced_mesh,
3571 : // but then be overridden when no displacements variables are
3572 : // provided in the Mesh block. If that happened, update the value
3573 : // of use_displaced_mesh appropriately for this DGKernel.
3574 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3575 0 : parameters.set<bool>("use_displaced_mesh") = false;
3576 : }
3577 :
3578 2542 : parameters.set<SubProblem *>("_subproblem") = this;
3579 3813 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3580 : }
3581 :
3582 1295 : logAdd("DGKernel", name, dg_kernel_name, parameters);
3583 1295 : _nl[nl_sys_num]->addDGKernel(dg_kernel_name, name, parameters);
3584 :
3585 1295 : _has_internal_edge_residual_objects = true;
3586 1295 : }
3587 :
3588 : void
3589 6609 : FEProblemBase::addFVKernel(const std::string & fv_kernel_name,
3590 : const std::string & name,
3591 : InputParameters & parameters)
3592 : {
3593 6609 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3594 : // FVElementalKernels are computed in the historically finite element threaded loops. They rely
3595 : // on Assembly data like _current_elem. When we call reinit on the FEProblemBase we will only
3596 : // reinit the DisplacedProblem and its associated Assembly objects if we mark this boolean as
3597 : // true
3598 0 : _reinit_displaced_elem = true;
3599 6609 : addObject<FVKernel>(fv_kernel_name, name, parameters);
3600 6603 : }
3601 :
3602 : void
3603 5855 : FEProblemBase::addFVBC(const std::string & fv_bc_name,
3604 : const std::string & name,
3605 : InputParameters & parameters)
3606 : {
3607 5855 : addObject<FVBoundaryCondition>(fv_bc_name, name, parameters);
3608 5855 : }
3609 :
3610 : void
3611 238 : FEProblemBase::addFVInterfaceKernel(const std::string & fv_ik_name,
3612 : const std::string & name,
3613 : InputParameters & parameters)
3614 : {
3615 : /// We assume that variable1 and variable2 can live on different systems, in this case
3616 : /// the user needs to create two interface kernels with flipped variables and parameters
3617 238 : addObject<FVInterfaceKernel>(
3618 : fv_ik_name, name, parameters, /*threaded=*/true, /*variable_param_name=*/"variable1");
3619 229 : }
3620 :
3621 : void
3622 2157 : FEProblemBase::addLinearFVKernel(const std::string & kernel_name,
3623 : const std::string & name,
3624 : InputParameters & parameters)
3625 : {
3626 2157 : addObject<LinearFVKernel>(kernel_name, name, parameters);
3627 2157 : }
3628 :
3629 : void
3630 2064 : FEProblemBase::addLinearFVBC(const std::string & bc_name,
3631 : const std::string & name,
3632 : InputParameters & parameters)
3633 : {
3634 2064 : addObject<LinearFVBoundaryCondition>(bc_name, name, parameters);
3635 2064 : }
3636 :
3637 : // InterfaceKernels ////
3638 :
3639 : void
3640 800 : FEProblemBase::addInterfaceKernel(const std::string & interface_kernel_name,
3641 : const std::string & name,
3642 : InputParameters & parameters)
3643 : {
3644 : parallel_object_only();
3645 :
3646 1600 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3647 797 : if (!isSolverSystemNonlinear(nl_sys_num))
3648 0 : mooseError("You are trying to add a InterfaceKernel to a linear variable/system, which is not "
3649 : "supported at the moment!");
3650 :
3651 797 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3652 : {
3653 24 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3654 24 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3655 12 : _reinit_displaced_neighbor = true;
3656 : }
3657 : else
3658 : {
3659 785 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3660 : {
3661 : // We allow InterfaceKernels to request that they use_displaced_mesh,
3662 : // but then be overridden when no displacements variables are
3663 : // provided in the Mesh block. If that happened, update the value
3664 : // of use_displaced_mesh appropriately for this InterfaceKernel.
3665 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3666 0 : parameters.set<bool>("use_displaced_mesh") = false;
3667 : }
3668 :
3669 1570 : parameters.set<SubProblem *>("_subproblem") = this;
3670 2355 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3671 : }
3672 :
3673 797 : logAdd("InterfaceKernel", name, interface_kernel_name, parameters);
3674 797 : _nl[nl_sys_num]->addInterfaceKernel(interface_kernel_name, name, parameters);
3675 :
3676 797 : _has_internal_edge_residual_objects = true;
3677 797 : }
3678 :
3679 : void
3680 33826 : FEProblemBase::checkICRestartError(const std::string & ic_name,
3681 : const std::string & name,
3682 : const VariableName & var_name)
3683 : {
3684 33826 : if (!_allow_ics_during_restart)
3685 : {
3686 33747 : std::string restart_method = "";
3687 33747 : if (_app.isRestarting())
3688 : restart_method =
3689 0 : "a checkpoint restart, by IC object '" + ic_name + "' for variable '" + name + "'";
3690 33747 : else if (_app.getExReaderForRestart())
3691 : {
3692 3 : std::vector<std::string> restarted_vars = _app.getExReaderForRestart()->get_elem_var_names();
3693 3 : const auto nodal_vars = _app.getExReaderForRestart()->get_nodal_var_names();
3694 3 : const auto global_vars = _app.getExReaderForRestart()->get_global_var_names();
3695 3 : restarted_vars.insert(restarted_vars.end(), nodal_vars.begin(), nodal_vars.end());
3696 3 : restarted_vars.insert(restarted_vars.end(), global_vars.begin(), global_vars.end());
3697 :
3698 3 : if (std::find(restarted_vars.begin(), restarted_vars.end(), var_name) != restarted_vars.end())
3699 6 : restart_method = "an Exodus restart, by IC object '" + ic_name + "' for variable '" + name +
3700 3 : "' that is also being restarted";
3701 3 : }
3702 33747 : if (!restart_method.empty())
3703 3 : mooseError(
3704 : "Initial conditions have been specified during ",
3705 : restart_method,
3706 : ".\nThis is only allowed if you specify 'allow_initial_conditions_with_restart' to "
3707 : "the [Problem], as initial conditions can override restarted fields");
3708 33744 : }
3709 33823 : }
3710 :
3711 : void
3712 31560 : FEProblemBase::addInitialCondition(const std::string & ic_name,
3713 : const std::string & name,
3714 : InputParameters & parameters)
3715 : {
3716 : parallel_object_only();
3717 :
3718 : // before we start to mess with the initial condition, we need to check parameters for errors.
3719 31560 : parameters.checkParams(name);
3720 31557 : const std::string & var_name = parameters.get<VariableName>("variable");
3721 :
3722 : // Forbid initial conditions on a restarted problem, as they would override the restart
3723 31557 : checkICRestartError(ic_name, name, var_name);
3724 :
3725 63108 : parameters.set<SubProblem *>("_subproblem") = this;
3726 :
3727 : // field IC
3728 31554 : if (hasVariable(var_name))
3729 : {
3730 63446 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3731 : {
3732 33452 : MooseVariableFEBase & var = getVariable(
3733 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3734 66904 : parameters.set<SystemBase *>("_sys") = &var.sys();
3735 33452 : std::shared_ptr<InitialConditionBase> ic;
3736 33452 : if (dynamic_cast<MooseVariable *>(&var))
3737 31331 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3738 2121 : else if (dynamic_cast<VectorMooseVariable *>(&var))
3739 250 : ic = _factory.create<VectorInitialCondition>(ic_name, name, parameters, tid);
3740 1871 : else if (dynamic_cast<ArrayMooseVariable *>(&var))
3741 1207 : ic = _factory.create<ArrayInitialCondition>(ic_name, name, parameters, tid);
3742 664 : else if (dynamic_cast<MooseVariableFVReal *>(&var))
3743 664 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3744 0 : else if (dynamic_cast<MooseLinearVariableFVReal *>(&var))
3745 0 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3746 : else
3747 0 : mooseError("Your FE variable in initial condition ",
3748 : name,
3749 : " must be either of scalar or vector type");
3750 33431 : logAdd("IC", name, ic_name, parameters);
3751 33431 : _ics.addObject(ic, tid);
3752 33416 : }
3753 : }
3754 :
3755 : // scalar IC
3756 1524 : else if (hasScalarVariable(var_name))
3757 : {
3758 1524 : MooseVariableScalar & var = getScalarVariable(0, var_name);
3759 3048 : parameters.set<SystemBase *>("_sys") = &var.sys();
3760 : std::shared_ptr<ScalarInitialCondition> ic =
3761 1524 : _factory.create<ScalarInitialCondition>(ic_name, name, parameters);
3762 1524 : logAdd("ScalarIC", name, ic_name, parameters);
3763 1524 : _scalar_ics.addObject(ic);
3764 1524 : }
3765 :
3766 : else
3767 0 : mooseError(
3768 : "Variable '", var_name, "' requested in initial condition '", name, "' does not exist.");
3769 31518 : }
3770 :
3771 : void
3772 2269 : FEProblemBase::addFVInitialCondition(const std::string & ic_name,
3773 : const std::string & name,
3774 : InputParameters & parameters)
3775 : {
3776 : parallel_object_only();
3777 :
3778 : // before we start to mess with the initial condition, we need to check parameters for errors.
3779 2269 : parameters.checkParams(name);
3780 2269 : const std::string & var_name = parameters.get<VariableName>("variable");
3781 :
3782 : // Forbid initial conditions on a restarted problem, as they would override the restart
3783 2269 : checkICRestartError(ic_name, name, var_name);
3784 :
3785 4538 : parameters.set<SubProblem *>("_subproblem") = this;
3786 :
3787 : // field IC
3788 2269 : if (hasVariable(var_name))
3789 : {
3790 4620 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3791 : {
3792 2351 : auto & var = getVariable(
3793 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3794 4702 : parameters.set<SystemBase *>("_sys") = &var.sys();
3795 2351 : std::shared_ptr<FVInitialConditionBase> ic;
3796 2351 : if (var.isFV())
3797 2351 : ic = _factory.create<FVInitialCondition>(ic_name, name, parameters, tid);
3798 : else
3799 0 : mooseError(
3800 : "Your variable for an FVInitialCondition needs to be an a finite volume variable!");
3801 2351 : _fv_ics.addObject(ic, tid);
3802 2351 : }
3803 : }
3804 : else
3805 0 : mooseError("Variable '",
3806 : var_name,
3807 : "' requested in finite volume initial condition '",
3808 : name,
3809 : "' does not exist.");
3810 2269 : }
3811 :
3812 : void
3813 56601 : FEProblemBase::projectSolution()
3814 : {
3815 283005 : TIME_SECTION("projectSolution", 2, "Projecting Initial Solutions")
3816 :
3817 56601 : FloatingPointExceptionGuard fpe_guard(_app);
3818 :
3819 56601 : ComputeInitialConditionThread cic(*this);
3820 56601 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cic);
3821 :
3822 56595 : if (haveFV())
3823 : {
3824 : using ElemInfoRange = StoredRange<MooseMesh::const_elem_info_iterator, const ElemInfo *>;
3825 3932 : ElemInfoRange elem_info_range(_mesh.ownedElemInfoBegin(), _mesh.ownedElemInfoEnd());
3826 :
3827 3932 : ComputeFVInitialConditionThread cfvic(*this);
3828 3932 : Threads::parallel_reduce(elem_info_range, cfvic);
3829 3932 : }
3830 :
3831 : // Need to close the solution vector here so that boundary ICs take precendence
3832 112431 : for (auto & nl : _nl)
3833 55836 : nl->solution().close();
3834 56595 : _aux->solution().close();
3835 :
3836 : // now run boundary-restricted initial conditions
3837 56595 : ComputeBoundaryInitialConditionThread cbic(*this);
3838 56595 : Threads::parallel_reduce(getCurrentAlgebraicBndNodeRange(), cbic);
3839 :
3840 112431 : for (auto & nl : _nl)
3841 55836 : nl->solution().close();
3842 56595 : _aux->solution().close();
3843 :
3844 : // Also, load values into the SCALAR dofs
3845 : // Note: We assume that all SCALAR dofs are on the
3846 : // processor with highest ID
3847 56595 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3848 : {
3849 567 : const auto & ics = _scalar_ics.getActiveObjects();
3850 1561 : for (const auto & ic : ics)
3851 : {
3852 994 : MooseVariableScalar & var = ic->variable();
3853 994 : var.reinit();
3854 :
3855 994 : DenseVector<Number> vals(var.order());
3856 994 : ic->compute(vals);
3857 :
3858 994 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3859 2323 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3860 : {
3861 1329 : const auto global_index = var.dofIndices()[i];
3862 1329 : var.sys().solution().set(global_index, vals(i));
3863 1329 : var.setValue(i, vals(i));
3864 : }
3865 994 : }
3866 : }
3867 :
3868 113431 : for (auto & sys : _solver_systems)
3869 : {
3870 56836 : sys->solution().close();
3871 56836 : sys->solution().localize(*sys->system().current_local_solution, sys->dofMap().get_send_list());
3872 : }
3873 :
3874 56595 : _aux->solution().close();
3875 56595 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3876 56595 : }
3877 :
3878 : void
3879 2041 : FEProblemBase::projectInitialConditionOnCustomRange(
3880 : ConstElemRange & elem_range,
3881 : ConstBndNodeRange & bnd_nodes,
3882 : const std::optional<std::set<VariableName>> & target_vars)
3883 : {
3884 2041 : if (target_vars)
3885 : {
3886 2041 : ComputeInitialConditionThread cic(*this, &(*target_vars));
3887 2041 : Threads::parallel_reduce(elem_range, cic);
3888 : }
3889 : else
3890 : {
3891 0 : ComputeInitialConditionThread cic(*this);
3892 0 : Threads::parallel_reduce(elem_range, cic);
3893 : }
3894 :
3895 : // Need to close the solution vector here so that boundary ICs take precendence
3896 4082 : for (auto & nl : _nl)
3897 2041 : nl->solution().close();
3898 2041 : _aux->solution().close();
3899 :
3900 2041 : if (target_vars)
3901 : {
3902 2041 : ComputeBoundaryInitialConditionThread cbic(*this, &(*target_vars));
3903 2041 : Threads::parallel_reduce(bnd_nodes, cbic);
3904 2041 : }
3905 : else
3906 : {
3907 0 : ComputeBoundaryInitialConditionThread cbic(*this);
3908 0 : Threads::parallel_reduce(bnd_nodes, cbic);
3909 0 : }
3910 :
3911 4082 : for (auto & nl : _nl)
3912 2041 : nl->solution().close();
3913 2041 : _aux->solution().close();
3914 :
3915 : // Also, load values into the SCALAR dofs
3916 : // Note: We assume that all SCALAR dofs are on the
3917 : // processor with highest ID
3918 2041 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3919 : {
3920 0 : const auto & ics = _scalar_ics.getActiveObjects();
3921 0 : for (const auto & ic : ics)
3922 : {
3923 0 : MooseVariableScalar & var = ic->variable();
3924 :
3925 0 : if (target_vars && !target_vars->count(var.name()))
3926 0 : continue;
3927 :
3928 0 : var.reinit();
3929 :
3930 0 : DenseVector<Number> vals(var.order());
3931 0 : ic->compute(vals);
3932 :
3933 0 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3934 0 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3935 : {
3936 0 : const auto global_index = var.dofIndices()[i];
3937 0 : var.sys().solution().set(global_index, vals(i));
3938 0 : var.setValue(i, vals(i));
3939 : }
3940 0 : }
3941 : }
3942 :
3943 4082 : for (auto & nl : _nl)
3944 : {
3945 2041 : nl->solution().close();
3946 2041 : nl->solution().localize(*nl->system().current_local_solution, nl->dofMap().get_send_list());
3947 : }
3948 :
3949 2041 : _aux->solution().close();
3950 2041 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3951 2041 : }
3952 :
3953 : void
3954 737 : FEProblemBase::projectFunctionOnCustomRange(ConstElemRange & elem_range,
3955 : Number (*func)(const Point &,
3956 : const libMesh::Parameters &,
3957 : const std::string &,
3958 : const std::string &),
3959 : Gradient (*func_grad)(const Point &,
3960 : const libMesh::Parameters &,
3961 : const std::string &,
3962 : const std::string &),
3963 : const libMesh::Parameters & params,
3964 : const std::vector<VariableName> & target_vars)
3965 : {
3966 : mooseAssert(!Threads::in_threads,
3967 : "We're performing a projection based on data from just the thread 0 variable, so any "
3968 : "modifications to the variable solution must have been thread joined already");
3969 :
3970 737 : std::unordered_map<unsigned int, std::vector<unsigned int>> sys_to_var_nums;
3971 :
3972 1474 : for (const auto & target_var : target_vars)
3973 : {
3974 737 : const auto sn = systemNumForVariable(target_var);
3975 737 : const auto & var = getStandardVariable(0, target_var);
3976 737 : sys_to_var_nums[sn].push_back(var.number());
3977 : }
3978 :
3979 1474 : for (const auto & [sys_num, var_nums] : sys_to_var_nums)
3980 : {
3981 737 : System & libmesh_sys = getSystemBase(sys_num).system();
3982 737 : libmesh_sys.project_solution(func, func_grad, params, elem_range, var_nums);
3983 : }
3984 737 : }
3985 :
3986 : std::shared_ptr<MaterialBase>
3987 252 : FEProblemBase::getMaterial(std::string name,
3988 : Moose::MaterialDataType type,
3989 : const THREAD_ID tid,
3990 : bool no_warn)
3991 : {
3992 252 : switch (type)
3993 : {
3994 65 : case Moose::NEIGHBOR_MATERIAL_DATA:
3995 65 : name += "_neighbor";
3996 65 : break;
3997 65 : case Moose::FACE_MATERIAL_DATA:
3998 65 : name += "_face";
3999 65 : break;
4000 122 : default:
4001 122 : break;
4002 : }
4003 :
4004 252 : std::shared_ptr<MaterialBase> material = _all_materials[type].getActiveObject(name, tid);
4005 657 : if (!no_warn && material->getParam<bool>("compute") && type == Moose::BLOCK_MATERIAL_DATA)
4006 3 : mooseWarning("You are retrieving a Material object (",
4007 3 : material->name(),
4008 : "), but its compute flag is set to true. This indicates that MOOSE is "
4009 : "computing this property which may not be desired and produce un-expected "
4010 : "results.");
4011 :
4012 246 : return material;
4013 : }
4014 :
4015 : MaterialData &
4016 33127121 : FEProblemBase::getMaterialData(Moose::MaterialDataType type,
4017 : const THREAD_ID tid,
4018 : const MooseObject * object) const
4019 : {
4020 33127121 : switch (type)
4021 : {
4022 850585 : case Moose::BLOCK_MATERIAL_DATA:
4023 850585 : if (object)
4024 260527 : _material_props.addConsumer(type, object);
4025 850585 : return _material_props.getMaterialData(tid);
4026 15162872 : case Moose::NEIGHBOR_MATERIAL_DATA:
4027 15162872 : if (object)
4028 22406 : _neighbor_material_props.addConsumer(type, object);
4029 15162872 : return _neighbor_material_props.getMaterialData(tid);
4030 17113664 : case Moose::BOUNDARY_MATERIAL_DATA:
4031 : case Moose::FACE_MATERIAL_DATA:
4032 : case Moose::INTERFACE_MATERIAL_DATA:
4033 17113664 : if (object)
4034 52151 : _bnd_material_props.addConsumer(type, object);
4035 17113664 : return _bnd_material_props.getMaterialData(tid);
4036 : }
4037 :
4038 0 : mooseError("FEProblemBase::getMaterialData(): Invalid MaterialDataType ", type);
4039 : }
4040 :
4041 : const std::set<const MooseObject *> &
4042 0 : FEProblemBase::getMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
4043 : {
4044 0 : switch (type)
4045 : {
4046 0 : case Moose::BLOCK_MATERIAL_DATA:
4047 0 : return _material_props.getConsumers(type);
4048 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
4049 0 : return _neighbor_material_props.getConsumers(type);
4050 0 : case Moose::BOUNDARY_MATERIAL_DATA:
4051 : case Moose::FACE_MATERIAL_DATA:
4052 : case Moose::INTERFACE_MATERIAL_DATA:
4053 0 : return _bnd_material_props.getConsumers(type);
4054 : }
4055 :
4056 0 : mooseError("FEProblemBase::getMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
4057 : type);
4058 : }
4059 :
4060 : void
4061 0 : FEProblemBase::setPreserveMatrixSparsityPattern(bool preserve)
4062 : {
4063 0 : if (_ignore_zeros_in_jacobian && preserve)
4064 0 : paramWarning(
4065 : "ignore_zeros_in_jacobian",
4066 : "We likely cannot preserve the sparsity pattern if ignoring zeros in the Jacobian, which "
4067 : "leads to removing those entries from the Jacobian sparsity pattern");
4068 0 : _preserve_matrix_sparsity_pattern = preserve;
4069 0 : }
4070 :
4071 : bool
4072 309779 : FEProblemBase::acceptInvalidSolution() const
4073 : {
4074 619473 : return allowInvalidSolution() || // invalid solutions are always allowed
4075 619473 : !_app.solutionInvalidity().hasInvalidSolutionError(); // if not allowed, check for errors
4076 : }
4077 :
4078 : void
4079 964 : FEProblemBase::addFunctorMaterial(const std::string & functor_material_name,
4080 : const std::string & name,
4081 : InputParameters & parameters)
4082 : {
4083 : parallel_object_only();
4084 :
4085 964 : auto add_functor_materials = [&](const auto & parameters, const auto & name)
4086 : {
4087 2011 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
4088 : {
4089 : // Create the general Block/Boundary MaterialBase object
4090 1047 : std::shared_ptr<MaterialBase> material =
4091 1047 : _factory.create<MaterialBase>(functor_material_name, name, parameters, tid);
4092 2094 : logAdd("FunctorMaterial", name, functor_material_name, parameters);
4093 1047 : _all_materials.addObject(material, tid);
4094 1047 : _materials.addObject(material, tid);
4095 : }
4096 964 : };
4097 :
4098 1928 : parameters.set<SubProblem *>("_subproblem") = this;
4099 964 : add_functor_materials(parameters, name);
4100 964 : if (_displaced_problem)
4101 : {
4102 0 : auto disp_params = parameters;
4103 0 : disp_params.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4104 0 : add_functor_materials(disp_params, name + "_displaced");
4105 0 : }
4106 964 : }
4107 :
4108 : void
4109 12139 : FEProblemBase::addMaterial(const std::string & mat_name,
4110 : const std::string & name,
4111 : InputParameters & parameters)
4112 : {
4113 24278 : addMaterialHelper({&_materials}, mat_name, name, parameters);
4114 12041 : }
4115 :
4116 : void
4117 340 : FEProblemBase::addInterfaceMaterial(const std::string & mat_name,
4118 : const std::string & name,
4119 : InputParameters & parameters)
4120 : {
4121 680 : addMaterialHelper({&_interface_materials}, mat_name, name, parameters);
4122 340 : }
4123 :
4124 : void
4125 13421 : FEProblemBase::addMaterialHelper(std::vector<MaterialWarehouse *> warehouses,
4126 : const std::string & mat_name,
4127 : const std::string & name,
4128 : InputParameters & parameters)
4129 : {
4130 : parallel_object_only();
4131 :
4132 13421 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4133 : {
4134 252 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4135 126 : _reinit_displaced_elem = _reinit_displaced_face = _reinit_displaced_neighbor = true;
4136 : }
4137 : else
4138 : {
4139 13295 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
4140 : {
4141 : // We allow Materials to request that they use_displaced_mesh,
4142 : // but then be overridden when no displacements variables are
4143 : // provided in the Mesh block. If that happened, update the value
4144 : // of use_displaced_mesh appropriately for this Material.
4145 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
4146 0 : parameters.set<bool>("use_displaced_mesh") = false;
4147 : }
4148 :
4149 39885 : parameters.set<SubProblem *>("_subproblem") = this;
4150 : }
4151 :
4152 13421 : unsigned int n_threads = libMesh::n_threads();
4153 :
4154 : #ifdef MOOSE_KOKKOS_ENABLED
4155 10010 : if (parameters.isKokkosObject())
4156 942 : n_threads = 1;
4157 : #endif
4158 :
4159 27924 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4160 : {
4161 : // Create the general Block/Boundary MaterialBase object
4162 : std::shared_ptr<MaterialBase> material =
4163 14609 : _factory.create<MaterialBase>(mat_name, name, parameters, tid);
4164 14503 : logAdd("Material", name, mat_name, parameters);
4165 29006 : bool discrete = !material->getParam<bool>("compute");
4166 :
4167 : // If the object is boundary restricted or if it is a functor material we do not create the
4168 : // neighbor and face objects
4169 14503 : if (material->boundaryRestricted() || dynamic_cast<FunctorMaterial *>(material.get()))
4170 : {
4171 2970 : _all_materials.addObject(material, tid);
4172 2970 : if (discrete)
4173 4 : _discrete_materials.addObject(material, tid);
4174 : else
4175 5932 : for (auto && warehouse : warehouses)
4176 2966 : warehouse->addObject(material, tid);
4177 : }
4178 :
4179 : // Non-boundary restricted require face and neighbor objects
4180 : else
4181 : {
4182 : // TODO: we only need to do this if we have needs for face materials (e.g.
4183 : // FV, DG, etc.) - but currently we always do it. Figure out how to fix
4184 : // this.
4185 :
4186 : // The name of the object being created, this is changed multiple times as objects are
4187 : // created below
4188 11533 : std::string object_name;
4189 :
4190 : // Create a copy of the supplied parameters to the setting for "_material_data_type" isn't
4191 : // used from a previous tid loop
4192 11533 : InputParameters current_parameters = parameters;
4193 :
4194 : // face material
4195 11533 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4196 : Moose::FACE_MATERIAL_DATA;
4197 11533 : object_name = name + "_face";
4198 : std::shared_ptr<MaterialBase> face_material =
4199 11533 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4200 :
4201 : // neighbor material
4202 23066 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4203 : Moose::NEIGHBOR_MATERIAL_DATA;
4204 11533 : current_parameters.set<bool>("_neighbor") = true;
4205 11533 : object_name = name + "_neighbor";
4206 : std::shared_ptr<MaterialBase> neighbor_material =
4207 11533 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4208 :
4209 : // Store the material objects
4210 11533 : _all_materials.addObjects(material, neighbor_material, face_material, tid);
4211 :
4212 11533 : if (discrete)
4213 73 : _discrete_materials.addObjects(material, neighbor_material, face_material, tid);
4214 : else
4215 22920 : for (auto && warehouse : warehouses)
4216 11460 : warehouse->addObjects(material, neighbor_material, face_material, tid);
4217 :
4218 : // Names of all controllable parameters for this Material object
4219 11533 : const std::string & base = parameters.getBase();
4220 34599 : MooseObjectParameterName name(MooseObjectName(base, material->name()), "*");
4221 : const auto param_names =
4222 11533 : _app.getInputParameterWarehouse().getControllableParameterNames(name);
4223 :
4224 : // Connect parameters of the primary Material object to those on the face and neighbor
4225 : // objects
4226 29328 : for (const auto & p_name : param_names)
4227 : {
4228 35590 : MooseObjectParameterName primary_name(MooseObjectName(base, material->name()),
4229 35590 : p_name.parameter());
4230 35590 : MooseObjectParameterName face_name(MooseObjectName(base, face_material->name()),
4231 35590 : p_name.parameter());
4232 35590 : MooseObjectParameterName neighbor_name(MooseObjectName(base, neighbor_material->name()),
4233 35590 : p_name.parameter());
4234 17795 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4235 : primary_name, face_name, false);
4236 17795 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4237 : primary_name, neighbor_name, false);
4238 17795 : }
4239 11533 : }
4240 14503 : }
4241 13315 : }
4242 :
4243 : void
4244 4722361 : FEProblemBase::prepareMaterials(const std::unordered_set<unsigned int> & consumer_needed_mat_props,
4245 : const SubdomainID blk_id,
4246 : const THREAD_ID tid)
4247 : {
4248 4722361 : std::set<MooseVariableFEBase *> needed_moose_vars;
4249 4722361 : std::unordered_set<unsigned int> needed_mat_props;
4250 :
4251 4722361 : if (_all_materials.hasActiveBlockObjects(blk_id, tid))
4252 : {
4253 586840 : _all_materials.updateVariableDependency(needed_moose_vars, tid);
4254 586840 : _all_materials.updateBlockMatPropDependency(blk_id, needed_mat_props, tid);
4255 : }
4256 :
4257 4722361 : const auto & ids = _mesh.getSubdomainBoundaryIds(blk_id);
4258 21836072 : for (const auto id : ids)
4259 : {
4260 17113711 : _materials.updateBoundaryVariableDependency(id, needed_moose_vars, tid);
4261 17113711 : _materials.updateBoundaryMatPropDependency(id, needed_mat_props, tid);
4262 : }
4263 :
4264 4722361 : const auto & current_active_elemental_moose_variables = getActiveElementalMooseVariables(tid);
4265 4722361 : needed_moose_vars.insert(current_active_elemental_moose_variables.begin(),
4266 : current_active_elemental_moose_variables.end());
4267 :
4268 4722361 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
4269 :
4270 4722361 : setActiveElementalMooseVariables(needed_moose_vars, tid);
4271 4722361 : setActiveMaterialProperties(needed_mat_props, tid);
4272 4722361 : }
4273 :
4274 : void
4275 363663998 : FEProblemBase::reinitMaterials(SubdomainID blk_id, const THREAD_ID tid, bool swap_stateful)
4276 : {
4277 363663998 : if (hasActiveMaterialProperties(tid))
4278 : {
4279 15111940 : auto && elem = _assembly[tid][0]->elem();
4280 15111940 : unsigned int n_points = _assembly[tid][0]->qRule()->n_points();
4281 :
4282 15111940 : auto & material_data = _material_props.getMaterialData(tid);
4283 15111940 : material_data.resize(n_points);
4284 :
4285 : // Only swap if requested
4286 15111940 : if (swap_stateful)
4287 15105398 : material_data.swap(*elem);
4288 :
4289 15111940 : if (_discrete_materials.hasActiveBlockObjects(blk_id, tid))
4290 2698 : material_data.reset(_discrete_materials.getActiveBlockObjects(blk_id, tid));
4291 :
4292 15111937 : if (_materials.hasActiveBlockObjects(blk_id, tid))
4293 15093398 : material_data.reinit(_materials.getActiveBlockObjects(blk_id, tid));
4294 : }
4295 363663943 : }
4296 :
4297 : void
4298 5106440 : FEProblemBase::reinitMaterialsFace(const SubdomainID blk_id,
4299 : const THREAD_ID tid,
4300 : const bool swap_stateful,
4301 : const std::deque<MaterialBase *> * const reinit_mats)
4302 : {
4303 : // we reinit more often than needed here because we dont have a way to check whether
4304 : // we need to compute the face materials on a particular (possibly external) face
4305 5106440 : if (hasActiveMaterialProperties(tid))
4306 : {
4307 1045875 : auto && elem = _assembly[tid][0]->elem();
4308 1045875 : unsigned int side = _assembly[tid][0]->side();
4309 1045875 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4310 :
4311 1045875 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4312 1045875 : bnd_material_data.resize(n_points);
4313 :
4314 1045875 : if (swap_stateful && !bnd_material_data.isSwapped())
4315 1024401 : bnd_material_data.swap(*elem, side);
4316 :
4317 1045875 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4318 0 : bnd_material_data.reset(
4319 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4320 :
4321 1045875 : if (reinit_mats)
4322 21474 : bnd_material_data.reinit(*reinit_mats);
4323 1024401 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4324 1024401 : bnd_material_data.reinit(
4325 1024401 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4326 : }
4327 5106440 : }
4328 :
4329 : void
4330 3813814 : FEProblemBase::reinitMaterialsFaceOnBoundary(const BoundaryID boundary_id,
4331 : const SubdomainID blk_id,
4332 : const THREAD_ID tid,
4333 : const bool swap_stateful,
4334 : const std::deque<MaterialBase *> * const reinit_mats)
4335 : {
4336 4144745 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4337 330931 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4338 330931 : needInternalNeighborSideMaterial(blk_id, tid)))
4339 : {
4340 407446 : const auto * const elem = _assembly[tid][0]->elem();
4341 407446 : unsigned int side = _assembly[tid][0]->side();
4342 407446 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4343 :
4344 407446 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4345 407446 : bnd_material_data.resize(n_points);
4346 :
4347 407446 : if (swap_stateful && !bnd_material_data.isSwapped())
4348 407446 : bnd_material_data.swap(*elem, side);
4349 :
4350 407446 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4351 0 : bnd_material_data.reset(
4352 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4353 :
4354 407446 : if (reinit_mats)
4355 0 : bnd_material_data.reinit(*reinit_mats);
4356 407446 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4357 393054 : bnd_material_data.reinit(
4358 393054 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4359 : }
4360 3813814 : }
4361 :
4362 : void
4363 43931 : FEProblemBase::reinitMaterialsNeighborOnBoundary(
4364 : const BoundaryID boundary_id,
4365 : const SubdomainID blk_id,
4366 : const THREAD_ID tid,
4367 : const bool swap_stateful,
4368 : const std::deque<MaterialBase *> * const reinit_mats)
4369 : {
4370 : // Since objects don't declare whether they need the face or neighbor (side) material properties,
4371 : // we use the same criteria for skipping material property computations as for face material
4372 : // properties This could be a future optimization.
4373 48141 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4374 4210 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4375 4210 : needInternalNeighborSideMaterial(blk_id, tid)))
4376 35986 : reinitMaterialsNeighbor(blk_id, tid, swap_stateful, reinit_mats);
4377 43931 : }
4378 :
4379 : void
4380 3974249 : FEProblemBase::reinitMaterialsNeighbor(const SubdomainID blk_id,
4381 : const THREAD_ID tid,
4382 : const bool swap_stateful,
4383 : const std::deque<MaterialBase *> * const reinit_mats)
4384 : {
4385 3974249 : if (hasActiveMaterialProperties(tid))
4386 : {
4387 : // NOTE: this will not work with h-adaptivity
4388 : // lindsayad: why not?
4389 :
4390 887183 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4391 887183 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(_assembly[tid][0]->elem());
4392 :
4393 : mooseAssert(neighbor, "neighbor should be non-null");
4394 : mooseAssert(blk_id == neighbor->subdomain_id(),
4395 : "The provided blk_id " << blk_id << " and neighbor subdomain ID "
4396 : << neighbor->subdomain_id() << " do not match.");
4397 :
4398 887183 : unsigned int n_points = _assembly[tid][0]->qRuleNeighbor()->n_points();
4399 :
4400 887183 : auto & neighbor_material_data = _neighbor_material_props.getMaterialData(tid);
4401 887183 : neighbor_material_data.resize(n_points);
4402 :
4403 : // Only swap if requested
4404 887183 : if (swap_stateful)
4405 865709 : neighbor_material_data.swap(*neighbor, neighbor_side);
4406 :
4407 887183 : if (_discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4408 0 : neighbor_material_data.reset(
4409 0 : _discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4410 :
4411 887183 : if (reinit_mats)
4412 21474 : neighbor_material_data.reinit(*reinit_mats);
4413 865709 : else if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4414 865446 : neighbor_material_data.reinit(
4415 865446 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4416 : }
4417 3974249 : }
4418 :
4419 : void
4420 4067906 : FEProblemBase::reinitMaterialsBoundary(const BoundaryID boundary_id,
4421 : const THREAD_ID tid,
4422 : const bool swap_stateful,
4423 : const std::deque<MaterialBase *> * const reinit_mats)
4424 : {
4425 4067906 : if (hasActiveMaterialProperties(tid) && needBoundaryMaterialOnSide(boundary_id, tid))
4426 : {
4427 285958 : auto && elem = _assembly[tid][0]->elem();
4428 285958 : unsigned int side = _assembly[tid][0]->side();
4429 285958 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4430 :
4431 285958 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4432 285958 : bnd_material_data.resize(n_points);
4433 :
4434 285958 : if (swap_stateful && !bnd_material_data.isSwapped())
4435 238273 : bnd_material_data.swap(*elem, side);
4436 :
4437 285958 : if (_discrete_materials.hasActiveBoundaryObjects(boundary_id, tid))
4438 0 : bnd_material_data.reset(_discrete_materials.getActiveBoundaryObjects(boundary_id, tid));
4439 :
4440 285958 : if (reinit_mats)
4441 21474 : bnd_material_data.reinit(*reinit_mats);
4442 264484 : else if (_materials.hasActiveBoundaryObjects(boundary_id, tid))
4443 23154 : bnd_material_data.reinit(_materials.getActiveBoundaryObjects(boundary_id, tid));
4444 : }
4445 4067906 : }
4446 :
4447 : void
4448 46788 : FEProblemBase::reinitMaterialsInterface(BoundaryID boundary_id,
4449 : const THREAD_ID tid,
4450 : bool swap_stateful)
4451 : {
4452 46788 : if (hasActiveMaterialProperties(tid) && needInterfaceMaterialOnSide(boundary_id, tid))
4453 : {
4454 37022 : const Elem * const & elem = _assembly[tid][0]->elem();
4455 37022 : unsigned int side = _assembly[tid][0]->side();
4456 37022 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4457 :
4458 37022 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4459 37022 : bnd_material_data.resize(n_points);
4460 :
4461 37022 : if (swap_stateful && !bnd_material_data.isSwapped())
4462 35527 : bnd_material_data.swap(*elem, side);
4463 :
4464 37022 : if (_interface_materials.hasActiveBoundaryObjects(boundary_id, tid))
4465 1171 : bnd_material_data.reinit(_interface_materials.getActiveBoundaryObjects(boundary_id, tid));
4466 : }
4467 46788 : }
4468 :
4469 : void
4470 362045176 : FEProblemBase::swapBackMaterials(const THREAD_ID tid)
4471 : {
4472 362045176 : auto && elem = _assembly[tid][0]->elem();
4473 362045176 : _material_props.getMaterialData(tid).swapBack(*elem);
4474 362045176 : }
4475 :
4476 : void
4477 8666151 : FEProblemBase::swapBackMaterialsFace(const THREAD_ID tid)
4478 : {
4479 8666151 : auto && elem = _assembly[tid][0]->elem();
4480 8666151 : unsigned int side = _assembly[tid][0]->side();
4481 8666151 : _bnd_material_props.getMaterialData(tid).swapBack(*elem, side);
4482 8666151 : }
4483 :
4484 : void
4485 3630872 : FEProblemBase::swapBackMaterialsNeighbor(const THREAD_ID tid)
4486 : {
4487 : // NOTE: this will not work with h-adaptivity
4488 3630872 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4489 : unsigned int neighbor_side =
4490 3630872 : neighbor ? neighbor->which_neighbor_am_i(_assembly[tid][0]->elem()) : libMesh::invalid_uint;
4491 :
4492 3630872 : if (!neighbor)
4493 : {
4494 0 : if (haveFV())
4495 : {
4496 : // If neighbor is null, then we're on the neighbor side of a mesh boundary, e.g. we're off
4497 : // the mesh in ghost-land. If we're using the finite volume method, then variable values and
4498 : // consequently material properties have well-defined values in this ghost region outside of
4499 : // the mesh and we really do want to reinit our neighbor materials in this case. Since we're
4500 : // off in ghost land it's safe to do swaps with `MaterialPropertyStorage` using the elem and
4501 : // elem_side keys
4502 0 : neighbor = _assembly[tid][0]->elem();
4503 0 : neighbor_side = _assembly[tid][0]->side();
4504 : mooseAssert(neighbor, "We should have an appropriate value for elem coming from Assembly");
4505 : }
4506 : else
4507 0 : mooseError("neighbor is null in Assembly!");
4508 : }
4509 :
4510 3630872 : _neighbor_material_props.getMaterialData(tid).swapBack(*neighbor, neighbor_side);
4511 3630872 : }
4512 :
4513 : void
4514 971908 : FEProblemBase::logAdd(const std::string & system,
4515 : const std::string & name,
4516 : const std::string & type,
4517 : const InputParameters & params) const
4518 : {
4519 971908 : if (_verbose_setup != "false")
4520 102 : _console << "[DBG] Adding " << system << " '" << name << "' of type " << type << std::endl;
4521 971908 : if (_verbose_setup == "extra")
4522 0 : _console << params << std::endl;
4523 971908 : }
4524 :
4525 : void
4526 136532 : FEProblemBase::addObjectParamsHelper(InputParameters & parameters,
4527 : const std::string & object_name,
4528 : const std::string & var_param_name)
4529 : {
4530 : // Due to objects like SolutionUserObject which manipulate libmesh objects
4531 : // and variables directly at the back end, we need a default option here
4532 : // which is going to be the pointer to the first solver system within this
4533 : // problem
4534 136532 : unsigned int sys_num = 0;
4535 136532 : if (parameters.isParamValid(var_param_name))
4536 : {
4537 57276 : const auto variable_name = parameters.varName(var_param_name, object_name);
4538 57276 : if (this->hasVariable(variable_name) || this->hasScalarVariable(variable_name))
4539 51060 : sys_num = getSystem(variable_name).number();
4540 57276 : }
4541 273064 : if (parameters.isParamValid("solver_sys"))
4542 : {
4543 3722 : const auto var_sys_num = sys_num;
4544 3722 : sys_num = getSystemBase(parameters.get<SolverSystemName>("solver_sys")).number();
4545 3722 : if (sys_num != var_sys_num && parameters.isParamValid(var_param_name))
4546 0 : mooseError("We dont support setting 'variable' to a variable that is not set to the same "
4547 : "system as the 'solver_sys' parameter");
4548 : }
4549 :
4550 137696 : if (_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4551 137696 : parameters.get<bool>("use_displaced_mesh"))
4552 : {
4553 1132 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4554 566 : if (sys_num == _aux->number())
4555 1104 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->systemBaseAuxiliary();
4556 : else
4557 594 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(sys_num);
4558 : }
4559 : else
4560 : {
4561 : // The object requested use_displaced_mesh, but it was overridden
4562 : // due to there being no displacements variables in the [Mesh] block.
4563 : // If that happened, update the value of use_displaced_mesh appropriately.
4564 220497 : if (!_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4565 220497 : parameters.get<bool>("use_displaced_mesh"))
4566 144 : parameters.set<bool>("use_displaced_mesh") = false;
4567 :
4568 271932 : parameters.set<SubProblem *>("_subproblem") = this;
4569 :
4570 135966 : if (sys_num == _aux->number())
4571 39945 : parameters.set<SystemBase *>("_sys") = _aux.get();
4572 : else
4573 367953 : parameters.set<SystemBase *>("_sys") = _solver_systems[sys_num].get();
4574 : }
4575 136532 : }
4576 :
4577 : void
4578 61804 : FEProblemBase::checkUserObjectNameCollision(const std::string & name,
4579 : const std::string & type) const
4580 : {
4581 61804 : if (hasUserObject(name))
4582 9 : mooseError("A ",
4583 9 : getUserObjectBase(name).typeAndName(),
4584 : " already exists. You may not add a ",
4585 : type,
4586 : " by the same name.");
4587 :
4588 : #ifdef MOOSE_KOKKOS_ENABLED
4589 46253 : if (hasKokkosUserObject(name))
4590 0 : mooseError("A ",
4591 0 : getKokkosUserObject<UserObjectBase>(name).typeAndName(),
4592 : " already exists. You may not add a ",
4593 : type,
4594 : " by the same name.");
4595 : #endif
4596 61795 : }
4597 :
4598 : void
4599 49122 : FEProblemBase::addPostprocessor(const std::string & pp_name,
4600 : const std::string & name,
4601 : InputParameters & parameters)
4602 : {
4603 49122 : checkUserObjectNameCollision(name, "Postprocessor");
4604 :
4605 49119 : addUserObject(pp_name, name, parameters);
4606 49090 : }
4607 :
4608 : void
4609 5683 : FEProblemBase::addVectorPostprocessor(const std::string & pp_name,
4610 : const std::string & name,
4611 : InputParameters & parameters)
4612 : {
4613 5683 : checkUserObjectNameCollision(name, "VectorPostprocessor");
4614 :
4615 5680 : addUserObject(pp_name, name, parameters);
4616 5650 : }
4617 :
4618 : void
4619 4719 : FEProblemBase::addReporter(const std::string & type,
4620 : const std::string & name,
4621 : InputParameters & parameters)
4622 : {
4623 4719 : checkUserObjectNameCollision(name, "Reporter");
4624 :
4625 4716 : addUserObject(type, name, parameters);
4626 4683 : }
4627 :
4628 : std::vector<std::shared_ptr<UserObject>>
4629 69599 : FEProblemBase::addUserObject(const std::string & user_object_name,
4630 : const std::string & name,
4631 : InputParameters & parameters)
4632 : {
4633 : parallel_object_only();
4634 :
4635 69599 : std::vector<std::shared_ptr<UserObject>> uos;
4636 :
4637 : // Add the _subproblem and _sys parameters depending on use_displaced_mesh
4638 69599 : addObjectParamsHelper(parameters, name);
4639 :
4640 112975 : for (const auto tid : make_range(libMesh::n_threads()))
4641 : {
4642 : // Create the UserObject
4643 : std::shared_ptr<UserObject> user_object =
4644 73364 : _factory.create<UserObject>(user_object_name, name, parameters, tid);
4645 73089 : logAdd("UserObject", name, user_object_name, parameters);
4646 73089 : uos.push_back(user_object);
4647 :
4648 73089 : if (tid != 0)
4649 3765 : user_object->setPrimaryThreadCopy(uos[0].get());
4650 :
4651 73089 : theWarehouse().add(user_object);
4652 :
4653 : // Attempt to create all the possible UserObject types
4654 73083 : auto euo = std::dynamic_pointer_cast<ElementUserObject>(user_object);
4655 73083 : auto suo = std::dynamic_pointer_cast<SideUserObject>(user_object);
4656 73083 : auto isuo = std::dynamic_pointer_cast<InternalSideUserObject>(user_object);
4657 73083 : auto iuo = std::dynamic_pointer_cast<InterfaceUserObjectBase>(user_object);
4658 73083 : auto nuo = std::dynamic_pointer_cast<NodalUserObject>(user_object);
4659 73083 : auto duo = std::dynamic_pointer_cast<DomainUserObject>(user_object);
4660 73083 : auto guo = std::dynamic_pointer_cast<GeneralUserObject>(user_object);
4661 73083 : auto tguo = std::dynamic_pointer_cast<ThreadedGeneralUserObject>(user_object);
4662 73083 : auto muo = std::dynamic_pointer_cast<MortarUserObject>(user_object);
4663 :
4664 : // Account for displaced mesh use
4665 73083 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4666 : {
4667 : // Whether to re-init or not depends on the attributes of the base classes.
4668 : // For example, InterfaceUOBase has "_current_side_elem" and "_neighbor_elem"
4669 : // so it needs to reinit on displaced neighbors and faces
4670 : // _reinit_displaced_elem -> _current_elem will be reinited
4671 : // _reinit_displaced_face -> _current_elem, lowerD if any and _current_side_elem to be
4672 : // reinited _reinit_displaced_neighbor -> _current_elem, lowerD if any and _current_neighbor
4673 : // to be reinited Note that as soon as you use materials on the displaced mesh, all three get
4674 : // turned on.
4675 615 : if (euo || nuo || duo)
4676 555 : _reinit_displaced_elem = true;
4677 615 : if (suo || duo || isuo || iuo)
4678 24 : _reinit_displaced_face = true;
4679 615 : if (iuo || duo || isuo)
4680 0 : _reinit_displaced_neighbor = true;
4681 : }
4682 :
4683 : // These objects only require one thread
4684 73083 : if ((guo && !tguo) || muo)
4685 29707 : break;
4686 340446 : }
4687 :
4688 : // Add as a Functor if it is one. We usually need to add the user object from thread 0 as the
4689 : // registered functor for all threads because when user objects are thread joined, generally only
4690 : // the primary thread copy ends up with all the data
4691 145880 : for (const auto tid : make_range(libMesh::n_threads()))
4692 : {
4693 76562 : const decltype(uos)::size_type uo_index = uos.front()->needThreadedCopy() ? tid : 0;
4694 76562 : if (const auto functor = dynamic_cast<Moose::FunctorBase<Real> *>(uos[uo_index].get()))
4695 : {
4696 59747 : this->addFunctor(name, *functor, tid);
4697 59747 : if (_displaced_problem)
4698 729 : _displaced_problem->addFunctor(name, *functor, tid);
4699 : }
4700 : }
4701 :
4702 69318 : return uos;
4703 2 : }
4704 :
4705 : void
4706 2002 : FEProblemBase::addFVInterpolationMethod(const std::string & method_type,
4707 : const std::string & name,
4708 : InputParameters & parameters)
4709 : {
4710 : parallel_object_only();
4711 :
4712 2002 : addObjectParamsHelper(parameters, name);
4713 :
4714 4004 : for (const auto tid : make_range(libMesh::n_threads()))
4715 : {
4716 2002 : auto method = _factory.create<FVInterpolationMethod>(method_type, name, parameters, tid);
4717 2002 : logAdd("FVInterpolationMethod", name, method_type, parameters);
4718 2002 : theWarehouse().add(method);
4719 2002 : }
4720 2002 : }
4721 :
4722 : const UserObject &
4723 285472 : FEProblemBase::getUserObjectBase(const std::string & name, const THREAD_ID tid /* = 0 */) const
4724 : {
4725 285472 : std::vector<UserObject *> objs;
4726 285472 : theWarehouse()
4727 570944 : .query()
4728 285472 : .condition<AttribSystem>("UserObject")
4729 285472 : .condition<AttribThread>(tid)
4730 285472 : .condition<AttribName>(name)
4731 285472 : .queryInto(objs);
4732 285472 : if (objs.empty())
4733 : {
4734 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_user_object"),
4735 : "A UserObject getter was called before UserObjects have been constructed. The "
4736 : "requested UserObject '" +
4737 : name + "' may exist in the input file, but UserObjects are not available yet.");
4738 :
4739 0 : mooseError("Unable to find user object with name '" + name + "'");
4740 : }
4741 : mooseAssert(objs.size() == 1, "Should only find one UO");
4742 570944 : return *(objs[0]);
4743 285472 : }
4744 :
4745 : const Positions &
4746 1142 : FEProblemBase::getPositionsObject(const std::string & name) const
4747 : {
4748 1142 : std::vector<Positions *> objs;
4749 1142 : theWarehouse()
4750 2284 : .query()
4751 1142 : .condition<AttribSystem>("UserObject")
4752 1142 : .condition<AttribName>(name)
4753 1142 : .queryInto(objs);
4754 1142 : if (objs.empty())
4755 0 : mooseError("Unable to find Positions object with name '" + name + "'");
4756 : mooseAssert(objs.size() == 1, "Should only find one Positions");
4757 2284 : return *(objs[0]);
4758 1142 : }
4759 :
4760 : bool
4761 90477 : FEProblemBase::hasUserObject(const std::string & name) const
4762 : {
4763 90477 : std::vector<UserObject *> objs;
4764 90477 : theWarehouse()
4765 90477 : .query()
4766 90477 : .condition<AttribSystem>("UserObject")
4767 180954 : .condition<AttribThread>(0)
4768 90477 : .condition<AttribName>(name)
4769 90477 : .queryInto(objs);
4770 180954 : return !objs.empty();
4771 90477 : }
4772 :
4773 : const FVInterpolationMethod &
4774 571 : FEProblemBase::getFVInterpolationMethod(const InterpolationMethodName & name,
4775 : const THREAD_ID tid) const
4776 : {
4777 571 : std::vector<FVInterpolationMethod *> methods;
4778 571 : theWarehouse()
4779 1142 : .query()
4780 571 : .condition<AttribSystem>("FVInterpolationMethod")
4781 571 : .condition<AttribThread>(tid)
4782 571 : .condition<AttribName>(name)
4783 571 : .queryInto(methods);
4784 :
4785 571 : if (methods.empty())
4786 : {
4787 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_interpolation_method"),
4788 : "An FVInterpolationMethod getter was called before FVInterpolationMethods have "
4789 : "been constructed. If you are attempting to access this object in the constructor "
4790 : "of another object then make sure that the FVInterpolationMethod is constructed "
4791 : "before the object using it.");
4792 :
4793 0 : mooseError("Unable to find FVInterpolationMethod with name '", name, "'");
4794 : }
4795 :
4796 : mooseAssert(methods.size() == 1, "Expected a single FVInterpolationMethod per thread");
4797 1142 : return *(methods[0]);
4798 571 : }
4799 :
4800 : const FVFaceInterpolationMethod &
4801 0 : FEProblemBase::getFVFaceInterpolationMethod(const InterpolationMethodName & name,
4802 : const THREAD_ID tid) const
4803 : {
4804 0 : const auto & method = getFVInterpolationMethod(name, tid);
4805 0 : const auto * face_method = dynamic_cast<const FVFaceInterpolationMethod *>(&method);
4806 :
4807 0 : if (!face_method)
4808 0 : mooseError("FVInterpolationMethod '",
4809 : name,
4810 : "' (",
4811 0 : method.type(),
4812 : ") is not a scalar face interpolation method.");
4813 :
4814 0 : return *face_method;
4815 : }
4816 :
4817 : const FVAdvectedInterpolationMethod &
4818 571 : FEProblemBase::getFVAdvectedInterpolationMethod(const InterpolationMethodName & name,
4819 : const THREAD_ID tid) const
4820 : {
4821 571 : const auto & method = getFVInterpolationMethod(name, tid);
4822 571 : const auto * advected_method = dynamic_cast<const FVAdvectedInterpolationMethod *>(&method);
4823 :
4824 571 : if (!advected_method)
4825 0 : mooseError("FVInterpolationMethod '",
4826 : name,
4827 : "' (",
4828 0 : method.type(),
4829 : ") is not an advected interpolation method.");
4830 :
4831 571 : return *advected_method;
4832 : }
4833 :
4834 : bool
4835 0 : FEProblemBase::hasFVInterpolationMethod(const InterpolationMethodName & name) const
4836 : {
4837 0 : std::vector<FVInterpolationMethod *> methods;
4838 0 : theWarehouse()
4839 0 : .query()
4840 0 : .condition<AttribSystem>("FVInterpolationMethod")
4841 0 : .condition<AttribThread>(0)
4842 0 : .condition<AttribName>(name)
4843 0 : .queryInto(methods);
4844 0 : return !methods.empty();
4845 0 : }
4846 :
4847 : bool
4848 308 : FEProblemBase::hasPostprocessorValueByName(const PostprocessorName & name) const
4849 : {
4850 308 : return _reporter_data.hasReporterValue<PostprocessorValue>(PostprocessorReporterName(name));
4851 : }
4852 :
4853 : const Postprocessor &
4854 2 : FEProblemBase::getPostprocessorObjectByName(const PostprocessorName & object_name,
4855 : const THREAD_ID tid) const
4856 : {
4857 2 : std::vector<Postprocessor *> objs;
4858 2 : theWarehouse()
4859 2 : .query()
4860 4 : .condition<AttribInterfaces>(Interfaces::Postprocessor)
4861 2 : .condition<AttribThread>(tid)
4862 2 : .condition<AttribName>(object_name)
4863 2 : .queryInto(objs);
4864 :
4865 2 : if (objs.empty())
4866 0 : mooseError("Unable to find Postprocessor with name '", object_name, "'");
4867 : mooseAssert(objs.size() == 1,
4868 : "We shouldn't find more than one postprocessor object for a given name");
4869 4 : return *(objs[0]);
4870 2 : }
4871 :
4872 : const PostprocessorValue &
4873 842336 : FEProblemBase::getPostprocessorValueByName(const PostprocessorName & name,
4874 : std::size_t t_index) const
4875 : {
4876 1684672 : return _reporter_data.getReporterValue<PostprocessorValue>(PostprocessorReporterName(name),
4877 1684672 : t_index);
4878 : }
4879 :
4880 : void
4881 577029 : FEProblemBase::setPostprocessorValueByName(const PostprocessorName & name,
4882 : const PostprocessorValue & value,
4883 : std::size_t t_index)
4884 : {
4885 577029 : _reporter_data.setReporterValue<PostprocessorValue>(
4886 1154058 : PostprocessorReporterName(name), value, t_index);
4887 577029 : }
4888 :
4889 : bool
4890 52 : FEProblemBase::hasPostprocessor(const std::string & name) const
4891 : {
4892 52 : mooseDeprecated("FEProblemBase::hasPostprocssor is being removed; use "
4893 : "hasPostprocessorValueByName instead.");
4894 52 : return hasPostprocessorValueByName(name);
4895 : }
4896 :
4897 : const VectorPostprocessorValue &
4898 50 : FEProblemBase::getVectorPostprocessorValueByName(const std::string & object_name,
4899 : const std::string & vector_name,
4900 : std::size_t t_index) const
4901 : {
4902 50 : return _reporter_data.getReporterValue<VectorPostprocessorValue>(
4903 100 : VectorPostprocessorReporterName(object_name, vector_name), t_index);
4904 : }
4905 :
4906 : void
4907 18 : FEProblemBase::setVectorPostprocessorValueByName(const std::string & object_name,
4908 : const std::string & vector_name,
4909 : const VectorPostprocessorValue & value,
4910 : std::size_t t_index)
4911 : {
4912 18 : _reporter_data.setReporterValue<VectorPostprocessorValue>(
4913 36 : VectorPostprocessorReporterName(object_name, vector_name), value, t_index);
4914 18 : }
4915 :
4916 : const VectorPostprocessor &
4917 9750 : FEProblemBase::getVectorPostprocessorObjectByName(const std::string & object_name,
4918 : const THREAD_ID tid) const
4919 : {
4920 9750 : std::vector<VectorPostprocessor *> objs;
4921 9750 : theWarehouse()
4922 9750 : .query()
4923 19500 : .condition<AttribInterfaces>(Interfaces::VectorPostprocessor)
4924 9750 : .condition<AttribThread>(tid)
4925 9750 : .condition<AttribName>(object_name)
4926 9750 : .queryInto(objs);
4927 :
4928 9750 : if (objs.empty())
4929 : {
4930 : mooseAssert(
4931 : getMooseApp().actionWarehouse().isTaskComplete("add_vector_postprocessor"),
4932 : "A VectorPostprocessor getter was called before VectorPostprocessors have been "
4933 : "constructed. The requested VectorPostprocessor '" +
4934 : object_name +
4935 : "' may exist in the input file, but VectorPostprocessors are not available yet.");
4936 :
4937 0 : mooseError("Unable to find VectorPostprocessor with name '", object_name, "'");
4938 : }
4939 : mooseAssert(objs.size() == 1,
4940 : "We shouldn't find more than one vector postprocessor object for a given name");
4941 19500 : return *(objs[0]);
4942 9750 : }
4943 :
4944 : void
4945 70 : FEProblemBase::parentOutputPositionChanged()
4946 : {
4947 1960 : for (const auto & it : _multi_apps)
4948 : {
4949 1890 : const auto & objects = it.second.getActiveObjects();
4950 1909 : for (const auto & obj : objects)
4951 19 : obj->parentOutputPositionChanged();
4952 : }
4953 70 : }
4954 :
4955 : void
4956 0 : FEProblemBase::computeIndicatorsAndMarkers()
4957 : {
4958 0 : computeIndicators();
4959 0 : computeMarkers();
4960 0 : }
4961 :
4962 : void
4963 224441 : FEProblemBase::computeIndicators()
4964 : {
4965 : // Initialize indicator aux variable fields
4966 224441 : if (_indicators.hasActiveObjects() || _internal_side_indicators.hasActiveObjects())
4967 : {
4968 11875 : TIME_SECTION("computeIndicators", 1, "Computing Indicators");
4969 :
4970 : // Internal side indicators may lead to creating a much larger sparsity pattern than dictated by
4971 : // the actual finite element scheme (e.g. CFEM)
4972 2375 : const auto old_do_derivatives = ADReal::do_derivatives;
4973 2375 : ADReal::do_derivatives = false;
4974 :
4975 2375 : std::vector<std::string> fields;
4976 :
4977 : // Indicator Fields
4978 2375 : const auto & indicators = _indicators.getActiveObjects();
4979 2521 : for (const auto & indicator : indicators)
4980 146 : fields.push_back(indicator->name());
4981 :
4982 : // InternalSideIndicator Fields
4983 2375 : const auto & internal_indicators = _internal_side_indicators.getActiveObjects();
4984 4706 : for (const auto & internal_indicator : internal_indicators)
4985 2331 : fields.push_back(internal_indicator->name());
4986 :
4987 2375 : _aux->zeroVariables(fields);
4988 :
4989 : // compute Indicators
4990 2375 : ComputeIndicatorThread cit(*this);
4991 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cit);
4992 2375 : _aux->solution().close();
4993 2375 : _aux->update();
4994 :
4995 2375 : ComputeIndicatorThread finalize_cit(*this, true);
4996 2375 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), finalize_cit);
4997 2375 : _aux->solution().close();
4998 2375 : _aux->update();
4999 :
5000 2375 : ADReal::do_derivatives = old_do_derivatives;
5001 2375 : }
5002 224441 : }
5003 :
5004 : void
5005 224463 : FEProblemBase::computeMarkers()
5006 : {
5007 224463 : if (_markers.hasActiveObjects())
5008 : {
5009 32550 : TIME_SECTION("computeMarkers", 1, "Computing Markers");
5010 :
5011 6510 : std::vector<std::string> fields;
5012 :
5013 : // Marker Fields
5014 6510 : const auto & markers = _markers.getActiveObjects();
5015 13999 : for (const auto & marker : markers)
5016 7489 : fields.push_back(marker->name());
5017 :
5018 6510 : _aux->zeroVariables(fields);
5019 :
5020 6510 : _adaptivity.updateErrorVectors();
5021 :
5022 13648 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5023 : {
5024 7141 : const auto & markers = _markers.getActiveObjects(tid);
5025 15353 : for (const auto & marker : markers)
5026 8215 : marker->markerSetup();
5027 : }
5028 :
5029 6507 : ComputeMarkerThread cmt(*this);
5030 6507 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cmt);
5031 :
5032 6507 : _aux->solution().close();
5033 6507 : _aux->update();
5034 6507 : }
5035 224460 : }
5036 :
5037 : const ExecFlagType &
5038 6209012 : FEProblemBase::getCurrentExecuteOnFlag() const
5039 : {
5040 6209012 : return _current_execute_on_flag;
5041 : }
5042 :
5043 : void
5044 6948451 : FEProblemBase::setCurrentExecuteOnFlag(const ExecFlagType & flag)
5045 : {
5046 6948451 : _current_execute_on_flag = flag;
5047 6948451 : }
5048 :
5049 : void
5050 72 : FEProblemBase::executeAllObjects(const ExecFlagType & /*exec_type*/)
5051 : {
5052 72 : }
5053 :
5054 : void
5055 1770040 : FEProblemBase::customSetup(const ExecFlagType & exec_type)
5056 : {
5057 1770040 : SubProblem::customSetup(exec_type);
5058 :
5059 1770040 : if (_line_search)
5060 0 : _line_search->customSetup(exec_type);
5061 :
5062 1770040 : unsigned int n_threads = libMesh::n_threads();
5063 3717674 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5064 : {
5065 1947634 : _all_materials.customSetup(exec_type, tid);
5066 1947634 : _functions.customSetup(exec_type, tid);
5067 : }
5068 :
5069 : #ifdef MOOSE_KOKKOS_ENABLED
5070 1294253 : _kokkos_functions.customSetup(exec_type);
5071 : #endif
5072 :
5073 1770040 : _aux->customSetup(exec_type);
5074 3566955 : for (auto & nl : _nl)
5075 1796915 : nl->customSetup(exec_type);
5076 :
5077 1770040 : if (_displaced_problem)
5078 145157 : _displaced_problem->customSetup(exec_type);
5079 :
5080 3717674 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
5081 : {
5082 1947634 : _internal_side_indicators.customSetup(exec_type, tid);
5083 1947634 : _indicators.customSetup(exec_type, tid);
5084 1947634 : _markers.customSetup(exec_type, tid);
5085 : }
5086 :
5087 1770040 : std::vector<UserObject *> userobjs;
5088 1770040 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
5089 4379476 : for (auto obj : userobjs)
5090 2609436 : obj->customSetup(exec_type);
5091 :
5092 : #ifdef MOOSE_KOKKOS_ENABLED
5093 : {
5094 1294253 : std::vector<UserObjectBase *> userobjs;
5095 1294253 : theWarehouse().query().condition<AttribSystem>("KokkosUserObject").queryIntoUnsorted(userobjs);
5096 1307714 : for (auto obj : userobjs)
5097 13461 : obj->customSetup(exec_type);
5098 1294253 : }
5099 : #endif
5100 :
5101 1770040 : _app.getOutputWarehouse().customSetup(exec_type);
5102 1770040 : }
5103 :
5104 : void
5105 2143037 : FEProblemBase::execute(const ExecFlagType & exec_type)
5106 : {
5107 : // Set the current flag
5108 2143037 : setCurrentExecuteOnFlag(exec_type);
5109 :
5110 2143037 : if (exec_type != EXEC_INITIAL)
5111 2087341 : executeControls(exec_type);
5112 :
5113 : // intentially call this after executing controls because the setups may rely on the controls
5114 : // FIXME: we skip the following flags because they have dedicated setup functions in
5115 : // SetupInterface and it may not be appropriate to call them here.
5116 3915530 : if (!(exec_type == EXEC_INITIAL || exec_type == EXEC_TIMESTEP_BEGIN ||
5117 1772511 : exec_type == EXEC_SUBDOMAIN || exec_type == EXEC_NONLINEAR || exec_type == EXEC_LINEAR))
5118 1770040 : customSetup(exec_type);
5119 :
5120 2143019 : executeSamplers(exec_type);
5121 :
5122 : // Pre-aux UserObjects
5123 2142995 : computeUserObjects(exec_type, Moose::PRE_AUX);
5124 :
5125 : // Systems (includes system time derivative and aux kernel calculations)
5126 2142995 : computeSystems(exec_type);
5127 : // With the auxiliary system solution computed, sync the displaced problem auxiliary solution
5128 : // before computation of post-aux user objects. The undisplaced auxiliary system current local
5129 : // solution is updated (via System::update) within the AuxiliarySystem class's variable
5130 : // computation methods (e.g. computeElementalVarsHelper, computeNodalVarsHelper), so it is safe to
5131 : // use it here
5132 2142962 : if (_displaced_problem)
5133 179606 : _displaced_problem->syncAuxSolution(*getAuxiliarySystem().currentSolution());
5134 :
5135 : // Post-aux UserObjects
5136 2142962 : computeUserObjects(exec_type, Moose::POST_AUX);
5137 :
5138 : // Return the current flag to None
5139 2142876 : setCurrentExecuteOnFlag(EXEC_NONE);
5140 :
5141 2142876 : if (_uo_aux_state_check && !_checking_uo_aux_state)
5142 : {
5143 : // we will only check aux variables and postprocessors
5144 : // checking more reporter data can be added in the future if needed
5145 559 : std::unique_ptr<NumericVector<Number>> x = _aux->currentSolution()->clone();
5146 559 : DenseVector<Real> pp_values = getReporterData().getAllRealReporterValues();
5147 :
5148 : // call THIS execute one more time for checking the possible states
5149 559 : _checking_uo_aux_state = true;
5150 559 : FEProblemBase::execute(exec_type);
5151 559 : _checking_uo_aux_state = false;
5152 :
5153 559 : const Real check_tol = 1e-8;
5154 :
5155 559 : const Real xnorm = x->l2_norm();
5156 559 : *x -= *_aux->currentSolution();
5157 559 : if (x->l2_norm() > check_tol * xnorm)
5158 : {
5159 3 : const auto & sys = _aux->system();
5160 3 : const unsigned int n_vars = sys.n_vars();
5161 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5162 15 : for (const auto i : make_range(n_vars))
5163 : {
5164 12 : const Real vnorm = sys.calculate_norm(*x, i, DISCRETE_L2);
5165 12 : ordered_map.emplace(vnorm, sys.variable_name(i));
5166 : }
5167 :
5168 3 : std::ostringstream oss;
5169 15 : for (const auto & [error_norm, var_name] : ordered_map)
5170 12 : oss << " {" << var_name << ", " << error_norm << "},\n";
5171 :
5172 3 : mooseError("Aux kernels, user objects appear to have states for aux variables on ",
5173 : exec_type,
5174 : ".\nVariable error norms in descending order:\n",
5175 3 : oss.str());
5176 0 : }
5177 :
5178 556 : const DenseVector<Real> new_pp_values = getReporterData().getAllRealReporterValues();
5179 556 : if (pp_values.size() != new_pp_values.size())
5180 0 : mooseError("Second execution for uo/aux state check should not change the number of "
5181 : "real reporter values");
5182 :
5183 556 : const Real ppnorm = pp_values.l2_norm();
5184 556 : pp_values -= new_pp_values;
5185 556 : if (pp_values.l2_norm() > check_tol * ppnorm)
5186 : {
5187 3 : const auto pp_names = getReporterData().getAllRealReporterFullNames();
5188 3 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
5189 12 : for (const auto i : index_range(pp_names))
5190 9 : ordered_map.emplace(std::abs(pp_values(i)), pp_names[i]);
5191 :
5192 3 : std::ostringstream oss;
5193 12 : for (const auto & [error_norm, pp_name] : ordered_map)
5194 9 : oss << " {" << pp_name << ", " << error_norm << "},\n";
5195 :
5196 3 : mooseError("Aux kernels, user objects appear to have states for real reporter values on ",
5197 : exec_type,
5198 : ".\nErrors of real reporter values in descending order:\n",
5199 3 : oss.str());
5200 0 : }
5201 553 : }
5202 2142870 : }
5203 :
5204 : // Finalize, threadJoin, and update PP values of Elemental/Nodal/Side/InternalSideUserObjects
5205 : void
5206 1342270 : FEProblemBase::joinAndFinalize(TheWarehouse::Query query, bool isgen)
5207 : {
5208 1342270 : std::vector<UserObject *> objs;
5209 1342270 : query.queryInto(objs);
5210 1342270 : if (!isgen)
5211 : {
5212 : // join all threaded user objects (i.e. not regular general user objects) to the primary
5213 : // thread
5214 1421523 : for (auto obj : objs)
5215 370776 : if (obj->primaryThreadCopy())
5216 30712 : obj->primaryThreadCopy()->threadJoin(*obj);
5217 : }
5218 :
5219 1342270 : query.condition<AttribThread>(0).queryInto(objs);
5220 :
5221 : // finalize objects and retrieve/store any postprocessor values
5222 1913529 : for (auto obj : objs)
5223 : {
5224 571339 : if (isgen && dynamic_cast<ThreadedGeneralUserObject *>(obj))
5225 133 : continue;
5226 571206 : if (isgen)
5227 : {
5228 : // general user objects are not run in their own threaded loop object - so run them here
5229 231142 : if (shouldPrintExecution(0))
5230 724 : _console << "[DBG] Initializing, executing & finalizing general UO '" << obj->name()
5231 724 : << "' on " << _current_execute_on_flag.name() << std::endl;
5232 231142 : obj->initialize();
5233 231142 : obj->execute();
5234 : }
5235 :
5236 571156 : obj->finalize();
5237 :
5238 : // These have to be stored piecemeal (with every call to this function) because general
5239 : // postprocessors (which run last after other userobjects have been completed) might depend on
5240 : // them being stored. This wouldn't be a problem if all userobjects satisfied the dependency
5241 : // resolver interface and could be sorted appropriately with the general userobjects, but they
5242 : // don't.
5243 571141 : auto pp = dynamic_cast<const Postprocessor *>(obj);
5244 571141 : if (pp)
5245 : {
5246 498740 : _reporter_data.finalize(obj->name());
5247 498740 : setPostprocessorValueByName(obj->name(), pp->getValue());
5248 : }
5249 :
5250 571132 : auto vpp = dynamic_cast<VectorPostprocessor *>(obj);
5251 571132 : if (vpp)
5252 13514 : _reporter_data.finalize(obj->name());
5253 :
5254 : // Update Reporter data
5255 571132 : auto reporter = dynamic_cast<Reporter *>(obj);
5256 571132 : if (reporter)
5257 5313 : _reporter_data.finalize(obj->name());
5258 : }
5259 1342190 : }
5260 :
5261 : TheWarehouse::Query
5262 19892401 : FEProblemBase::getUOQuery(const std::string & system,
5263 : const ExecFlagType & type,
5264 : const Moose::AuxGroup & group) const
5265 : {
5266 : TheWarehouse::Query query =
5267 19892401 : theWarehouse().query().condition<AttribSystem>(system).condition<AttribExecOns>(type);
5268 :
5269 19892401 : if (group == Moose::PRE_IC)
5270 96967 : query.condition<AttribPreIC>(true);
5271 19795434 : else if (group == Moose::PRE_AUX)
5272 9897338 : query.condition<AttribPreAux>(type);
5273 9898096 : else if (group == Moose::POST_AUX)
5274 9898028 : query.condition<AttribPostAux>(type);
5275 :
5276 19892401 : return query;
5277 0 : }
5278 :
5279 : void
5280 19892401 : FEProblemBase::getUOExecutionGroups(TheWarehouse::Query & query,
5281 : std::set<int> & execution_groups) const
5282 : {
5283 19892401 : std::vector<UserObjectBase *> uos;
5284 19892401 : query.queryIntoUnsorted(uos);
5285 20496876 : for (const auto & uo : uos)
5286 1813425 : execution_groups.insert(uo->getParam<int>("execution_order_group"));
5287 19892401 : }
5288 :
5289 : void
5290 56831 : FEProblemBase::computeUserObjectByName(const ExecFlagType & type,
5291 : const Moose::AuxGroup & group,
5292 : const std::string & name)
5293 : {
5294 56831 : const auto old_exec_flag = _current_execute_on_flag;
5295 56831 : _current_execute_on_flag = type;
5296 :
5297 56831 : std::set<int> execution_groups;
5298 :
5299 : #ifdef MOOSE_KOKKOS_ENABLED
5300 : TheWarehouse::Query kokkos_query =
5301 40978 : getUOQuery("KokkosUserObject", type, group).condition<AttribName>(name);
5302 40978 : getUOExecutionGroups(kokkos_query, execution_groups);
5303 : #endif
5304 :
5305 56831 : TheWarehouse::Query query = getUOQuery("UserObject", type, group).condition<AttribName>(name);
5306 56831 : getUOExecutionGroups(query, execution_groups);
5307 :
5308 82782 : for (const auto execution_group : execution_groups)
5309 : {
5310 : #ifdef MOOSE_KOKKOS_ENABLED
5311 18721 : computeKokkosUserObjectsInternal(
5312 18721 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5313 : #endif
5314 :
5315 25951 : computeUserObjectsInternal(type,
5316 25951 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5317 : }
5318 :
5319 56831 : _current_execute_on_flag = old_exec_flag;
5320 56831 : }
5321 :
5322 : void
5323 11448825 : FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGroup & group)
5324 : {
5325 11448825 : std::set<int> execution_groups;
5326 :
5327 : #ifdef MOOSE_KOKKOS_ENABLED
5328 8345767 : TheWarehouse::Query kokkos_query = getUOQuery("KokkosUserObject", type, group);
5329 8345767 : getUOExecutionGroups(kokkos_query, execution_groups);
5330 : #endif
5331 :
5332 11448825 : TheWarehouse::Query query = getUOQuery("UserObject", type, group);
5333 11448825 : getUOExecutionGroups(query, execution_groups);
5334 :
5335 11716033 : for (const auto execution_group : execution_groups)
5336 : {
5337 : #ifdef MOOSE_KOKKOS_ENABLED
5338 194654 : computeKokkosUserObjectsInternal(
5339 194654 : type, kokkos_query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5340 : #endif
5341 :
5342 267294 : computeUserObjectsInternal(type,
5343 267294 : query.clone().condition<AttribExecutionOrderGroup>(execution_group));
5344 : }
5345 11448739 : }
5346 :
5347 : void
5348 293245 : FEProblemBase::computeUserObjectsInternal(const ExecFlagType & type, TheWarehouse::Query & query)
5349 : {
5350 : try
5351 : {
5352 1466225 : TIME_SECTION("computeUserObjects", 1, "Computing User Objects");
5353 :
5354 293245 : std::vector<GeneralUserObject *> genobjs;
5355 293245 : query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject).queryInto(genobjs);
5356 :
5357 293245 : std::vector<UserObject *> userobjs;
5358 293245 : query.clone()
5359 586490 : .condition<AttribInterfaces>(Interfaces::ElementUserObject | Interfaces::SideUserObject |
5360 : Interfaces::InternalSideUserObject |
5361 586490 : Interfaces::InterfaceUserObject | Interfaces::DomainUserObject)
5362 293245 : .queryInto(userobjs);
5363 :
5364 293245 : std::vector<UserObject *> tgobjs;
5365 293245 : query.clone()
5366 586490 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5367 293245 : .queryInto(tgobjs);
5368 :
5369 293245 : std::vector<UserObject *> nodal;
5370 293245 : query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).queryInto(nodal);
5371 :
5372 293245 : std::vector<MortarUserObject *> mortar;
5373 293245 : query.clone().condition<AttribInterfaces>(Interfaces::MortarUserObject).queryInto(mortar);
5374 :
5375 293245 : if (userobjs.empty() && genobjs.empty() && tgobjs.empty() && nodal.empty() && mortar.empty())
5376 1710 : return;
5377 :
5378 : // Start the timer here since we have at least one active user object
5379 291535 : std::string compute_uo_tag = "computeUserObjects(" + Moose::stringify(type) + ")";
5380 :
5381 : // Perform Residual/Jacobian setups
5382 291535 : if (type == EXEC_LINEAR)
5383 : {
5384 125261 : for (auto obj : userobjs)
5385 71388 : obj->residualSetup();
5386 58331 : for (auto obj : nodal)
5387 4458 : obj->residualSetup();
5388 53873 : for (auto obj : mortar)
5389 0 : obj->residualSetup();
5390 53882 : for (auto obj : tgobjs)
5391 9 : obj->residualSetup();
5392 65556 : for (auto obj : genobjs)
5393 11683 : obj->residualSetup();
5394 : }
5395 237662 : else if (type == EXEC_NONLINEAR)
5396 : {
5397 14080 : for (auto obj : userobjs)
5398 4420 : obj->jacobianSetup();
5399 10052 : for (auto obj : nodal)
5400 392 : obj->jacobianSetup();
5401 9660 : for (auto obj : mortar)
5402 0 : obj->jacobianSetup();
5403 9663 : for (auto obj : tgobjs)
5404 3 : obj->jacobianSetup();
5405 26470 : for (auto obj : genobjs)
5406 16810 : obj->jacobianSetup();
5407 : }
5408 :
5409 641596 : for (auto obj : userobjs)
5410 350061 : obj->initialize();
5411 :
5412 : // Execute Side/InternalSide/Interface/Elemental/DomainUserObjects
5413 291535 : if (!userobjs.empty())
5414 : {
5415 : // non-nodal user objects have to be run separately before the nodal user objects run
5416 : // because some nodal user objects (NodalNormal related) depend on elemental user objects
5417 : // :-(
5418 206856 : ComputeUserObjectsThread cppt(*this, query);
5419 206856 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), cppt);
5420 :
5421 : // There is one instance in rattlesnake where an elemental user object's finalize depends
5422 : // on a side user object having been finalized first :-(
5423 206850 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::SideUserObject));
5424 206847 : joinAndFinalize(
5425 413694 : query.clone().condition<AttribInterfaces>(Interfaces::InternalSideUserObject));
5426 206847 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::InterfaceUserObject));
5427 206847 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::ElementUserObject));
5428 206844 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::DomainUserObject));
5429 206844 : }
5430 :
5431 : // if any elemental user object may have written to variables we need to close the aux solution
5432 641496 : for (const auto & uo : userobjs)
5433 350006 : if (auto euo = dynamic_cast<const ElementUserObject *>(uo);
5434 350006 : euo && euo->hasWritableCoupledVariables())
5435 : {
5436 33 : _aux->solution().close();
5437 33 : _aux->system().update();
5438 33 : break;
5439 : }
5440 :
5441 : // Execute NodalUserObjects
5442 : // BISON has an axial reloc elemental user object that has a finalize func that depends on a
5443 : // nodal user object's prev value. So we can't initialize this until after elemental objects
5444 : // have been finalized :-(
5445 311829 : for (auto obj : nodal)
5446 20306 : obj->initialize();
5447 291523 : if (query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).count() > 0)
5448 : {
5449 16379 : ComputeNodalUserObjectsThread cnppt(*this, query);
5450 16379 : Threads::parallel_reduce(getCurrentAlgebraicNodeRange(), cnppt);
5451 16379 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject));
5452 16379 : }
5453 :
5454 : // if any nodal user object may have written to variables we need to close the aux solution
5455 311793 : for (const auto & uo : nodal)
5456 20292 : if (auto nuo = dynamic_cast<const NodalUserObject *>(uo);
5457 20292 : nuo && nuo->hasWritableCoupledVariables())
5458 : {
5459 22 : _aux->solution().close();
5460 22 : _aux->system().update();
5461 22 : break;
5462 : }
5463 :
5464 : // Execute MortarUserObjects
5465 : {
5466 291545 : for (auto obj : mortar)
5467 22 : obj->initialize();
5468 291523 : if (!mortar.empty())
5469 : {
5470 33 : auto create_and_run_mortar_functors = [this, type, &mortar](const bool displaced)
5471 : {
5472 : // go over mortar interfaces and construct functors
5473 33 : const auto & mortar_interfaces = getMortarInterfaces(displaced);
5474 55 : for (const auto & [primary_secondary_boundary_pair, mortar_generation_ptr] :
5475 88 : mortar_interfaces)
5476 : {
5477 : auto mortar_uos_to_execute =
5478 22 : getMortarUserObjects(primary_secondary_boundary_pair.first,
5479 22 : primary_secondary_boundary_pair.second,
5480 : displaced,
5481 22 : mortar);
5482 :
5483 : auto * const subproblem = displaced
5484 22 : ? static_cast<SubProblem *>(_displaced_problem.get())
5485 22 : : static_cast<SubProblem *>(this);
5486 : MortarUserObjectThread muot(mortar_uos_to_execute,
5487 22 : *mortar_generation_ptr,
5488 : *subproblem,
5489 : *this,
5490 : displaced,
5491 22 : subproblem->assembly(0, 0));
5492 :
5493 22 : muot();
5494 22 : }
5495 55 : };
5496 :
5497 22 : create_and_run_mortar_functors(false);
5498 22 : if (_displaced_problem)
5499 11 : create_and_run_mortar_functors(true);
5500 22 : }
5501 291545 : for (auto obj : mortar)
5502 22 : obj->finalize();
5503 : }
5504 :
5505 : // Execute threaded general user objects
5506 291964 : for (auto obj : tgobjs)
5507 441 : obj->initialize();
5508 291523 : std::vector<GeneralUserObject *> tguos_zero;
5509 291523 : query.clone()
5510 291523 : .condition<AttribThread>(0)
5511 583046 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5512 291523 : .queryInto(tguos_zero);
5513 291656 : for (auto obj : tguos_zero)
5514 : {
5515 133 : std::vector<GeneralUserObject *> tguos;
5516 133 : auto q = query.clone()
5517 133 : .condition<AttribName>(obj->name())
5518 133 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject);
5519 133 : q.queryInto(tguos);
5520 :
5521 133 : ComputeThreadedGeneralUserObjectsThread ctguot(*this);
5522 :
5523 : // Force one thread per ThreadedGeneralUserObject via grainsize
5524 266 : Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(),
5525 133 : tguos.end(),
5526 : /*grainsize=*/1),
5527 : ctguot);
5528 133 : joinAndFinalize(q);
5529 133 : }
5530 :
5531 : // Execute general user objects
5532 291523 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject), true);
5533 301709 : }
5534 0 : catch (...)
5535 : {
5536 0 : handleException("computeUserObjectsInternal");
5537 0 : }
5538 : }
5539 :
5540 : void
5541 5700359 : FEProblemBase::executeControls(const ExecFlagType & exec_type)
5542 : {
5543 5700359 : if (_control_warehouse[exec_type].hasActiveObjects())
5544 : {
5545 40205 : TIME_SECTION("executeControls", 1, "Executing Controls");
5546 :
5547 8041 : DependencyResolver<std::shared_ptr<Control>> resolver;
5548 :
5549 8041 : auto controls_wh = _control_warehouse[exec_type];
5550 : // Add all of the dependencies into the resolver and sort them
5551 20003 : for (const auto & it : controls_wh.getActiveObjects())
5552 : {
5553 : // Make sure an item with no dependencies comes out too!
5554 11965 : resolver.addItem(it);
5555 :
5556 11965 : std::vector<std::string> & dependent_controls = it->getDependencies();
5557 14857 : for (const auto & depend_name : dependent_controls)
5558 : {
5559 2895 : if (controls_wh.hasActiveObject(depend_name))
5560 : {
5561 2892 : auto dep_control = controls_wh.getActiveObject(depend_name);
5562 2892 : resolver.addEdge(dep_control, it);
5563 2892 : }
5564 : else
5565 3 : mooseError("The Control \"",
5566 : depend_name,
5567 : "\" was not created, did you make a "
5568 : "spelling mistake or forget to include it "
5569 : "in your input file?");
5570 : }
5571 : }
5572 :
5573 8038 : const auto & ordered_controls = resolver.getSortedValues();
5574 :
5575 8038 : if (!ordered_controls.empty())
5576 : {
5577 : // already called by initialSetup when exec_type == EXEC_INITIAL
5578 8038 : if (exec_type != EXEC_INITIAL)
5579 7130 : _control_warehouse.setup(exec_type);
5580 :
5581 : // Run the controls in the proper order
5582 19958 : for (const auto & control : ordered_controls)
5583 11962 : control->execute();
5584 : }
5585 7996 : }
5586 5700314 : }
5587 :
5588 : void
5589 2143019 : FEProblemBase::executeSamplers(const ExecFlagType & exec_type)
5590 : {
5591 : // TODO: This should be done in a threaded loop, but this should be super quick so for now
5592 : // do a serial loop.
5593 4500569 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5594 : {
5595 2357574 : std::vector<Sampler *> objects;
5596 2357574 : theWarehouse()
5597 4715148 : .query()
5598 2357574 : .condition<AttribSystem>("Sampler")
5599 2357574 : .condition<AttribThread>(tid)
5600 2357574 : .condition<AttribExecOns>(exec_type)
5601 2357574 : .queryInto(objects);
5602 :
5603 2357574 : if (!objects.empty())
5604 : {
5605 1475 : TIME_SECTION("executeSamplers", 1, "Executing Samplers");
5606 295 : FEProblemBase::objectSetupHelper<Sampler>(objects, exec_type);
5607 295 : FEProblemBase::objectExecuteHelper<Sampler>(objects);
5608 271 : }
5609 2357550 : }
5610 2142995 : }
5611 :
5612 : void
5613 312085 : FEProblemBase::updateActiveObjects()
5614 : {
5615 1560425 : TIME_SECTION("updateActiveObjects", 5, "Updating Active Objects");
5616 :
5617 655055 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5618 : {
5619 688468 : for (auto & nl : _nl)
5620 345498 : nl->updateActive(tid);
5621 342970 : _aux->updateActive(tid);
5622 342970 : _indicators.updateActive(tid);
5623 342970 : _internal_side_indicators.updateActive(tid);
5624 342970 : _markers.updateActive(tid);
5625 342970 : _all_materials.updateActive(tid);
5626 342970 : _materials.updateActive(tid);
5627 342970 : _discrete_materials.updateActive(tid);
5628 : }
5629 :
5630 312085 : _control_warehouse.updateActive();
5631 312085 : _multi_apps.updateActive();
5632 312085 : _transient_multi_apps.updateActive();
5633 312085 : _transfers.updateActive();
5634 312085 : _to_multi_app_transfers.updateActive();
5635 312085 : _from_multi_app_transfers.updateActive();
5636 312085 : _between_multi_app_transfers.updateActive();
5637 :
5638 : #ifdef MOOSE_KOKKOS_ENABLED
5639 227591 : _kokkos_materials.updateActive();
5640 : #endif
5641 312085 : }
5642 :
5643 : void
5644 0 : FEProblemBase::reportMooseObjectDependency(MooseObject * /*a*/, MooseObject * /*b*/)
5645 : {
5646 : //<< "Object " << a->name() << " -> " << b->name() << std::endl;
5647 0 : }
5648 :
5649 : void
5650 67501 : FEProblemBase::reinitBecauseOfGhostingOrNewGeomObjects(const bool mortar_changed)
5651 : {
5652 337505 : TIME_SECTION("reinitBecauseOfGhostingOrNewGeomObjects",
5653 : 3,
5654 : "Reinitializing Because of Geometric Search Objects");
5655 :
5656 : // Need to see if _any_ processor has ghosted elems or geometry objects.
5657 67501 : bool needs_reinit = !_ghosted_elems.empty();
5658 134496 : needs_reinit = needs_reinit || !_geometric_search_data._nearest_node_locators.empty() ||
5659 66995 : (_mortar_data->hasObjects() && mortar_changed);
5660 67501 : needs_reinit =
5661 136195 : needs_reinit || (_displaced_problem &&
5662 4423 : (!_displaced_problem->geomSearchData()._nearest_node_locators.empty() ||
5663 68173 : (_mortar_data->hasDisplacedObjects() && mortar_changed)));
5664 67501 : _communicator.max(needs_reinit);
5665 :
5666 67501 : if (needs_reinit)
5667 : {
5668 : // Call reinit to get the ghosted vectors correct now that some geometric search has been done
5669 2211 : es().reinit();
5670 :
5671 2211 : if (_displaced_mesh)
5672 1607 : _displaced_problem->es().reinit();
5673 : }
5674 67501 : }
5675 :
5676 : void
5677 177 : FEProblemBase::addDamper(const std::string & damper_name,
5678 : const std::string & name,
5679 : InputParameters & parameters)
5680 : {
5681 : parallel_object_only();
5682 :
5683 : const auto nl_sys_num =
5684 177 : parameters.isParamValid("variable")
5685 588 : ? determineSolverSystem(parameters.varName("variable", name), true).second
5686 174 : : (unsigned int)0;
5687 :
5688 174 : if (!isSolverSystemNonlinear(nl_sys_num))
5689 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
5690 : "supported at the moment!");
5691 :
5692 348 : parameters.set<SubProblem *>("_subproblem") = this;
5693 348 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
5694 :
5695 174 : _has_dampers = true;
5696 174 : logAdd("Damper", name, damper_name, parameters);
5697 174 : _nl[nl_sys_num]->addDamper(damper_name, name, parameters);
5698 174 : }
5699 :
5700 : void
5701 162 : FEProblemBase::setupDampers()
5702 : {
5703 324 : for (auto & nl : _nl)
5704 162 : nl->setupDampers();
5705 162 : }
5706 :
5707 : void
5708 659 : FEProblemBase::addIndicator(const std::string & indicator_name,
5709 : const std::string & name,
5710 : InputParameters & parameters)
5711 : {
5712 : parallel_object_only();
5713 :
5714 659 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5715 : {
5716 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5717 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5718 0 : _reinit_displaced_elem = true;
5719 : }
5720 : else
5721 : {
5722 659 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5723 : {
5724 : // We allow Indicators to request that they use_displaced_mesh,
5725 : // but then be overridden when no displacements variables are
5726 : // provided in the Mesh block. If that happened, update the value
5727 : // of use_displaced_mesh appropriately for this Indicator.
5728 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5729 0 : parameters.set<bool>("use_displaced_mesh") = false;
5730 : }
5731 :
5732 1318 : parameters.set<SubProblem *>("_subproblem") = this;
5733 1977 : parameters.set<SystemBase *>("_sys") = _aux.get();
5734 : }
5735 :
5736 1382 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5737 : {
5738 : std::shared_ptr<Indicator> indicator =
5739 723 : _factory.create<Indicator>(indicator_name, name, parameters, tid);
5740 723 : logAdd("Indicator", name, indicator_name, parameters);
5741 : std::shared_ptr<InternalSideIndicatorBase> isi =
5742 723 : std::dynamic_pointer_cast<InternalSideIndicatorBase>(indicator);
5743 723 : if (isi)
5744 622 : _internal_side_indicators.addObject(isi, tid);
5745 : else
5746 101 : _indicators.addObject(indicator, tid);
5747 723 : }
5748 659 : }
5749 :
5750 : void
5751 1962 : FEProblemBase::addMarker(const std::string & marker_name,
5752 : const std::string & name,
5753 : InputParameters & parameters)
5754 : {
5755 : parallel_object_only();
5756 :
5757 1962 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5758 : {
5759 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5760 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5761 0 : _reinit_displaced_elem = true;
5762 : }
5763 : else
5764 : {
5765 1962 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5766 : {
5767 : // We allow Markers to request that they use_displaced_mesh,
5768 : // but then be overridden when no displacements variables are
5769 : // provided in the Mesh block. If that happened, update the value
5770 : // of use_displaced_mesh appropriately for this Marker.
5771 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5772 0 : parameters.set<bool>("use_displaced_mesh") = false;
5773 : }
5774 :
5775 3924 : parameters.set<SubProblem *>("_subproblem") = this;
5776 5886 : parameters.set<SystemBase *>("_sys") = _aux.get();
5777 : }
5778 :
5779 4099 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5780 : {
5781 2140 : std::shared_ptr<Marker> marker = _factory.create<Marker>(marker_name, name, parameters, tid);
5782 2137 : logAdd("Marker", name, marker_name, parameters);
5783 2137 : _markers.addObject(marker, tid);
5784 2137 : }
5785 1959 : }
5786 :
5787 : void
5788 8151 : FEProblemBase::addMultiApp(const std::string & multi_app_name,
5789 : const std::string & name,
5790 : InputParameters & parameters)
5791 : {
5792 : parallel_object_only();
5793 :
5794 16302 : parameters.set<MPI_Comm>("_mpi_comm") = _communicator.get();
5795 :
5796 8151 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5797 : {
5798 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5799 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5800 0 : _reinit_displaced_elem = true;
5801 : }
5802 : else
5803 : {
5804 8151 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5805 : {
5806 : // We allow MultiApps to request that they use_displaced_mesh,
5807 : // but then be overridden when no displacements variables are
5808 : // provided in the Mesh block. If that happened, update the value
5809 : // of use_displaced_mesh appropriately for this MultiApp.
5810 42 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5811 84 : parameters.set<bool>("use_displaced_mesh") = false;
5812 : }
5813 :
5814 16302 : parameters.set<SubProblem *>("_subproblem") = this;
5815 24453 : parameters.set<SystemBase *>("_sys") = _aux.get();
5816 : }
5817 :
5818 8151 : std::shared_ptr<MultiApp> multi_app = _factory.create<MultiApp>(multi_app_name, name, parameters);
5819 8139 : logAdd("MultiApp", name, multi_app_name, parameters);
5820 8139 : multi_app->setupPositions();
5821 :
5822 8052 : _multi_apps.addObject(multi_app);
5823 :
5824 : // Store TransientMultiApp objects in another container, this is needed for calling computeDT
5825 : std::shared_ptr<TransientMultiApp> trans_multi_app =
5826 8052 : std::dynamic_pointer_cast<TransientMultiApp>(multi_app);
5827 8052 : if (trans_multi_app)
5828 5391 : _transient_multi_apps.addObject(trans_multi_app);
5829 8052 : }
5830 :
5831 : bool
5832 112534 : FEProblemBase::hasMultiApps(ExecFlagType type) const
5833 : {
5834 112534 : return _multi_apps[type].hasActiveObjects();
5835 : }
5836 :
5837 : bool
5838 26593 : FEProblemBase::hasMultiApp(const std::string & multi_app_name) const
5839 : {
5840 26593 : return _multi_apps.hasActiveObject(multi_app_name);
5841 : }
5842 :
5843 : std::shared_ptr<MultiApp>
5844 26593 : FEProblemBase::getMultiApp(const std::string & multi_app_name) const
5845 : {
5846 26593 : if (!hasMultiApp(multi_app_name))
5847 : mooseAssert(getMooseApp().actionWarehouse().isTaskComplete("add_multi_app"),
5848 : "A MultiApp getter was called before MultiApps have been constructed. "
5849 : "If you are attempting to access this object in the constructor of another object "
5850 : "then make sure that the MultiApp is constructed before the object using it.");
5851 :
5852 26593 : return _multi_apps.getObject(multi_app_name);
5853 : }
5854 :
5855 : void
5856 14458488 : FEProblemBase::execMultiAppTransfers(ExecFlagType type, Transfer::DIRECTION direction)
5857 : {
5858 14458488 : bool to_multiapp = direction == MultiAppTransfer::TO_MULTIAPP;
5859 14458488 : bool from_multiapp = direction == MultiAppTransfer::FROM_MULTIAPP;
5860 14458488 : std::string string_direction;
5861 14458488 : if (to_multiapp)
5862 4819550 : string_direction = " To ";
5863 9638938 : else if (from_multiapp)
5864 4819424 : string_direction = " From ";
5865 : else
5866 4819514 : string_direction = " Between ";
5867 :
5868 14458650 : const MooseObjectWarehouse<Transfer> & wh = to_multiapp ? _to_multi_app_transfers[type]
5869 24097336 : : from_multiapp ? _from_multi_app_transfers[type]
5870 33736436 : : _between_multi_app_transfers[type];
5871 :
5872 14458488 : if (wh.hasActiveObjects())
5873 : {
5874 546320 : TIME_SECTION("execMultiAppTransfers", 1, "Executing Transfers");
5875 :
5876 109264 : const auto & transfers = wh.getActiveObjects();
5877 :
5878 109264 : if (_verbose_multiapps)
5879 : {
5880 1120 : _console << COLOR_CYAN << "\nTransfers on " << Moose::stringify(type) << string_direction
5881 1120 : << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
5882 :
5883 : VariadicTable<std::string, std::string, std::string, std::string> table(
5884 2240 : {"Name", "Type", "From", "To"});
5885 :
5886 : // Build Table of Transfer Info
5887 3060 : for (const auto & transfer : transfers)
5888 : {
5889 1940 : auto multiapp_transfer = dynamic_cast<MultiAppTransfer *>(transfer.get());
5890 :
5891 1940 : table.addRow(multiapp_transfer->name(),
5892 1940 : multiapp_transfer->type(),
5893 3880 : multiapp_transfer->getFromName(),
5894 3880 : multiapp_transfer->getToName());
5895 : }
5896 :
5897 : // Print it
5898 1120 : table.print(_console);
5899 1120 : }
5900 :
5901 223569 : for (const auto & transfer : transfers)
5902 : {
5903 114402 : transfer->setCurrentDirection(direction);
5904 114402 : transfer->execute();
5905 : }
5906 :
5907 109167 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5908 :
5909 109167 : if (_verbose_multiapps)
5910 2228 : _console << COLOR_CYAN << "Transfers on " << Moose::stringify(type) << " Are Finished\n"
5911 1114 : << COLOR_DEFAULT << std::endl;
5912 109167 : }
5913 14349224 : else if (_multi_apps[type].getActiveObjects().size())
5914 : {
5915 110526 : if (_verbose_multiapps)
5916 7076 : _console << COLOR_CYAN << "\nNo Transfers on " << Moose::stringify(type) << string_direction
5917 3538 : << "MultiApps\n"
5918 3538 : << COLOR_DEFAULT << std::endl;
5919 : }
5920 14458391 : }
5921 :
5922 : std::vector<std::shared_ptr<Transfer>>
5923 12 : FEProblemBase::getTransfers(ExecFlagType type, Transfer::DIRECTION direction) const
5924 : {
5925 12 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5926 12 : return _to_multi_app_transfers[type].getActiveObjects();
5927 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5928 0 : return _from_multi_app_transfers[type].getActiveObjects();
5929 : else
5930 0 : return _between_multi_app_transfers[type].getActiveObjects();
5931 : }
5932 :
5933 : std::vector<std::shared_ptr<Transfer>>
5934 0 : FEProblemBase::getTransfers(Transfer::DIRECTION direction) const
5935 : {
5936 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5937 0 : return _to_multi_app_transfers.getActiveObjects();
5938 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5939 0 : return _from_multi_app_transfers.getActiveObjects();
5940 : else
5941 0 : return _between_multi_app_transfers.getActiveObjects();
5942 : }
5943 :
5944 : const ExecuteMooseObjectWarehouse<Transfer> &
5945 0 : FEProblemBase::getMultiAppTransferWarehouse(Transfer::DIRECTION direction) const
5946 : {
5947 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5948 0 : return _to_multi_app_transfers;
5949 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5950 0 : return _from_multi_app_transfers;
5951 : else
5952 0 : return _between_multi_app_transfers;
5953 : }
5954 :
5955 : bool
5956 4819553 : FEProblemBase::execMultiApps(ExecFlagType type, bool auto_advance)
5957 : {
5958 : // Active MultiApps
5959 : const std::vector<MooseSharedPointer<MultiApp>> & multi_apps =
5960 4819553 : _multi_apps[type].getActiveObjects();
5961 :
5962 : // Do anything that needs to be done to Apps before transfers
5963 4884774 : for (const auto & multi_app : multi_apps)
5964 65224 : multi_app->preTransfer(_dt, _time);
5965 :
5966 : // Execute Transfers _to_ MultiApps
5967 4819550 : execMultiAppTransfers(type, MultiAppTransfer::TO_MULTIAPP);
5968 :
5969 : // Execute Transfers _between_ Multiapps
5970 4819514 : execMultiAppTransfers(type, MultiAppTransfer::BETWEEN_MULTIAPP);
5971 :
5972 : // Execute MultiApps
5973 4819514 : if (multi_apps.size())
5974 : {
5975 319820 : TIME_SECTION("execMultiApps", 1, "Executing MultiApps", false);
5976 :
5977 63964 : if (_verbose_multiapps)
5978 2988 : _console << COLOR_CYAN << "\nExecuting MultiApps on " << Moose::stringify(type)
5979 1494 : << COLOR_DEFAULT << std::endl;
5980 :
5981 63964 : bool success = true;
5982 :
5983 129060 : for (const auto & multi_app : multi_apps)
5984 : {
5985 65179 : success = multi_app->solveStep(_dt, _time, auto_advance);
5986 : // no need to finish executing the subapps if one fails
5987 65170 : if (!success)
5988 74 : break;
5989 : }
5990 :
5991 63955 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5992 :
5993 63955 : _communicator.min(success);
5994 :
5995 63955 : if (!success)
5996 81 : return false;
5997 :
5998 63874 : if (_verbose_multiapps)
5999 2988 : _console << COLOR_CYAN << "Finished Executing MultiApps on " << Moose::stringify(type) << "\n"
6000 1494 : << COLOR_DEFAULT << std::endl;
6001 63955 : }
6002 :
6003 : // Execute Transfers _from_ MultiApps
6004 4819424 : execMultiAppTransfers(type, MultiAppTransfer::FROM_MULTIAPP);
6005 :
6006 : // If we made it here then everything passed
6007 4819363 : return true;
6008 : }
6009 :
6010 : void
6011 49171 : FEProblemBase::finalizeMultiApps()
6012 : {
6013 49171 : const auto & multi_apps = _multi_apps.getActiveObjects();
6014 :
6015 56207 : for (const auto & multi_app : multi_apps)
6016 7036 : multi_app->finalize();
6017 49171 : }
6018 :
6019 : void
6020 50711 : FEProblemBase::postExecute()
6021 : {
6022 50711 : const auto & multi_apps = _multi_apps.getActiveObjects();
6023 :
6024 58134 : for (const auto & multi_app : multi_apps)
6025 7423 : multi_app->postExecute();
6026 50711 : }
6027 :
6028 : void
6029 668220 : FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
6030 : {
6031 668220 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6032 :
6033 668220 : if (multi_apps.size())
6034 27809 : for (const auto & multi_app : multi_apps)
6035 14076 : multi_app->incrementTStep(_time);
6036 668220 : }
6037 :
6038 : void
6039 36370 : FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
6040 : {
6041 36370 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6042 :
6043 36370 : if (multi_apps.size())
6044 : {
6045 8584 : if (_verbose_multiapps)
6046 270 : _console << COLOR_CYAN << "\nAdvancing MultiApps on " << type.name() << COLOR_DEFAULT
6047 270 : << std::endl;
6048 :
6049 17174 : for (const auto & multi_app : multi_apps)
6050 8590 : multi_app->finishStep(recurse_through_multiapp_levels);
6051 :
6052 8584 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6053 :
6054 8584 : if (_verbose_multiapps)
6055 270 : _console << COLOR_CYAN << "Finished Advancing MultiApps on " << type.name() << "\n"
6056 270 : << COLOR_DEFAULT << std::endl;
6057 : }
6058 36370 : }
6059 :
6060 : void
6061 1119840 : FEProblemBase::backupMultiApps(ExecFlagType type)
6062 : {
6063 1119840 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6064 :
6065 1119840 : if (multi_apps.size())
6066 : {
6067 105845 : TIME_SECTION("backupMultiApps", 5, "Backing Up MultiApp");
6068 :
6069 21169 : if (_verbose_multiapps)
6070 798 : _console << COLOR_CYAN << "\nBacking Up MultiApps on " << type.name() << COLOR_DEFAULT
6071 798 : << std::endl;
6072 :
6073 43487 : for (const auto & multi_app : multi_apps)
6074 22318 : multi_app->backup();
6075 :
6076 21169 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6077 :
6078 21169 : if (_verbose_multiapps)
6079 798 : _console << COLOR_CYAN << "Finished Backing Up MultiApps on " << type.name() << "\n"
6080 798 : << COLOR_DEFAULT << std::endl;
6081 21169 : }
6082 1119840 : }
6083 :
6084 : void
6085 104694 : FEProblemBase::restoreMultiApps(ExecFlagType type, bool force)
6086 : {
6087 104694 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
6088 :
6089 104694 : if (multi_apps.size())
6090 : {
6091 43043 : if (_verbose_multiapps)
6092 : {
6093 528 : if (force)
6094 0 : _console << COLOR_CYAN << "\nRestoring Multiapps on " << type.name()
6095 0 : << " because of solve failure!" << COLOR_DEFAULT << std::endl;
6096 : else
6097 528 : _console << COLOR_CYAN << "\nRestoring MultiApps on " << type.name() << COLOR_DEFAULT
6098 528 : << std::endl;
6099 : }
6100 :
6101 86128 : for (const auto & multi_app : multi_apps)
6102 43088 : multi_app->restore(force);
6103 :
6104 43040 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
6105 :
6106 43040 : if (_verbose_multiapps)
6107 528 : _console << COLOR_CYAN << "Finished Restoring MultiApps on " << type.name() << "\n"
6108 528 : << COLOR_DEFAULT << std::endl;
6109 : }
6110 104691 : }
6111 :
6112 : Real
6113 692752 : FEProblemBase::computeMultiAppsDT(ExecFlagType type)
6114 : {
6115 692752 : const auto & multi_apps = _transient_multi_apps[type].getActiveObjects();
6116 :
6117 692752 : Real smallest_dt = std::numeric_limits<Real>::max();
6118 :
6119 710543 : for (const auto & multi_app : multi_apps)
6120 17791 : smallest_dt = std::min(smallest_dt, multi_app->computeDT());
6121 :
6122 692752 : return smallest_dt;
6123 : }
6124 :
6125 : void
6126 4765705 : FEProblemBase::execTransfers(ExecFlagType type)
6127 : {
6128 4765705 : if (_transfers[type].hasActiveObjects())
6129 : {
6130 0 : TIME_SECTION("execTransfers", 3, "Executing Transfers");
6131 :
6132 0 : const auto & transfers = _transfers[type].getActiveObjects();
6133 :
6134 0 : for (const auto & transfer : transfers)
6135 0 : transfer->execute();
6136 0 : }
6137 4765705 : }
6138 :
6139 : void
6140 13211 : FEProblemBase::addTransfer(const std::string & transfer_name,
6141 : const std::string & name,
6142 : InputParameters & parameters)
6143 : {
6144 : parallel_object_only();
6145 :
6146 13211 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
6147 : {
6148 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
6149 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
6150 0 : _reinit_displaced_elem = true;
6151 : }
6152 : else
6153 : {
6154 13211 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
6155 : {
6156 : // We allow Transfers to request that they use_displaced_mesh,
6157 : // but then be overridden when no displacements variables are
6158 : // provided in the Mesh block. If that happened, update the value
6159 : // of use_displaced_mesh appropriately for this Transfer.
6160 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
6161 0 : parameters.set<bool>("use_displaced_mesh") = false;
6162 : }
6163 :
6164 26422 : parameters.set<SubProblem *>("_subproblem") = this;
6165 39633 : parameters.set<SystemBase *>("_sys") = _aux.get();
6166 : }
6167 :
6168 : // Handle the "SAME_AS_MULTIAPP" execute option. The get method is used to test for the
6169 : // flag so the set by user flag is not reset, calling set with the true flag causes the set
6170 : // by user status to be reset, which should only be done if the EXEC_SAME_AS_MULTIAPP is
6171 : // being applied to the object.
6172 13211 : if (parameters.get<ExecFlagEnum>("execute_on").isValueSet(EXEC_SAME_AS_MULTIAPP))
6173 : {
6174 11084 : ExecFlagEnum & exec_enum = parameters.set<ExecFlagEnum>("execute_on", true);
6175 11084 : std::shared_ptr<MultiApp> multiapp;
6176 22168 : if (parameters.isParamValid("multi_app"))
6177 50 : multiapp = getMultiApp(parameters.get<MultiAppName>("multi_app"));
6178 : // This catches the sibling transfer case, where we want to be executing only as often as the
6179 : // receiving application. A transfer 'to' a multiapp is executed before that multiapp
6180 22068 : else if (parameters.isParamValid("to_multi_app"))
6181 5746 : multiapp = getMultiApp(parameters.get<MultiAppName>("to_multi_app"));
6182 10576 : else if (parameters.isParamValid("from_multi_app"))
6183 5282 : multiapp = getMultiApp(parameters.get<MultiAppName>("from_multi_app"));
6184 : // else do nothing because the user has provided invalid input. They should get a nice error
6185 : // about this during transfer construction. This necessitates checking for null in this next
6186 : // line, however
6187 11084 : if (multiapp)
6188 33234 : exec_enum = multiapp->getParam<ExecFlagEnum>("execute_on");
6189 11084 : }
6190 :
6191 : // Create the Transfer objects
6192 13211 : std::shared_ptr<Transfer> transfer = _factory.create<Transfer>(transfer_name, name, parameters);
6193 13154 : logAdd("Transfer", name, transfer_name, parameters);
6194 :
6195 : // Add MultiAppTransfer object
6196 : std::shared_ptr<MultiAppTransfer> multi_app_transfer =
6197 13154 : std::dynamic_pointer_cast<MultiAppTransfer>(transfer);
6198 13154 : if (multi_app_transfer)
6199 : {
6200 13154 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::TO_MULTIAPP))
6201 5265 : _to_multi_app_transfers.addObject(multi_app_transfer);
6202 13154 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::FROM_MULTIAPP))
6203 6419 : _from_multi_app_transfers.addObject(multi_app_transfer);
6204 13154 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::BETWEEN_MULTIAPP))
6205 1470 : _between_multi_app_transfers.addObject(multi_app_transfer);
6206 : }
6207 : else
6208 0 : _transfers.addObject(transfer);
6209 13154 : }
6210 :
6211 : bool
6212 1429603 : FEProblemBase::hasVariable(const std::string & var_name) const
6213 : {
6214 2334348 : for (auto & sys : _solver_systems)
6215 1434966 : if (sys->hasVariable(var_name))
6216 530221 : return true;
6217 899382 : if (_aux->hasVariable(var_name))
6218 829638 : return true;
6219 :
6220 69744 : return false;
6221 : }
6222 :
6223 : bool
6224 65 : FEProblemBase::hasSolverVariable(const std::string & var_name) const
6225 : {
6226 78 : for (auto & sys : _solver_systems)
6227 65 : if (sys->hasVariable(var_name))
6228 52 : return true;
6229 :
6230 13 : return false;
6231 : }
6232 :
6233 : const MooseVariableFieldBase &
6234 4294140 : FEProblemBase::getVariable(const THREAD_ID tid,
6235 : const std::string & var_name,
6236 : Moose::VarKindType expected_var_type,
6237 : Moose::VarFieldType expected_var_field_type) const
6238 : {
6239 12882414 : return getVariableHelper(
6240 4294140 : tid, var_name, expected_var_type, expected_var_field_type, _solver_systems, *_aux);
6241 : }
6242 :
6243 : MooseVariable &
6244 7551 : FEProblemBase::getStandardVariable(const THREAD_ID tid, const std::string & var_name)
6245 : {
6246 11429 : for (auto & sys : _solver_systems)
6247 7551 : if (sys->hasVariable(var_name))
6248 3673 : return sys->getFieldVariable<Real>(tid, var_name);
6249 3878 : if (_aux->hasVariable(var_name))
6250 3875 : return _aux->getFieldVariable<Real>(tid, var_name);
6251 :
6252 3 : mooseError("Unknown variable " + var_name);
6253 : }
6254 :
6255 : MooseVariableFieldBase &
6256 581 : FEProblemBase::getActualFieldVariable(const THREAD_ID tid, const std::string & var_name)
6257 : {
6258 771 : for (auto & sys : _solver_systems)
6259 581 : if (sys->hasVariable(var_name))
6260 391 : return sys->getActualFieldVariable<Real>(tid, var_name);
6261 190 : if (_aux->hasVariable(var_name))
6262 190 : return _aux->getActualFieldVariable<Real>(tid, var_name);
6263 :
6264 0 : mooseError("Unknown variable " + var_name);
6265 : }
6266 :
6267 : VectorMooseVariable &
6268 0 : FEProblemBase::getVectorVariable(const THREAD_ID tid, const std::string & var_name)
6269 : {
6270 0 : for (auto & sys : _solver_systems)
6271 0 : if (sys->hasVariable(var_name))
6272 0 : return sys->getFieldVariable<RealVectorValue>(tid, var_name);
6273 0 : if (_aux->hasVariable(var_name))
6274 0 : return _aux->getFieldVariable<RealVectorValue>(tid, var_name);
6275 :
6276 0 : mooseError("Unknown variable " + var_name);
6277 : }
6278 :
6279 : ArrayMooseVariable &
6280 351 : FEProblemBase::getArrayVariable(const THREAD_ID tid, const std::string & var_name)
6281 : {
6282 611 : for (auto & sys : _solver_systems)
6283 351 : if (sys->hasVariable(var_name))
6284 91 : return sys->getFieldVariable<RealEigenVector>(tid, var_name);
6285 260 : if (_aux->hasVariable(var_name))
6286 260 : return _aux->getFieldVariable<RealEigenVector>(tid, var_name);
6287 :
6288 0 : mooseError("Unknown variable " + var_name);
6289 : }
6290 :
6291 : bool
6292 179319 : FEProblemBase::hasScalarVariable(const std::string & var_name) const
6293 : {
6294 337477 : for (auto & sys : _solver_systems)
6295 179919 : if (sys->hasScalarVariable(var_name))
6296 21761 : return true;
6297 157558 : if (_aux->hasScalarVariable(var_name))
6298 10613 : return true;
6299 :
6300 146945 : return false;
6301 : }
6302 :
6303 : MooseVariableScalar &
6304 44342 : FEProblemBase::getScalarVariable(const THREAD_ID tid, const std::string & var_name)
6305 : {
6306 56683 : for (auto & sys : _solver_systems)
6307 44342 : if (sys->hasScalarVariable(var_name))
6308 32001 : return sys->getScalarVariable(tid, var_name);
6309 12341 : if (_aux->hasScalarVariable(var_name))
6310 12341 : return _aux->getScalarVariable(tid, var_name);
6311 :
6312 0 : mooseError("Unknown variable " + var_name);
6313 : }
6314 :
6315 : System &
6316 58527 : FEProblemBase::getSystem(const std::string & var_name)
6317 : {
6318 58527 : const auto [var_in_sys, sys_num] = determineSolverSystem(var_name);
6319 58527 : if (var_in_sys)
6320 39748 : return _solver_systems[sys_num]->system();
6321 18779 : else if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
6322 18779 : return _aux->system();
6323 : else
6324 0 : mooseError("Unable to find a system containing the variable " + var_name);
6325 : }
6326 :
6327 : const RestartableEquationSystems &
6328 0 : FEProblemBase::getRestartableEquationSystems() const
6329 : {
6330 0 : return _req.get();
6331 : }
6332 :
6333 : void
6334 464188 : FEProblemBase::setActiveFEVariableCoupleableMatrixTags(std::set<TagID> & mtags, const THREAD_ID tid)
6335 : {
6336 464188 : SubProblem::setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6337 :
6338 464188 : if (_displaced_problem)
6339 98260 : _displaced_problem->setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6340 464188 : }
6341 :
6342 : void
6343 6236823 : FEProblemBase::setActiveFEVariableCoupleableVectorTags(std::set<TagID> & vtags, const THREAD_ID tid)
6344 : {
6345 6236823 : SubProblem::setActiveFEVariableCoupleableVectorTags(vtags, tid);
6346 :
6347 6236823 : if (_displaced_problem)
6348 340076 : _displaced_problem->setActiveFEVariableCoupleableVectorTags(vtags, tid);
6349 6236823 : }
6350 :
6351 : void
6352 47814 : FEProblemBase::setActiveScalarVariableCoupleableMatrixTags(std::set<TagID> & mtags,
6353 : const THREAD_ID tid)
6354 : {
6355 47814 : SubProblem::setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6356 :
6357 47814 : if (_displaced_problem)
6358 0 : _displaced_problem->setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6359 47814 : }
6360 :
6361 : void
6362 47814 : FEProblemBase::setActiveScalarVariableCoupleableVectorTags(std::set<TagID> & vtags,
6363 : const THREAD_ID tid)
6364 : {
6365 47814 : SubProblem::setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6366 :
6367 47814 : if (_displaced_problem)
6368 0 : _displaced_problem->setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6369 47814 : }
6370 :
6371 : void
6372 9561276 : FEProblemBase::setActiveElementalMooseVariables(const std::set<MooseVariableFEBase *> & moose_vars,
6373 : const THREAD_ID tid)
6374 : {
6375 9561276 : SubProblem::setActiveElementalMooseVariables(moose_vars, tid);
6376 :
6377 9561276 : if (_displaced_problem)
6378 480807 : _displaced_problem->setActiveElementalMooseVariables(moose_vars, tid);
6379 9561276 : }
6380 :
6381 : void
6382 3833118 : FEProblemBase::clearActiveElementalMooseVariables(const THREAD_ID tid)
6383 : {
6384 3833118 : SubProblem::clearActiveElementalMooseVariables(tid);
6385 :
6386 3833118 : if (_displaced_problem)
6387 170463 : _displaced_problem->clearActiveElementalMooseVariables(tid);
6388 3833118 : }
6389 :
6390 : void
6391 206661 : FEProblemBase::clearActiveFEVariableCoupleableMatrixTags(const THREAD_ID tid)
6392 : {
6393 206661 : SubProblem::clearActiveFEVariableCoupleableMatrixTags(tid);
6394 :
6395 206661 : if (_displaced_problem)
6396 41849 : _displaced_problem->clearActiveFEVariableCoupleableMatrixTags(tid);
6397 206661 : }
6398 :
6399 : void
6400 206661 : FEProblemBase::clearActiveFEVariableCoupleableVectorTags(const THREAD_ID tid)
6401 : {
6402 206661 : SubProblem::clearActiveFEVariableCoupleableVectorTags(tid);
6403 :
6404 206661 : if (_displaced_problem)
6405 41849 : _displaced_problem->clearActiveFEVariableCoupleableVectorTags(tid);
6406 206661 : }
6407 :
6408 : void
6409 47814 : FEProblemBase::clearActiveScalarVariableCoupleableMatrixTags(const THREAD_ID tid)
6410 : {
6411 47814 : SubProblem::clearActiveScalarVariableCoupleableMatrixTags(tid);
6412 :
6413 47814 : if (_displaced_problem)
6414 0 : _displaced_problem->clearActiveScalarVariableCoupleableMatrixTags(tid);
6415 47814 : }
6416 :
6417 : void
6418 47814 : FEProblemBase::clearActiveScalarVariableCoupleableVectorTags(const THREAD_ID tid)
6419 : {
6420 47814 : SubProblem::clearActiveScalarVariableCoupleableVectorTags(tid);
6421 :
6422 47814 : if (_displaced_problem)
6423 0 : _displaced_problem->clearActiveScalarVariableCoupleableVectorTags(tid);
6424 47814 : }
6425 :
6426 : void
6427 4902403 : FEProblemBase::setActiveMaterialProperties(const std::unordered_set<unsigned int> & mat_prop_ids,
6428 : const THREAD_ID tid)
6429 : {
6430 : // mark active properties in every material
6431 6006268 : for (auto & mat : _all_materials.getObjects(tid))
6432 1103865 : mat->setActiveProperties(mat_prop_ids);
6433 5781399 : for (auto & mat : _all_materials[Moose::FACE_MATERIAL_DATA].getObjects(tid))
6434 878996 : mat->setActiveProperties(mat_prop_ids);
6435 5781399 : for (auto & mat : _all_materials[Moose::NEIGHBOR_MATERIAL_DATA].getObjects(tid))
6436 878996 : mat->setActiveProperties(mat_prop_ids);
6437 :
6438 4902403 : _has_active_material_properties[tid] = !mat_prop_ids.empty();
6439 4902403 : }
6440 :
6441 : bool
6442 382402572 : FEProblemBase::hasActiveMaterialProperties(const THREAD_ID tid) const
6443 : {
6444 382402572 : return _has_active_material_properties[tid];
6445 : }
6446 :
6447 : void
6448 3928739 : FEProblemBase::clearActiveMaterialProperties(const THREAD_ID tid)
6449 : {
6450 3928739 : _has_active_material_properties[tid] = 0;
6451 3928739 : }
6452 :
6453 : void
6454 59930 : FEProblemBase::addAnyRedistributers()
6455 : {
6456 : #ifdef LIBMESH_ENABLE_AMR
6457 62195 : if ((_adaptivity.isOn() || _num_grid_steps) &&
6458 2265 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
6459 2202 : _neighbor_material_props.hasStatefulProperties()))
6460 : {
6461 : // Even on a serialized Mesh, we don't keep our material
6462 : // properties serialized, so we'll rely on the callback to
6463 : // redistribute() to redistribute properties at the same time
6464 : // libMesh is redistributing elements.
6465 63 : auto add_redistributer = [this](MooseMesh & mesh,
6466 : const std::string & redistributer_name,
6467 : const bool use_displaced_mesh)
6468 : {
6469 63 : InputParameters redistribute_params = RedistributeProperties::validParams();
6470 63 : redistribute_params.set<MooseApp *>(MooseBase::app_param) = &_app;
6471 126 : redistribute_params.set<std::string>("for_whom") = this->name();
6472 189 : redistribute_params.set<MooseMesh *>("mesh") = &mesh;
6473 63 : redistribute_params.set<Moose::RelationshipManagerType>("rm_type") =
6474 : Moose::RelationshipManagerType::GEOMETRIC;
6475 126 : redistribute_params.set<bool>("use_displaced_mesh") = use_displaced_mesh;
6476 63 : redistribute_params.setHitNode(*parameters().getHitNode(), {});
6477 :
6478 : std::shared_ptr<RedistributeProperties> redistributer =
6479 63 : _factory.create<RedistributeProperties>(
6480 126 : "RedistributeProperties", redistributer_name, redistribute_params);
6481 :
6482 63 : if (_material_props.hasStatefulProperties())
6483 63 : redistributer->addMaterialPropertyStorage(_material_props);
6484 :
6485 63 : if (_bnd_material_props.hasStatefulProperties())
6486 63 : redistributer->addMaterialPropertyStorage(_bnd_material_props);
6487 :
6488 63 : if (_neighbor_material_props.hasStatefulProperties())
6489 63 : redistributer->addMaterialPropertyStorage(_neighbor_material_props);
6490 :
6491 63 : mesh.getMesh().add_ghosting_functor(redistributer);
6492 126 : };
6493 :
6494 63 : add_redistributer(_mesh, "mesh_property_redistributer", false);
6495 63 : if (_displaced_problem)
6496 0 : add_redistributer(_displaced_problem->mesh(), "displaced_mesh_property_redistributer", true);
6497 : }
6498 : #endif // LIBMESH_ENABLE_AMR
6499 59930 : }
6500 :
6501 : void
6502 61806 : FEProblemBase::updateMaxQps()
6503 : {
6504 : // Find the maximum number of quadrature points
6505 : {
6506 61806 : MaxQpsThread mqt(*this);
6507 61806 : Threads::parallel_reduce(getCurrentAlgebraicElementRange(), mqt);
6508 61806 : _max_qps = mqt.max();
6509 :
6510 : // If we have more shape functions or more quadrature points on
6511 : // another processor, then we may need to handle those elements
6512 : // ourselves later after repartitioning.
6513 61806 : _communicator.max(_max_qps);
6514 : }
6515 :
6516 61806 : unsigned int max_qpts = getMaxQps();
6517 61806 : if (max_qpts > Moose::constMaxQpsPerElem)
6518 0 : mooseError("Max quadrature points per element assumptions made in some code (e.g. Coupleable ",
6519 : "and MaterialPropertyInterface classes) have been violated.\n",
6520 : "Complain to Moose developers to have constMaxQpsPerElem increased from ",
6521 : Moose::constMaxQpsPerElem,
6522 : " to ",
6523 : max_qpts);
6524 130211 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6525 : {
6526 : // the highest available order in libMesh is 43
6527 68405 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
6528 68405 : _zero[tid].resize(max_qpts, 0);
6529 68405 : _ad_zero[tid].resize(max_qpts, 0);
6530 68405 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
6531 68405 : _ad_grad_zero[tid].resize(max_qpts, ADRealGradient(0));
6532 68405 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
6533 68405 : _ad_second_zero[tid].resize(max_qpts, ADRealTensorValue(0));
6534 68405 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
6535 68405 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
6536 : }
6537 61806 : }
6538 :
6539 : void
6540 78 : FEProblemBase::bumpVolumeQRuleOrder(Order order, SubdomainID block)
6541 : {
6542 168 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6543 180 : for (const auto i : index_range(_nl))
6544 90 : _assembly[tid][i]->bumpVolumeQRuleOrder(order, block);
6545 :
6546 78 : if (_displaced_problem)
6547 0 : _displaced_problem->bumpVolumeQRuleOrder(order, block);
6548 :
6549 78 : updateMaxQps();
6550 78 : }
6551 :
6552 : void
6553 13 : FEProblemBase::bumpAllQRuleOrder(Order order, SubdomainID block)
6554 : {
6555 28 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6556 30 : for (const auto i : index_range(_nl))
6557 15 : _assembly[tid][i]->bumpAllQRuleOrder(order, block);
6558 :
6559 13 : if (_displaced_problem)
6560 0 : _displaced_problem->bumpAllQRuleOrder(order, block);
6561 :
6562 13 : updateMaxQps();
6563 13 : }
6564 :
6565 : void
6566 61715 : FEProblemBase::createQRules(QuadratureType type,
6567 : Order order,
6568 : Order volume_order,
6569 : Order face_order,
6570 : SubdomainID block,
6571 : const bool allow_negative_qweights)
6572 : {
6573 61715 : if (order == INVALID_ORDER)
6574 : {
6575 : // automatically determine the integration order
6576 61154 : order = _solver_systems[0]->getMinQuadratureOrder();
6577 61419 : for (const auto i : make_range(std::size_t(1), _solver_systems.size()))
6578 265 : if (order < _solver_systems[i]->getMinQuadratureOrder())
6579 0 : order = _solver_systems[i]->getMinQuadratureOrder();
6580 61154 : if (order < _aux->getMinQuadratureOrder())
6581 6083 : order = _aux->getMinQuadratureOrder();
6582 : }
6583 :
6584 61715 : if (volume_order == INVALID_ORDER)
6585 61563 : volume_order = order;
6586 :
6587 61715 : if (face_order == INVALID_ORDER)
6588 61563 : face_order = order;
6589 :
6590 130015 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6591 136885 : for (const auto i : index_range(_solver_systems))
6592 68585 : _assembly[tid][i]->createQRules(
6593 : type, order, volume_order, face_order, block, allow_negative_qweights);
6594 :
6595 61715 : if (_displaced_problem)
6596 2022 : _displaced_problem->createQRules(
6597 : type, order, volume_order, face_order, block, allow_negative_qweights);
6598 :
6599 61715 : updateMaxQps();
6600 61715 : }
6601 :
6602 : void
6603 19549 : FEProblemBase::setCoupling(Moose::CouplingType type)
6604 : {
6605 19549 : if (_trust_user_coupling_matrix)
6606 : {
6607 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6608 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6609 : "haven't been provided a coupling matrix!");
6610 :
6611 : // We've been told to trust the user coupling matrix, so we're going to leave things alone
6612 3 : return;
6613 : }
6614 :
6615 19546 : _coupling = type;
6616 : }
6617 :
6618 : void
6619 0 : FEProblemBase::setCouplingMatrix(CouplingMatrix * cm, const unsigned int i)
6620 : {
6621 : // TODO: Deprecate method
6622 0 : setCoupling(Moose::COUPLING_CUSTOM);
6623 0 : _cm[i].reset(cm);
6624 0 : }
6625 :
6626 : void
6627 13230 : FEProblemBase::setCouplingMatrix(std::unique_ptr<CouplingMatrix> cm, const unsigned int i)
6628 : {
6629 13230 : setCoupling(Moose::COUPLING_CUSTOM);
6630 13230 : _cm[i] = std::move(cm);
6631 13230 : }
6632 :
6633 : void
6634 3 : FEProblemBase::trustUserCouplingMatrix()
6635 : {
6636 3 : if (_coupling != Moose::COUPLING_CUSTOM)
6637 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6638 : "haven't been provided a coupling matrix!");
6639 :
6640 3 : _trust_user_coupling_matrix = true;
6641 3 : }
6642 :
6643 : void
6644 63 : FEProblemBase::setNonlocalCouplingMatrix()
6645 : {
6646 315 : TIME_SECTION("setNonlocalCouplingMatrix", 5, "Setting Nonlocal Coupling Matrix");
6647 :
6648 63 : if (_nl.size() > 1)
6649 0 : mooseError("Nonlocal kernels are weirdly stored on the FEProblem so we don't currently support "
6650 : "multiple nonlinear systems with nonlocal kernels.");
6651 :
6652 126 : for (const auto nl_sys_num : index_range(_nl))
6653 : {
6654 63 : auto & nl = _nl[nl_sys_num];
6655 63 : auto & nonlocal_cm = _nonlocal_cm[nl_sys_num];
6656 63 : unsigned int n_vars = nl->nVariables();
6657 63 : nonlocal_cm.resize(n_vars);
6658 63 : const auto & vars = nl->getVariables(0);
6659 63 : const auto & nonlocal_kernel = _nonlocal_kernels.getObjects();
6660 63 : const auto & nonlocal_integrated_bc = _nonlocal_integrated_bcs.getObjects();
6661 189 : for (const auto & ivar : vars)
6662 : {
6663 196 : for (const auto & kernel : nonlocal_kernel)
6664 : {
6665 140 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6666 70 : if (i == kernel->variable().number())
6667 105 : for (const auto & jvar : vars)
6668 : {
6669 70 : const auto it = _var_dof_map.find(jvar->name());
6670 70 : if (it != _var_dof_map.end())
6671 : {
6672 62 : unsigned int j = jvar->number();
6673 62 : nonlocal_cm(i, j) = 1;
6674 : }
6675 : }
6676 : }
6677 182 : for (const auto & integrated_bc : nonlocal_integrated_bc)
6678 : {
6679 112 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6680 56 : if (i == integrated_bc->variable().number())
6681 84 : for (const auto & jvar : vars)
6682 : {
6683 56 : const auto it = _var_dof_map.find(jvar->name());
6684 56 : if (it != _var_dof_map.end())
6685 : {
6686 28 : unsigned int j = jvar->number();
6687 28 : nonlocal_cm(i, j) = 1;
6688 : }
6689 : }
6690 : }
6691 : }
6692 : }
6693 63 : }
6694 :
6695 : bool
6696 624 : FEProblemBase::areCoupled(const unsigned int ivar,
6697 : const unsigned int jvar,
6698 : const unsigned int nl_sys) const
6699 : {
6700 624 : return (*_cm[nl_sys])(ivar, jvar);
6701 : }
6702 :
6703 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6704 15127529 : FEProblemBase::couplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6705 : {
6706 15127529 : return _assembly[tid][nl_sys]->couplingEntries();
6707 : }
6708 :
6709 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6710 4162 : FEProblemBase::nonlocalCouplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6711 : {
6712 4162 : return _assembly[tid][nl_sys]->nonlocalCouplingEntries();
6713 : }
6714 :
6715 : void
6716 60424 : FEProblemBase::init()
6717 : {
6718 60424 : if (_initialized)
6719 0 : return;
6720 :
6721 302120 : TIME_SECTION("init", 2, "Initializing");
6722 :
6723 : // call executioner's preProblemInit so that it can do some setups before problem init
6724 60424 : _app.getExecutioner()->preProblemInit();
6725 :
6726 : // If we have AD and we are doing global AD indexing, then we should by default set the matrix
6727 : // coupling to full. If the user has told us to trust their coupling matrix, then this call will
6728 : // not do anything
6729 60424 : if (haveADObjects() && Moose::globalADIndexing())
6730 6226 : setCoupling(Moose::COUPLING_FULL);
6731 :
6732 120075 : for (const auto i : index_range(_nl))
6733 : {
6734 59651 : auto & nl = _nl[i];
6735 59651 : auto & cm = _cm[i];
6736 :
6737 59651 : unsigned int n_vars = nl->nVariables();
6738 : {
6739 298255 : TIME_SECTION("fillCouplingMatrix", 3, "Filling Coupling Matrix");
6740 :
6741 59651 : switch (_coupling)
6742 : {
6743 44750 : case Moose::COUPLING_DIAG:
6744 44750 : cm = std::make_unique<CouplingMatrix>(n_vars);
6745 81381 : for (unsigned int i = 0; i < n_vars; i++)
6746 36631 : (*cm)(i, i) = 1;
6747 44750 : break;
6748 :
6749 : // for full jacobian
6750 6358 : case Moose::COUPLING_FULL:
6751 6358 : cm = std::make_unique<CouplingMatrix>(n_vars);
6752 16247 : for (unsigned int i = 0; i < n_vars; i++)
6753 28654 : for (unsigned int j = 0; j < n_vars; j++)
6754 18765 : (*cm)(i, j) = 1;
6755 6358 : break;
6756 :
6757 8543 : case Moose::COUPLING_CUSTOM:
6758 : // do nothing, _cm was already set through couplingMatrix() call
6759 8543 : break;
6760 : }
6761 59651 : }
6762 :
6763 59651 : nl->dofMap()._dof_coupling = cm.get();
6764 :
6765 : // If there are no variables, make sure to pass a nullptr coupling
6766 : // matrix, to avoid warnings about non-nullptr yet empty
6767 : // CouplingMatrices.
6768 59651 : if (n_vars == 0)
6769 13824 : nl->dofMap()._dof_coupling = nullptr;
6770 :
6771 59651 : nl->dofMap().attach_extra_sparsity_function(&extraSparsity, nl.get());
6772 59651 : nl->dofMap().attach_extra_send_list_function(&extraSendList, nl.get());
6773 59651 : _aux->dofMap().attach_extra_send_list_function(&extraSendList, _aux.get());
6774 :
6775 59651 : if (!_skip_nl_system_check && _solve && n_vars == 0)
6776 0 : mooseError("No variables specified in nonlinear system '", nl->name(), "'.");
6777 : }
6778 :
6779 60424 : ghostGhostedBoundaries(); // We do this again right here in case new boundaries have been added
6780 :
6781 : // We may have added element/nodes to the mesh in ghostGhostedBoundaries so we need to update
6782 : // all of our mesh information. We need to make sure that mesh information is up-to-date before
6783 : // EquationSystems::init because that will call through to updateGeomSearch (for sparsity
6784 : // augmentation) and if we haven't added back boundary node information before that latter call,
6785 : // then we're screwed. We'll get things like "Unable to find closest node!"
6786 60424 : _mesh.meshChanged();
6787 60424 : if (_displaced_problem)
6788 2022 : _displaced_mesh->meshChanged();
6789 :
6790 60424 : if (_mesh.doingPRefinement())
6791 : {
6792 240 : preparePRefinement();
6793 240 : if (_displaced_problem)
6794 0 : _displaced_problem->preparePRefinement();
6795 : }
6796 :
6797 : // do not assemble system matrix for JFNK solve
6798 120075 : for (auto & nl : _nl)
6799 59651 : if (solverParams(nl->number())._type == Moose::ST_JFNK)
6800 120 : nl->turnOffJacobian();
6801 :
6802 121107 : for (auto & sys : _solver_systems)
6803 60683 : sys->preInit();
6804 60424 : _aux->preInit();
6805 :
6806 : // Build the mortar segment meshes, if they haven't been already, for a couple reasons:
6807 : // 1) Get the ghosting correct for both static and dynamic meshes
6808 : // 2) Make sure the mortar mesh is built for mortar constraints that live on the static mesh
6809 : //
6810 : // It is worth-while to note that mortar meshes that live on a dynamic mesh will be built
6811 : // during residual and Jacobian evaluation because when displacements are solution variables
6812 : // the mortar mesh will move and change during the course of a non-linear solve. We DO NOT
6813 : // redo ghosting during non-linear solve, so for purpose 1) the below call has to be made
6814 60424 : if (!_mortar_data->initialized())
6815 45081 : updateMortarMesh();
6816 :
6817 : {
6818 302120 : TIME_SECTION("EquationSystems::Init", 2, "Initializing Equation Systems");
6819 60424 : es().init();
6820 60424 : }
6821 :
6822 121107 : for (auto & sys : _solver_systems)
6823 60683 : sys->postInit();
6824 60424 : _aux->postInit();
6825 :
6826 : // Now that the equation system and the dof distribution is done, we can generate the
6827 : // finite volume-related parts if needed.
6828 60424 : if (haveFV())
6829 4129 : _mesh.setupFiniteVolumeMeshData();
6830 :
6831 121107 : for (auto & sys : _solver_systems)
6832 60683 : sys->update();
6833 60424 : _aux->update();
6834 :
6835 127002 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
6836 132401 : for (const auto i : index_range(_nl))
6837 : {
6838 : mooseAssert(
6839 : _cm[i],
6840 : "Coupling matrix not set for system "
6841 : << i
6842 : << ". This should only happen if a preconditioner was not setup for this system");
6843 65823 : _assembly[tid][i]->init(_cm[i].get());
6844 : }
6845 :
6846 60424 : if (_displaced_problem)
6847 2022 : _displaced_problem->init();
6848 :
6849 : #ifdef MOOSE_KOKKOS_ENABLED
6850 45679 : if (_has_kokkos_objects)
6851 2265 : initKokkos();
6852 : #endif
6853 :
6854 60424 : _initialized = true;
6855 60424 : }
6856 :
6857 : unsigned int
6858 11949 : FEProblemBase::nlSysNum(const NonlinearSystemName & nl_sys_name) const
6859 : {
6860 11949 : std::istringstream ss(nl_sys_name);
6861 : unsigned int nl_sys_num;
6862 11949 : if (!(ss >> nl_sys_num) || !ss.eof())
6863 11949 : nl_sys_num = libmesh_map_find(_nl_sys_name_to_num, nl_sys_name);
6864 :
6865 11949 : return nl_sys_num;
6866 11949 : }
6867 :
6868 : unsigned int
6869 78396 : FEProblemBase::linearSysNum(const LinearSystemName & linear_sys_name) const
6870 : {
6871 78396 : std::istringstream ss(linear_sys_name);
6872 : unsigned int linear_sys_num;
6873 78396 : if (!(ss >> linear_sys_num) || !ss.eof())
6874 78396 : linear_sys_num = libmesh_map_find(_linear_sys_name_to_num, linear_sys_name);
6875 :
6876 78396 : return linear_sys_num;
6877 78396 : }
6878 :
6879 : unsigned int
6880 124419 : FEProblemBase::solverSysNum(const SolverSystemName & solver_sys_name) const
6881 : {
6882 124419 : std::istringstream ss(solver_sys_name);
6883 : unsigned int solver_sys_num;
6884 124419 : if (!(ss >> solver_sys_num) || !ss.eof())
6885 : {
6886 124419 : const auto & search = _solver_sys_name_to_num.find(solver_sys_name);
6887 124419 : if (search == _solver_sys_name_to_num.end())
6888 0 : mooseError("The solver system number was requested for system '" + solver_sys_name,
6889 : "' but this system does not exist in the Problem. Systems can be added to the "
6890 : "problem using the 'nl_sys_names'/'linear_sys_names' parameter.\nSystems in the "
6891 0 : "Problem: " +
6892 0 : Moose::stringify(_solver_sys_names));
6893 124419 : solver_sys_num = search->second;
6894 : }
6895 :
6896 124419 : return solver_sys_num;
6897 124419 : }
6898 :
6899 : unsigned int
6900 1644 : FEProblemBase::systemNumForVariable(const VariableName & variable_name) const
6901 : {
6902 1746 : for (const auto & solver_sys : _solver_systems)
6903 1644 : if (solver_sys->hasVariable(variable_name))
6904 1542 : return solver_sys->number();
6905 : mooseAssert(_aux, "Should have an auxiliary system");
6906 102 : if (_aux->hasVariable(variable_name))
6907 102 : return _aux->number();
6908 :
6909 0 : mooseError("Variable '",
6910 : variable_name,
6911 : "' was not found in any solver (nonlinear/linear) or auxiliary system");
6912 : }
6913 :
6914 : void
6915 321809 : FEProblemBase::solve(const unsigned int nl_sys_num)
6916 : {
6917 1609045 : TIME_SECTION("solve", 1, "Solving", false);
6918 :
6919 321809 : setCurrentNonlinearSystem(nl_sys_num);
6920 :
6921 : // This prevents stale dof indices from lingering around and possibly leading to invalid reads
6922 : // and writes. Dof indices may be made stale through operations like mesh adaptivity
6923 321809 : clearAllDofIndices();
6924 321809 : if (_displaced_problem)
6925 32837 : _displaced_problem->clearAllDofIndices();
6926 :
6927 : // Setup the output system for printing linear/nonlinear iteration information and some solver
6928 : // settings, including setting matrix prefixes. This must occur before petscSetOptions
6929 321809 : initPetscOutputAndSomeSolverSettings();
6930 :
6931 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6932 : Moose::PetscSupport::petscSetOptions(
6933 : _petsc_options, _solver_params); // Make sure the PETSc options are setup for this app
6934 : #else
6935 : // Now this database will be the default
6936 : // Each app should have only one database
6937 321809 : if (!_app.isUltimateMaster())
6938 85520 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6939 : // We did not add PETSc options to database yet
6940 321809 : if (!_is_petsc_options_inserted)
6941 : {
6942 : // Insert options for all systems all at once
6943 39103 : Moose::PetscSupport::petscSetOptions(_petsc_options, _solver_params, this);
6944 39100 : _is_petsc_options_inserted = true;
6945 : }
6946 : #endif
6947 :
6948 : // set up DM which is required if use a field split preconditioner
6949 : // We need to setup DM every "solve()" because libMesh destroy SNES after solve()
6950 : // Do not worry, DM setup is very cheap
6951 321806 : _current_nl_sys->setupDM();
6952 :
6953 321806 : possiblyRebuildGeomSearchPatches();
6954 :
6955 : // reset flag so that residual evaluation does not get skipped
6956 : // and the next non-linear iteration does not automatically fail with
6957 : // "DIVERGED_NANORINF", when we throw an exception and stop solve
6958 321806 : _fail_next_system_convergence_check = false;
6959 :
6960 321806 : if (_solve)
6961 : {
6962 288217 : _current_nl_sys->solve();
6963 288162 : _current_nl_sys->update();
6964 : }
6965 :
6966 : // sync solutions in displaced problem
6967 321751 : if (_displaced_problem)
6968 32834 : _displaced_problem->syncSolutions();
6969 :
6970 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6971 321751 : if (!_app.isUltimateMaster())
6972 85520 : LibmeshPetscCall(PetscOptionsPop());
6973 : #endif
6974 321751 : }
6975 :
6976 : void
6977 199 : FEProblemBase::setException(const std::string & message)
6978 : {
6979 199 : _has_exception = true;
6980 199 : _exception_message = message;
6981 199 : }
6982 :
6983 : void
6984 19913510 : FEProblemBase::checkExceptionAndStopSolve(bool print_message)
6985 : {
6986 19913510 : if (_skip_exception_check)
6987 216 : return;
6988 :
6989 59739882 : TIME_SECTION("checkExceptionAndStopSolve", 5);
6990 :
6991 : // See if any processor had an exception. If it did, get back the
6992 : // processor that the exception occurred on.
6993 : unsigned int processor_id;
6994 :
6995 19913294 : _communicator.maxloc(_has_exception, processor_id);
6996 :
6997 19913294 : if (_has_exception)
6998 : {
6999 302 : _communicator.broadcast(_exception_message, processor_id);
7000 :
7001 432 : if (_current_execute_on_flag == EXEC_LINEAR || _current_execute_on_flag == EXEC_NONLINEAR ||
7002 130 : _current_execute_on_flag == EXEC_POSTCHECK)
7003 : {
7004 : // Print the message
7005 302 : if (_communicator.rank() == 0 && print_message)
7006 : {
7007 184 : _console << "\n" << _exception_message << "\n";
7008 184 : if (isTransient())
7009 : _console
7010 : << "To recover, the solution will fail and then be re-attempted with a reduced time "
7011 172 : "step.\n"
7012 172 : << std::endl;
7013 : }
7014 :
7015 : // Stop the solve -- this entails setting
7016 : // SNESSetFunctionDomainError() or directly inserting NaNs in the
7017 : // residual vector to let PETSc >= 3.6 return DIVERGED_NANORINF.
7018 302 : if (_current_nl_sys)
7019 302 : _current_nl_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7020 :
7021 302 : if (_current_linear_sys)
7022 0 : _current_linear_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
7023 :
7024 : // and close Aux system (we MUST do this here; see #11525)
7025 302 : _aux->solution().close();
7026 :
7027 : // We've handled this exception, so we no longer have one.
7028 302 : _has_exception = false;
7029 :
7030 : // Force the next non-linear convergence check to fail (and all further residual evaluation
7031 : // to be skipped).
7032 302 : _fail_next_system_convergence_check = true;
7033 :
7034 : // Repropagate the exception, so it can be caught at a higher level, typically
7035 : // this is NonlinearSystem::computeResidual().
7036 302 : throw MooseException(_exception_message);
7037 : }
7038 : else
7039 0 : mooseError("The following parallel-communicated exception was detected during " +
7040 0 : Moose::stringify(_current_execute_on_flag) + " evaluation:\n" +
7041 0 : _exception_message +
7042 : "\nBecause this did not occur during residual evaluation, there"
7043 : " is no way to handle this, so the solution is aborting.\n");
7044 : }
7045 19913294 : }
7046 :
7047 : void
7048 3533804 : FEProblemBase::resetState()
7049 : {
7050 : // Our default state is to allow computing derivatives
7051 3533804 : ADReal::do_derivatives = true;
7052 3533804 : _current_execute_on_flag = EXEC_NONE;
7053 :
7054 : // Clear the VectorTags and MatrixTags
7055 3533804 : clearCurrentResidualVectorTags();
7056 3533804 : clearCurrentJacobianMatrixTags();
7057 :
7058 3533804 : _safe_access_tagged_vectors = true;
7059 3533804 : _safe_access_tagged_matrices = true;
7060 :
7061 3533804 : setCurrentlyComputingResidual(false);
7062 3533804 : setCurrentlyComputingJacobian(false);
7063 3533804 : setCurrentlyComputingResidualAndJacobian(false);
7064 3533804 : if (_displaced_problem)
7065 : {
7066 145291 : _displaced_problem->setCurrentlyComputingResidual(false);
7067 145291 : _displaced_problem->setCurrentlyComputingJacobian(false);
7068 145291 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(false);
7069 : }
7070 3533804 : }
7071 :
7072 : void
7073 26086 : FEProblemBase::solveLinearSystem(const unsigned int linear_sys_num,
7074 : const Moose::PetscSupport::PetscOptions * po)
7075 : {
7076 130430 : TIME_SECTION("solve", 1, "Solving", false);
7077 :
7078 26086 : setCurrentLinearSystem(linear_sys_num);
7079 :
7080 26086 : const Moose::PetscSupport::PetscOptions & options = po ? *po : _petsc_options;
7081 26086 : auto & solver_params = _solver_params[numNonlinearSystems() + linear_sys_num];
7082 :
7083 : // Set custom convergence criteria
7084 26086 : Moose::PetscSupport::petscSetDefaults(*this);
7085 :
7086 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
7087 : LibmeshPetscCall(Moose::PetscSupport::petscSetOptions(
7088 : options, solver_params)); // Make sure the PETSc options are setup for this app
7089 : #else
7090 : // Now this database will be the default
7091 : // Each app should have only one database
7092 26086 : if (!_app.isUltimateMaster())
7093 195 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
7094 :
7095 : // We did not add PETSc options to database yet
7096 26086 : if (!_is_petsc_options_inserted)
7097 : {
7098 942 : Moose::PetscSupport::petscSetOptions(options, solver_params, this);
7099 942 : _is_petsc_options_inserted = true;
7100 : }
7101 : #endif
7102 :
7103 26086 : if (_solve)
7104 26076 : _current_linear_sys->solve();
7105 :
7106 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
7107 26086 : if (!_app.isUltimateMaster())
7108 195 : LibmeshPetscCall(PetscOptionsPop());
7109 : #endif
7110 26086 : }
7111 :
7112 : bool
7113 335423 : FEProblemBase::solverSystemConverged(const unsigned int sys_num)
7114 : {
7115 335423 : if (_solve)
7116 335401 : return _solver_systems[sys_num]->converged();
7117 : else
7118 22 : return true;
7119 : }
7120 :
7121 : unsigned int
7122 4901 : FEProblemBase::nNonlinearIterations(const unsigned int nl_sys_num) const
7123 : {
7124 4901 : return _nl[nl_sys_num]->nNonlinearIterations();
7125 : }
7126 :
7127 : unsigned int
7128 4051 : FEProblemBase::nLinearIterations(const unsigned int nl_sys_num) const
7129 : {
7130 4051 : return _nl[nl_sys_num]->nLinearIterations();
7131 : }
7132 :
7133 : Real
7134 242 : FEProblemBase::finalNonlinearResidual(const unsigned int nl_sys_num) const
7135 : {
7136 242 : return _nl[nl_sys_num]->finalNonlinearResidual();
7137 : }
7138 :
7139 : bool
7140 760320 : FEProblemBase::computingPreSMOResidual(const unsigned int nl_sys_num) const
7141 : {
7142 760320 : return _nl[nl_sys_num]->computingPreSMOResidual();
7143 : }
7144 :
7145 : void
7146 55197 : FEProblemBase::copySolutionsBackwards()
7147 : {
7148 275985 : TIME_SECTION("copySolutionsBackwards", 3, "Copying Solutions Backward");
7149 :
7150 110635 : for (auto & sys : _solver_systems)
7151 55438 : sys->copySolutionsBackwards();
7152 55197 : _aux->copySolutionsBackwards();
7153 55197 : }
7154 :
7155 : void
7156 152 : FEProblemBase::skipNextForwardSolutionCopyToOld()
7157 : {
7158 324 : for (auto & sys : _solver_systems)
7159 172 : sys->skipNextSolutionToOldCopy();
7160 152 : _aux->skipNextSolutionToOldCopy();
7161 152 : }
7162 :
7163 : void
7164 233070 : FEProblemBase::advanceState()
7165 : {
7166 1165350 : TIME_SECTION("advanceState", 5, "Advancing State");
7167 :
7168 469487 : for (auto & sys : _solver_systems)
7169 236417 : sys->copyOldSolutions();
7170 233070 : _aux->copyOldSolutions();
7171 :
7172 233070 : if (_displaced_problem)
7173 : {
7174 61858 : for (const auto i : index_range(_solver_systems))
7175 30929 : _displaced_problem->solverSys(i).copyOldSolutions();
7176 30929 : _displaced_problem->auxSys().copyOldSolutions();
7177 : }
7178 :
7179 233070 : _reporter_data.copyValuesBack();
7180 :
7181 233070 : getMooseApp().getChainControlDataSystem().copyValuesBack();
7182 :
7183 233070 : if (_material_props.hasStatefulProperties())
7184 1760 : _material_props.shift();
7185 :
7186 233070 : if (_bnd_material_props.hasStatefulProperties())
7187 1612 : _bnd_material_props.shift();
7188 :
7189 233070 : if (_neighbor_material_props.hasStatefulProperties())
7190 1496 : _neighbor_material_props.shift();
7191 :
7192 : #ifdef MOOSE_KOKKOS_ENABLED
7193 171065 : if (_kokkos_material_props.hasStatefulProperties())
7194 566 : _kokkos_material_props.shift();
7195 :
7196 171065 : if (_kokkos_bnd_material_props.hasStatefulProperties())
7197 641 : _kokkos_bnd_material_props.shift();
7198 :
7199 171065 : if (_kokkos_neighbor_material_props.hasStatefulProperties())
7200 566 : _kokkos_neighbor_material_props.shift();
7201 : #endif
7202 233070 : }
7203 :
7204 : void
7205 3341 : FEProblemBase::restoreSolutions()
7206 : {
7207 16705 : TIME_SECTION("restoreSolutions", 5, "Restoring Solutions");
7208 :
7209 3341 : if (!_not_zeroed_tagged_vectors.empty())
7210 0 : paramError("not_zeroed_tag_vectors",
7211 : "There is currently no way to restore not-zeroed vectors.");
7212 :
7213 6684 : for (auto & sys : _solver_systems)
7214 : {
7215 3343 : if (_verbose_restore)
7216 18 : _console << "Restoring solutions on system " << sys->name() << "..." << std::endl;
7217 3343 : sys->restoreSolutions();
7218 : }
7219 :
7220 3341 : if (_verbose_restore)
7221 18 : _console << "Restoring solutions on Auxiliary system..." << std::endl;
7222 3341 : _aux->restoreSolutions();
7223 :
7224 3341 : if (_verbose_restore)
7225 18 : _console << "Restoring postprocessor, vector-postprocessor, and reporter data..." << std::endl;
7226 3341 : _reporter_data.restoreState(_verbose_restore);
7227 :
7228 3341 : if (_displaced_problem)
7229 132 : _displaced_problem->updateMesh();
7230 3341 : }
7231 :
7232 : void
7233 100 : FEProblemBase::saveOldSolutions()
7234 : {
7235 500 : TIME_SECTION("saveOldSolutions", 5, "Saving Old Solutions");
7236 :
7237 200 : for (auto & sys : _solver_systems)
7238 100 : sys->saveOldSolutions();
7239 100 : _aux->saveOldSolutions();
7240 100 : }
7241 :
7242 : void
7243 100 : FEProblemBase::restoreOldSolutions()
7244 : {
7245 500 : TIME_SECTION("restoreOldSolutions", 5, "Restoring Old Solutions");
7246 :
7247 200 : for (auto & sys : _solver_systems)
7248 100 : sys->restoreOldSolutions();
7249 100 : _aux->restoreOldSolutions();
7250 100 : }
7251 :
7252 : void
7253 1238800 : FEProblemBase::outputStep(ExecFlagType type)
7254 : {
7255 6194000 : TIME_SECTION("outputStep", 1, "Outputting");
7256 :
7257 1238800 : setCurrentExecuteOnFlag(type);
7258 :
7259 2491754 : for (auto & sys : _solver_systems)
7260 1252954 : sys->update();
7261 1238800 : _aux->update();
7262 :
7263 1238800 : if (_displaced_problem)
7264 126253 : _displaced_problem->syncSolutions();
7265 1238800 : _app.getOutputWarehouse().outputStep(type);
7266 :
7267 1238791 : setCurrentExecuteOnFlag(EXEC_NONE);
7268 1238791 : }
7269 :
7270 : void
7271 83883 : FEProblemBase::allowOutput(bool state)
7272 : {
7273 83883 : _app.getOutputWarehouse().allowOutput(state);
7274 83883 : }
7275 :
7276 : void
7277 23 : FEProblemBase::forceOutput()
7278 : {
7279 23 : _app.getOutputWarehouse().forceOutput();
7280 23 : }
7281 :
7282 : void
7283 340834 : FEProblemBase::initPetscOutputAndSomeSolverSettings()
7284 : {
7285 340834 : _app.getOutputWarehouse().solveSetup();
7286 340834 : Moose::PetscSupport::petscSetDefaults(*this);
7287 340834 : }
7288 :
7289 : void
7290 239529 : FEProblemBase::onTimestepBegin()
7291 : {
7292 718587 : TIME_SECTION("onTimestepBegin", 2);
7293 :
7294 482099 : for (auto & nl : _nl)
7295 242570 : nl->onTimestepBegin();
7296 239529 : }
7297 :
7298 : void
7299 310222 : FEProblemBase::onTimestepEnd()
7300 : {
7301 310222 : }
7302 :
7303 : Real
7304 4628031 : FEProblemBase::getTimeFromStateArg(const Moose::StateArg & state) const
7305 : {
7306 4628031 : if (state.iteration_type != Moose::SolutionIterationType::Time)
7307 : // If we are any iteration type other than time (e.g. nonlinear), then temporally we are still
7308 : // in the present time
7309 0 : return time();
7310 :
7311 4628031 : switch (state.state)
7312 : {
7313 4628031 : case 0:
7314 4628031 : return time();
7315 :
7316 0 : case 1:
7317 0 : return timeOld();
7318 :
7319 0 : default:
7320 0 : mooseError("Unhandled state ", state.state, " in FEProblemBase::getTimeFromStateArg");
7321 : }
7322 : }
7323 :
7324 : void
7325 30418 : FEProblemBase::addTimeIntegrator(const std::string & type,
7326 : const std::string & name,
7327 : InputParameters & parameters)
7328 : {
7329 : parallel_object_only();
7330 :
7331 91254 : parameters.set<SubProblem *>("_subproblem") = this;
7332 30418 : logAdd("TimeIntegrator", name, type, parameters);
7333 30418 : _aux->addTimeIntegrator(type, name + ":aux", parameters);
7334 60863 : for (auto & sys : _solver_systems)
7335 30445 : sys->addTimeIntegrator(type, name + ":" + sys->name(), parameters);
7336 30418 : _has_time_integrator = true;
7337 :
7338 : // add vectors to store u_dot, u_dotdot, udot_old, u_dotdot_old and
7339 : // solution vectors older than 2 time steps, if requested by the time
7340 : // integrator
7341 30418 : _aux->addDotVectors();
7342 60832 : for (auto & nl : _nl)
7343 : {
7344 30414 : nl->addDotVectors();
7345 :
7346 30414 : auto tag_udot = nl->getTimeIntegrators()[0]->uDotFactorTag();
7347 30414 : if (!nl->hasVector(tag_udot))
7348 30378 : nl->associateVectorToTag(*nl->solutionUDot(), tag_udot);
7349 30414 : auto tag_udotdot = nl->getTimeIntegrators()[0]->uDotDotFactorTag();
7350 30414 : if (!nl->hasVector(tag_udotdot) && uDotDotRequested())
7351 150 : nl->associateVectorToTag(*nl->solutionUDotDot(), tag_udotdot);
7352 : }
7353 :
7354 30418 : if (_displaced_problem)
7355 : // Time integrator does not exist when displaced problem is created.
7356 1728 : _displaced_problem->addTimeIntegrator();
7357 30418 : }
7358 :
7359 : void
7360 22 : FEProblemBase::addPredictor(const std::string & type,
7361 : const std::string & name,
7362 : InputParameters & parameters)
7363 : {
7364 : parallel_object_only();
7365 :
7366 22 : if (!numNonlinearSystems() && numLinearSystems())
7367 0 : mooseError("Vector bounds cannot be used with LinearSystems!");
7368 :
7369 44 : parameters.set<SubProblem *>("_subproblem") = this;
7370 22 : std::shared_ptr<Predictor> predictor = _factory.create<Predictor>(type, name, parameters);
7371 22 : logAdd("Predictor", name, type, parameters);
7372 :
7373 44 : for (auto & nl : _nl)
7374 22 : nl->setPredictor(predictor);
7375 22 : }
7376 :
7377 : Real
7378 62579 : FEProblemBase::computeResidualL2Norm(NonlinearSystemBase & sys)
7379 : {
7380 62579 : _current_nl_sys = &sys;
7381 62579 : computeResidual(*sys.currentSolution(), sys.RHS(), sys.number());
7382 62579 : return sys.RHS().l2_norm();
7383 : }
7384 :
7385 : Real
7386 84 : FEProblemBase::computeResidualL2Norm(LinearSystem & sys)
7387 : {
7388 84 : _current_linear_sys = &sys;
7389 :
7390 : // We assemble the current system to check the current residual
7391 84 : computeLinearSystemSys(sys.linearImplicitSystem(),
7392 84 : *sys.linearImplicitSystem().matrix,
7393 84 : *sys.linearImplicitSystem().rhs,
7394 : /*compute fresh gradients*/ true);
7395 :
7396 : // Unfortunate, but we have to allocate a new vector for the residual
7397 84 : auto residual = sys.linearImplicitSystem().rhs->clone();
7398 84 : residual->scale(-1.0);
7399 84 : residual->add_vector(*sys.currentSolution(), *sys.linearImplicitSystem().matrix);
7400 168 : return residual->l2_norm();
7401 84 : }
7402 :
7403 : Real
7404 62633 : FEProblemBase::computeResidualL2Norm()
7405 : {
7406 313165 : TIME_SECTION("computeResidualL2Norm", 2, "Computing L2 Norm of Residual");
7407 :
7408 : // We use sum the squared norms of the individual systems and then take the square root of it
7409 62633 : Real l2_norm = 0.0;
7410 125212 : for (auto sys : _nl)
7411 : {
7412 62579 : const auto norm = computeResidualL2Norm(*sys);
7413 62579 : l2_norm += norm * norm;
7414 62579 : }
7415 :
7416 62717 : for (auto sys : _linear_systems)
7417 : {
7418 84 : const auto norm = computeResidualL2Norm(*sys);
7419 84 : l2_norm += norm * norm;
7420 84 : }
7421 :
7422 125266 : return std::sqrt(l2_norm);
7423 62633 : }
7424 :
7425 : void
7426 2944915 : FEProblemBase::computeResidualSys(NonlinearImplicitSystem & sys,
7427 : const NumericVector<Number> & soln,
7428 : NumericVector<Number> & residual)
7429 : {
7430 : parallel_object_only();
7431 :
7432 8834745 : TIME_SECTION("computeResidualSys", 5);
7433 : // Reset before residual setup, calculation & execution
7434 2944915 : _app.solutionInvalidity().resetIterationOccurences();
7435 :
7436 2944915 : computeResidual(soln, residual, sys.number());
7437 2944885 : }
7438 :
7439 : void
7440 0 : FEProblemBase::computeResidual(NonlinearImplicitSystem & sys,
7441 : const NumericVector<Number> & soln,
7442 : NumericVector<Number> & residual)
7443 : {
7444 0 : mooseDeprecated("Please use computeResidualSys");
7445 :
7446 0 : computeResidualSys(sys, soln, residual);
7447 0 : }
7448 :
7449 : void
7450 3004316 : FEProblemBase::computeResidual(const NumericVector<Number> & soln,
7451 : NumericVector<Number> & residual,
7452 : const unsigned int nl_sys_num)
7453 : {
7454 3004316 : setCurrentNonlinearSystem(nl_sys_num);
7455 :
7456 : // We associate the residual tag with the given residual vector to make sure we
7457 : // don't filter it out below
7458 3004316 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7459 3004316 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7460 :
7461 : mooseAssert(_fe_vector_tags.empty(), "This should be empty indicating a clean starting state");
7462 : // We filter out tags which do not have associated vectors in the current nonlinear
7463 : // system. This is essential to be able to use system-dependent residual tags.
7464 3004316 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7465 :
7466 3004316 : computeResidualInternal(soln, residual, _fe_vector_tags);
7467 3004286 : _fe_vector_tags.clear();
7468 3004286 : }
7469 :
7470 : void
7471 9899 : FEProblemBase::computeResidualAndJacobian(const NumericVector<Number> & soln,
7472 : NumericVector<Number> & residual,
7473 : SparseMatrix<Number> & jacobian)
7474 : {
7475 : try
7476 : {
7477 : try
7478 : {
7479 : // vector tags
7480 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7481 9899 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7482 :
7483 : mooseAssert(_fe_vector_tags.empty(),
7484 : "This should be empty indicating a clean starting state");
7485 : // We filter out tags which do not have associated vectors in the current nonlinear
7486 : // system. This is essential to be able to use system-dependent residual tags.
7487 9899 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7488 :
7489 9899 : setCurrentResidualVectorTags(_fe_vector_tags);
7490 :
7491 : // matrix tags
7492 : {
7493 9899 : _fe_matrix_tags.clear();
7494 :
7495 9899 : auto & tags = getMatrixTags();
7496 29697 : for (auto & tag : tags)
7497 19798 : _fe_matrix_tags.insert(tag.second);
7498 : }
7499 :
7500 9899 : _current_nl_sys->setSolution(soln);
7501 :
7502 9899 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7503 9899 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7504 :
7505 29697 : for (const auto tag : _fe_matrix_tags)
7506 19798 : if (_current_nl_sys->hasMatrix(tag))
7507 : {
7508 9899 : auto & matrix = _current_nl_sys->getMatrix(tag);
7509 9899 : matrix.zero();
7510 9899 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7511 : // PETSc algorithms require diagonal allocations regardless of whether there is non-zero
7512 : // diagonal dependence. With global AD indexing we only add non-zero
7513 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7514 123018 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7515 120004 : matrix.add(index, index, 0);
7516 : }
7517 :
7518 9899 : _aux->zeroVariablesForResidual();
7519 :
7520 9899 : unsigned int n_threads = libMesh::n_threads();
7521 :
7522 9899 : _current_execute_on_flag = EXEC_LINEAR;
7523 :
7524 : // Random interface objects
7525 9899 : for (const auto & it : _random_data_objects)
7526 0 : it.second->updateSeeds(EXEC_LINEAR);
7527 :
7528 9899 : setCurrentlyComputingResidual(true);
7529 9899 : setCurrentlyComputingJacobian(true);
7530 9899 : setCurrentlyComputingResidualAndJacobian(true);
7531 9899 : if (_displaced_problem)
7532 : {
7533 1057 : _displaced_problem->setCurrentlyComputingResidual(true);
7534 1057 : _displaced_problem->setCurrentlyComputingJacobian(true);
7535 1057 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(true);
7536 : }
7537 :
7538 9899 : execTransfers(EXEC_LINEAR);
7539 :
7540 9899 : execMultiApps(EXEC_LINEAR);
7541 :
7542 21234 : for (unsigned int tid = 0; tid < n_threads; tid++)
7543 11335 : reinitScalars(tid);
7544 :
7545 9899 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7546 :
7547 9899 : _aux->residualSetup();
7548 :
7549 9899 : if (_displaced_problem)
7550 : {
7551 1057 : computeSystems(EXEC_PRE_DISPLACE);
7552 1057 : _displaced_problem->updateMesh();
7553 1057 : if (_mortar_data->hasDisplacedObjects())
7554 1057 : updateMortarMesh();
7555 : }
7556 :
7557 21234 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7558 : {
7559 11335 : _all_materials.residualSetup(tid);
7560 11335 : _functions.residualSetup(tid);
7561 : }
7562 :
7563 : #ifdef MOOSE_KOKKOS_ENABLED
7564 8839 : _kokkos_functions.residualSetup();
7565 : #endif
7566 :
7567 9899 : computeSystems(EXEC_LINEAR);
7568 :
7569 9899 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7570 :
7571 9899 : executeControls(EXEC_LINEAR);
7572 :
7573 9899 : _app.getOutputWarehouse().residualSetup();
7574 :
7575 9899 : _safe_access_tagged_vectors = false;
7576 9899 : _safe_access_tagged_matrices = false;
7577 :
7578 9899 : _current_nl_sys->computeResidualAndJacobianTags(_fe_vector_tags, _fe_matrix_tags);
7579 :
7580 9899 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7581 9899 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7582 : }
7583 0 : catch (...)
7584 : {
7585 0 : handleException("computeResidualAndJacobian");
7586 0 : }
7587 : }
7588 0 : catch (const MooseException &)
7589 : {
7590 : // The buck stops here, we have already handled the exception by
7591 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7592 : // "diverged" reason during the next solve.
7593 0 : }
7594 0 : catch (...)
7595 : {
7596 0 : mooseError("Unexpected exception type");
7597 0 : }
7598 :
7599 9899 : resetState();
7600 9899 : _fe_vector_tags.clear();
7601 9899 : _fe_matrix_tags.clear();
7602 9899 : }
7603 :
7604 : void
7605 0 : FEProblemBase::computeResidualTag(const NumericVector<Number> & soln,
7606 : NumericVector<Number> & residual,
7607 : TagID tag)
7608 : {
7609 : try
7610 : {
7611 0 : _current_nl_sys->setSolution(soln);
7612 :
7613 0 : _current_nl_sys->associateVectorToTag(residual, tag);
7614 :
7615 0 : computeResidualTags({tag});
7616 :
7617 0 : _current_nl_sys->disassociateVectorFromTag(residual, tag);
7618 : }
7619 0 : catch (MooseException & e)
7620 : {
7621 : // If a MooseException propagates all the way to here, it means
7622 : // that it was thrown from a MOOSE system where we do not
7623 : // (currently) properly support the throwing of exceptions, and
7624 : // therefore we have no choice but to error out. It may be
7625 : // *possible* to handle exceptions from other systems, but in the
7626 : // meantime, we don't want to silently swallow any unhandled
7627 : // exceptions here.
7628 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7629 : "contact the MOOSE team for assistance.");
7630 0 : }
7631 0 : }
7632 :
7633 : void
7634 3007900 : FEProblemBase::computeResidualInternal(const NumericVector<Number> & soln,
7635 : NumericVector<Number> & residual,
7636 : const std::set<TagID> & tags)
7637 : {
7638 : parallel_object_only();
7639 :
7640 9023700 : TIME_SECTION("computeResidualInternal", 1);
7641 :
7642 : try
7643 : {
7644 3007900 : _current_nl_sys->setSolution(soln);
7645 :
7646 3007900 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7647 :
7648 3007900 : computeResidualTags(tags);
7649 :
7650 3007870 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7651 : }
7652 0 : catch (MooseException & e)
7653 : {
7654 : // If a MooseException propagates all the way to here, it means
7655 : // that it was thrown from a MOOSE system where we do not
7656 : // (currently) properly support the throwing of exceptions, and
7657 : // therefore we have no choice but to error out. It may be
7658 : // *possible* to handle exceptions from other systems, but in the
7659 : // meantime, we don't want to silently swallow any unhandled
7660 : // exceptions here.
7661 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7662 : "contact the MOOSE team for assistance.");
7663 0 : }
7664 3007870 : }
7665 :
7666 : void
7667 0 : FEProblemBase::computeResidualType(const NumericVector<Number> & soln,
7668 : NumericVector<Number> & residual,
7669 : TagID tag)
7670 : {
7671 0 : TIME_SECTION("computeResidualType", 5);
7672 :
7673 : try
7674 : {
7675 0 : _current_nl_sys->setSolution(soln);
7676 :
7677 0 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7678 :
7679 0 : computeResidualTags({tag, _current_nl_sys->residualVectorTag()});
7680 :
7681 0 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7682 : }
7683 0 : catch (MooseException & e)
7684 : {
7685 : // If a MooseException propagates all the way to here, it means
7686 : // that it was thrown from a MOOSE system where we do not
7687 : // (currently) properly support the throwing of exceptions, and
7688 : // therefore we have no choice but to error out. It may be
7689 : // *possible* to handle exceptions from other systems, but in the
7690 : // meantime, we don't want to silently swallow any unhandled
7691 : // exceptions here.
7692 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7693 : "contact the MOOSE team for assistance.");
7694 0 : }
7695 0 : }
7696 :
7697 : void
7698 3 : FEProblemBase::handleException(const std::string & calling_method)
7699 : {
7700 : auto create_exception_message =
7701 3 : [&calling_method](const std::string & exception_type, const auto & exception)
7702 : {
7703 : return std::string("A " + exception_type + " was raised during FEProblemBase::" +
7704 6 : calling_method + "\n" + std::string(exception.what()));
7705 3 : };
7706 :
7707 : try
7708 : {
7709 3 : throw;
7710 : }
7711 3 : catch (const MooseException & e)
7712 : {
7713 0 : setException(create_exception_message("MooseException", e));
7714 0 : }
7715 0 : catch (const MetaPhysicL::LogicError & e)
7716 : {
7717 0 : moose::translateMetaPhysicLError(e);
7718 0 : }
7719 3 : catch (const libMesh::PetscSolverException & e)
7720 : {
7721 : // One PETSc solver exception that we cannot currently recover from are new nonzero errors. In
7722 : // particular I have observed the following scenario in a parallel test:
7723 : // - Both processes throw because of a new nonzero during MOOSE's computeJacobianTags
7724 : // - We potentially handle the exceptions nicely here
7725 : // - When the matrix is closed in libMesh's libmesh_petsc_snes_solver, there is a new nonzero
7726 : // throw which we do not catch here in MOOSE and the simulation terminates. This only appears
7727 : // in parallel (and not all the time; a test I was examining threw with distributed mesh, but
7728 : // not with replicated). In serial there are no new throws from libmesh_petsc_snes_solver.
7729 : // So for uniformity of behavior across serial/parallel, we will choose to abort here and always
7730 : // produce a non-zero exit code
7731 6 : mooseError(create_exception_message("libMesh::PetscSolverException", e));
7732 0 : }
7733 0 : catch (const std::exception & e)
7734 : {
7735 : // This might be libMesh detecting a degenerate Jacobian or matrix
7736 0 : if (strstr(e.what(), "Jacobian") || strstr(e.what(), "singular") ||
7737 0 : strstr(e.what(), "det != 0"))
7738 0 : setException(create_exception_message("libMesh DegenerateMap", e));
7739 : else
7740 : {
7741 0 : const auto message = create_exception_message("std::exception", e);
7742 0 : if (_regard_general_exceptions_as_errors)
7743 0 : mooseError(message);
7744 : else
7745 0 : setException(message);
7746 0 : }
7747 0 : }
7748 :
7749 0 : checkExceptionAndStopSolve();
7750 0 : }
7751 :
7752 : void
7753 3043303 : FEProblemBase::computeResidualTags(const std::set<TagID> & tags)
7754 : {
7755 : parallel_object_only();
7756 :
7757 : try
7758 : {
7759 : try
7760 : {
7761 15216515 : TIME_SECTION("computeResidualTags", 5, "Computing Residual");
7762 :
7763 3043303 : ADReal::do_derivatives = false;
7764 :
7765 3043303 : setCurrentResidualVectorTags(tags);
7766 :
7767 3043303 : _aux->zeroVariablesForResidual();
7768 :
7769 3043303 : unsigned int n_threads = libMesh::n_threads();
7770 :
7771 3043303 : _current_execute_on_flag = EXEC_LINEAR;
7772 :
7773 : // Random interface objects
7774 3056185 : for (const auto & it : _random_data_objects)
7775 12882 : it.second->updateSeeds(EXEC_LINEAR);
7776 :
7777 3043303 : execTransfers(EXEC_LINEAR);
7778 :
7779 3043303 : execMultiApps(EXEC_LINEAR);
7780 :
7781 6394471 : for (unsigned int tid = 0; tid < n_threads; tid++)
7782 3351168 : reinitScalars(tid);
7783 :
7784 3043303 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7785 :
7786 3043303 : _aux->residualSetup();
7787 :
7788 3043303 : if (_displaced_problem)
7789 : {
7790 123090 : computeSystems(EXEC_PRE_DISPLACE);
7791 123090 : _displaced_problem->updateMesh();
7792 123090 : if (_mortar_data->hasDisplacedObjects())
7793 2211 : updateMortarMesh();
7794 : }
7795 :
7796 6394471 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7797 : {
7798 3351168 : _all_materials.residualSetup(tid);
7799 3351168 : _functions.residualSetup(tid);
7800 : }
7801 :
7802 : #ifdef MOOSE_KOKKOS_ENABLED
7803 2212209 : _kokkos_functions.residualSetup();
7804 : #endif
7805 :
7806 3043303 : computeSystems(EXEC_LINEAR);
7807 :
7808 3043303 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7809 :
7810 3043303 : executeControls(EXEC_LINEAR);
7811 :
7812 3043303 : _app.getOutputWarehouse().residualSetup();
7813 :
7814 3043303 : _safe_access_tagged_vectors = false;
7815 3043303 : _current_nl_sys->computeResidualTags(tags);
7816 3043273 : }
7817 0 : catch (...)
7818 : {
7819 0 : handleException("computeResidualTags");
7820 0 : }
7821 : }
7822 0 : catch (const MooseException &)
7823 : {
7824 : // The buck stops here, we have already handled the exception by
7825 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7826 : // "diverged" reason during the next solve.
7827 0 : }
7828 0 : catch (...)
7829 : {
7830 0 : mooseError("Unexpected exception type");
7831 0 : }
7832 :
7833 3043273 : resetState();
7834 3043273 : }
7835 :
7836 : void
7837 472241 : FEProblemBase::computeJacobianSys(NonlinearImplicitSystem & sys,
7838 : const NumericVector<Number> & soln,
7839 : SparseMatrix<Number> & jacobian)
7840 : {
7841 : // Reset before Jacobian setup, calculation & execution
7842 472241 : _app.solutionInvalidity().resetIterationOccurences();
7843 472241 : computeJacobian(soln, jacobian, sys.number());
7844 472228 : }
7845 :
7846 : void
7847 4467 : FEProblemBase::computeJacobianTag(const NumericVector<Number> & soln,
7848 : SparseMatrix<Number> & jacobian,
7849 : TagID tag)
7850 : {
7851 4467 : _current_nl_sys->setSolution(soln);
7852 :
7853 4467 : _current_nl_sys->associateMatrixToTag(jacobian, tag);
7854 :
7855 8934 : computeJacobianTags({tag});
7856 :
7857 4467 : _current_nl_sys->disassociateMatrixFromTag(jacobian, tag);
7858 4467 : }
7859 :
7860 : void
7861 471397 : FEProblemBase::computeJacobian(const NumericVector<Number> & soln,
7862 : SparseMatrix<Number> & jacobian,
7863 : const unsigned int nl_sys_num)
7864 : {
7865 471397 : setCurrentNonlinearSystem(nl_sys_num);
7866 :
7867 471397 : _fe_matrix_tags.clear();
7868 :
7869 471397 : auto & tags = getMatrixTags();
7870 1414453 : for (auto & tag : tags)
7871 943056 : _fe_matrix_tags.insert(tag.second);
7872 :
7873 471397 : computeJacobianInternal(soln, jacobian, _fe_matrix_tags);
7874 471384 : }
7875 :
7876 : void
7877 471397 : FEProblemBase::computeJacobianInternal(const NumericVector<Number> & soln,
7878 : SparseMatrix<Number> & jacobian,
7879 : const std::set<TagID> & tags)
7880 : {
7881 1414191 : TIME_SECTION("computeJacobianInternal", 1);
7882 :
7883 471397 : _current_nl_sys->setSolution(soln);
7884 :
7885 471397 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7886 :
7887 471397 : computeJacobianTags(tags);
7888 :
7889 471384 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7890 471384 : }
7891 :
7892 : void
7893 480645 : FEProblemBase::computeJacobianTags(const std::set<TagID> & tags)
7894 : {
7895 : try
7896 : {
7897 : try
7898 : {
7899 480645 : if (!_has_jacobian || !_const_jacobian)
7900 : {
7901 2370950 : TIME_SECTION("computeJacobianTags", 5, "Computing Jacobian");
7902 :
7903 1415240 : for (auto tag : tags)
7904 941050 : if (_current_nl_sys->hasMatrix(tag))
7905 : {
7906 476108 : auto & matrix = _current_nl_sys->getMatrix(tag);
7907 476108 : if (_restore_original_nonzero_pattern)
7908 7033 : matrix.restore_original_nonzero_pattern();
7909 : else
7910 469075 : matrix.zero();
7911 476108 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7912 : // PETSc algorithms require diagonal allocations regardless of whether there is
7913 : // non-zero diagonal dependence. With global AD indexing we only add non-zero
7914 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7915 4688718 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7916 4645468 : matrix.add(index, index, 0);
7917 : }
7918 :
7919 474190 : _aux->zeroVariablesForJacobian();
7920 :
7921 474190 : unsigned int n_threads = libMesh::n_threads();
7922 :
7923 : // Random interface objects
7924 476212 : for (const auto & it : _random_data_objects)
7925 2022 : it.second->updateSeeds(EXEC_NONLINEAR);
7926 :
7927 474190 : _current_execute_on_flag = EXEC_NONLINEAR;
7928 474190 : _currently_computing_jacobian = true;
7929 474190 : if (_displaced_problem)
7930 21147 : _displaced_problem->setCurrentlyComputingJacobian(true);
7931 :
7932 474190 : execTransfers(EXEC_NONLINEAR);
7933 474190 : execMultiApps(EXEC_NONLINEAR);
7934 :
7935 998504 : for (unsigned int tid = 0; tid < n_threads; tid++)
7936 524314 : reinitScalars(tid);
7937 :
7938 474190 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7939 :
7940 474190 : _aux->jacobianSetup();
7941 :
7942 474190 : if (_displaced_problem)
7943 : {
7944 21147 : computeSystems(EXEC_PRE_DISPLACE);
7945 21147 : _displaced_problem->updateMesh();
7946 : }
7947 :
7948 998497 : for (unsigned int tid = 0; tid < n_threads; tid++)
7949 : {
7950 524310 : _all_materials.jacobianSetup(tid);
7951 524310 : _functions.jacobianSetup(tid);
7952 : }
7953 :
7954 : #ifdef MOOSE_KOKKOS_ENABLED
7955 345703 : _kokkos_functions.jacobianSetup();
7956 : #endif
7957 :
7958 474187 : computeSystems(EXEC_NONLINEAR);
7959 :
7960 474187 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7961 :
7962 474187 : executeControls(EXEC_NONLINEAR);
7963 :
7964 474187 : _app.getOutputWarehouse().jacobianSetup();
7965 :
7966 474187 : _safe_access_tagged_matrices = false;
7967 :
7968 474187 : _current_nl_sys->computeJacobianTags(tags);
7969 :
7970 : // For explicit Euler calculations for example we often compute the Jacobian one time and
7971 : // then re-use it over and over. If we're performing automatic scaling, we don't want to
7972 : // use that kernel, diagonal-block only Jacobian for our actual matrix when performing
7973 : // solves!
7974 474177 : if (!_current_nl_sys->computingScalingJacobian())
7975 473684 : _has_jacobian = true;
7976 474180 : }
7977 : }
7978 3 : catch (...)
7979 : {
7980 3 : handleException("computeJacobianTags");
7981 0 : }
7982 : }
7983 0 : catch (const MooseException &)
7984 : {
7985 : // The buck stops here, we have already handled the exception by
7986 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7987 : // "diverged" reason during the next solve.
7988 0 : }
7989 0 : catch (...)
7990 : {
7991 0 : mooseError("Unexpected exception type");
7992 0 : }
7993 :
7994 480632 : resetState();
7995 480632 : }
7996 :
7997 : void
7998 263 : FEProblemBase::computeJacobianBlocks(std::vector<JacobianBlock *> & blocks,
7999 : const unsigned int nl_sys_num)
8000 : {
8001 789 : TIME_SECTION("computeTransientImplicitJacobian", 2);
8002 263 : setCurrentNonlinearSystem(nl_sys_num);
8003 :
8004 263 : if (_displaced_problem)
8005 : {
8006 0 : computeSystems(EXEC_PRE_DISPLACE);
8007 0 : _displaced_problem->updateMesh();
8008 : }
8009 :
8010 263 : computeSystems(EXEC_NONLINEAR);
8011 :
8012 263 : _currently_computing_jacobian = true;
8013 263 : _current_nl_sys->computeJacobianBlocks(blocks);
8014 263 : _currently_computing_jacobian = false;
8015 263 : }
8016 :
8017 : void
8018 0 : FEProblemBase::computeJacobianBlock(SparseMatrix<Number> & jacobian,
8019 : libMesh::System & precond_system,
8020 : unsigned int ivar,
8021 : unsigned int jvar)
8022 : {
8023 0 : JacobianBlock jac_block(precond_system, jacobian, ivar, jvar);
8024 0 : std::vector<JacobianBlock *> blocks = {&jac_block};
8025 : mooseAssert(_current_nl_sys, "This should be non-null");
8026 0 : computeJacobianBlocks(blocks, _current_nl_sys->number());
8027 0 : }
8028 :
8029 : void
8030 714 : FEProblemBase::computeBounds(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8031 : NumericVector<Number> & lower,
8032 : NumericVector<Number> & upper)
8033 : {
8034 : try
8035 : {
8036 : try
8037 : {
8038 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8039 : "I expect these system numbers to be the same");
8040 :
8041 3570 : if (!_current_nl_sys->hasVector("lower_bound") || !_current_nl_sys->hasVector("upper_bound"))
8042 0 : return;
8043 :
8044 3570 : TIME_SECTION("computeBounds", 1, "Computing Bounds");
8045 :
8046 1428 : NumericVector<Number> & _lower = _current_nl_sys->getVector("lower_bound");
8047 1428 : NumericVector<Number> & _upper = _current_nl_sys->getVector("upper_bound");
8048 714 : _lower.swap(lower);
8049 714 : _upper.swap(upper);
8050 1492 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
8051 778 : _all_materials.residualSetup(tid);
8052 :
8053 714 : _aux->residualSetup();
8054 714 : computeSystems(EXEC_LINEAR);
8055 714 : _lower.swap(lower);
8056 714 : _upper.swap(upper);
8057 714 : }
8058 0 : catch (...)
8059 : {
8060 0 : handleException("computeBounds");
8061 0 : }
8062 : }
8063 0 : catch (MooseException & e)
8064 : {
8065 0 : mooseError("Irrecoverable exception: " + std::string(e.what()));
8066 0 : }
8067 0 : catch (...)
8068 : {
8069 0 : mooseError("Unexpected exception type");
8070 0 : }
8071 : }
8072 :
8073 : void
8074 26160 : FEProblemBase::computeLinearSystemSys(LinearImplicitSystem & sys,
8075 : SparseMatrix<Number> & system_matrix,
8076 : NumericVector<Number> & rhs,
8077 : const bool compute_gradients)
8078 : {
8079 78480 : TIME_SECTION("computeLinearSystemSys", 5);
8080 :
8081 26160 : setCurrentLinearSystem(linearSysNum(sys.name()));
8082 :
8083 26160 : _current_linear_sys->associateVectorToTag(rhs, _current_linear_sys->rightHandSideVectorTag());
8084 26160 : _current_linear_sys->associateMatrixToTag(system_matrix, _current_linear_sys->systemMatrixTag());
8085 :
8086 : // We are using the residual tag system for right hand sides so we fetch everything
8087 26160 : const auto & vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
8088 :
8089 : // We filter out tags which do not have associated vectors in the current
8090 : // system. This is essential to be able to use system-dependent vector tags.
8091 26160 : selectVectorTagsFromSystem(*_current_linear_sys, vector_tags, _linear_vector_tags);
8092 26160 : selectMatrixTagsFromSystem(*_current_linear_sys, getMatrixTags(), _linear_matrix_tags);
8093 :
8094 26160 : computeLinearSystemTags(*(_current_linear_sys->currentSolution()),
8095 26160 : _linear_vector_tags,
8096 26160 : _linear_matrix_tags,
8097 : compute_gradients);
8098 :
8099 26160 : _current_linear_sys->disassociateMatrixFromTag(system_matrix,
8100 26160 : _current_linear_sys->systemMatrixTag());
8101 26160 : _current_linear_sys->disassociateVectorFromTag(rhs,
8102 26160 : _current_linear_sys->rightHandSideVectorTag());
8103 : // We reset the tags to the default containers for further operations
8104 26160 : _current_linear_sys->associateVectorToTag(_current_linear_sys->getRightHandSideVector(),
8105 26160 : _current_linear_sys->rightHandSideVectorTag());
8106 26160 : _current_linear_sys->associateMatrixToTag(_current_linear_sys->getSystemMatrix(),
8107 26160 : _current_linear_sys->systemMatrixTag());
8108 26160 : }
8109 :
8110 : void
8111 26160 : FEProblemBase::computeLinearSystemTags(const NumericVector<Number> & soln,
8112 : const std::set<TagID> & vector_tags,
8113 : const std::set<TagID> & matrix_tags,
8114 : const bool compute_gradients)
8115 : {
8116 130800 : TIME_SECTION("computeLinearSystemTags", 5, "Computing Linear System");
8117 :
8118 26160 : _current_linear_sys->setSolution(soln);
8119 :
8120 52340 : for (auto tag : matrix_tags)
8121 : {
8122 26180 : auto & matrix = _current_linear_sys->getMatrix(tag);
8123 26180 : matrix.zero();
8124 : }
8125 :
8126 26160 : unsigned int n_threads = libMesh::n_threads();
8127 :
8128 26160 : _current_execute_on_flag = EXEC_NONLINEAR;
8129 :
8130 : // Random interface objects
8131 26160 : for (const auto & it : _random_data_objects)
8132 0 : it.second->updateSeeds(EXEC_NONLINEAR);
8133 :
8134 26160 : execTransfers(EXEC_NONLINEAR);
8135 26160 : execMultiApps(EXEC_NONLINEAR);
8136 :
8137 26160 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
8138 :
8139 26160 : _aux->jacobianSetup();
8140 :
8141 52320 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
8142 : {
8143 26160 : _functions.jacobianSetup(tid);
8144 : }
8145 :
8146 : #ifdef MOOSE_KOKKOS_ENABLED
8147 17995 : _kokkos_functions.jacobianSetup();
8148 : #endif
8149 :
8150 : try
8151 : {
8152 26160 : computeSystems(EXEC_NONLINEAR);
8153 : }
8154 0 : catch (MooseException & e)
8155 : {
8156 0 : _console << "\nA MooseException was raised during Auxiliary variable computation.\n"
8157 0 : << "The next solve will fail, the timestep will be reduced, and we will try again.\n"
8158 0 : << std::endl;
8159 :
8160 : // We know the next solve is going to fail, so there's no point in
8161 : // computing anything else after this. Plus, using incompletely
8162 : // computed AuxVariables in subsequent calculations could lead to
8163 : // other errors or unhandled exceptions being thrown.
8164 0 : return;
8165 0 : }
8166 :
8167 26160 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
8168 26160 : executeControls(EXEC_NONLINEAR);
8169 :
8170 26160 : _app.getOutputWarehouse().jacobianSetup();
8171 :
8172 26160 : _current_linear_sys->computeLinearSystemTags(vector_tags, matrix_tags, compute_gradients);
8173 :
8174 : // Reset execution flag as after this point we are no longer on LINEAR
8175 26160 : _current_execute_on_flag = EXEC_NONE;
8176 :
8177 : // These are the relevant parts of resetState()
8178 26160 : _safe_access_tagged_vectors = true;
8179 26160 : _safe_access_tagged_matrices = true;
8180 26160 : }
8181 :
8182 : void
8183 295346 : FEProblemBase::computeNearNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8184 : std::vector<NumericVector<Number> *> & sp)
8185 : {
8186 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8187 : "I expect these system numbers to be the same");
8188 :
8189 295346 : sp.clear();
8190 886038 : for (unsigned int i = 0; i < subspaceDim("NearNullSpace"); ++i)
8191 : {
8192 0 : std::stringstream postfix;
8193 0 : postfix << "_" << i;
8194 0 : std::string modename = "NearNullSpace" + postfix.str();
8195 0 : sp.push_back(&_current_nl_sys->getVector(modename));
8196 0 : }
8197 295346 : }
8198 :
8199 : void
8200 295346 : FEProblemBase::computeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8201 : std::vector<NumericVector<Number> *> & sp)
8202 : {
8203 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8204 : "I expect these system numbers to be the same");
8205 295346 : sp.clear();
8206 886104 : for (unsigned int i = 0; i < subspaceDim("NullSpace"); ++i)
8207 : {
8208 22 : std::stringstream postfix;
8209 22 : postfix << "_" << i;
8210 22 : sp.push_back(&_current_nl_sys->getVector("NullSpace" + postfix.str()));
8211 22 : }
8212 295346 : }
8213 :
8214 : void
8215 295346 : FEProblemBase::computeTransposeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
8216 : std::vector<NumericVector<Number> *> & sp)
8217 : {
8218 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8219 : "I expect these system numbers to be the same");
8220 295346 : sp.clear();
8221 886071 : for (unsigned int i = 0; i < subspaceDim("TransposeNullSpace"); ++i)
8222 : {
8223 11 : std::stringstream postfix;
8224 11 : postfix << "_" << i;
8225 11 : sp.push_back(&_current_nl_sys->getVector("TransposeNullSpace" + postfix.str()));
8226 11 : }
8227 295346 : }
8228 :
8229 : void
8230 2128 : FEProblemBase::computePostCheck(NonlinearImplicitSystem & sys,
8231 : const NumericVector<Number> & old_soln,
8232 : NumericVector<Number> & search_direction,
8233 : NumericVector<Number> & new_soln,
8234 : bool & changed_search_direction,
8235 : bool & changed_new_soln)
8236 : {
8237 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
8238 : "I expect these system numbers to be the same");
8239 :
8240 : // This function replaces the old PetscSupport::dampedCheck() function.
8241 : //
8242 : // 1.) Recreate code in PetscSupport::dampedCheck() for constructing
8243 : // ghosted "soln" and "update" vectors.
8244 : // 2.) Call FEProblemBase::computeDamping() with these ghost vectors.
8245 : // 3.) Recreate the code in PetscSupport::dampedCheck() to actually update
8246 : // the solution vector based on the damping, and set the "changed" flags
8247 : // appropriately.
8248 :
8249 10640 : TIME_SECTION("computePostCheck", 2, "Computing Post Check");
8250 :
8251 2128 : _current_execute_on_flag = EXEC_POSTCHECK;
8252 :
8253 : // MOOSE's FEProblemBase doesn't update the solution during the
8254 : // postcheck, but FEProblemBase-derived classes might.
8255 2128 : if (_has_dampers || shouldUpdateSolution())
8256 : {
8257 : // We need ghosted versions of new_soln and search_direction (the
8258 : // ones we get from libmesh/PETSc are PARALLEL vectors. To make
8259 : // our lives simpler, we use the same ghosting pattern as the
8260 : // system's current_local_solution to create new ghosted vectors.
8261 :
8262 : // Construct zeroed-out clones with the same ghosted dofs as the
8263 : // System's current_local_solution.
8264 : std::unique_ptr<NumericVector<Number>> ghosted_solution =
8265 1625 : sys.current_local_solution->zero_clone(),
8266 : ghosted_search_direction =
8267 1625 : sys.current_local_solution->zero_clone();
8268 :
8269 : // Copy values from input vectors into clones with ghosted values.
8270 1625 : *ghosted_solution = new_soln;
8271 1625 : *ghosted_search_direction = search_direction;
8272 :
8273 1625 : if (_has_dampers)
8274 : {
8275 : // Compute the damping coefficient using the ghosted vectors
8276 1625 : Real damping = computeDamping(*ghosted_solution, *ghosted_search_direction);
8277 :
8278 : // If some non-trivial damping was computed, update the new_soln
8279 : // vector accordingly.
8280 1625 : if (damping < 1.0)
8281 : {
8282 1221 : new_soln = old_soln;
8283 1221 : new_soln.add(-damping, search_direction);
8284 1221 : changed_new_soln = true;
8285 : }
8286 : }
8287 :
8288 1625 : if (shouldUpdateSolution())
8289 : {
8290 : // Update the ghosted copy of the new solution, if necessary.
8291 0 : if (changed_new_soln)
8292 0 : *ghosted_solution = new_soln;
8293 :
8294 0 : bool updated_solution = updateSolution(new_soln, *ghosted_solution);
8295 0 : if (updated_solution)
8296 0 : changed_new_soln = true;
8297 : }
8298 1625 : }
8299 :
8300 2128 : if (vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
8301 : {
8302 503 : _current_nl_sys->setPreviousNewtonSolution(old_soln);
8303 503 : _aux->copyCurrentIntoPreviousNL();
8304 : }
8305 :
8306 : // MOOSE doesn't change the search_direction
8307 2128 : changed_search_direction = false;
8308 :
8309 2128 : _current_execute_on_flag = EXEC_NONE;
8310 2128 : }
8311 :
8312 : Real
8313 1625 : FEProblemBase::computeDamping(const NumericVector<Number> & soln,
8314 : const NumericVector<Number> & update)
8315 : {
8316 : // Default to no damping
8317 1625 : Real damping = 1.0;
8318 :
8319 1625 : if (_has_dampers)
8320 : {
8321 8125 : TIME_SECTION("computeDamping", 1, "Computing Damping");
8322 :
8323 : // Save pointer to the current solution
8324 1625 : const NumericVector<Number> * _saved_current_solution = _current_nl_sys->currentSolution();
8325 :
8326 1625 : _current_nl_sys->setSolution(soln);
8327 : // For now, do not re-compute auxiliary variables. Doing so allows a wild solution increment
8328 : // to get to the material models, which may not be able to cope with drastically different
8329 : // values. Once more complete dependency checking is in place, auxiliary variables (and
8330 : // material properties) will be computed as needed by dampers.
8331 : // _aux.compute();
8332 1625 : damping = _current_nl_sys->computeDamping(soln, update);
8333 :
8334 : // restore saved solution
8335 1625 : _current_nl_sys->setSolution(*_saved_current_solution);
8336 1625 : }
8337 :
8338 1625 : return damping;
8339 : }
8340 :
8341 : bool
8342 289942 : FEProblemBase::shouldUpdateSolution()
8343 : {
8344 289942 : return false;
8345 : }
8346 :
8347 : bool
8348 0 : FEProblemBase::updateSolution(NumericVector<Number> & /*vec_solution*/,
8349 : NumericVector<Number> & /*ghosted_solution*/)
8350 : {
8351 0 : return false;
8352 : }
8353 :
8354 : void
8355 203 : FEProblemBase::predictorCleanup(NumericVector<Number> & /*ghosted_solution*/)
8356 : {
8357 203 : }
8358 :
8359 : void
8360 2022 : FEProblemBase::addDisplacedProblem(std::shared_ptr<DisplacedProblem> displaced_problem)
8361 : {
8362 : parallel_object_only();
8363 :
8364 2022 : _displaced_mesh = &displaced_problem->mesh();
8365 2022 : _displaced_problem = displaced_problem;
8366 2022 : }
8367 :
8368 : void
8369 120526 : FEProblemBase::updateGeomSearch(GeometricSearchData::GeometricSearchType type)
8370 : {
8371 602630 : TIME_SECTION("updateGeometricSearch", 3, "Updating Geometric Search");
8372 :
8373 120526 : _geometric_search_data.update(type);
8374 :
8375 120526 : if (_displaced_problem)
8376 4209 : _displaced_problem->updateGeomSearch(type);
8377 120526 : }
8378 :
8379 : void
8380 63809 : FEProblemBase::updateMortarMesh()
8381 : {
8382 319045 : TIME_SECTION("updateMortarMesh", 5, "Updating Mortar Mesh");
8383 :
8384 63809 : FloatingPointExceptionGuard fpe_guard(_app);
8385 :
8386 63809 : _mortar_data->update();
8387 63806 : }
8388 :
8389 : void
8390 1329 : FEProblemBase::createMortarInterface(
8391 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8392 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8393 : bool on_displaced,
8394 : bool periodic,
8395 : const bool debug,
8396 : const bool correct_edge_dropping,
8397 : const Real minimum_projection_angle)
8398 : {
8399 1329 : _has_mortar = true;
8400 :
8401 1329 : if (on_displaced)
8402 254 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8403 : primary_secondary_subdomain_pair,
8404 127 : *_displaced_problem,
8405 : on_displaced,
8406 : periodic,
8407 : debug,
8408 : correct_edge_dropping,
8409 127 : minimum_projection_angle);
8410 : else
8411 1202 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8412 : primary_secondary_subdomain_pair,
8413 : *this,
8414 : on_displaced,
8415 : periodic,
8416 : debug,
8417 : correct_edge_dropping,
8418 1202 : minimum_projection_angle);
8419 : }
8420 :
8421 : const AutomaticMortarGeneration &
8422 0 : FEProblemBase::getMortarInterface(
8423 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8424 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8425 : bool on_displaced) const
8426 : {
8427 0 : return _mortar_data->getMortarInterface(
8428 0 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8429 : }
8430 :
8431 : AutomaticMortarGeneration &
8432 165402 : FEProblemBase::getMortarInterface(
8433 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8434 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8435 : bool on_displaced)
8436 : {
8437 165402 : return _mortar_data->getMortarInterface(
8438 165402 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8439 : }
8440 :
8441 : void
8442 321806 : FEProblemBase::possiblyRebuildGeomSearchPatches()
8443 : {
8444 321806 : if (_displaced_problem) // Only need to do this if things are moving...
8445 : {
8446 164185 : TIME_SECTION("possiblyRebuildGeomSearchPatches", 5, "Rebuilding Geometric Search Patches");
8447 :
8448 32837 : switch (_mesh.getPatchUpdateStrategy())
8449 : {
8450 31742 : case Moose::Never:
8451 31742 : break;
8452 365 : case Moose::Iteration:
8453 : // Update the list of ghosted elements at the start of the time step
8454 365 : _geometric_search_data.updateGhostedElems();
8455 365 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8456 :
8457 365 : _displaced_problem->geomSearchData().updateGhostedElems();
8458 365 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8459 :
8460 : // The commands below ensure that the sparsity of the Jacobian matrix is
8461 : // augmented at the start of the time step using neighbor nodes from the end
8462 : // of the previous time step.
8463 :
8464 365 : reinitBecauseOfGhostingOrNewGeomObjects();
8465 :
8466 : // This is needed to reinitialize PETSc output
8467 365 : initPetscOutputAndSomeSolverSettings();
8468 :
8469 365 : break;
8470 :
8471 331 : case Moose::Auto:
8472 : {
8473 331 : Real max = _displaced_problem->geomSearchData().maxPatchPercentage();
8474 331 : _communicator.max(max);
8475 :
8476 : // If we haven't moved very far through the patch
8477 331 : if (max < 0.4)
8478 298 : break;
8479 : }
8480 : libmesh_fallthrough();
8481 :
8482 : // Let this fall through if things do need to be updated...
8483 : case Moose::Always:
8484 : // Flush output here to see the message before the reinitialization, which could take a
8485 : // while
8486 432 : _console << "\n\nUpdating geometric search patches\n" << std::endl;
8487 :
8488 432 : _geometric_search_data.clearNearestNodeLocators();
8489 432 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8490 :
8491 432 : _displaced_problem->geomSearchData().clearNearestNodeLocators();
8492 432 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8493 :
8494 432 : reinitBecauseOfGhostingOrNewGeomObjects();
8495 :
8496 : // This is needed to reinitialize PETSc output
8497 432 : initPetscOutputAndSomeSolverSettings();
8498 : }
8499 32837 : }
8500 321806 : }
8501 :
8502 : #ifdef LIBMESH_ENABLE_AMR
8503 : void
8504 55695 : FEProblemBase::initialAdaptMesh()
8505 : {
8506 55695 : unsigned int n = adaptivity().getInitialSteps();
8507 55695 : _cycles_completed = 0;
8508 55695 : if (n)
8509 : {
8510 629 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8511 3 : mooseError("HFEM does not support mesh adaptivity currently.");
8512 :
8513 3130 : TIME_SECTION("initialAdaptMesh", 2, "Performing Initial Adaptivity");
8514 :
8515 1506 : for (unsigned int i = 0; i < n; i++)
8516 : {
8517 1055 : computeIndicators();
8518 1055 : computeMarkers();
8519 :
8520 1055 : if (_adaptivity.initialAdaptMesh())
8521 : {
8522 880 : meshChanged(
8523 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8524 :
8525 : // reproject the initial condition
8526 880 : projectSolution();
8527 :
8528 880 : _cycles_completed++;
8529 : }
8530 : else
8531 : {
8532 175 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8533 175 : return;
8534 : }
8535 : }
8536 626 : }
8537 : }
8538 :
8539 : bool
8540 167971 : FEProblemBase::adaptMesh()
8541 : {
8542 : // reset cycle counter
8543 167971 : _cycles_completed = 0;
8544 :
8545 167971 : if (!_adaptivity.isAdaptivityDue())
8546 163394 : return false;
8547 :
8548 22885 : TIME_SECTION("adaptMesh", 3, "Adapting Mesh");
8549 :
8550 4577 : unsigned int cycles_per_step = _adaptivity.getCyclesPerStep();
8551 :
8552 4577 : bool mesh_changed = false;
8553 :
8554 8056 : for (unsigned int i = 0; i < cycles_per_step; ++i)
8555 : {
8556 4729 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8557 0 : mooseError("HFEM does not support mesh adaptivity currently.");
8558 :
8559 : // Markers were already computed once by Executioner
8560 4729 : if (_adaptivity.getRecomputeMarkersFlag() && i > 0)
8561 22 : computeMarkers();
8562 :
8563 : bool mesh_changed_this_step;
8564 4729 : mesh_changed_this_step = _adaptivity.adaptMesh();
8565 :
8566 4729 : if (mesh_changed_this_step)
8567 : {
8568 3479 : mesh_changed = true;
8569 :
8570 3479 : meshChanged(
8571 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8572 3479 : _cycles_completed++;
8573 : }
8574 : else
8575 : {
8576 : // If the mesh didn't change, we still need to update the displaced mesh
8577 : // to undo the undisplacement performed in Adaptivity::adaptMesh
8578 1250 : if (_displaced_problem)
8579 44 : _displaced_problem->updateMesh();
8580 :
8581 1250 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8582 1250 : break;
8583 : }
8584 :
8585 : // Show adaptivity progress
8586 3479 : _console << std::flush;
8587 : }
8588 :
8589 : // We're done with all intermediate changes; now get systems ready
8590 : // for real if necessary.
8591 4577 : if (mesh_changed)
8592 3338 : es().reinit_systems();
8593 :
8594 : // Execute multi-apps that need to run after adaptivity, but before the next timestep.
8595 4577 : execMultiApps(EXEC_POST_ADAPTIVITY);
8596 :
8597 4577 : return mesh_changed;
8598 4577 : }
8599 : #endif // LIBMESH_ENABLE_AMR
8600 :
8601 : void
8602 0 : FEProblemBase::initXFEM(std::shared_ptr<XFEMInterface> xfem)
8603 : {
8604 0 : _xfem = xfem;
8605 0 : _xfem->setMesh(&_mesh);
8606 0 : if (_displaced_mesh)
8607 0 : _xfem->setDisplacedMesh(_displaced_mesh);
8608 :
8609 0 : auto fill_data = [](auto & storage)
8610 : {
8611 0 : std::vector<MaterialData *> data(libMesh::n_threads());
8612 0 : for (const auto tid : make_range(libMesh::n_threads()))
8613 0 : data[tid] = &storage.getMaterialData(tid);
8614 0 : return data;
8615 0 : };
8616 0 : _xfem->setMaterialData(fill_data(_material_props));
8617 0 : _xfem->setBoundaryMaterialData(fill_data(_bnd_material_props));
8618 :
8619 0 : unsigned int n_threads = libMesh::n_threads();
8620 0 : for (unsigned int i = 0; i < n_threads; ++i)
8621 0 : for (const auto nl_sys_num : index_range(_nl))
8622 : {
8623 0 : _assembly[i][nl_sys_num]->setXFEM(_xfem);
8624 0 : if (_displaced_problem)
8625 0 : _displaced_problem->assembly(i, nl_sys_num).setXFEM(_xfem);
8626 : }
8627 0 : }
8628 :
8629 : bool
8630 0 : FEProblemBase::updateMeshXFEM()
8631 : {
8632 0 : TIME_SECTION("updateMeshXFEM", 5, "Updating XFEM");
8633 :
8634 0 : bool updated = false;
8635 0 : if (haveXFEM())
8636 : {
8637 0 : if (_xfem->updateHeal())
8638 : // XFEM exodiff tests rely on a given numbering because they cannot use map = true due to
8639 : // having coincident elements. While conceptually speaking we do not need to contract the
8640 : // mesh, we need its call to renumber_nodes_and_elements in order to preserve these tests
8641 0 : meshChanged(
8642 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8643 :
8644 0 : updated = _xfem->update(_time, _nl, *_aux);
8645 0 : if (updated)
8646 : {
8647 0 : meshChanged(
8648 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8649 0 : _xfem->initSolution(_nl, *_aux);
8650 0 : restoreSolutions();
8651 0 : _console << "\nXFEM update complete: Mesh modified" << std::endl;
8652 : }
8653 : else
8654 0 : _console << "\nXFEM update complete: Mesh not modified" << std::endl;
8655 : }
8656 0 : return updated;
8657 0 : }
8658 :
8659 : void
8660 7022 : FEProblemBase::meshChanged(const bool intermediate_change,
8661 : const bool contract_mesh,
8662 : const bool clean_refinement_flags)
8663 : {
8664 35110 : TIME_SECTION("meshChanged", 3, "Handling Mesh Changes");
8665 :
8666 13836 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8667 6814 : _neighbor_material_props.hasStatefulProperties())
8668 208 : _mesh.cacheChangedLists(); // Currently only used with adaptivity and stateful material
8669 : // properties
8670 :
8671 : // Clear these out because they corresponded to the old mesh
8672 7022 : _ghosted_elems.clear();
8673 7022 : ghostGhostedBoundaries();
8674 :
8675 : // The mesh changed. We notify the MooseMesh first, because
8676 : // callbacks (e.g. for sparsity calculations) triggered by the
8677 : // EquationSystems reinit may require up-to-date MooseMesh caches.
8678 7022 : _mesh.meshChanged();
8679 :
8680 : // If we're just going to alter the mesh again, all we need to
8681 : // handle here is AMR and projections, not full system reinit
8682 7022 : if (intermediate_change)
8683 3510 : es().reinit_solutions();
8684 : else
8685 3512 : es().reinit();
8686 :
8687 7022 : if (contract_mesh)
8688 : // Once vectors are restricted, we can delete children of coarsened elements
8689 4450 : _mesh.getMesh().contract();
8690 7022 : if (clean_refinement_flags)
8691 : {
8692 : // Finally clear refinement flags so that if someone tries to project vectors again without
8693 : // an intervening mesh refinement to clear flags they won't run into trouble
8694 4450 : MeshRefinement refinement(_mesh.getMesh());
8695 4450 : refinement.clean_refinement_flags();
8696 4450 : }
8697 :
8698 7022 : if (!intermediate_change)
8699 : {
8700 : // Since the mesh has changed, we need to make sure that we update any of our
8701 : // MOOSE-system specific data.
8702 7024 : for (auto & sys : _solver_systems)
8703 3512 : sys->reinit();
8704 3512 : _aux->reinit();
8705 : }
8706 :
8707 : // Updating MooseMesh first breaks other adaptivity code, unless we
8708 : // then *again* update the MooseMesh caches. E.g. the definition of
8709 : // "active" and "local" may have been *changed* by refinement and
8710 : // repartitioning done in EquationSystems::reinit().
8711 7022 : _mesh.meshChanged();
8712 :
8713 : // If we have finite volume variables, we will need to recompute additional elemental/face
8714 : // quantities
8715 7022 : if (haveFV() && _mesh.isFiniteVolumeInfoDirty())
8716 351 : _mesh.setupFiniteVolumeMeshData();
8717 :
8718 : // Let the meshChangedInterface notify the mesh changed event before we update the active
8719 : // semilocal nodes, because the set of ghosted elements may potentially be updated during a mesh
8720 : // changed event.
8721 125463 : for (const auto & mci : _notify_when_mesh_changes)
8722 118441 : mci->meshChanged();
8723 :
8724 : // Since the Mesh changed, update the PointLocator object used by DiracKernels.
8725 7022 : _dirac_kernel_info.updatePointLocator(_mesh);
8726 :
8727 : // Need to redo ghosting
8728 7022 : _geometric_search_data.reinit();
8729 :
8730 7022 : if (_displaced_problem)
8731 : {
8732 569 : _displaced_problem->meshChanged(contract_mesh, clean_refinement_flags);
8733 569 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8734 : }
8735 :
8736 7022 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8737 :
8738 7022 : _evaluable_local_elem_range.reset();
8739 7022 : _nl_evaluable_local_elem_range.reset();
8740 :
8741 : // Just like we reinitialized our geometric search objects, we also need to reinitialize our
8742 : // mortar meshes. Note that this needs to happen after DisplacedProblem::meshChanged because the
8743 : // mortar mesh discretization will depend necessarily on the displaced mesh being re-displaced
8744 7022 : _mortar_data->meshChanged();
8745 :
8746 : // Nonlinear systems hold the mortar mesh functors. The domains of definition of the mortar
8747 : // functors might have changed when the mesh changed.
8748 14034 : for (auto & nl_sys : _nl)
8749 7012 : nl_sys->reinitMortarFunctors();
8750 :
8751 7022 : reinitBecauseOfGhostingOrNewGeomObjects(/*mortar_changed=*/true);
8752 :
8753 : // We need to create new storage for newly active elements, and copy
8754 : // stateful properties from the old elements.
8755 7230 : if (_has_initialized_stateful &&
8756 208 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties()))
8757 : {
8758 208 : if (havePRefinement())
8759 66 : _mesh.buildPRefinementAndCoarseningMaps(_assembly[0][0].get());
8760 :
8761 : // Prolong properties onto newly refined elements' children
8762 : {
8763 : ProjectMaterialProperties pmp(
8764 208 : /* refine = */ true, *this, _material_props, _bnd_material_props, _assembly);
8765 208 : const auto & range = *_mesh.refinedElementRange();
8766 208 : Threads::parallel_reduce(range, pmp);
8767 :
8768 : // Concurrent erasure from the shared hash map is not safe while we are reading from it in
8769 : // ProjectMaterialProperties, so we handle erasure here. Moreover, erasure based on key is
8770 : // not thread safe in and of itself because it is a read-write operation. Note that we do not
8771 : // do the erasure for p-refinement because the coarse level element is the same as our active
8772 : // refined level element
8773 208 : if (!doingPRefinement())
8774 3272 : for (const auto & elem : range)
8775 : {
8776 3130 : _material_props.eraseProperty(elem);
8777 3130 : _bnd_material_props.eraseProperty(elem);
8778 3130 : _neighbor_material_props.eraseProperty(elem);
8779 : }
8780 208 : }
8781 :
8782 : // Restrict properties onto newly coarsened elements
8783 : {
8784 : ProjectMaterialProperties pmp(
8785 208 : /* refine = */ false, *this, _material_props, _bnd_material_props, _assembly);
8786 208 : const auto & range = *_mesh.coarsenedElementRange();
8787 208 : Threads::parallel_reduce(range, pmp);
8788 : // Note that we do not do the erasure for p-refinement because the coarse level element is the
8789 : // same as our active refined level element
8790 208 : if (!doingPRefinement())
8791 1322 : for (const auto & elem : range)
8792 : {
8793 1180 : auto && coarsened_children = _mesh.coarsenedElementChildren(elem);
8794 7732 : for (auto && child : coarsened_children)
8795 : {
8796 6552 : _material_props.eraseProperty(child);
8797 6552 : _bnd_material_props.eraseProperty(child);
8798 6552 : _neighbor_material_props.eraseProperty(child);
8799 : }
8800 : }
8801 208 : }
8802 : }
8803 :
8804 7022 : if (_calculate_jacobian_in_uo)
8805 0 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
8806 :
8807 7022 : _has_jacobian = false; // we have to recompute jacobian when mesh changed
8808 :
8809 : // Now for backwards compatibility with user code that overrode the old no-arg meshChanged we must
8810 : // call it here
8811 7022 : meshChanged();
8812 7022 : }
8813 :
8814 : void
8815 917304 : FEProblemBase::notifyWhenMeshChanges(MeshChangedInterface * mci)
8816 : {
8817 917304 : _notify_when_mesh_changes.push_back(mci);
8818 917304 : }
8819 :
8820 : void
8821 76092 : FEProblemBase::notifyWhenMeshDisplaces(MeshDisplacedInterface * mdi)
8822 : {
8823 76092 : _notify_when_mesh_displaces.push_back(mdi);
8824 76092 : }
8825 :
8826 : void
8827 63048 : FEProblemBase::meshDisplaced()
8828 : {
8829 91725 : for (const auto & mdi : _notify_when_mesh_displaces)
8830 28677 : mdi->meshDisplaced();
8831 63048 : }
8832 :
8833 : void
8834 9679 : FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const bool threaded)
8835 : {
8836 : ComputeMaterialsObjectThread cmt(
8837 9679 : *this, _material_props, _bnd_material_props, _neighbor_material_props, _assembly);
8838 9679 : if (threaded)
8839 9679 : Threads::parallel_reduce(elem_range, cmt);
8840 : else
8841 0 : cmt(elem_range, true);
8842 :
8843 : #ifdef MOOSE_KOKKOS_ENABLED
8844 7291 : if (_has_kokkos_objects)
8845 781 : initKokkosStatefulProps();
8846 : #endif
8847 9679 : }
8848 :
8849 : void
8850 60372 : FEProblemBase::checkProblemIntegrity()
8851 : {
8852 181116 : TIME_SECTION("checkProblemIntegrity", 5);
8853 :
8854 : // Subdomains specified by the "Problem/block" parameter
8855 120744 : const auto & subdomain_names = getParam<std::vector<SubdomainName>>("block");
8856 60372 : auto mesh_subdomains_vec = MooseMeshUtils::getSubdomainIDs(_mesh, subdomain_names);
8857 60372 : std::set<SubdomainID> mesh_subdomains(mesh_subdomains_vec.begin(), mesh_subdomains_vec.end());
8858 :
8859 : // Check kernel coverage of subdomains (blocks) in the mesh
8860 60372 : if (!_skip_nl_system_check && _solve && _kernel_coverage_check != CoverageCheckMode::FALSE &&
8861 41339 : _kernel_coverage_check != CoverageCheckMode::OFF)
8862 : {
8863 41327 : std::set<SubdomainID> blocks;
8864 41327 : if (_kernel_coverage_check == CoverageCheckMode::TRUE ||
8865 231 : _kernel_coverage_check == CoverageCheckMode::ON)
8866 41096 : blocks = mesh_subdomains;
8867 231 : else if (_kernel_coverage_check == CoverageCheckMode::SKIP_LIST)
8868 : {
8869 12 : blocks = mesh_subdomains;
8870 24 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8871 : {
8872 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8873 12 : if (id == Moose::INVALID_BLOCK_ID)
8874 0 : paramError("kernel_coverage_block_list",
8875 : "Subdomain \"",
8876 : subdomain_name,
8877 : "\" not found in mesh.");
8878 12 : blocks.erase(id);
8879 : }
8880 : }
8881 219 : else if (_kernel_coverage_check == CoverageCheckMode::ONLY_LIST)
8882 438 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8883 : {
8884 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8885 219 : if (id == Moose::INVALID_BLOCK_ID)
8886 0 : paramError("kernel_coverage_block_list",
8887 : "Subdomain \"",
8888 : subdomain_name,
8889 : "\" not found in mesh.");
8890 219 : blocks.insert(id);
8891 : }
8892 41327 : if (!blocks.empty())
8893 81856 : for (auto & nl : _nl)
8894 40541 : nl->checkKernelCoverage(blocks);
8895 41315 : }
8896 :
8897 : // Check materials
8898 : {
8899 : #ifdef LIBMESH_ENABLE_AMR
8900 62625 : if ((_adaptivity.isOn() || _num_grid_steps) &&
8901 2265 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8902 2202 : _neighbor_material_props.hasStatefulProperties()))
8903 : {
8904 63 : _console << "Using EXPERIMENTAL Stateful Material Property projection with Adaptivity!\n"
8905 63 : << std::flush;
8906 : }
8907 : #endif
8908 :
8909 60360 : std::set<SubdomainID> local_mesh_subs(mesh_subdomains);
8910 :
8911 60360 : if (_material_coverage_check != CoverageCheckMode::FALSE &&
8912 60276 : _material_coverage_check != CoverageCheckMode::OFF)
8913 : {
8914 : /**
8915 : * If a material is specified for any block in the simulation, then all blocks must
8916 : * have a material specified.
8917 : */
8918 60276 : bool check_material_coverage = false;
8919 60276 : std::set<SubdomainID> ids = _all_materials.getActiveBlocks();
8920 72813 : for (const auto & id : ids)
8921 : {
8922 12537 : local_mesh_subs.erase(id);
8923 12537 : check_material_coverage = true;
8924 : }
8925 :
8926 : // did the user limit the subdomains to be checked?
8927 60276 : if (_material_coverage_check == CoverageCheckMode::SKIP_LIST)
8928 : {
8929 24 : for (const auto & subdomain_name : _material_coverage_blocks)
8930 : {
8931 12 : const auto id = _mesh.getSubdomainID(subdomain_name);
8932 12 : if (id == Moose::INVALID_BLOCK_ID)
8933 0 : paramError("material_coverage_block_list",
8934 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8935 12 : local_mesh_subs.erase(id);
8936 : }
8937 : }
8938 60264 : else if (_material_coverage_check == CoverageCheckMode::ONLY_LIST)
8939 : {
8940 219 : std::set<SubdomainID> blocks(local_mesh_subs);
8941 438 : for (const auto & subdomain_name : _material_coverage_blocks)
8942 : {
8943 219 : const auto id = _mesh.getSubdomainID(subdomain_name);
8944 219 : if (id == Moose::INVALID_BLOCK_ID)
8945 0 : paramError("material_coverage_block_list",
8946 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8947 219 : blocks.erase(id);
8948 : }
8949 231 : for (const auto id : blocks)
8950 12 : local_mesh_subs.erase(id);
8951 219 : }
8952 :
8953 : // also exclude mortar spaces from the material check
8954 60276 : auto && mortar_subdomain_ids = _mortar_data->getMortarSubdomainIDs();
8955 62130 : for (auto subdomain_id : mortar_subdomain_ids)
8956 1854 : local_mesh_subs.erase(subdomain_id);
8957 :
8958 : // Check Material Coverage
8959 60276 : if (check_material_coverage && !local_mesh_subs.empty())
8960 : {
8961 6 : std::stringstream extra_subdomain_ids;
8962 : /// unsigned int is necessary to print SubdomainIDs in the statement below
8963 6 : std::copy(local_mesh_subs.begin(),
8964 : local_mesh_subs.end(),
8965 12 : std::ostream_iterator<unsigned int>(extra_subdomain_ids, " "));
8966 : /// vector is necessary to get the subdomain names
8967 : std::vector<SubdomainID> local_mesh_subs_vec(local_mesh_subs.begin(),
8968 6 : local_mesh_subs.end());
8969 :
8970 18 : mooseError("The following blocks from your input mesh do not contain an active material: " +
8971 12 : extra_subdomain_ids.str() +
8972 18 : "(names: " + Moose::stringify(_mesh.getSubdomainNames(local_mesh_subs_vec)) +
8973 : ")\nWhen ANY mesh block contains a Material object, "
8974 : "all blocks must contain a Material object.\n");
8975 0 : }
8976 60270 : }
8977 :
8978 : // Check material properties on blocks and boundaries
8979 60354 : checkBlockMatProps();
8980 60322 : checkBoundaryMatProps();
8981 :
8982 : // Check that material properties exist when requested by other properties on a given block
8983 60313 : const auto & materials = _all_materials.getActiveObjects();
8984 74271 : for (const auto & material : materials)
8985 13958 : material->checkStatefulSanity();
8986 :
8987 : // auto mats_to_check = _materials.getActiveBlockObjects();
8988 : // const auto & discrete_materials = _discrete_materials.getActiveBlockObjects();
8989 : // for (const auto & map_it : discrete_materials)
8990 : // for (const auto & container_element : map_it.second)
8991 : // mats_to_check[map_it.first].push_back(container_element);
8992 60313 : if (_material_dependency_check)
8993 60289 : checkDependMaterialsHelper(_all_materials.getActiveBlockObjects());
8994 60300 : }
8995 :
8996 60300 : checkUserObjects();
8997 :
8998 : // Verify that we don't have any Element type/Coordinate Type conflicts
8999 60300 : checkCoordinateSystems();
9000 :
9001 : // Coordinate transforms are only intended for use with MultiApps at this time. If you are not
9002 : // using multiapps but still require these, contact a moose developer
9003 60437 : if (_mesh.coordTransform().hasScalingOrRotationTransformation() && _app.isUltimateMaster() &&
9004 140 : !hasMultiApps())
9005 3 : mooseError("Coordinate transformation parameters, listed below, are only to be used in the "
9006 : "context of application to application field transfers at this time. The mesh is "
9007 : "not modified by these parameters within an application.\n"
9008 : "You should likely use a 'TransformGenerator' in the [Mesh] block to achieve the "
9009 : "desired mesh modification.\n\n",
9010 3 : Moose::stringify(MooseAppCoordTransform::validParams()));
9011 :
9012 : // If using displacements, verify that the order of the displacement
9013 : // variables matches the order of the elements in the displaced
9014 : // mesh.
9015 60294 : checkDisplacementOrders();
9016 :
9017 : // Check for postprocessor names with same name as a scalar variable
9018 60291 : checkDuplicatePostprocessorVariableNames();
9019 60291 : }
9020 :
9021 : void
9022 60294 : FEProblemBase::checkDisplacementOrders()
9023 : {
9024 60294 : if (_displaced_problem)
9025 : {
9026 2022 : bool mesh_has_second_order_elements = false;
9027 4044 : for (const auto & elem : as_range(_displaced_mesh->activeLocalElementsBegin(),
9028 464948 : _displaced_mesh->activeLocalElementsEnd()))
9029 : {
9030 229785 : if (elem->default_order() == SECOND)
9031 : {
9032 344 : mesh_has_second_order_elements = true;
9033 344 : break;
9034 : }
9035 2022 : }
9036 :
9037 : // We checked our local elements, so take the max over all processors.
9038 2022 : _displaced_mesh->comm().max(mesh_has_second_order_elements);
9039 :
9040 : // If the Mesh has second order elements, make sure the
9041 : // displacement variables are second-order.
9042 2022 : if (mesh_has_second_order_elements)
9043 : {
9044 : const std::vector<std::string> & displacement_variables =
9045 344 : _displaced_problem->getDisplacementVarNames();
9046 :
9047 1133 : for (const auto & var_name : displacement_variables)
9048 : {
9049 : MooseVariableFEBase & mv =
9050 792 : _displaced_problem->getVariable(/*tid=*/0,
9051 : var_name,
9052 : Moose::VarKindType::VAR_ANY,
9053 : Moose::VarFieldType::VAR_FIELD_STANDARD);
9054 792 : if (mv.order() != SECOND)
9055 3 : mooseError("Error: mesh has SECOND order elements, so all displacement variables must be "
9056 : "SECOND order.");
9057 : }
9058 : }
9059 : }
9060 60291 : }
9061 :
9062 : void
9063 60300 : FEProblemBase::checkUserObjects()
9064 : {
9065 : // Check user_objects block coverage
9066 60300 : std::set<SubdomainID> mesh_subdomains = _mesh.meshSubdomains();
9067 60300 : std::set<SubdomainID> user_objects_blocks;
9068 :
9069 : // gather names of all user_objects that were defined in the input file
9070 : // and the blocks that they are defined on
9071 60300 : std::set<std::string> names;
9072 :
9073 60300 : std::vector<UserObjectBase *> objects;
9074 60300 : theWarehouse().query().condition<AttribInterfaces>(Interfaces::UserObject).queryInto(objects);
9075 :
9076 134098 : for (const auto & obj : objects)
9077 73798 : names.insert(obj->name());
9078 :
9079 : // See if all referenced blocks are covered
9080 60300 : std::set<SubdomainID> difference;
9081 60300 : std::set_difference(user_objects_blocks.begin(),
9082 : user_objects_blocks.end(),
9083 : mesh_subdomains.begin(),
9084 : mesh_subdomains.end(),
9085 : std::inserter(difference, difference.end()));
9086 :
9087 60300 : if (!difference.empty())
9088 : {
9089 0 : std::ostringstream oss;
9090 0 : oss << "One or more UserObjects is referencing a nonexistent block:\n";
9091 0 : for (const auto & id : difference)
9092 0 : oss << id << "\n";
9093 0 : mooseError(oss.str());
9094 0 : }
9095 60300 : }
9096 :
9097 : void
9098 60289 : FEProblemBase::checkDependMaterialsHelper(
9099 : const std::map<SubdomainID, std::vector<std::shared_ptr<MaterialBase>>> & materials_map)
9100 : {
9101 72768 : for (const auto & it : materials_map)
9102 : {
9103 : /// These two sets are used to make sure that all dependent props on a block are actually supplied
9104 12487 : std::set<std::string> block_depend_props, block_supplied_props;
9105 :
9106 31006 : for (const auto & mat1 : it.second)
9107 : {
9108 18519 : auto & alldeps = mat1->getMatPropDependencies(); // includes requested stateful props
9109 20838 : for (auto & dep : alldeps)
9110 2319 : block_depend_props.insert(_material_prop_registry.getName(dep));
9111 :
9112 : // See if any of the active materials supply this property
9113 57064 : for (const auto & mat2 : it.second)
9114 : {
9115 38545 : const std::set<std::string> & supplied_props = mat2->MaterialBase::getSuppliedItems();
9116 38545 : block_supplied_props.insert(supplied_props.begin(), supplied_props.end());
9117 : }
9118 : }
9119 :
9120 : // Add zero material properties specific to this block and unrestricted
9121 12487 : block_supplied_props.insert(_zero_block_material_props[it.first].begin(),
9122 12487 : _zero_block_material_props[it.first].end());
9123 :
9124 : // Error check to make sure all properties consumed by materials are supplied on this block
9125 12487 : std::set<std::string> difference;
9126 12487 : std::set_difference(block_depend_props.begin(),
9127 : block_depend_props.end(),
9128 : block_supplied_props.begin(),
9129 : block_supplied_props.end(),
9130 : std::inserter(difference, difference.end()));
9131 :
9132 12487 : if (!difference.empty())
9133 : {
9134 8 : std::ostringstream oss;
9135 8 : oss << "One or more Material Properties were not supplied on block ";
9136 8 : const std::string & subdomain_name = _mesh.getSubdomainName(it.first);
9137 8 : if (subdomain_name.length() > 0)
9138 0 : oss << subdomain_name << " (" << it.first << ")";
9139 : else
9140 8 : oss << it.first;
9141 8 : oss << ":\n";
9142 16 : for (const auto & name : difference)
9143 8 : oss << name << "\n";
9144 8 : mooseError(oss.str());
9145 0 : }
9146 12479 : }
9147 :
9148 : // This loop checks that materials are not supplied by multiple Material objects
9149 72755 : for (const auto & it : materials_map)
9150 : {
9151 12479 : const auto & materials = it.second;
9152 12479 : std::set<std::string> inner_supplied, outer_supplied;
9153 :
9154 30978 : for (const auto & outer_mat : materials)
9155 : {
9156 : // Storage for properties for this material (outer) and all other materials (inner)
9157 18504 : outer_supplied = outer_mat->getSuppliedItems();
9158 18504 : inner_supplied.clear();
9159 :
9160 : // Property to material map for error reporting
9161 18504 : std::map<std::string, std::set<std::string>> prop_to_mat;
9162 39287 : for (const auto & name : outer_supplied)
9163 20783 : prop_to_mat[name].insert(outer_mat->name());
9164 :
9165 57025 : for (const auto & inner_mat : materials)
9166 : {
9167 38521 : if (outer_mat == inner_mat)
9168 18504 : continue;
9169 :
9170 : // Check whether these materials are an AD pair
9171 20017 : auto outer_mat_type = outer_mat->type();
9172 20017 : auto inner_mat_type = inner_mat->type();
9173 40034 : removeSubstring(outer_mat_type, "<RESIDUAL>");
9174 40034 : removeSubstring(outer_mat_type, "<JACOBIAN>");
9175 40034 : removeSubstring(inner_mat_type, "<RESIDUAL>");
9176 20017 : removeSubstring(inner_mat_type, "<JACOBIAN>");
9177 20017 : if (outer_mat_type == inner_mat_type && outer_mat_type != outer_mat->type() &&
9178 0 : inner_mat_type != inner_mat->type())
9179 0 : continue;
9180 :
9181 20017 : inner_supplied.insert(inner_mat->getSuppliedItems().begin(),
9182 20017 : inner_mat->getSuppliedItems().end());
9183 :
9184 115069 : for (const auto & inner_supplied_name : inner_supplied)
9185 95052 : prop_to_mat[inner_supplied_name].insert(inner_mat->name());
9186 20017 : }
9187 :
9188 : // Test that a property isn't supplied on multiple blocks
9189 18504 : std::set<std::string> intersection;
9190 18504 : std::set_intersection(outer_supplied.begin(),
9191 : outer_supplied.end(),
9192 : inner_supplied.begin(),
9193 : inner_supplied.end(),
9194 : std::inserter(intersection, intersection.end()));
9195 :
9196 18504 : if (!intersection.empty())
9197 : {
9198 5 : std::ostringstream oss;
9199 5 : oss << "The following material properties are declared on block " << it.first
9200 5 : << " by multiple materials:\n";
9201 10 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << "Material Property"
9202 5 : << "Material Objects\n";
9203 20 : for (const auto & outer_name : intersection)
9204 : {
9205 15 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << outer_name;
9206 45 : for (const auto & inner_name : prop_to_mat[outer_name])
9207 30 : oss << inner_name << " ";
9208 15 : oss << '\n';
9209 : }
9210 :
9211 5 : mooseError(oss.str());
9212 : break;
9213 0 : }
9214 18499 : }
9215 12474 : }
9216 60276 : }
9217 :
9218 : void
9219 60300 : FEProblemBase::checkCoordinateSystems()
9220 : {
9221 60300 : _mesh.checkCoordinateSystems();
9222 60297 : }
9223 :
9224 : void
9225 467 : FEProblemBase::setRestartFile(const std::string & file_name)
9226 : {
9227 467 : if (_app.isRecovering())
9228 : {
9229 23 : mooseInfo("Restart file ", file_name, " is NOT being used since we are performing recovery.");
9230 : }
9231 : else
9232 : {
9233 444 : _app.setRestart(true);
9234 444 : _app.setRestartRecoverFileBase(file_name);
9235 444 : mooseInfo("Using ", file_name, " for restart.");
9236 : }
9237 467 : }
9238 :
9239 : std::vector<VariableName>
9240 357572 : FEProblemBase::getVariableNames()
9241 : {
9242 357572 : std::vector<VariableName> names;
9243 :
9244 719302 : for (auto & sys : _solver_systems)
9245 : {
9246 361730 : const std::vector<VariableName> & var_names = sys->getVariableNames();
9247 361730 : names.insert(names.end(), var_names.begin(), var_names.end());
9248 : }
9249 :
9250 357572 : const std::vector<VariableName> & aux_var_names = _aux->getVariableNames();
9251 357572 : names.insert(names.end(), aux_var_names.begin(), aux_var_names.end());
9252 :
9253 357572 : return names;
9254 0 : }
9255 :
9256 : SolverParams &
9257 1623873 : FEProblemBase::solverParams(const unsigned int solver_sys_num)
9258 : {
9259 : mooseAssert(solver_sys_num < numSolverSystems(),
9260 : "Solver system number '" << solver_sys_num << "' is out of bounds. We have '"
9261 : << numSolverSystems() << "' solver systems");
9262 1623873 : return _solver_params[solver_sys_num];
9263 : }
9264 :
9265 : const SolverParams &
9266 14652 : FEProblemBase::solverParams(const unsigned int solver_sys_num) const
9267 : {
9268 14652 : return const_cast<FEProblemBase *>(this)->solverParams(solver_sys_num);
9269 : }
9270 :
9271 : void
9272 371 : FEProblemBase::registerRandomInterface(RandomInterface & random_interface, const std::string & name)
9273 : {
9274 371 : auto insert_pair = moose_try_emplace(
9275 371 : _random_data_objects, name, std::make_unique<RandomData>(*this, random_interface));
9276 :
9277 371 : auto random_data_ptr = insert_pair.first->second.get();
9278 371 : random_interface.setRandomDataPointer(random_data_ptr);
9279 371 : }
9280 :
9281 : bool
9282 1601219 : FEProblemBase::needBoundaryMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9283 : {
9284 1601219 : if (_bnd_mat_side_cache[tid].find(bnd_id) == _bnd_mat_side_cache[tid].end())
9285 : {
9286 28050 : auto & bnd_mat_side_cache = _bnd_mat_side_cache[tid][bnd_id];
9287 28050 : bnd_mat_side_cache = false;
9288 :
9289 : // Check systems
9290 28050 : if (_aux->needMaterialOnSide(bnd_id))
9291 : {
9292 506 : bnd_mat_side_cache = true;
9293 506 : return true;
9294 : }
9295 52846 : for (auto & nl : _nl)
9296 27386 : if (nl->needBoundaryMaterialOnSide(bnd_id, tid))
9297 : {
9298 2084 : bnd_mat_side_cache = true;
9299 2084 : return true;
9300 : }
9301 :
9302 : // TODO: these objects should be checked for whether they actually consume materials
9303 : // NOTE: InterfaceUO can use use boundary properties too
9304 25460 : if (theWarehouse()
9305 50920 : .query()
9306 25460 : .condition<AttribThread>(tid)
9307 25460 : .condition<AttribInterfaces>(Interfaces::SideUserObject | Interfaces::DomainUserObject |
9308 : Interfaces::InterfaceUserObject)
9309 25460 : .condition<AttribBoundaries>(bnd_id)
9310 25460 : .count() > 0)
9311 : {
9312 560 : bnd_mat_side_cache = true;
9313 560 : return true;
9314 : }
9315 : }
9316 :
9317 1598069 : return _bnd_mat_side_cache[tid][bnd_id];
9318 : }
9319 :
9320 : bool
9321 386695 : FEProblemBase::needInterfaceMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9322 : {
9323 386695 : if (_interface_mat_side_cache[tid].find(bnd_id) == _interface_mat_side_cache[tid].end())
9324 : {
9325 2700 : auto & interface_mat_side_cache = _interface_mat_side_cache[tid][bnd_id];
9326 2700 : interface_mat_side_cache = false;
9327 :
9328 : // Aux-system has not needed interface materials so far
9329 5184 : for (auto & nl : _nl)
9330 2700 : if (nl->needInterfaceMaterialOnSide(bnd_id, tid))
9331 : {
9332 216 : interface_mat_side_cache = true;
9333 216 : return true;
9334 : }
9335 :
9336 : // TODO: these objects should be checked for whether they actually consume materials
9337 2484 : if (theWarehouse()
9338 4968 : .query()
9339 2484 : .condition<AttribThread>(tid)
9340 2484 : .condition<AttribInterfaces>(Interfaces::InterfaceUserObject |
9341 : Interfaces::DomainUserObject)
9342 2484 : .condition<AttribBoundaries>(bnd_id)
9343 2484 : .count() > 0)
9344 : {
9345 79 : interface_mat_side_cache = true;
9346 79 : return true;
9347 : }
9348 2405 : else if (_interface_materials.hasActiveBoundaryObjects(bnd_id, tid))
9349 : {
9350 9 : interface_mat_side_cache = true;
9351 9 : return true;
9352 : }
9353 : }
9354 386391 : return _interface_mat_side_cache[tid][bnd_id];
9355 : }
9356 :
9357 : bool
9358 422665 : FEProblemBase::needInternalNeighborSideMaterial(SubdomainID subdomain_id, const THREAD_ID tid)
9359 : {
9360 422665 : if (_block_mat_side_cache[tid].find(subdomain_id) == _block_mat_side_cache[tid].end())
9361 : {
9362 12121 : _block_mat_side_cache[tid][subdomain_id] = false;
9363 :
9364 23791 : for (auto & nl : _nl)
9365 12080 : if (nl->needInternalNeighborSideMaterial(subdomain_id, tid))
9366 : {
9367 410 : _block_mat_side_cache[tid][subdomain_id] = true;
9368 410 : return true;
9369 : }
9370 :
9371 : // TODO: these objects should be checked for whether they actually consume materials
9372 11711 : if (theWarehouse()
9373 23422 : .query()
9374 11711 : .condition<AttribThread>(tid)
9375 11711 : .condition<AttribInterfaces>(Interfaces::InternalSideUserObject |
9376 : Interfaces::DomainUserObject)
9377 11711 : .condition<AttribSubdomains>(subdomain_id)
9378 11711 : .count() > 0)
9379 : {
9380 33 : _block_mat_side_cache[tid][subdomain_id] = true;
9381 33 : return true;
9382 : }
9383 : }
9384 :
9385 422222 : return _block_mat_side_cache[tid][subdomain_id];
9386 : }
9387 :
9388 : bool
9389 287814 : FEProblemBase::needsPreviousNewtonIteration() const
9390 : {
9391 287814 : return vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG);
9392 : }
9393 :
9394 : void
9395 76 : FEProblemBase::needsPreviousNewtonIteration(bool state)
9396 : {
9397 76 : if (state && !vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
9398 0 : mooseError("Previous nonlinear solution is required but not added through "
9399 : "Problem/previous_nl_solution_required=true");
9400 76 : }
9401 :
9402 : void
9403 52 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(bool needed,
9404 : const unsigned int solver_sys_num)
9405 : {
9406 52 : _previous_multiapp_fp_nl_solution_required[solver_sys_num] = needed;
9407 52 : }
9408 :
9409 : bool
9410 58231 : FEProblemBase::needsPreviousMultiAppFixedPointIterationSolution(
9411 : const unsigned int solver_sys_num) const
9412 : {
9413 58231 : return _previous_multiapp_fp_nl_solution_required[solver_sys_num];
9414 : }
9415 :
9416 : void
9417 13 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary(bool state)
9418 : {
9419 13 : _previous_multiapp_fp_aux_solution_required = state;
9420 13 : }
9421 :
9422 : bool
9423 58095 : FEProblemBase::needsPreviousMultiAppFixedPointIterationAuxiliary() const
9424 : {
9425 58095 : return _previous_multiapp_fp_aux_solution_required;
9426 : }
9427 :
9428 : bool
9429 7561278 : FEProblemBase::hasJacobian() const
9430 : {
9431 7561278 : return _has_jacobian;
9432 : }
9433 :
9434 : bool
9435 7076750 : FEProblemBase::constJacobian() const
9436 : {
9437 7076750 : return _const_jacobian;
9438 : }
9439 :
9440 : void
9441 287513 : FEProblemBase::addOutput(const std::string & object_type,
9442 : const std::string & object_name,
9443 : InputParameters & parameters)
9444 : {
9445 : parallel_object_only();
9446 :
9447 : // Get a reference to the OutputWarehouse
9448 287513 : OutputWarehouse & output_warehouse = _app.getOutputWarehouse();
9449 :
9450 : // Reject the reserved names for objects not built by MOOSE
9451 287513 : if (!parameters.get<bool>("_built_by_moose") && output_warehouse.isReservedName(object_name))
9452 6 : mooseError("The name '", object_name, "' is a reserved name for output objects");
9453 :
9454 : // Check that an object by the same name does not already exist; this must be done before the
9455 : // object is created to avoid getting misleading errors from the Parser
9456 287507 : if (output_warehouse.hasOutput(object_name))
9457 3 : mooseError("An output object named '", object_name, "' already exists");
9458 :
9459 : // Add a pointer to the FEProblemBase class
9460 575008 : parameters.addPrivateParam<FEProblemBase *>("_fe_problem_base", this);
9461 :
9462 : // --show-input should enable the display of the input file on the screen
9463 696839 : if (object_type == "Console" && _app.getParam<bool>("show_input") &&
9464 287531 : parameters.get<bool>("output_screen"))
9465 54 : parameters.set<ExecFlagEnum>("execute_input_on") = EXEC_INITIAL;
9466 :
9467 : // Apply only user-set parameters from the common [Outputs] block so that
9468 : // each output type's own defaults are not overridden by common defaults.
9469 287504 : const InputParameters * common = output_warehouse.getCommonParameters();
9470 287504 : if (common)
9471 287504 : parameters.applyCommonUserSetParameters(*common);
9472 :
9473 : // Set the correct value for the binary flag for XDA/XDR output
9474 287504 : if (object_type == "XDR")
9475 120 : parameters.set<bool>("_binary") = true;
9476 287444 : else if (object_type == "XDA")
9477 244 : parameters.set<bool>("_binary") = false;
9478 :
9479 : // Adjust the checkpoint suffix if auto recovery was enabled
9480 287504 : if (object_name == "auto_recovery_checkpoint")
9481 0 : parameters.set<std::string>("suffix") = "auto_recovery";
9482 :
9483 : // Create the object and add it to the warehouse
9484 287504 : std::shared_ptr<Output> output = _factory.create<Output>(object_type, object_name, parameters);
9485 287492 : logAdd("Output", object_name, object_type, parameters);
9486 287492 : output_warehouse.addOutput(output);
9487 287492 : }
9488 :
9489 : void
9490 22788 : FEProblemBase::haveADObjects(const bool have_ad_objects)
9491 : {
9492 22788 : _have_ad_objects = have_ad_objects;
9493 22788 : if (_displaced_problem)
9494 227 : _displaced_problem->SubProblem::haveADObjects(have_ad_objects);
9495 22788 : }
9496 :
9497 : const SystemBase &
9498 0 : FEProblemBase::getSystemBase(const unsigned int sys_num) const
9499 : {
9500 0 : if (sys_num < _solver_systems.size())
9501 0 : return *_solver_systems[sys_num];
9502 :
9503 0 : return *_aux;
9504 : }
9505 :
9506 : SystemBase &
9507 3722 : FEProblemBase::getSystemBase(const std::string & sys_name)
9508 : {
9509 3722 : if (std::find(_solver_sys_names.begin(), _solver_sys_names.end(), sys_name) !=
9510 7444 : _solver_sys_names.end())
9511 3722 : return getSystemBase(solverSysNum(sys_name));
9512 0 : else if (sys_name == "aux0")
9513 0 : return *_aux;
9514 : else
9515 0 : mooseError("System '" + sys_name + "' was requested from problem but does not exist.");
9516 : }
9517 :
9518 : SystemBase &
9519 5366 : FEProblemBase::getSystemBase(const unsigned int sys_num)
9520 : {
9521 5366 : if (sys_num < _solver_systems.size())
9522 5264 : return *_solver_systems[sys_num];
9523 :
9524 102 : return *_aux;
9525 : }
9526 :
9527 : const SystemBase &
9528 12422 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num) const
9529 : {
9530 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9531 12422 : return *_nl[sys_num];
9532 : }
9533 :
9534 : SystemBase &
9535 1261167 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num)
9536 : {
9537 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9538 1261167 : return *_nl[sys_num];
9539 : }
9540 :
9541 : const SystemBase &
9542 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num) const
9543 : {
9544 : mooseAssert(sys_num < _linear_systems.size(),
9545 : "System number greater than the number of linear systems");
9546 0 : return *_linear_systems[sys_num];
9547 : }
9548 :
9549 : SystemBase &
9550 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num)
9551 : {
9552 : mooseAssert(sys_num < _linear_systems.size(),
9553 : "System number greater than the number of linear systems");
9554 0 : return *_linear_systems[sys_num];
9555 : }
9556 :
9557 : const SystemBase &
9558 0 : FEProblemBase::systemBaseSolver(const unsigned int sys_num) const
9559 : {
9560 : mooseAssert(sys_num < _solver_systems.size(),
9561 : "System number greater than the number of solver systems");
9562 0 : return *_solver_systems[sys_num];
9563 : }
9564 :
9565 : SystemBase &
9566 6351547 : FEProblemBase::systemBaseSolver(const unsigned int sys_num)
9567 : {
9568 : mooseAssert(sys_num < _solver_systems.size(),
9569 : "System number greater than the number of solver systems");
9570 6351547 : return *_solver_systems[sys_num];
9571 : }
9572 :
9573 : const SystemBase &
9574 417 : FEProblemBase::systemBaseAuxiliary() const
9575 : {
9576 417 : return *_aux;
9577 : }
9578 :
9579 : SystemBase &
9580 8159924 : FEProblemBase::systemBaseAuxiliary()
9581 : {
9582 8159924 : return *_aux;
9583 : }
9584 :
9585 : void
9586 3894587 : FEProblemBase::computingNonlinearResid(bool computing_nonlinear_residual)
9587 : {
9588 : parallel_object_only();
9589 :
9590 3894587 : if (_displaced_problem)
9591 192292 : _displaced_problem->computingNonlinearResid(computing_nonlinear_residual);
9592 3894587 : _computing_nonlinear_residual = computing_nonlinear_residual;
9593 3894587 : }
9594 :
9595 : void
9596 9630426 : FEProblemBase::setCurrentlyComputingResidual(bool currently_computing_residual)
9597 : {
9598 9630426 : if (_displaced_problem)
9599 392528 : _displaced_problem->setCurrentlyComputingResidual(currently_computing_residual);
9600 9630426 : _currently_computing_residual = currently_computing_residual;
9601 9630426 : }
9602 :
9603 : void
9604 50 : FEProblemBase::uniformRefine()
9605 : {
9606 : // ResetDisplacedMeshThread::onNode looks up the reference mesh by ID, so we need to make sure
9607 : // we undisplace before adapting the reference mesh
9608 50 : if (_displaced_problem)
9609 34 : _displaced_problem->undisplaceMesh();
9610 :
9611 50 : Adaptivity::uniformRefine(&_mesh, 1);
9612 50 : if (_displaced_problem)
9613 34 : Adaptivity::uniformRefine(&_displaced_problem->mesh(), 1);
9614 :
9615 50 : meshChanged(
9616 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
9617 50 : }
9618 :
9619 : void
9620 60154 : FEProblemBase::automaticScaling(bool automatic_scaling)
9621 : {
9622 60154 : if (_displaced_problem)
9623 2022 : _displaced_problem->automaticScaling(automatic_scaling);
9624 :
9625 60154 : SubProblem::automaticScaling(automatic_scaling);
9626 60154 : }
9627 :
9628 : void
9629 254092 : FEProblemBase::reinitElemFaceRef(const Elem * elem,
9630 : unsigned int side,
9631 : Real tolerance,
9632 : const std::vector<Point> * const pts,
9633 : const std::vector<Real> * const weights,
9634 : const THREAD_ID tid)
9635 : {
9636 254092 : SubProblem::reinitElemFaceRef(elem, side, tolerance, pts, weights, tid);
9637 :
9638 254092 : if (_displaced_problem)
9639 20096 : _displaced_problem->reinitElemFaceRef(
9640 20096 : _displaced_mesh->elemPtr(elem->id()), side, tolerance, pts, weights, tid);
9641 254092 : }
9642 :
9643 : void
9644 254092 : FEProblemBase::reinitNeighborFaceRef(const Elem * neighbor_elem,
9645 : unsigned int neighbor_side,
9646 : Real tolerance,
9647 : const std::vector<Point> * const pts,
9648 : const std::vector<Real> * const weights,
9649 : const THREAD_ID tid)
9650 : {
9651 254092 : SubProblem::reinitNeighborFaceRef(neighbor_elem, neighbor_side, tolerance, pts, weights, tid);
9652 :
9653 254092 : if (_displaced_problem)
9654 20096 : _displaced_problem->reinitNeighborFaceRef(
9655 20096 : _displaced_mesh->elemPtr(neighbor_elem->id()), neighbor_side, tolerance, pts, weights, tid);
9656 254092 : }
9657 :
9658 : void
9659 3035869 : FEProblemBase::getFVMatsAndDependencies(
9660 : const SubdomainID blk_id,
9661 : std::vector<std::shared_ptr<MaterialBase>> & face_materials,
9662 : std::vector<std::shared_ptr<MaterialBase>> & neighbor_materials,
9663 : std::set<MooseVariableFieldBase *> & variables,
9664 : const THREAD_ID tid)
9665 : {
9666 3035869 : if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9667 : {
9668 : auto & this_face_mats =
9669 3544 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9670 7268 : for (std::shared_ptr<MaterialBase> face_mat : this_face_mats)
9671 3724 : if (face_mat->ghostable())
9672 : {
9673 3724 : face_materials.push_back(face_mat);
9674 3724 : auto & var_deps = face_mat->getMooseVariableDependencies();
9675 4084 : for (auto * var : var_deps)
9676 : {
9677 360 : if (!var->isFV())
9678 0 : mooseError(
9679 : "Ghostable materials should only have finite volume variables coupled into them.");
9680 360 : else if (face_mat->hasStatefulProperties())
9681 0 : mooseError("Finite volume materials do not currently support stateful properties.");
9682 360 : variables.insert(var);
9683 : }
9684 3724 : }
9685 : }
9686 :
9687 3035869 : if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9688 : {
9689 : auto & this_neighbor_mats =
9690 3544 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9691 7268 : for (std::shared_ptr<MaterialBase> neighbor_mat : this_neighbor_mats)
9692 3724 : if (neighbor_mat->ghostable())
9693 : {
9694 3724 : neighbor_materials.push_back(neighbor_mat);
9695 : #ifndef NDEBUG
9696 : auto & var_deps = neighbor_mat->getMooseVariableDependencies();
9697 : for (auto * var : var_deps)
9698 : {
9699 : if (!var->isFV())
9700 : mooseError(
9701 : "Ghostable materials should only have finite volume variables coupled into them.");
9702 : else if (neighbor_mat->hasStatefulProperties())
9703 : mooseError("Finite volume materials do not currently support stateful properties.");
9704 : auto pr = variables.insert(var);
9705 : mooseAssert(!pr.second,
9706 : "We should not have inserted any new variables dependencies from our "
9707 : "neighbor materials that didn't exist for our face materials");
9708 : }
9709 : #endif
9710 3724 : }
9711 : }
9712 3035869 : }
9713 :
9714 : void
9715 31842697 : FEProblemBase::resizeMaterialData(const Moose::MaterialDataType data_type,
9716 : const unsigned int nqp,
9717 : const THREAD_ID tid)
9718 : {
9719 31842697 : getMaterialData(data_type, tid).resize(nqp);
9720 31842697 : }
9721 :
9722 : void
9723 60094 : FEProblemBase::setNonlinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9724 : {
9725 60094 : if (convergence_names.size() != numNonlinearSystems())
9726 0 : paramError("nonlinear_convergence",
9727 : "There must be one convergence object per nonlinear system");
9728 60094 : _nonlinear_convergence_names = convergence_names;
9729 60094 : }
9730 :
9731 : void
9732 61536 : FEProblemBase::setMultiAppFixedPointConvergenceName(const ConvergenceName & convergence_name)
9733 : {
9734 61536 : _multiapp_fixed_point_convergence_name = convergence_name;
9735 61536 : }
9736 :
9737 : void
9738 30352 : FEProblemBase::setSteadyStateConvergenceName(const ConvergenceName & convergence_name)
9739 : {
9740 30352 : _steady_state_convergence_name = convergence_name;
9741 30352 : }
9742 :
9743 : const std::vector<ConvergenceName> &
9744 985277 : FEProblemBase::getNonlinearConvergenceNames() const
9745 : {
9746 985277 : if (_nonlinear_convergence_names)
9747 985277 : return *_nonlinear_convergence_names;
9748 0 : mooseError("The nonlinear system convergence name(s) have not been set.");
9749 : }
9750 :
9751 : bool
9752 26452 : FEProblemBase::hasLinearConvergenceObjects() const
9753 : {
9754 : // If false,this means we have not set one, not that we are querying this too early
9755 : // TODO: once there is a default linear CV object, error on the 'not set' case
9756 26452 : return _linear_convergence_names.has_value();
9757 : }
9758 :
9759 : void
9760 134 : FEProblemBase::setLinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9761 : {
9762 134 : if (convergence_names.size() != numLinearSystems())
9763 0 : paramError("linear_convergence", "There must be one convergence object per linear system");
9764 134 : _linear_convergence_names = convergence_names;
9765 134 : }
9766 :
9767 : const std::vector<ConvergenceName> &
9768 4635 : FEProblemBase::getLinearConvergenceNames() const
9769 : {
9770 4635 : if (_linear_convergence_names)
9771 4635 : return *_linear_convergence_names;
9772 0 : mooseError("The linear convergence name(s) have not been set.");
9773 : }
9774 :
9775 : const ConvergenceName &
9776 250161 : FEProblemBase::getMultiAppFixedPointConvergenceName() const
9777 : {
9778 250161 : if (_multiapp_fixed_point_convergence_name)
9779 250161 : return _multiapp_fixed_point_convergence_name.value();
9780 : else
9781 0 : mooseError("The fixed point convergence name has not been set.");
9782 : }
9783 :
9784 : const ConvergenceName &
9785 103148 : FEProblemBase::getSteadyStateConvergenceName() const
9786 : {
9787 103148 : if (_steady_state_convergence_name)
9788 103148 : return _steady_state_convergence_name.value();
9789 : else
9790 0 : mooseError("The steady convergence name has not been set.");
9791 : }
9792 :
9793 : void
9794 3053298 : FEProblemBase::residualSetup()
9795 : {
9796 3053298 : SubProblem::residualSetup();
9797 : // We need to setup all the nonlinear systems other than our current one which actually called
9798 : // this method (so we have to make sure we don't go in a circle)
9799 6192474 : for (const auto i : make_range(numNonlinearSystems()))
9800 3139176 : if (i != currentNlSysNum())
9801 85878 : _nl[i]->residualSetup();
9802 : // We don't setup the aux sys because that's been done elsewhere
9803 3053298 : if (_displaced_problem)
9804 124147 : _displaced_problem->residualSetup();
9805 3053298 : }
9806 :
9807 : void
9808 474187 : FEProblemBase::jacobianSetup()
9809 : {
9810 474187 : SubProblem::jacobianSetup();
9811 : // We need to setup all the nonlinear systems other than our current one which actually called
9812 : // this method (so we have to make sure we don't go in a circle)
9813 963154 : for (const auto i : make_range(numNonlinearSystems()))
9814 488967 : if (i != currentNlSysNum())
9815 14780 : _nl[i]->jacobianSetup();
9816 : // We don't setup the aux sys because that's been done elsewhere
9817 474187 : if (_displaced_problem)
9818 21144 : _displaced_problem->jacobianSetup();
9819 474187 : }
9820 :
9821 : MooseAppCoordTransform &
9822 96018 : FEProblemBase::coordTransform()
9823 : {
9824 96018 : return mesh().coordTransform();
9825 : }
9826 :
9827 : unsigned int
9828 476767753 : FEProblemBase::currentNlSysNum() const
9829 : {
9830 : // If we don't have nonlinear systems this should be an invalid number
9831 476767753 : unsigned int current_nl_sys_num = libMesh::invalid_uint;
9832 476767753 : if (_nl.size())
9833 476765641 : current_nl_sys_num = currentNonlinearSystem().number();
9834 :
9835 476767753 : return current_nl_sys_num;
9836 : }
9837 :
9838 : unsigned int
9839 0 : FEProblemBase::currentLinearSysNum() const
9840 : {
9841 : // If we don't have linear systems this should be an invalid number
9842 0 : unsigned int current_linear_sys_num = libMesh::invalid_uint;
9843 0 : if (_linear_systems.size())
9844 0 : current_linear_sys_num = currentLinearSystem().number();
9845 :
9846 0 : return current_linear_sys_num;
9847 : }
9848 :
9849 : bool
9850 123240922 : FEProblemBase::shouldPrintExecution(const THREAD_ID tid) const
9851 : {
9852 : // For now, only support printing from thread 0
9853 123240922 : if (tid != 0)
9854 519480 : return false;
9855 :
9856 245205502 : if (_print_execution_on.isValueSet(_current_execute_on_flag) ||
9857 122484060 : _print_execution_on.isValueSet(EXEC_ALWAYS))
9858 337814 : return true;
9859 : else
9860 122383628 : return false;
9861 : }
9862 :
9863 : std::vector<MortarUserObject *>
9864 252918 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9865 : const BoundaryID secondary_boundary_id,
9866 : const bool displaced,
9867 : const std::vector<MortarUserObject *> & mortar_uo_superset)
9868 : {
9869 252918 : std::vector<MortarUserObject *> mortar_uos;
9870 252918 : auto * const subproblem = displaced ? static_cast<SubProblem *>(_displaced_problem.get())
9871 252918 : : static_cast<SubProblem *>(this);
9872 252940 : for (auto * const obj : mortar_uo_superset)
9873 44 : if (obj->onInterface(primary_boundary_id, secondary_boundary_id) &&
9874 22 : (&obj->getSubProblem() == subproblem))
9875 22 : mortar_uos.push_back(obj);
9876 :
9877 252918 : return mortar_uos;
9878 0 : }
9879 :
9880 : std::vector<MortarUserObject *>
9881 252896 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9882 : const BoundaryID secondary_boundary_id,
9883 : const bool displaced)
9884 : {
9885 252896 : std::vector<MortarUserObject *> mortar_uos;
9886 252896 : theWarehouse()
9887 252896 : .query()
9888 505792 : .condition<AttribInterfaces>(Interfaces::MortarUserObject)
9889 252896 : .queryInto(mortar_uos);
9890 505792 : return getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced, mortar_uos);
9891 252896 : }
9892 :
9893 : void
9894 252896 : FEProblemBase::reinitMortarUserObjects(const BoundaryID primary_boundary_id,
9895 : const BoundaryID secondary_boundary_id,
9896 : const bool displaced)
9897 : {
9898 : const auto mortar_uos =
9899 252896 : getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced);
9900 252896 : for (auto * const mortar_uo : mortar_uos)
9901 : {
9902 0 : mortar_uo->setNormals();
9903 0 : mortar_uo->reinit();
9904 : }
9905 252896 : }
9906 :
9907 : void
9908 0 : FEProblemBase::setVerboseProblem(bool verbose)
9909 : {
9910 0 : _verbose_setup = verbose ? "true" : "false";
9911 0 : _verbose_multiapps = verbose;
9912 0 : _verbose_restore = verbose;
9913 0 : }
9914 :
9915 : void
9916 111832 : FEProblemBase::setCurrentLowerDElem(const Elem * const lower_d_elem, const THREAD_ID tid)
9917 : {
9918 111832 : SubProblem::setCurrentLowerDElem(lower_d_elem, tid);
9919 111832 : if (_displaced_problem)
9920 27619 : _displaced_problem->setCurrentLowerDElem(
9921 0 : lower_d_elem ? _displaced_mesh->elemPtr(lower_d_elem->id()) : nullptr, tid);
9922 111832 : }
9923 :
9924 : void
9925 121628976 : FEProblemBase::setCurrentBoundaryID(BoundaryID bid, const THREAD_ID tid)
9926 : {
9927 121628976 : SubProblem::setCurrentBoundaryID(bid, tid);
9928 121628976 : if (_displaced_problem)
9929 8992635 : _displaced_problem->setCurrentBoundaryID(bid, tid);
9930 121628976 : }
9931 :
9932 : void
9933 7412094 : FEProblemBase::setCurrentNonlinearSystem(const unsigned int nl_sys_num)
9934 : {
9935 : mooseAssert(nl_sys_num < _nl.size(),
9936 : "System number greater than the number of nonlinear systems");
9937 7412094 : _current_nl_sys = _nl[nl_sys_num].get();
9938 7412094 : _current_solver_sys = _current_nl_sys;
9939 7412094 : }
9940 :
9941 : void
9942 78406 : FEProblemBase::setCurrentLinearSystem(const unsigned int sys_num)
9943 : {
9944 : mooseAssert(sys_num < _linear_systems.size(),
9945 : "System number greater than the number of linear systems");
9946 78406 : _current_linear_sys = _linear_systems[sys_num].get();
9947 78406 : _current_solver_sys = _current_linear_sys;
9948 78406 : }
9949 :
9950 : void
9951 5843139 : FEProblemBase::computeSystems(const ExecFlagType & type)
9952 : {
9953 : // When performing an adjoint solve in the optimization module, the current solver system is the
9954 : // adjoint. However, the adjoint solve requires having accurate time derivative calculations for
9955 : // the forward system. The cleanest way to handle such uses is just to compute the time
9956 : // derivatives for all solver systems instead of trying to guess which ones we need and don't need
9957 11829994 : for (auto & solver_sys : _solver_systems)
9958 5986855 : solver_sys->compute(type);
9959 :
9960 5843139 : _aux->compute(type);
9961 5843106 : }
9962 :
9963 : const ConstElemRange &
9964 3934897 : FEProblemBase::getCurrentAlgebraicElementRange()
9965 : {
9966 3934897 : if (!_current_algebraic_elem_range)
9967 3934897 : return *_mesh.getActiveLocalElementRange();
9968 :
9969 0 : return *_current_algebraic_elem_range;
9970 : }
9971 : const ConstNodeRange &
9972 95995 : FEProblemBase::getCurrentAlgebraicNodeRange()
9973 : {
9974 95995 : if (!_current_algebraic_node_range)
9975 95995 : return *_mesh.getLocalNodeRange();
9976 :
9977 0 : return *_current_algebraic_node_range;
9978 : }
9979 : const ConstBndNodeRange &
9980 3483433 : FEProblemBase::getCurrentAlgebraicBndNodeRange()
9981 : {
9982 3483433 : if (!_current_algebraic_bnd_node_range)
9983 3483433 : return *_mesh.getBoundaryNodeRange();
9984 :
9985 0 : return *_current_algebraic_bnd_node_range;
9986 : }
9987 :
9988 : void
9989 0 : FEProblemBase::setCurrentAlgebraicElementRange(ConstElemRange * range)
9990 : {
9991 0 : if (!range)
9992 : {
9993 0 : _current_algebraic_elem_range = nullptr;
9994 0 : return;
9995 : }
9996 :
9997 0 : _current_algebraic_elem_range = std::make_unique<ConstElemRange>(*range);
9998 : }
9999 : void
10000 0 : FEProblemBase::setCurrentAlgebraicNodeRange(ConstNodeRange * range)
10001 : {
10002 0 : if (!range)
10003 : {
10004 0 : _current_algebraic_node_range = nullptr;
10005 0 : return;
10006 : }
10007 :
10008 0 : _current_algebraic_node_range = std::make_unique<ConstNodeRange>(*range);
10009 : }
10010 : void
10011 0 : FEProblemBase::setCurrentAlgebraicBndNodeRange(ConstBndNodeRange * range)
10012 : {
10013 0 : if (!range)
10014 : {
10015 0 : _current_algebraic_bnd_node_range = nullptr;
10016 0 : return;
10017 : }
10018 :
10019 0 : _current_algebraic_bnd_node_range = std::make_unique<ConstBndNodeRange>(*range);
10020 : }
10021 :
10022 : unsigned short
10023 58741 : FEProblemBase::getCurrentICState()
10024 : {
10025 58741 : return _current_ic_state;
10026 : }
10027 :
10028 : std::string
10029 53459 : FEProblemBase::solverTypeString(const unsigned int solver_sys_num)
10030 : {
10031 53459 : return Moose::stringify(solverParams(solver_sys_num)._type);
10032 : }
10033 :
10034 : SolverParams
10035 1032 : FEProblemBase::makeLinearSolverParams()
10036 : {
10037 1032 : SolverParams solver_params;
10038 1032 : solver_params._type = Moose::SolveType::ST_LINEAR;
10039 1032 : solver_params._line_search = Moose::LineSearchType::LS_NONE;
10040 1032 : return solver_params;
10041 : }
10042 :
10043 : const libMesh::CouplingMatrix &
10044 71601 : FEProblemBase::nonlocalCouplingMatrix(const unsigned i) const
10045 : {
10046 71601 : return _nonlocal_cm[i];
10047 : }
10048 :
10049 : bool
10050 90569590 : FEProblemBase::checkNonlocalCouplingRequirement() const
10051 : {
10052 90569590 : return _requires_nonlocal_coupling;
10053 : }
10054 :
10055 : const std::unordered_map<std::pair<BoundaryID, BoundaryID>,
10056 : std::unique_ptr<AutomaticMortarGeneration>> &
10057 117889 : FEProblemBase::getMortarInterfaces(bool on_displaced) const
10058 : {
10059 117889 : return _mortar_data->getMortarInterfaces(on_displaced);
10060 : }
|
