Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #ifdef MOOSE_KOKKOS_ENABLED
11 : #include "KokkosMaterialPropertyStorage.h"
12 : #endif
13 :
14 : #include "FEProblemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialPropertyStorage.h"
17 : #include "MooseEnum.h"
18 : #include "Factory.h"
19 : #include "MooseUtils.h"
20 : #include "DisplacedProblem.h"
21 : #include "SystemBase.h"
22 : #include "MaterialData.h"
23 : #include "ComputeUserObjectsThread.h"
24 : #include "ComputeNodalUserObjectsThread.h"
25 : #include "ComputeThreadedGeneralUserObjectsThread.h"
26 : #include "ComputeMaterialsObjectThread.h"
27 : #include "ProjectMaterialProperties.h"
28 : #include "ComputeIndicatorThread.h"
29 : #include "ComputeMarkerThread.h"
30 : #include "ComputeInitialConditionThread.h"
31 : #include "ComputeFVInitialConditionThread.h"
32 : #include "ComputeBoundaryInitialConditionThread.h"
33 : #include "MaxQpsThread.h"
34 : #include "ActionWarehouse.h"
35 : #include "Conversion.h"
36 : #include "Material.h"
37 : #include "FunctorMaterial.h"
38 : #include "ConstantIC.h"
39 : #include "Parser.h"
40 : #include "ElementH1Error.h"
41 : #include "Function.h"
42 : #include "Convergence.h"
43 : #include "NonlinearSystem.h"
44 : #include "LinearSystem.h"
45 : #include "SolverSystem.h"
46 : #include "Distribution.h"
47 : #include "Sampler.h"
48 : #include "PetscSupport.h"
49 : #include "RandomInterface.h"
50 : #include "RandomData.h"
51 : #include "MooseEigenSystem.h"
52 : #include "MooseParsedFunction.h"
53 : #include "MeshChangedInterface.h"
54 : #include "MeshDisplacedInterface.h"
55 : #include "ComputeJacobianBlocksThread.h"
56 : #include "ScalarInitialCondition.h"
57 : #include "FVInitialConditionTempl.h"
58 : #include "ElementPostprocessor.h"
59 : #include "NodalPostprocessor.h"
60 : #include "SidePostprocessor.h"
61 : #include "InternalSidePostprocessor.h"
62 : #include "InterfacePostprocessor.h"
63 : #include "GeneralPostprocessor.h"
64 : #include "ElementVectorPostprocessor.h"
65 : #include "NodalVectorPostprocessor.h"
66 : #include "SideVectorPostprocessor.h"
67 : #include "InternalSideVectorPostprocessor.h"
68 : #include "GeneralVectorPostprocessor.h"
69 : #include "Positions.h"
70 : #include "Indicator.h"
71 : #include "Marker.h"
72 : #include "MultiApp.h"
73 : #include "MultiAppTransfer.h"
74 : #include "TransientMultiApp.h"
75 : #include "ElementUserObject.h"
76 : #include "DomainUserObject.h"
77 : #include "NodalUserObject.h"
78 : #include "SideUserObject.h"
79 : #include "InternalSideUserObject.h"
80 : #include "InterfaceUserObject.h"
81 : #include "GeneralUserObject.h"
82 : #include "ThreadedGeneralUserObject.h"
83 : #include "InternalSideIndicatorBase.h"
84 : #include "Transfer.h"
85 : #include "MultiAppTransfer.h"
86 : #include "MultiMooseEnum.h"
87 : #include "Predictor.h"
88 : #include "Assembly.h"
89 : #include "Control.h"
90 : #include "XFEMInterface.h"
91 : #include "ConsoleUtils.h"
92 : #include "NonlocalKernel.h"
93 : #include "NonlocalIntegratedBC.h"
94 : #include "ShapeElementUserObject.h"
95 : #include "ShapeSideUserObject.h"
96 : #include "MooseVariableFE.h"
97 : #include "MooseVariableScalar.h"
98 : #include "InputParameterWarehouse.h"
99 : #include "TimeIntegrator.h"
100 : #include "LineSearch.h"
101 : #include "FloatingPointExceptionGuard.h"
102 : #include "MaxVarNDofsPerElem.h"
103 : #include "MaxVarNDofsPerNode.h"
104 : #include "FVKernel.h"
105 : #include "LinearFVKernel.h"
106 : #include "FVTimeKernel.h"
107 : #include "MooseVariableFV.h"
108 : #include "MooseLinearVariableFV.h"
109 : #include "FVBoundaryCondition.h"
110 : #include "LinearFVBoundaryCondition.h"
111 : #include "FVInterfaceKernel.h"
112 : #include "Reporter.h"
113 : #include "ADUtils.h"
114 : #include "Executioner.h"
115 : #include "VariadicTable.h"
116 : #include "BoundaryNodeIntegrityCheckThread.h"
117 : #include "BoundaryElemIntegrityCheckThread.h"
118 : #include "NodalBCBase.h"
119 : #include "MortarUserObject.h"
120 : #include "MortarUserObjectThread.h"
121 : #include "RedistributeProperties.h"
122 : #include "Checkpoint.h"
123 : #include "MortarInterfaceWarehouse.h"
124 : #include "AutomaticMortarGeneration.h"
125 :
126 : #include "libmesh/exodusII_io.h"
127 : #include "libmesh/quadrature.h"
128 : #include "libmesh/coupling_matrix.h"
129 : #include "libmesh/nonlinear_solver.h"
130 : #include "libmesh/sparse_matrix.h"
131 : #include "libmesh/string_to_enum.h"
132 : #include "libmesh/fe_interface.h"
133 : #include "libmesh/enum_norm_type.h"
134 : #include "libmesh/petsc_solver_exception.h"
135 :
136 : #include "metaphysicl/dualnumber.h"
137 :
138 : using namespace libMesh;
139 :
140 : // Anonymous namespace for helper function
141 : namespace
142 : {
143 : /**
144 : * Method for sorting the MooseVariableFEBases based on variable numbers
145 : */
146 : bool
147 70 : sortMooseVariables(const MooseVariableFEBase * a, const MooseVariableFEBase * b)
148 : {
149 70 : return a->number() < b->number();
150 : }
151 : } // namespace
152 :
153 : Threads::spin_mutex get_function_mutex;
154 :
155 : InputParameters
156 354527 : FEProblemBase::validParams()
157 : {
158 354527 : InputParameters params = SubProblem::validParams();
159 1418108 : params.addParam<unsigned int>("null_space_dimension", 0, "The dimension of the nullspace");
160 1063581 : params.addParam<unsigned int>(
161 709054 : "transpose_null_space_dimension", 0, "The dimension of the transpose nullspace");
162 1063581 : params.addParam<unsigned int>(
163 709054 : "near_null_space_dimension", 0, "The dimension of the near nullspace");
164 1063581 : params.addParam<bool>("solve",
165 709054 : true,
166 : "Whether or not to actually solve the Nonlinear system. "
167 : "This is handy in the case that all you want to do is "
168 : "execute AuxKernels, Transfers, etc. without actually "
169 : "solving anything");
170 1063581 : params.addParam<bool>("use_nonlinear",
171 709054 : true,
172 : "Determines whether to use a Nonlinear vs a "
173 : "Eigenvalue system (Automatically determined based "
174 : "on executioner)");
175 1418108 : params.addParam<bool>("error_on_jacobian_nonzero_reallocation",
176 : "This causes PETSc to error if it had to reallocate memory in the Jacobian "
177 : "matrix due to not having enough nonzeros");
178 1063581 : params.addParam<bool>("ignore_zeros_in_jacobian",
179 709054 : false,
180 : "Do not explicitly store zero values in "
181 : "the Jacobian matrix if true");
182 1063581 : params.addParam<bool>("force_restart",
183 709054 : false,
184 : "EXPERIMENTAL: If true, a sub_app may use a "
185 : "restart file instead of using of using the master "
186 : "backup file");
187 1772635 : params.addDeprecatedParam<bool>("skip_additional_restart_data",
188 709054 : false,
189 : "True to skip additional data in equation system for restart.",
190 : "This parameter is no longer used, as we do not load additional "
191 : "vectors by default with restart");
192 1063581 : params.addParam<bool>("skip_nl_system_check",
193 709054 : false,
194 : "True to skip the NonlinearSystem check for work to do (e.g. Make sure "
195 : "that there are variables to solve for).");
196 1063581 : params.addParam<bool>("allow_initial_conditions_with_restart",
197 709054 : false,
198 : "True to allow the user to specify initial conditions when restarting. "
199 : "Initial conditions can override any restarted field");
200 :
201 709054 : auto coverage_check_description = [](std::string scope, std::string list_param_name)
202 : {
203 1418108 : return "Controls, if and how a " + scope +
204 : " subdomain coverage check is performed. "
205 : "With 'TRUE' or 'ON' all subdomains are checked (the default). Setting 'FALSE' or 'OFF' "
206 : "will disable the check for all subdomains. "
207 : "To exclude a predefined set of subdomains 'SKIP_LIST' is to "
208 1418108 : "be used, while the subdomains to skip are to be defined in the parameter '" +
209 1418108 : list_param_name +
210 : "'. To limit the check to a list of subdomains, 'ONLY_LIST' is to "
211 2127162 : "be used (again, using the parameter '" +
212 1418108 : list_param_name + "').";
213 : };
214 :
215 1772635 : params.addParam<std::vector<SubdomainName>>(
216 : "block",
217 : {"ANY_BLOCK_ID"},
218 : "List of subdomains for kernel coverage and material coverage checks. Setting this parameter "
219 : "is equivalent to setting 'kernel_coverage_block_list' and 'material_coverage_block_list' as "
220 : "well as using 'ONLY_LIST' as the coverage check mode.");
221 :
222 1418108 : MooseEnum kernel_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
223 354527 : params.addParam<MooseEnum>("kernel_coverage_check",
224 : kernel_coverage_check_modes,
225 1772635 : coverage_check_description("kernel", "kernel_coverage_block_list"));
226 1418108 : params.addParam<std::vector<SubdomainName>>(
227 : "kernel_coverage_block_list",
228 : {},
229 : "List of subdomains for kernel coverage check. The meaning of this list is controlled by the "
230 : "parameter 'kernel_coverage_check' (whether this is the list of subdomains to be checked, "
231 : "not to be checked or not taken into account).");
232 1063581 : params.addParam<bool>(
233 : "boundary_restricted_node_integrity_check",
234 709054 : true,
235 : "Set to false to disable checking of boundary restricted nodal object variable dependencies, "
236 : "e.g. are the variable dependencies defined on the selected boundaries?");
237 1063581 : params.addParam<bool>("boundary_restricted_elem_integrity_check",
238 709054 : true,
239 : "Set to false to disable checking of boundary restricted elemental object "
240 : "variable dependencies, e.g. are the variable dependencies defined on the "
241 : "selected boundaries?");
242 1418108 : MooseEnum material_coverage_check_modes("FALSE TRUE OFF ON SKIP_LIST ONLY_LIST", "TRUE");
243 354527 : params.addParam<MooseEnum>(
244 : "material_coverage_check",
245 : material_coverage_check_modes,
246 1772635 : coverage_check_description("material", "material_coverage_block_list"));
247 1418108 : params.addParam<std::vector<SubdomainName>>(
248 : "material_coverage_block_list",
249 : {},
250 : "List of subdomains for material coverage check. The meaning of this list is controlled by "
251 : "the parameter 'material_coverage_check' (whether this is the list of subdomains to be "
252 : "checked, not to be checked or not taken into account).");
253 :
254 1063581 : params.addParam<bool>("fv_bcs_integrity_check",
255 709054 : true,
256 : "Set to false to disable checking of overlapping Dirichlet and Flux BCs "
257 : "and/or multiple DirichletBCs per sideset");
258 :
259 1063581 : params.addParam<bool>(
260 709054 : "material_dependency_check", true, "Set to false to disable material dependency check");
261 1063581 : params.addParam<bool>("parallel_barrier_messaging",
262 709054 : false,
263 : "Displays messaging from parallel "
264 : "barrier notifications when executing "
265 : "or transferring to/from Multiapps "
266 : "(default: false)");
267 :
268 1418108 : MooseEnum verbosity("false true extra", "false");
269 1418108 : params.addParam<MooseEnum>("verbose_setup",
270 : verbosity,
271 : "Set to 'true' to have the problem report on any object created. Set "
272 : "to 'extra' to also display all parameters.");
273 1063581 : params.addParam<bool>("verbose_multiapps",
274 709054 : false,
275 : "Set to True to enable verbose screen printing related to MultiApps");
276 1063581 : params.addParam<bool>(
277 : "verbose_restore",
278 709054 : false,
279 : "Set to True to enable verbose screen printing related to solution restoration");
280 :
281 1418108 : params.addParam<FileNameNoExtension>("restart_file_base",
282 : "File base name used for restart (e.g. "
283 : "<path>/<filebase> or <path>/LATEST to "
284 : "grab the latest file available)");
285 :
286 1418108 : params.addParam<std::vector<std::vector<TagName>>>(
287 : "extra_tag_vectors",
288 : {},
289 : "Extra vectors to add to the system that can be filled by objects which compute residuals "
290 : "and Jacobians (Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
291 : "nonlinear system the extra tag vectors should be added for");
292 :
293 1418108 : params.addParam<std::vector<std::vector<TagName>>>(
294 : "not_zeroed_tag_vectors",
295 : {},
296 : "Extra vector tags which the sytem will not zero when other vector tags are zeroed. "
297 : "The outer index is for which nonlinear system the extra tag vectors should be added for");
298 :
299 1418108 : params.addParam<std::vector<std::vector<TagName>>>(
300 : "extra_tag_matrices",
301 : {},
302 : "Extra matrices to add to the system that can be filled "
303 : "by objects which compute residuals and Jacobians "
304 : "(Kernels, BCs, etc.) by setting tags on them. The outer index is for which "
305 : "nonlinear system the extra tag vectors should be added for");
306 :
307 1418108 : params.addParam<std::vector<TagName>>(
308 : "extra_tag_solutions",
309 : {},
310 : "Extra solution vectors to add to the system that can be used by "
311 : "objects for coupling variable values stored in them.");
312 :
313 1063581 : params.addParam<bool>("previous_nl_solution_required",
314 709054 : false,
315 : "True to indicate that this calculation requires a solution vector for "
316 : "storing the previous nonlinear iteration.");
317 :
318 1063581 : params.addParam<std::vector<NonlinearSystemName>>(
319 1418108 : "nl_sys_names", std::vector<NonlinearSystemName>{"nl0"}, "The nonlinear system names");
320 :
321 1418108 : params.addParam<std::vector<LinearSystemName>>("linear_sys_names", {}, "The linear system names");
322 :
323 1063581 : params.addParam<bool>("check_uo_aux_state",
324 709054 : false,
325 : "True to turn on a check that no state presents during the evaluation of "
326 : "user objects and aux kernels");
327 :
328 354527 : params.addPrivateParam<MooseMesh *>("mesh");
329 :
330 1063581 : params.declareControllable("solve");
331 :
332 1063581 : params.addParam<bool>(
333 : "allow_invalid_solution",
334 709054 : false,
335 : "Set to true to allow convergence even though the solution has been marked as 'invalid'");
336 1063581 : params.addParam<bool>("show_invalid_solution_console",
337 709054 : true,
338 : "Set to true to show the invalid solution occurance summary in console");
339 1063581 : params.addParam<bool>("immediately_print_invalid_solution",
340 709054 : false,
341 : "Whether or not to report invalid solution warnings at the time the "
342 : "warning is produced instead of after the calculation");
343 :
344 1063581 : params.addParam<bool>(
345 : "identify_variable_groups_in_nl",
346 709054 : true,
347 : "Whether to identify variable groups in nonlinear systems. This affects dof ordering");
348 :
349 1063581 : params.addParam<bool>(
350 : "regard_general_exceptions_as_errors",
351 709054 : false,
352 : "If we catch an exception during residual/Jacobian evaluaton for which we don't have "
353 : "specific handling, immediately error instead of allowing the time step to be cut");
354 :
355 1063581 : params.addParam<bool>("use_hash_table_matrix_assembly",
356 709054 : false,
357 : "Whether to assemble matrices using hash tables instead of preallocating "
358 : "matrix memory. This can be a good option if the sparsity pattern changes "
359 : "throughout the course of the simulation.");
360 1418108 : params.addParam<bool>(
361 : "restore_original_nonzero_pattern",
362 : "Whether we should reset matrix memory for every Jacobian evaluation. This option is useful "
363 : "if the sparsity pattern is constantly changing and you are using hash table assembly or if "
364 : "you wish to continually restore the matrix to the originally preallocated sparsity pattern "
365 : "computed by relationship managers.");
366 :
367 1418108 : params.addParamNamesToGroup(
368 : "skip_nl_system_check kernel_coverage_check kernel_coverage_block_list "
369 : "boundary_restricted_node_integrity_check "
370 : "boundary_restricted_elem_integrity_check material_coverage_check "
371 : "material_coverage_block_list fv_bcs_integrity_check "
372 : "material_dependency_check check_uo_aux_state error_on_jacobian_nonzero_reallocation",
373 : "Simulation checks");
374 1418108 : params.addParamNamesToGroup("use_nonlinear previous_nl_solution_required nl_sys_names "
375 : "ignore_zeros_in_jacobian identify_variable_groups_in_nl "
376 : "use_hash_table_matrix_assembly restore_original_nonzero_pattern",
377 : "Nonlinear system(s)");
378 1418108 : params.addParamNamesToGroup(
379 : "restart_file_base force_restart allow_initial_conditions_with_restart", "Restart");
380 1418108 : params.addParamNamesToGroup(
381 : "verbose_setup verbose_multiapps verbose_restore parallel_barrier_messaging", "Verbosity");
382 1418108 : params.addParamNamesToGroup(
383 : "null_space_dimension transpose_null_space_dimension near_null_space_dimension",
384 : "Null space removal");
385 1418108 : params.addParamNamesToGroup(
386 : "extra_tag_vectors extra_tag_matrices extra_tag_solutions not_zeroed_tag_vectors",
387 : "Contribution to tagged field data");
388 1063581 : params.addParamNamesToGroup(
389 : "allow_invalid_solution show_invalid_solution_console immediately_print_invalid_solution",
390 : "Solution validity control");
391 :
392 709054 : return params;
393 1063581 : }
394 :
395 64824 : FEProblemBase::FEProblemBase(const InputParameters & parameters)
396 : : SubProblem(parameters),
397 : Restartable(this, "FEProblemBase"),
398 259296 : _mesh(*getCheckedPointerParam<MooseMesh *>("mesh")),
399 129648 : _req(declareManagedRestartableDataWithContext<RestartableEquationSystems>(
400 : "equation_systems", nullptr, _mesh)),
401 64824 : _initialized(false),
402 129648 : _solve(getParam<bool>("solve")),
403 64824 : _transient(false),
404 129648 : _time(declareRestartableData<Real>("time")),
405 129648 : _time_old(declareRestartableData<Real>("time_old")),
406 129648 : _t_step(declareRecoverableData<int>("t_step")),
407 129648 : _dt(declareRestartableData<Real>("dt")),
408 129648 : _dt_old(declareRestartableData<Real>("dt_old")),
409 64824 : _need_to_add_default_nonlinear_convergence(false),
410 64824 : _need_to_add_default_multiapp_fixed_point_convergence(false),
411 64824 : _need_to_add_default_steady_state_convergence(false),
412 129648 : _linear_sys_names(getParam<std::vector<LinearSystemName>>("linear_sys_names")),
413 64824 : _num_linear_sys(_linear_sys_names.size()),
414 129648 : _linear_systems(_num_linear_sys, nullptr),
415 64824 : _current_linear_sys(nullptr),
416 129648 : _using_default_nl(!isParamSetByUser("nl_sys_names")),
417 191918 : _nl_sys_names(!_using_default_nl || (_using_default_nl && !_linear_sys_names.size())
418 64824 : ? getParam<std::vector<NonlinearSystemName>>("nl_sys_names")
419 : : std::vector<NonlinearSystemName>()),
420 64824 : _num_nl_sys(_nl_sys_names.size()),
421 129648 : _nl(_num_nl_sys, nullptr),
422 64824 : _current_nl_sys(nullptr),
423 129648 : _solver_systems(_num_nl_sys + _num_linear_sys, nullptr),
424 64824 : _aux(nullptr),
425 64824 : _coupling(Moose::COUPLING_DIAG),
426 : #ifdef MOOSE_KOKKOS_ENABLED
427 43245 : _kokkos_assembly(*this),
428 : #endif
429 64824 : _mesh_divisions(/*threaded=*/true),
430 64824 : _material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
431 64824 : "material_props", &_mesh, _material_prop_registry, *this)),
432 64824 : _bnd_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
433 64824 : "bnd_material_props", &_mesh, _material_prop_registry, *this)),
434 64824 : _neighbor_material_props(declareRestartableDataWithContext<MaterialPropertyStorage>(
435 64824 : "neighbor_material_props", &_mesh, _material_prop_registry, *this)),
436 : #ifdef MOOSE_KOKKOS_ENABLED
437 43245 : _kokkos_material_props(
438 43245 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
439 43245 : "kokkos_material_props", &_mesh, _material_prop_registry, *this)),
440 43245 : _kokkos_bnd_material_props(
441 43245 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
442 43245 : "kokkos_bnd_material_props", &_mesh, _material_prop_registry, *this)),
443 43245 : _kokkos_neighbor_material_props(
444 43245 : declareRestartableDataWithContext<Moose::Kokkos::MaterialPropertyStorage>(
445 43245 : "kokkos_neighbor_material_props", &_mesh, _material_prop_registry, *this)),
446 : #endif
447 64824 : _reporter_data(_app),
448 : // TODO: delete the following line after apps have been updated to not call getUserObjects
449 64824 : _all_user_objects(_app.getExecuteOnEnum()),
450 64824 : _multi_apps(_app.getExecuteOnEnum()),
451 64824 : _transient_multi_apps(_app.getExecuteOnEnum()),
452 64824 : _transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
453 64824 : _to_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
454 64824 : _from_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
455 64824 : _between_multi_app_transfers(_app.getExecuteOnEnum(), /*threaded=*/false),
456 : #ifdef LIBMESH_ENABLE_AMR
457 64824 : _adaptivity(*this),
458 64824 : _cycles_completed(0),
459 : #endif
460 64824 : _displaced_mesh(nullptr),
461 64824 : _geometric_search_data(*this, _mesh),
462 64824 : _mortar_data(std::make_unique<MortarInterfaceWarehouse>(*this)),
463 64824 : _reinit_displaced_elem(false),
464 64824 : _reinit_displaced_face(false),
465 64824 : _reinit_displaced_neighbor(false),
466 64824 : _input_file_saved(false),
467 64824 : _has_dampers(false),
468 64824 : _has_constraints(false),
469 64824 : _snesmf_reuse_base(true),
470 64824 : _skip_exception_check(false),
471 64824 : _snesmf_reuse_base_set_by_user(false),
472 64824 : _has_initialized_stateful(false),
473 64824 : _const_jacobian(false),
474 64824 : _has_jacobian(false),
475 64824 : _needs_old_newton_iter(false),
476 129648 : _previous_nl_solution_required(getParam<bool>("previous_nl_solution_required")),
477 64824 : _has_nonlocal_coupling(false),
478 64824 : _calculate_jacobian_in_uo(false),
479 64824 : _kernel_coverage_check(
480 129648 : getParam<MooseEnum>("kernel_coverage_check").getEnum<CoverageCheckMode>()),
481 129648 : _kernel_coverage_blocks(getParam<std::vector<SubdomainName>>("kernel_coverage_block_list")),
482 64824 : _boundary_restricted_node_integrity_check(
483 129648 : getParam<bool>("boundary_restricted_node_integrity_check")),
484 64824 : _boundary_restricted_elem_integrity_check(
485 129648 : getParam<bool>("boundary_restricted_elem_integrity_check")),
486 64824 : _material_coverage_check(
487 129648 : getParam<MooseEnum>("material_coverage_check").getEnum<CoverageCheckMode>()),
488 129648 : _material_coverage_blocks(getParam<std::vector<SubdomainName>>("material_coverage_block_list")),
489 129648 : _fv_bcs_integrity_check(getParam<bool>("fv_bcs_integrity_check")),
490 129648 : _material_dependency_check(getParam<bool>("material_dependency_check")),
491 129648 : _uo_aux_state_check(getParam<bool>("check_uo_aux_state")),
492 64824 : _max_qps(std::numeric_limits<unsigned int>::max()),
493 64824 : _max_scalar_order(INVALID_ORDER),
494 64824 : _has_time_integrator(false),
495 64824 : _has_exception(false),
496 129648 : _parallel_barrier_messaging(getParam<bool>("parallel_barrier_messaging")),
497 129648 : _verbose_setup(getParam<MooseEnum>("verbose_setup")),
498 129648 : _verbose_multiapps(getParam<bool>("verbose_multiapps")),
499 129648 : _verbose_restore(getParam<bool>("verbose_restore")),
500 64824 : _current_execute_on_flag(EXEC_NONE),
501 64824 : _control_warehouse(_app.getExecuteOnEnum(), /*threaded=*/false),
502 64824 : _is_petsc_options_inserted(false),
503 64824 : _line_search(nullptr),
504 64824 : _using_ad_mat_props(false),
505 64824 : _current_ic_state(0),
506 129648 : _use_hash_table_matrix_assembly(getParam<bool>("use_hash_table_matrix_assembly")),
507 64824 : _error_on_jacobian_nonzero_reallocation(
508 129648 : isParamValid("error_on_jacobian_nonzero_reallocation")
509 130254 : ? getParam<bool>("error_on_jacobian_nonzero_reallocation")
510 64218 : : _app.errorOnJacobianNonzeroReallocation()),
511 129648 : _restore_original_nonzero_pattern(isParamValid("restore_original_nonzero_pattern")
512 129648 : ? getParam<bool>("restore_original_nonzero_pattern")
513 64824 : : _use_hash_table_matrix_assembly),
514 129648 : _ignore_zeros_in_jacobian(getParam<bool>("ignore_zeros_in_jacobian")),
515 64824 : _preserve_matrix_sparsity_pattern(true),
516 129648 : _force_restart(getParam<bool>("force_restart")),
517 129648 : _allow_ics_during_restart(getParam<bool>("allow_initial_conditions_with_restart")),
518 129648 : _skip_nl_system_check(getParam<bool>("skip_nl_system_check")),
519 64824 : _fail_next_system_convergence_check(false),
520 129648 : _allow_invalid_solution(getParam<bool>("allow_invalid_solution")),
521 129648 : _show_invalid_solution_console(getParam<bool>("show_invalid_solution_console")),
522 129648 : _immediately_print_invalid_solution(getParam<bool>("immediately_print_invalid_solution")),
523 64824 : _started_initial_setup(false),
524 64824 : _has_internal_edge_residual_objects(false),
525 64824 : _u_dot_requested(false),
526 64824 : _u_dotdot_requested(false),
527 64824 : _u_dot_old_requested(false),
528 64824 : _u_dotdot_old_requested(false),
529 64824 : _has_mortar(false),
530 64824 : _num_grid_steps(0),
531 64824 : _print_execution_on(),
532 129648 : _identify_variable_groups_in_nl(getParam<bool>("identify_variable_groups_in_nl")),
533 64824 : _regard_general_exceptions_as_errors(getParam<bool>("regard_general_exceptions_as_errors")),
534 1231656 : _requires_nonlocal_coupling(false)
535 : {
536 : auto checkCoverageCheckConflict =
537 129648 : [this](const std::string & coverage_check,
538 : const CoverageCheckMode & coverage_check_mode,
539 : const std::vector<SubdomainName> & coverage_blocks) -> void
540 : {
541 129648 : if (coverage_check_mode != CoverageCheckMode::FALSE &&
542 124888 : coverage_check_mode != CoverageCheckMode::OFF)
543 124867 : if (coverage_blocks.size() > 1)
544 0 : if (std::find(coverage_blocks.begin(), coverage_blocks.end(), "ANY_BLOCK_ID") !=
545 0 : coverage_blocks.end())
546 0 : paramError(coverage_check,
547 : "The list of blocks used for ",
548 : coverage_check,
549 : " cannot contain 'ANY_BLOCK_ID' along with other blocks. ");
550 129648 : };
551 :
552 64824 : checkCoverageCheckConflict(
553 64824 : "kernel_coverage_check", _kernel_coverage_check, _kernel_coverage_blocks);
554 64824 : checkCoverageCheckConflict(
555 64824 : "material_coverage_check", _material_coverage_check, _material_coverage_blocks);
556 :
557 : // Initialize static do_derivatives member. We initialize this to true so that all the
558 : // default AD things that we setup early in the simulation actually get their derivative
559 : // vectors initalized. We will toggle this to false when doing residual evaluations
560 64824 : ADReal::do_derivatives = true;
561 :
562 : // Disable refinement/coarsening in EquationSystems::reinit because we already do this ourselves
563 64824 : es().disable_refine_in_reinit();
564 :
565 64824 : _solver_params.reserve(_num_nl_sys + _num_linear_sys);
566 : // Default constructor fine for nonlinear because it will be populated later by framework
567 : // executioner/solve object parameters
568 64824 : _solver_params.resize(_num_nl_sys);
569 128611 : for (const auto i : index_range(_nl_sys_names))
570 : {
571 63787 : const auto & name = _nl_sys_names[i];
572 63787 : _nl_sys_name_to_num[name] = i;
573 63787 : _solver_sys_name_to_num[name] = i;
574 63787 : _solver_sys_names.push_back(name);
575 : }
576 :
577 66164 : for (const auto i : index_range(_linear_sys_names))
578 : {
579 1340 : const auto & name = _linear_sys_names[i];
580 1340 : _linear_sys_name_to_num[name] = i;
581 1340 : _solver_sys_name_to_num[name] = i + _num_nl_sys;
582 1340 : _solver_sys_names.push_back(name);
583 : // Unlike for nonlinear these are basically dummy parameters
584 1340 : _solver_params.push_back(makeLinearSolverParams());
585 : }
586 :
587 64824 : _nonlocal_cm.resize(numSolverSystems());
588 64824 : _cm.resize(numSolverSystems());
589 :
590 64824 : _time = 0.0;
591 64824 : _time_old = 0.0;
592 64824 : _t_step = 0;
593 64824 : _dt = 0;
594 64824 : _dt_old = _dt;
595 :
596 64824 : unsigned int n_threads = libMesh::n_threads();
597 :
598 64824 : _real_zero.resize(n_threads, 0.);
599 64824 : _scalar_zero.resize(n_threads);
600 64824 : _zero.resize(n_threads);
601 64824 : _phi_zero.resize(n_threads);
602 64824 : _ad_zero.resize(n_threads);
603 64824 : _grad_zero.resize(n_threads);
604 64824 : _ad_grad_zero.resize(n_threads);
605 64824 : _grad_phi_zero.resize(n_threads);
606 64824 : _second_zero.resize(n_threads);
607 64824 : _ad_second_zero.resize(n_threads);
608 64824 : _second_phi_zero.resize(n_threads);
609 64824 : _point_zero.resize(n_threads);
610 64824 : _vector_zero.resize(n_threads);
611 64824 : _vector_curl_zero.resize(n_threads);
612 64824 : _uo_jacobian_moose_vars.resize(n_threads);
613 :
614 64824 : _has_active_material_properties.resize(n_threads, 0);
615 :
616 64824 : _block_mat_side_cache.resize(n_threads);
617 64824 : _bnd_mat_side_cache.resize(n_threads);
618 64824 : _interface_mat_side_cache.resize(n_threads);
619 :
620 129648 : es().parameters.set<FEProblemBase *>("_fe_problem_base") = this;
621 :
622 194472 : if (isParamValid("restart_file_base"))
623 : {
624 1018 : std::string restart_file_base = getParam<FileNameNoExtension>("restart_file_base");
625 :
626 : // This check reverts to old behavior of providing "restart_file_base=" to mean
627 : // don't restart... BISON currently relies on this. It could probably be removed.
628 : // The new MooseUtils::convertLatestCheckpoint will error out if a checkpoint file
629 : // is not found, which I think makes sense. Which means, without this, if you
630 : // set "restart_file_base=", you'll get a "No checkpoint file found" error
631 509 : if (restart_file_base.size())
632 : {
633 509 : restart_file_base = MooseUtils::convertLatestCheckpoint(restart_file_base);
634 509 : setRestartFile(restart_file_base);
635 : }
636 509 : }
637 :
638 : // // Generally speaking, the mesh is prepared for use, and consequently remote elements are deleted
639 : // // well before our Problem(s) are constructed. Historically, in MooseMesh we have a bunch of
640 : // // needs_prepare type flags that make it so we never call prepare_for_use (and consequently
641 : // // delete_remote_elements) again. So the below line, historically, has had no impact. HOWEVER:
642 : // // I've added some code in SetupMeshCompleteAction for deleting remote elements post
643 : // // EquationSystems::init. If I execute that code without default ghosting, then I get > 40 MOOSE
644 : // // test failures, so we clearly have some simulations that are not yet covered properly by
645 : // // relationship managers. Until that is resolved, I am going to retain default geometric ghosting
646 : // if (!_default_ghosting)
647 : // _mesh.getMesh().remove_ghosting_functor(_mesh.getMesh().default_ghosting());
648 :
649 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
650 : // Main app should hold the default database to handle system petsc options
651 64824 : if (!_app.isUltimateMaster())
652 12471 : LibmeshPetscCall(PetscOptionsCreate(&_petsc_option_data_base));
653 : #endif
654 :
655 64824 : if (!_solve)
656 : {
657 : // If we are not solving, we do not care about seeing unused petsc options
658 47427 : Moose::PetscSupport::setSinglePetscOption("-options_left", "0");
659 : // We don't want petscSetOptions being called in solve and clearing the option that was just set
660 15809 : _is_petsc_options_inserted = true;
661 : }
662 64824 : }
663 :
664 : const MooseMesh &
665 322586 : FEProblemBase::mesh(bool use_displaced) const
666 : {
667 322586 : if (use_displaced && !_displaced_problem)
668 0 : mooseWarning("Displaced mesh was requested but the displaced problem does not exist. "
669 : "Regular mesh will be returned");
670 322586 : return ((use_displaced && _displaced_problem) ? _displaced_problem->mesh() : mesh());
671 : }
672 :
673 : void
674 64824 : FEProblemBase::createTagVectors()
675 : {
676 : // add vectors and their tags to system
677 129648 : auto & vectors = getParam<std::vector<std::vector<TagName>>>("extra_tag_vectors");
678 65549 : for (const auto sys_num : index_range(vectors))
679 1694 : for (auto & vector : vectors[sys_num])
680 : {
681 969 : auto tag = addVectorTag(vector);
682 969 : _solver_systems[sys_num]->addVector(tag, false, libMesh::GHOSTED);
683 : }
684 :
685 129648 : auto & not_zeroed_vectors = getParam<std::vector<std::vector<TagName>>>("not_zeroed_tag_vectors");
686 64836 : for (const auto sys_num : index_range(not_zeroed_vectors))
687 24 : for (auto & vector : not_zeroed_vectors[sys_num])
688 : {
689 12 : auto tag = addVectorTag(vector);
690 12 : _solver_systems[sys_num]->addVector(tag, false, GHOSTED);
691 12 : addNotZeroedVectorTag(tag);
692 : }
693 64824 : }
694 :
695 : void
696 63946 : FEProblemBase::createTagMatrices(CreateTaggedMatrixKey)
697 : {
698 127892 : auto & matrices = getParam<std::vector<std::vector<TagName>>>("extra_tag_matrices");
699 64194 : for (const auto sys_num : index_range(matrices))
700 692 : for (auto & matrix : matrices[sys_num])
701 : {
702 444 : auto tag = addMatrixTag(matrix);
703 444 : _solver_systems[sys_num]->addMatrix(tag);
704 : }
705 :
706 128167 : for (auto & sys : _solver_systems)
707 64221 : sys->sizeVariableMatrixData();
708 63946 : _aux->sizeVariableMatrixData();
709 63946 : }
710 :
711 : void
712 64824 : FEProblemBase::createTagSolutions()
713 : {
714 194503 : for (auto & vector : getParam<std::vector<TagName>>("extra_tag_solutions"))
715 : {
716 31 : auto tag = addVectorTag(vector, Moose::VECTOR_TAG_SOLUTION);
717 62 : for (auto & sys : _solver_systems)
718 31 : sys->addVector(tag, false, libMesh::GHOSTED);
719 31 : _aux->addVector(tag, false, libMesh::GHOSTED);
720 : }
721 :
722 64824 : if (_previous_nl_solution_required)
723 : {
724 : // We'll populate the zeroth state of the nonlinear iterations with the current solution for
725 : // ease of use in doing things like copying solutions backwards. We're just storing pointers in
726 : // the solution states containers so populating the zeroth state does not cost us the memory of
727 : // a new vector
728 94 : needSolutionState(1, Moose::SolutionIterationType::Nonlinear);
729 : }
730 :
731 64824 : auto tag = addVectorTag(Moose::SOLUTION_TAG, Moose::VECTOR_TAG_SOLUTION);
732 129951 : for (auto & sys : _solver_systems)
733 65127 : sys->associateVectorToTag(*sys->system().current_local_solution.get(), tag);
734 64824 : _aux->associateVectorToTag(*_aux->system().current_local_solution.get(), tag);
735 64824 : }
736 :
737 : void
738 150 : FEProblemBase::needSolutionState(unsigned int state, Moose::SolutionIterationType iteration_type)
739 : {
740 300 : for (auto & sys : _solver_systems)
741 150 : sys->needSolutionState(state, iteration_type);
742 150 : _aux->needSolutionState(state, iteration_type);
743 150 : }
744 :
745 : void
746 64824 : FEProblemBase::newAssemblyArray(std::vector<std::shared_ptr<SolverSystem>> & solver_systems)
747 : {
748 64824 : unsigned int n_threads = libMesh::n_threads();
749 :
750 64824 : _assembly.resize(n_threads);
751 135986 : for (const auto i : make_range(n_threads))
752 : {
753 71162 : _assembly[i].resize(solver_systems.size());
754 142651 : for (const auto j : index_range(solver_systems))
755 71489 : _assembly[i][j] = std::make_unique<Assembly>(*solver_systems[j], i);
756 : }
757 64824 : }
758 :
759 : void
760 62844 : FEProblemBase::initNullSpaceVectors(const InputParameters & parameters,
761 : std::vector<std::shared_ptr<NonlinearSystemBase>> & nls)
762 : {
763 314220 : TIME_SECTION("initNullSpaceVectors", 5, "Initializing Null Space Vectors");
764 :
765 62844 : unsigned int dimNullSpace = parameters.get<unsigned int>("null_space_dimension");
766 : unsigned int dimTransposeNullSpace =
767 62844 : parameters.get<unsigned int>("transpose_null_space_dimension");
768 62844 : unsigned int dimNearNullSpace = parameters.get<unsigned int>("near_null_space_dimension");
769 62870 : for (unsigned int i = 0; i < dimNullSpace; ++i)
770 : {
771 26 : std::ostringstream oss;
772 26 : oss << "_" << i;
773 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
774 : // builder might march over all nodes
775 52 : for (auto & nl : nls)
776 26 : nl->addVector("NullSpace" + oss.str(), false, libMesh::GHOSTED);
777 26 : }
778 125688 : _subspace_dim["NullSpace"] = dimNullSpace;
779 62857 : for (unsigned int i = 0; i < dimTransposeNullSpace; ++i)
780 : {
781 13 : std::ostringstream oss;
782 13 : oss << "_" << i;
783 : // do not project, since this will be recomputed, but make it ghosted, since the near nullspace
784 : // builder might march over all nodes
785 26 : for (auto & nl : nls)
786 13 : nl->addVector("TransposeNullSpace" + oss.str(), false, libMesh::GHOSTED);
787 13 : }
788 125688 : _subspace_dim["TransposeNullSpace"] = dimTransposeNullSpace;
789 62844 : for (unsigned int i = 0; i < dimNearNullSpace; ++i)
790 : {
791 0 : std::ostringstream oss;
792 0 : oss << "_" << i;
793 : // do not project, since this will be recomputed, but make it ghosted, since the near-nullspace
794 : // builder might march over all semilocal nodes
795 0 : for (auto & nl : nls)
796 0 : nl->addVector("NearNullSpace" + oss.str(), false, libMesh::GHOSTED);
797 0 : }
798 125688 : _subspace_dim["NearNullSpace"] = dimNearNullSpace;
799 62844 : }
800 :
801 181080 : FEProblemBase::~FEProblemBase()
802 : {
803 : // Flush the Console stream, the underlying call to Console::mooseConsole
804 : // relies on a call to Output::checkInterval that has references to
805 : // _time, etc. If it is not flushed here memory problems arise if you have
806 : // an unflushed stream and start destructing things.
807 60360 : _console << std::flush;
808 :
809 60360 : unsigned int n_threads = libMesh::n_threads();
810 125929 : for (unsigned int i = 0; i < n_threads; i++)
811 : {
812 65569 : _zero[i].release();
813 65569 : _phi_zero[i].release();
814 65569 : _scalar_zero[i].release();
815 65569 : _grad_zero[i].release();
816 65569 : _grad_phi_zero[i].release();
817 65569 : _second_zero[i].release();
818 65569 : _second_phi_zero[i].release();
819 65569 : _vector_zero[i].release();
820 65569 : _vector_curl_zero[i].release();
821 65569 : _ad_zero[i].release();
822 65569 : _ad_grad_zero[i].release();
823 65569 : _ad_second_zero[i].release();
824 : }
825 :
826 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
827 60360 : if (!_app.isUltimateMaster())
828 : {
829 11081 : auto ierr = PetscOptionsDestroy(&_petsc_option_data_base);
830 : // Don't throw on destruction
831 11081 : CHKERRABORT(this->comm().get(), ierr);
832 : }
833 : #endif
834 60360 : }
835 :
836 : void
837 0 : FEProblemBase::setCoordSystem(const std::vector<SubdomainName> & blocks,
838 : const MultiMooseEnum & coord_sys)
839 : {
840 0 : TIME_SECTION("setCoordSystem", 5, "Setting Coordinate System");
841 0 : _mesh.setCoordSystem(blocks, coord_sys);
842 0 : }
843 :
844 : void
845 0 : FEProblemBase::setAxisymmetricCoordAxis(const MooseEnum & rz_coord_axis)
846 : {
847 0 : _mesh.setAxisymmetricCoordAxis(rz_coord_axis);
848 0 : }
849 :
850 : const ConstElemRange &
851 1457 : FEProblemBase::getEvaluableElementRange()
852 : {
853 1457 : if (!_evaluable_local_elem_range)
854 : {
855 688 : std::vector<const DofMap *> dof_maps(es().n_systems());
856 2064 : for (const auto i : make_range(es().n_systems()))
857 : {
858 1376 : const auto & sys = es().get_system(i);
859 1376 : dof_maps[i] = &sys.get_dof_map();
860 : }
861 : _evaluable_local_elem_range =
862 1376 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
863 2064 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
864 688 : }
865 1457 : return *_evaluable_local_elem_range;
866 : }
867 :
868 : const ConstElemRange &
869 234 : FEProblemBase::getNonlinearEvaluableElementRange()
870 : {
871 234 : if (!_nl_evaluable_local_elem_range)
872 : {
873 234 : std::vector<const DofMap *> dof_maps(_nl.size());
874 468 : for (const auto i : index_range(dof_maps))
875 234 : dof_maps[i] = &_nl[i]->dofMap();
876 : _nl_evaluable_local_elem_range =
877 468 : std::make_unique<ConstElemRange>(_mesh.getMesh().multi_evaluable_elements_begin(dof_maps),
878 702 : _mesh.getMesh().multi_evaluable_elements_end(dof_maps));
879 234 : }
880 :
881 234 : return *_nl_evaluable_local_elem_range;
882 : }
883 :
884 : void
885 62192 : FEProblemBase::initialSetup()
886 : {
887 310960 : TIME_SECTION("initialSetup", 2, "Performing Initial Setup");
888 :
889 62192 : SubProblem::initialSetup();
890 :
891 62192 : if (_app.isRecovering() + _app.isRestarting() + bool(_app.getExReaderForRestart()) > 1)
892 0 : mooseError("Checkpoint recovery and restart and exodus restart are all mutually exclusive.");
893 :
894 62192 : if (_skip_exception_check)
895 10 : mooseWarning("MOOSE may fail to catch an exception when the \"skip_exception_check\" parameter "
896 : "is used. If you receive a terse MPI error during execution, remove this "
897 : "parameter and rerun your simulation");
898 :
899 : // set state flag indicating that we are in or beyond initialSetup.
900 : // This can be used to throw errors in methods that _must_ be called at construction time.
901 62192 : _started_initial_setup = true;
902 62192 : setCurrentExecuteOnFlag(EXEC_INITIAL);
903 :
904 : // Setup the solution states (current, old, etc) in each system based on
905 : // its default and the states requested of each of its variables
906 124659 : for (const auto i : index_range(_solver_systems))
907 : {
908 62467 : _solver_systems[i]->initSolutionState();
909 62467 : if (getDisplacedProblem())
910 2234 : getDisplacedProblem()->solverSys(i).initSolutionState();
911 : }
912 62192 : _aux->initSolutionState();
913 62192 : if (getDisplacedProblem())
914 2234 : getDisplacedProblem()->auxSys().initSolutionState();
915 :
916 : // always execute to get the max number of DoF per element and node needed to initialize phi_zero
917 : // variables
918 62192 : dof_id_type global_max_var_n_dofs_per_elem = 0;
919 124659 : for (const auto i : index_range(_solver_systems))
920 : {
921 62467 : auto & sys = *_solver_systems[i];
922 : dof_id_type max_var_n_dofs_per_elem;
923 : dof_id_type max_var_n_dofs_per_node;
924 : {
925 312335 : TIME_SECTION("computingMaxDofs", 3, "Computing Max Dofs Per Element");
926 :
927 62467 : MaxVarNDofsPerElem mvndpe(*this, sys);
928 62467 : Threads::parallel_reduce(*_mesh.getActiveLocalElementRange(), mvndpe);
929 62467 : max_var_n_dofs_per_elem = mvndpe.max();
930 62467 : _communicator.max(max_var_n_dofs_per_elem);
931 :
932 62467 : MaxVarNDofsPerNode mvndpn(*this, sys);
933 62467 : Threads::parallel_reduce(*_mesh.getLocalNodeRange(), mvndpn);
934 62467 : max_var_n_dofs_per_node = mvndpn.max();
935 62467 : _communicator.max(max_var_n_dofs_per_node);
936 62467 : global_max_var_n_dofs_per_elem =
937 62467 : std::max(global_max_var_n_dofs_per_elem, max_var_n_dofs_per_elem);
938 62467 : }
939 :
940 : {
941 312335 : TIME_SECTION("assignMaxDofs", 5, "Assigning Maximum Dofs Per Elem");
942 :
943 62467 : sys.assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
944 62467 : auto displaced_problem = getDisplacedProblem();
945 62467 : if (displaced_problem)
946 2234 : displaced_problem->solverSys(i).assignMaxVarNDofsPerElem(max_var_n_dofs_per_elem);
947 :
948 62467 : sys.assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
949 62467 : if (displaced_problem)
950 2234 : displaced_problem->solverSys(i).assignMaxVarNDofsPerNode(max_var_n_dofs_per_node);
951 62467 : }
952 : }
953 :
954 : {
955 310960 : TIME_SECTION("resizingVarValues", 5, "Resizing Variable Values");
956 :
957 130120 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
958 : {
959 135856 : _phi_zero[tid].resize(global_max_var_n_dofs_per_elem, std::vector<Real>(getMaxQps(), 0.));
960 135856 : _grad_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
961 135856 : std::vector<RealGradient>(getMaxQps(), RealGradient(0.)));
962 135856 : _second_phi_zero[tid].resize(global_max_var_n_dofs_per_elem,
963 135856 : std::vector<RealTensor>(getMaxQps(), RealTensor(0.)));
964 : }
965 62192 : }
966 :
967 : // Set up stateful material property redistribution, if we suspect
968 : // it may be necessary later.
969 62192 : addAnyRedistributers();
970 :
971 62192 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
972 : {
973 : // Only load all of the vectors if we're recovering
974 4601 : _req.set().setLoadAllVectors(_app.isRecovering());
975 :
976 : // This forces stateful material property loading to be an exact one-to-one match
977 4601 : if (_app.isRecovering())
978 : {
979 16088 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
980 12066 : props->setRecovering();
981 :
982 : #ifdef MOOSE_KOKKOS_ENABLED
983 15868 : for (auto props :
984 19835 : {&_kokkos_material_props, &_kokkos_bnd_material_props, &_kokkos_neighbor_material_props})
985 11901 : props->setRecovering();
986 : #endif
987 : }
988 :
989 23005 : TIME_SECTION("restore", 3, "Restoring from backup");
990 :
991 : // We could have a cached backup when this app is a sub-app and has been given a Backup
992 4601 : if (!_app.hasInitialBackup())
993 3604 : _app.restore(_app.restartFolderBase(_app.getRestartRecoverFileBase()), _app.isRestarting());
994 : else
995 997 : _app.restoreFromInitialBackup(_app.isRestarting());
996 :
997 : /**
998 : * If this is a restart run, the user may want to override the start time, which we already set
999 : * in the constructor. "_time" however will have been "restored" from the restart file. We need
1000 : * to honor the original request of the developer now that the restore has been completed.
1001 : */
1002 4554 : if (_app.isRestarting())
1003 : {
1004 532 : if (_app.hasStartTime())
1005 178 : _time = _time_old = _app.getStartTime();
1006 : else
1007 354 : _time_old = _time;
1008 : }
1009 4554 : }
1010 : else
1011 : {
1012 57591 : libMesh::ExodusII_IO * reader = _app.getExReaderForRestart();
1013 :
1014 57591 : if (reader)
1015 : {
1016 1445 : TIME_SECTION("copyingFromExodus", 3, "Copying Variables From Exodus");
1017 :
1018 586 : for (auto & sys : _solver_systems)
1019 301 : sys->copyVars(*reader);
1020 285 : _aux->copyVars(*reader);
1021 285 : }
1022 : else
1023 : {
1024 57302 : if (_solver_systems[0]->hasVarCopy() || _aux->hasVarCopy())
1025 0 : mooseError("Need Exodus reader to restart variables but the reader is not available\n"
1026 : "Use either FileMesh with an Exodus mesh file or FileMeshGenerator with an "
1027 : "Exodus mesh file and with use_for_exodus_restart equal to true");
1028 : }
1029 : }
1030 :
1031 : // Perform output related setups
1032 62141 : _app.getOutputWarehouse().initialSetup();
1033 :
1034 : // Flush all output to _console that occur during construction and initialization of objects
1035 62073 : _app.getOutputWarehouse().mooseConsole();
1036 :
1037 : // Build Refinement and Coarsening maps for stateful material projections if necessary
1038 64493 : if ((_adaptivity.isOn() || _num_grid_steps) &&
1039 2420 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1040 2349 : _neighbor_material_props.hasStatefulProperties()))
1041 : {
1042 71 : if (_has_internal_edge_residual_objects)
1043 8 : mooseError("Stateful neighbor material properties do not work with mesh adaptivity");
1044 :
1045 63 : _mesh.buildRefinementAndCoarseningMaps(_assembly[0][0].get());
1046 : }
1047 :
1048 62065 : if (!_app.isRecovering())
1049 : {
1050 : /**
1051 : * If we are not recovering but we are doing restart (_app.getExodusFileRestart() == true) with
1052 : * additional uniform refinements. We have to delay the refinement until this point
1053 : * in time so that the equation systems are initialized and projections can be performed.
1054 : */
1055 58043 : if (_mesh.uniformRefineLevel() > 0 && _app.getExodusFileRestart())
1056 : {
1057 11 : if (!_app.isUltimateMaster())
1058 0 : mooseError(
1059 : "Doing extra refinements when restarting is NOT supported for sub-apps of a MultiApp");
1060 :
1061 11 : adaptivity().uniformRefineWithProjection();
1062 : }
1063 : }
1064 :
1065 62065 : unsigned int n_threads = libMesh::n_threads();
1066 :
1067 : // Convergence initial setup
1068 : {
1069 310325 : TIME_SECTION("convergenceInitialSetup", 5, "Initializing Convergence objects");
1070 :
1071 129807 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1072 67762 : _convergences.initialSetup(tid);
1073 62045 : }
1074 :
1075 : // UserObject initialSetup
1076 62045 : std::set<std::string> depend_objects_ic = _ics.getDependObjects();
1077 62045 : std::set<std::string> depend_objects_aux = _aux->getDependObjects();
1078 :
1079 : // This replaces all prior updateDependObjects calls on the old user object warehouses.
1080 62045 : TheWarehouse::Query uo_query = theWarehouse().query().condition<AttribSystem>("UserObject");
1081 62045 : std::vector<UserObject *> userobjs;
1082 62045 : uo_query.queryInto(userobjs);
1083 62045 : groupUserObjects(
1084 62045 : theWarehouse(), getAuxiliarySystem(), _app.getExecuteOnEnum(), userobjs, depend_objects_ic);
1085 :
1086 62045 : std::map<int, std::vector<UserObject *>> group_userobjs;
1087 144039 : for (auto obj : userobjs)
1088 245982 : group_userobjs[obj->getParam<int>("execution_order_group")].push_back(obj);
1089 :
1090 94628 : for (auto & [group, objs] : group_userobjs)
1091 114537 : for (auto obj : objs)
1092 81954 : obj->initialSetup();
1093 :
1094 : // check if jacobian calculation is done in userobject
1095 129562 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1096 67617 : checkUserObjectJacobianRequirement(tid);
1097 :
1098 : // Check whether nonlocal couling is required or not
1099 61945 : checkNonlocalCoupling();
1100 61945 : if (_requires_nonlocal_coupling)
1101 81 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
1102 :
1103 : {
1104 309725 : TIME_SECTION("initializingFunctions", 5, "Initializing Functions");
1105 :
1106 : // Call the initialSetup methods for functions
1107 129522 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1108 : {
1109 67609 : reinitScalars(tid); // initialize scalars so they are properly sized for use as input into
1110 : // ParsedFunctions
1111 67609 : _functions.initialSetup(tid);
1112 : }
1113 :
1114 : #ifdef MOOSE_KOKKOS_ENABLED
1115 41601 : _kokkos_functions.initialSetup();
1116 : #endif
1117 61913 : }
1118 :
1119 : {
1120 309565 : TIME_SECTION("initializingRandomObjects", 5, "Initializing Random Objects");
1121 :
1122 : // Random interface objects
1123 62306 : for (const auto & it : _random_data_objects)
1124 393 : it.second->updateSeeds(EXEC_INITIAL);
1125 61913 : }
1126 :
1127 61913 : if (!_app.isRecovering())
1128 : {
1129 57891 : computeUserObjects(EXEC_INITIAL, Moose::PRE_IC);
1130 :
1131 : {
1132 289455 : TIME_SECTION("ICinitialSetup", 5, "Setting Up Initial Conditions");
1133 :
1134 121393 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1135 63510 : _ics.initialSetup(tid);
1136 :
1137 57883 : _scalar_ics.initialSetup();
1138 57883 : }
1139 :
1140 57883 : projectSolution();
1141 : }
1142 :
1143 : // Materials
1144 61897 : if (_all_materials.hasActiveObjects(0))
1145 : {
1146 44605 : TIME_SECTION("materialInitialSetup", 3, "Setting Up Materials");
1147 :
1148 18488 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1149 : {
1150 : // Sort the Material objects, these will be actually computed by MOOSE in reinit methods.
1151 9591 : _materials.sort(tid);
1152 9587 : _interface_materials.sort(tid);
1153 :
1154 : // Call initialSetup on all material objects
1155 9587 : _all_materials.initialSetup(tid);
1156 :
1157 : // Discrete materials may insert additional dependencies on materials during the initial
1158 : // setup. Therefore we resolve the dependencies once more, now with the additional
1159 : // dependencies due to discrete materials.
1160 9567 : if (_discrete_materials.hasActiveObjects())
1161 : {
1162 61 : _materials.sort(tid);
1163 61 : _interface_materials.sort(tid);
1164 : }
1165 : }
1166 :
1167 : #ifdef MOOSE_KOKKOS_ENABLED
1168 5986 : _kokkos_materials.sort(0, true);
1169 : #endif
1170 :
1171 : {
1172 44475 : TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");
1173 :
1174 8895 : initElementStatefulProps(*_mesh.getActiveLocalElementRange(), true);
1175 :
1176 17028 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1177 8133 : _neighbor_material_props.hasStatefulProperties())
1178 762 : _has_initialized_stateful = true;
1179 : #ifdef MOOSE_KOKKOS_ENABLED
1180 5984 : if (_kokkos_material_props.hasStatefulProperties() ||
1181 11816 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1182 5832 : _kokkos_neighbor_material_props.hasStatefulProperties())
1183 152 : _has_initialized_stateful = true;
1184 : #endif
1185 8895 : }
1186 8895 : }
1187 :
1188 : // setRestartInPlace() is set because the property maps have now been setup and we can
1189 : // dataLoad() them directly in place
1190 : // setRecovering() is set because from now on we require a one-to-one mapping of
1191 : // stateful properties because we shouldn't be declaring any more
1192 247484 : for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
1193 : {
1194 185613 : props->setRestartInPlace();
1195 185613 : props->setRecovering();
1196 : }
1197 :
1198 129395 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1199 : {
1200 67524 : _internal_side_indicators.initialSetup(tid);
1201 67524 : _indicators.initialSetup(tid);
1202 67524 : _markers.sort(tid);
1203 67524 : _markers.initialSetup(tid);
1204 : }
1205 :
1206 : #ifdef LIBMESH_ENABLE_AMR
1207 :
1208 61871 : if (!_app.isRecovering())
1209 : {
1210 57849 : unsigned int n = adaptivity().getInitialSteps();
1211 57849 : if (n && !_app.isUltimateMaster() && _app.isRestarting())
1212 0 : mooseError("Cannot perform initial adaptivity during restart on sub-apps of a MultiApp!");
1213 :
1214 57849 : initialAdaptMesh();
1215 : }
1216 :
1217 : #endif // LIBMESH_ENABLE_AMR
1218 :
1219 61867 : if (!_app.isRecovering() && !_app.isRestarting())
1220 : {
1221 : // During initial setup the solution is copied to the older solution states (old, older, etc)
1222 57313 : copySolutionsBackwards();
1223 :
1224 : // Check if there are old state initial conditions
1225 57313 : auto ics = _ics.getActiveObjects();
1226 57313 : auto fv_ics = _fv_ics.getActiveObjects();
1227 57313 : auto scalar_ics = _scalar_ics.getActiveObjects();
1228 57313 : unsigned short ic_state_max = 0;
1229 :
1230 171939 : auto findMax = [&ic_state_max](const auto & obj_list)
1231 : {
1232 202525 : for (auto ic : obj_list.getActiveObjects())
1233 30586 : ic_state_max = std::max(ic_state_max, ic->getState());
1234 229252 : };
1235 57313 : findMax(_ics);
1236 57313 : findMax(_fv_ics);
1237 57313 : findMax(_scalar_ics);
1238 :
1239 : // if there are old state ICs, compute them and write to old states accordingly
1240 57313 : if (ic_state_max > 0)
1241 : {
1242 : // state 0 copy (we'll overwrite current state when evaluating ICs and need to restore it once
1243 : // we're done with the old/older state ICs)
1244 0 : std::vector<std::unique_ptr<NumericVector<Real>>> state0_sys_buffers(_solver_systems.size());
1245 0 : std::unique_ptr<NumericVector<Real>> state0_aux_buffer;
1246 :
1247 : // save state 0
1248 0 : for (const auto i : index_range(_solver_systems))
1249 0 : state0_sys_buffers[i] = _solver_systems[i]->solutionState(0).clone();
1250 :
1251 0 : state0_aux_buffer = _aux->solutionState(0).clone();
1252 :
1253 : // compute old state ICs
1254 0 : for (_current_ic_state = 1; _current_ic_state <= ic_state_max; _current_ic_state++)
1255 : {
1256 0 : projectSolution();
1257 :
1258 0 : for (auto & sys : _solver_systems)
1259 0 : sys->solutionState(_current_ic_state) = sys->solutionState(0);
1260 :
1261 0 : _aux->solutionState(_current_ic_state) = _aux->solutionState(0);
1262 : }
1263 0 : _current_ic_state = 0;
1264 :
1265 : // recover state 0
1266 0 : for (const auto i : index_range(_solver_systems))
1267 : {
1268 0 : _solver_systems[i]->solutionState(0) = *state0_sys_buffers[i];
1269 0 : _solver_systems[i]->solutionState(0).close();
1270 0 : _solver_systems[i]->update();
1271 : }
1272 0 : _aux->solutionState(0) = *state0_aux_buffer;
1273 0 : _aux->solutionState(0).close();
1274 0 : _aux->update();
1275 0 : }
1276 57313 : }
1277 :
1278 61867 : if (!_app.isRecovering())
1279 : {
1280 57845 : if (haveXFEM())
1281 0 : updateMeshXFEM();
1282 : }
1283 :
1284 : // Call initialSetup on the solver systems
1285 124009 : for (auto & sys : _solver_systems)
1286 62142 : sys->initialSetup();
1287 :
1288 : // Auxilary variable initialSetup calls
1289 61867 : _aux->initialSetup();
1290 :
1291 61863 : if (_displaced_problem)
1292 : // initialSetup for displaced systems
1293 2234 : _displaced_problem->initialSetup();
1294 :
1295 124001 : for (auto & sys : _solver_systems)
1296 62138 : sys->setSolution(*(sys->system().current_local_solution.get()));
1297 :
1298 : // Update the nearest node searches (has to be called after the problem is all set up)
1299 : // We do this here because this sets up the Element's DoFs to ghost
1300 61863 : updateGeomSearch(GeometricSearchData::NEAREST_NODE);
1301 :
1302 61863 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
1303 61863 : if (_displaced_mesh)
1304 2234 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
1305 :
1306 : // We need to move the mesh in order to build a map between mortar secondary and primary
1307 : // interfaces. This map will then be used by the AgumentSparsityOnInterface ghosting functor to
1308 : // know which dofs we need ghosted when we call EquationSystems::reinit
1309 61863 : if (_displaced_problem && _mortar_data->hasDisplacedObjects())
1310 : {
1311 124 : _displaced_problem->updateMesh();
1312 : // if displacements were applied to the mesh, the mortar mesh should be updated too
1313 124 : updateMortarMesh();
1314 : }
1315 :
1316 : // Possibly reinit one more time to get ghosting correct
1317 61863 : reinitBecauseOfGhostingOrNewGeomObjects();
1318 :
1319 61863 : if (_displaced_mesh)
1320 2234 : _displaced_problem->updateMesh();
1321 :
1322 61863 : updateGeomSearch(); // Call all of the rest of the geometric searches
1323 :
1324 123997 : for (auto & sys : _solver_systems)
1325 : {
1326 62138 : const auto & tis = sys->getTimeIntegrators();
1327 :
1328 : {
1329 310690 : TIME_SECTION("timeIntegratorInitialSetup", 5, "Initializing Time Integrator");
1330 93342 : for (auto & ti : tis)
1331 31208 : ti->initialSetup();
1332 62134 : }
1333 : }
1334 :
1335 : // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ main-app restartable data
1336 : // has been restored
1337 :
1338 : // Call initialSetup on the MultiApps
1339 61859 : if (_multi_apps.hasObjects())
1340 : {
1341 32955 : TIME_SECTION("initialSetupMultiApps", 2, "Initializing MultiApps", false);
1342 6591 : _multi_apps.initialSetup();
1343 6583 : }
1344 :
1345 : // Call initialSetup on the transfers
1346 : {
1347 309255 : TIME_SECTION("initialSetupTransfers", 2, "Initializing Transfers");
1348 :
1349 61851 : _transfers.initialSetup();
1350 :
1351 : // Call initialSetup on the MultiAppTransfers to be executed on TO_MULTIAPP
1352 61851 : const auto & to_multi_app_objects = _to_multi_app_transfers.getActiveObjects();
1353 67015 : for (const auto & transfer : to_multi_app_objects)
1354 : {
1355 5228 : transfer->setCurrentDirection(Transfer::DIRECTION::TO_MULTIAPP);
1356 5228 : transfer->initialSetup();
1357 : }
1358 :
1359 : // Call initialSetup on the MultiAppTransfers to be executed on FROM_MULTIAPP
1360 61787 : const auto & from_multi_app_objects = _from_multi_app_transfers.getActiveObjects();
1361 67522 : for (const auto & transfer : from_multi_app_objects)
1362 : {
1363 5791 : transfer->setCurrentDirection(Transfer::DIRECTION::FROM_MULTIAPP);
1364 5791 : transfer->initialSetup();
1365 : }
1366 :
1367 : // Call initialSetup on the MultiAppTransfers to be executed on BETWEEN_MULTIAPP
1368 61731 : const auto & between_multi_app_objects = _between_multi_app_transfers.getActiveObjects();
1369 62927 : for (const auto & transfer : between_multi_app_objects)
1370 : {
1371 1196 : transfer->setCurrentDirection(Transfer::DIRECTION::BETWEEN_MULTIAPP);
1372 1196 : transfer->initialSetup();
1373 : }
1374 61731 : }
1375 :
1376 61731 : if (_boundary_restricted_node_integrity_check)
1377 : {
1378 185154 : TIME_SECTION("BoundaryRestrictedNodeIntegrityCheck", 5);
1379 :
1380 : // check that variables are defined along boundaries of boundary restricted nodal objects
1381 61718 : ConstBndNodeRange & bnd_nodes = *mesh().getBoundaryNodeRange();
1382 61718 : BoundaryNodeIntegrityCheckThread bnict(*this, uo_query);
1383 61718 : Threads::parallel_reduce(bnd_nodes, bnict);
1384 :
1385 : // Nodal bcs aren't threaded
1386 61706 : const auto & node_to_elem_map = _mesh.nodeToActiveSemilocalElemMap();
1387 4696754 : for (const auto & bnode : bnd_nodes)
1388 : {
1389 4635060 : const auto boundary_id = bnode->_bnd_id;
1390 4635060 : const Node * const node = bnode->_node;
1391 :
1392 4635060 : if (node->processor_id() != this->processor_id())
1393 879497 : continue;
1394 :
1395 : // Only check vertices. Variables may not be defined on non-vertex nodes (think first order
1396 : // Lagrange on a second order mesh) and user-code can often handle that
1397 : const Elem * const an_elem =
1398 3755563 : _mesh.getMesh().elem_ptr(libmesh_map_find(node_to_elem_map, node->id()).front());
1399 3755563 : if (!an_elem->is_vertex(an_elem->get_node_index(node)))
1400 464237 : continue;
1401 :
1402 3291326 : const auto & bnd_name = _mesh.getBoundaryName(boundary_id);
1403 :
1404 6536360 : for (auto & nl : _nl)
1405 : {
1406 3245046 : const auto & nodal_bcs = nl->getNodalBCWarehouse();
1407 3245046 : if (!nodal_bcs.hasBoundaryObjects(boundary_id, 0))
1408 2329865 : continue;
1409 :
1410 915181 : const auto & bnd_objects = nodal_bcs.getBoundaryObjects(boundary_id, 0);
1411 1920743 : for (const auto & bnd_object : bnd_objects)
1412 : // Skip if this object uses geometric search because coupled variables may be defined on
1413 : // paired boundaries instead of the boundary this node is on
1414 2011148 : if (!bnd_object->requiresGeometricSearch() &&
1415 1005574 : bnd_object->checkVariableBoundaryIntegrity())
1416 : {
1417 : std::set<MooseVariableFieldBase *> vars_to_omit = {
1418 : &static_cast<MooseVariableFieldBase &>(
1419 2011148 : const_cast<MooseVariableBase &>(bnd_object->variable()))};
1420 :
1421 2011148 : boundaryIntegrityCheckError(
1422 2011136 : *bnd_object, bnd_object->checkAllVariables(*node, vars_to_omit), bnd_name);
1423 1005562 : }
1424 : }
1425 : }
1426 61694 : }
1427 :
1428 61707 : if (_boundary_restricted_elem_integrity_check)
1429 : {
1430 185019 : TIME_SECTION("BoundaryRestrictedElemIntegrityCheck", 5);
1431 :
1432 : // check that variables are defined along boundaries of boundary restricted elemental objects
1433 61673 : ConstBndElemRange & bnd_elems = *mesh().getBoundaryElementRange();
1434 61673 : BoundaryElemIntegrityCheckThread beict(*this, uo_query);
1435 61673 : Threads::parallel_reduce(bnd_elems, beict);
1436 61657 : }
1437 :
1438 61691 : if (!_app.isRecovering())
1439 : {
1440 57669 : execTransfers(EXEC_INITIAL);
1441 :
1442 57669 : bool converged = execMultiApps(EXEC_INITIAL);
1443 57661 : if (!converged)
1444 4 : mooseError("failed to converge initial MultiApp");
1445 :
1446 : // We'll backup the Multiapp here
1447 57657 : backupMultiApps(EXEC_INITIAL);
1448 :
1449 120876 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1450 63219 : reinitScalars(tid);
1451 :
1452 57657 : execute(EXEC_INITIAL);
1453 :
1454 : // The FEProblemBase::execute method doesn't call all the systems on EXEC_INITIAL, but it does
1455 : // set/unset the current flag. Therefore, this resets the current flag to EXEC_INITIAL so that
1456 : // subsequent calls (e.g., executeControls) have the proper flag.
1457 57588 : setCurrentExecuteOnFlag(EXEC_INITIAL);
1458 : }
1459 :
1460 : // Here we will initialize the stateful properties once more since they may have been updated
1461 : // during initialSetup by calls to computeProperties.
1462 : //
1463 : // It's really bad that we don't allow this during restart. It means that we can't add new
1464 : // stateful materials
1465 : // during restart. This is only happening because this _has_ to be below initial userobject
1466 : // execution.
1467 : // Otherwise this could be done up above... _before_ restoring restartable data... which would
1468 : // allow you to have
1469 : // this happen during restart. I honestly have no idea why this has to happen after initial user
1470 : // object computation.
1471 : // THAT is something we should fix... so I've opened this ticket: #5804
1472 61610 : if (!_app.isRecovering() && !_app.isRestarting())
1473 : {
1474 113466 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
1475 56410 : _neighbor_material_props.hasStatefulProperties())
1476 : {
1477 3230 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1478 :
1479 646 : initElementStatefulProps(*_mesh.getActiveLocalElementRange(), true);
1480 646 : }
1481 : #ifdef MOOSE_KOKKOS_ENABLED
1482 37126 : if (_kokkos_material_props.hasStatefulProperties() ||
1483 74142 : _kokkos_bnd_material_props.hasStatefulProperties() ||
1484 37016 : _kokkos_neighbor_material_props.hasStatefulProperties())
1485 : {
1486 550 : TIME_SECTION("computeMaterials", 2, "Computing Initial Material Properties");
1487 :
1488 110 : initElementStatefulProps(*_mesh.getActiveLocalElementRange(), true);
1489 110 : }
1490 : #endif
1491 : }
1492 :
1493 : // Control Logic
1494 61610 : executeControls(EXEC_INITIAL);
1495 :
1496 : // Scalar variables need to reinited for the initial conditions to be available for output
1497 128728 : for (unsigned int tid = 0; tid < n_threads; tid++)
1498 67154 : reinitScalars(tid);
1499 :
1500 61574 : if (_displaced_mesh)
1501 2234 : _displaced_problem->syncSolutions();
1502 :
1503 : // Writes all calls to _console from initialSetup() methods
1504 61574 : _app.getOutputWarehouse().mooseConsole();
1505 :
1506 61574 : if (_requires_nonlocal_coupling)
1507 : {
1508 81 : setNonlocalCouplingMatrix();
1509 162 : for (THREAD_ID tid = 0; tid < n_threads; ++tid)
1510 162 : for (auto & assembly : _assembly[tid])
1511 81 : assembly->initNonlocalCoupling();
1512 : }
1513 :
1514 : {
1515 307870 : TIME_SECTION("lineSearchInitialSetup", 5, "Initializing Line Search");
1516 :
1517 61574 : if (_line_search)
1518 0 : _line_search->initialSetup();
1519 61574 : }
1520 :
1521 : // Perform Reporter get/declare check
1522 61574 : _reporter_data.check();
1523 :
1524 : // We do this late to allow objects to get late restartable data
1525 61574 : if (_app.isRestarting() || _app.isRecovering() || _force_restart)
1526 4554 : _app.finalizeRestore();
1527 :
1528 61574 : setCurrentExecuteOnFlag(EXEC_NONE);
1529 61574 : }
1530 :
1531 : void
1532 62622 : FEProblemBase::checkDuplicatePostprocessorVariableNames()
1533 : {
1534 118018 : for (const auto & pp : _reporter_data.getPostprocessorNames())
1535 55396 : if (hasScalarVariable(pp))
1536 0 : mooseError("Postprocessor \"" + pp +
1537 62622 : "\" has the same name as a scalar variable in the system.");
1538 62622 : }
1539 :
1540 : void
1541 290288 : FEProblemBase::timestepSetup()
1542 : {
1543 290288 : SubProblem::timestepSetup();
1544 :
1545 290288 : if (_t_step > 1 && _num_grid_steps)
1546 : {
1547 35 : libMesh::MeshRefinement mesh_refinement(_mesh);
1548 35 : std::unique_ptr<libMesh::MeshRefinement> displaced_mesh_refinement(nullptr);
1549 35 : if (_displaced_mesh)
1550 25 : displaced_mesh_refinement = std::make_unique<libMesh::MeshRefinement>(*_displaced_mesh);
1551 :
1552 70 : for (MooseIndex(_num_grid_steps) i = 0; i < _num_grid_steps; ++i)
1553 : {
1554 35 : if (_displaced_problem)
1555 : // If the DisplacedProblem is active, undisplace the DisplacedMesh in preparation for
1556 : // refinement. We can't safely refine the DisplacedMesh directly, since the Hilbert keys
1557 : // computed on the inconsistenly-displaced Mesh are different on different processors,
1558 : // leading to inconsistent Hilbert keys. We must do this before the undisplaced Mesh is
1559 : // coarsensed, so that the element and node numbering is still consistent. We also have to
1560 : // make sure this is done during every step of coarsening otherwise different partitions
1561 : // will be generated for the reference and displaced meshes (even for replicated)
1562 25 : _displaced_problem->undisplaceMesh();
1563 :
1564 35 : mesh_refinement.uniformly_coarsen();
1565 35 : if (_displaced_mesh)
1566 25 : displaced_mesh_refinement->uniformly_coarsen();
1567 :
1568 : // Mark this as an intermediate change because we do not yet want to reinit_systems. E.g. we
1569 : // need things to happen in the following order for the undisplaced problem:
1570 : // u1) EquationSystems::reinit_solutions. This will restrict the solution vectors and then
1571 : // contract the mesh
1572 : // u2) MooseMesh::meshChanged. This will update the node/side lists and other
1573 : // things which needs to happen after the contraction
1574 : // u3) GeometricSearchData::reinit. Once the node/side lists are updated we can perform our
1575 : // geometric searches which will aid in determining sparsity patterns
1576 : //
1577 : // We do these things for the displaced problem (if it exists)
1578 : // d1) EquationSystems::reinit. Restrict the displaced problem vector copies and then contract
1579 : // the mesh. It's safe to do a full reinit with the displaced because there are no
1580 : // matrices that sparsity pattern calculations will be conducted for
1581 : // d2) MooseMesh::meshChanged. This will update the node/side lists and other
1582 : // things which needs to happen after the contraction
1583 : // d3) UpdateDisplacedMeshThread::operator(). Re-displace the mesh using the *displaced*
1584 : // solution vector copy because we don't know the state of the reference solution vector.
1585 : // It's safe to use the displaced copy because we are outside of a non-linear solve,
1586 : // and there is no concern about differences between solution and current_local_solution
1587 : // d4) GeometricSearchData::reinit. With the node/side lists updated and the mesh
1588 : // re-displaced, we can perform our geometric searches, which will aid in determining the
1589 : // sparsity pattern of the matrix held by the libMesh::ImplicitSystem held by the
1590 : // NonlinearSystem held by this
1591 35 : meshChanged(
1592 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
1593 : }
1594 :
1595 : // u4) Now that all the geometric searches have been done (both undisplaced and displaced),
1596 : // we're ready to update the sparsity pattern
1597 35 : es().reinit_systems();
1598 35 : }
1599 :
1600 290288 : if (_line_search)
1601 0 : _line_search->timestepSetup();
1602 :
1603 : // Random interface objects
1604 292001 : for (const auto & it : _random_data_objects)
1605 1713 : it.second->updateSeeds(EXEC_TIMESTEP_BEGIN);
1606 :
1607 290288 : unsigned int n_threads = libMesh::n_threads();
1608 606293 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1609 : {
1610 316005 : _all_materials.timestepSetup(tid);
1611 316005 : _functions.timestepSetup(tid);
1612 : }
1613 :
1614 : #ifdef MOOSE_KOKKOS_ENABLED
1615 191401 : _kokkos_functions.timestepSetup();
1616 : #endif
1617 :
1618 290288 : _aux->timestepSetup();
1619 584351 : for (auto & sys : _solver_systems)
1620 294063 : sys->timestepSetup();
1621 :
1622 290288 : if (_displaced_problem)
1623 : // timestepSetup for displaced systems
1624 33515 : _displaced_problem->timestepSetup();
1625 :
1626 606293 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
1627 : {
1628 316005 : _internal_side_indicators.timestepSetup(tid);
1629 316005 : _indicators.timestepSetup(tid);
1630 316005 : _markers.timestepSetup(tid);
1631 : }
1632 :
1633 290288 : std::vector<UserObject *> userobjs;
1634 290288 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
1635 655189 : for (auto obj : userobjs)
1636 364901 : obj->timestepSetup();
1637 :
1638 : // Timestep setup of output objects
1639 290288 : _app.getOutputWarehouse().timestepSetup();
1640 :
1641 290288 : if (_requires_nonlocal_coupling)
1642 123 : if (_nonlocal_kernels.hasActiveObjects() || _nonlocal_integrated_bcs.hasActiveObjects())
1643 123 : _has_nonlocal_coupling = true;
1644 290288 : }
1645 :
1646 : unsigned int
1647 835029 : FEProblemBase::getMaxQps() const
1648 : {
1649 835029 : if (_max_qps == std::numeric_limits<unsigned int>::max())
1650 0 : mooseError("Max QPS uninitialized");
1651 835029 : return _max_qps;
1652 : }
1653 :
1654 : Order
1655 56 : FEProblemBase::getMaxScalarOrder() const
1656 : {
1657 56 : return _max_scalar_order;
1658 : }
1659 :
1660 : void
1661 61945 : FEProblemBase::checkNonlocalCoupling()
1662 : {
1663 309725 : TIME_SECTION("checkNonlocalCoupling", 5, "Checking Nonlocal Coupling");
1664 :
1665 129562 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
1666 134186 : for (auto & nl : _nl)
1667 : {
1668 66569 : const auto & all_kernels = nl->getKernelWarehouse();
1669 66569 : const auto & kernels = all_kernels.getObjects(tid);
1670 157720 : for (const auto & kernel : kernels)
1671 : {
1672 : std::shared_ptr<NonlocalKernel> nonlocal_kernel =
1673 91151 : std::dynamic_pointer_cast<NonlocalKernel>(kernel);
1674 91151 : if (nonlocal_kernel)
1675 : {
1676 45 : if (_calculate_jacobian_in_uo)
1677 45 : _requires_nonlocal_coupling = true;
1678 45 : _nonlocal_kernels.addObject(kernel, tid);
1679 : }
1680 91151 : }
1681 : const MooseObjectWarehouse<IntegratedBCBase> & all_integrated_bcs =
1682 66569 : nl->getIntegratedBCWarehouse();
1683 66569 : const auto & integrated_bcs = all_integrated_bcs.getObjects(tid);
1684 76384 : for (const auto & integrated_bc : integrated_bcs)
1685 : {
1686 : std::shared_ptr<NonlocalIntegratedBC> nonlocal_integrated_bc =
1687 9815 : std::dynamic_pointer_cast<NonlocalIntegratedBC>(integrated_bc);
1688 9815 : if (nonlocal_integrated_bc)
1689 : {
1690 36 : if (_calculate_jacobian_in_uo)
1691 36 : _requires_nonlocal_coupling = true;
1692 36 : _nonlocal_integrated_bcs.addObject(integrated_bc, tid);
1693 : }
1694 9815 : }
1695 : }
1696 61945 : }
1697 :
1698 : void
1699 67617 : FEProblemBase::checkUserObjectJacobianRequirement(THREAD_ID tid)
1700 : {
1701 67617 : std::set<const MooseVariableFEBase *> uo_jacobian_moose_vars;
1702 : {
1703 67617 : std::vector<ShapeElementUserObject *> objs;
1704 67617 : theWarehouse()
1705 67617 : .query()
1706 135234 : .condition<AttribInterfaces>(Interfaces::ShapeElementUserObject)
1707 67617 : .condition<AttribThread>(tid)
1708 67617 : .queryInto(objs);
1709 :
1710 67673 : for (const auto & uo : objs)
1711 : {
1712 56 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1713 56 : const auto & mv_deps = uo->jacobianMooseVariables();
1714 56 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1715 : }
1716 67617 : }
1717 : {
1718 67617 : std::vector<ShapeSideUserObject *> objs;
1719 67617 : theWarehouse()
1720 67617 : .query()
1721 135234 : .condition<AttribInterfaces>(Interfaces::ShapeSideUserObject)
1722 67617 : .condition<AttribThread>(tid)
1723 67617 : .queryInto(objs);
1724 67689 : for (const auto & uo : objs)
1725 : {
1726 72 : _calculate_jacobian_in_uo = uo->computeJacobianFlag();
1727 72 : const auto & mv_deps = uo->jacobianMooseVariables();
1728 72 : uo_jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
1729 : }
1730 67617 : }
1731 :
1732 67617 : _uo_jacobian_moose_vars[tid].assign(uo_jacobian_moose_vars.begin(), uo_jacobian_moose_vars.end());
1733 135234 : std::sort(
1734 135234 : _uo_jacobian_moose_vars[tid].begin(), _uo_jacobian_moose_vars[tid].end(), sortMooseVariables);
1735 67617 : }
1736 :
1737 : void
1738 81 : FEProblemBase::setVariableAllDoFMap(const std::vector<const MooseVariableFEBase *> & moose_vars)
1739 : {
1740 197 : for (unsigned int i = 0; i < moose_vars.size(); ++i)
1741 : {
1742 116 : VariableName var_name = moose_vars[i]->name();
1743 116 : auto & sys = _solver_systems[moose_vars[i]->sys().number()];
1744 116 : sys->setVariableGlobalDoFs(var_name);
1745 116 : _var_dof_map[var_name] = sys->getVariableGlobalDoFs();
1746 116 : }
1747 81 : }
1748 :
1749 : void
1750 431268659 : FEProblemBase::prepare(const Elem * elem, const THREAD_ID tid)
1751 : {
1752 865336278 : for (const auto i : index_range(_solver_systems))
1753 : {
1754 434067619 : _assembly[tid][i]->reinit(elem);
1755 434067619 : _solver_systems[i]->prepare(tid);
1756 :
1757 434067619 : if (i < _num_nl_sys)
1758 : {
1759 : // This method is called outside of residual/Jacobian callbacks during initial condition
1760 : // evaluation
1761 433394789 : if ((!_has_jacobian || !_const_jacobian) && currentlyComputingJacobian())
1762 56171636 : _assembly[tid][i]->prepareJacobianBlock();
1763 433394789 : _assembly[tid][i]->prepareResidual();
1764 433394789 : if (_has_nonlocal_coupling && currentlyComputingJacobian())
1765 11792 : _assembly[tid][i]->prepareNonlocal();
1766 : }
1767 : }
1768 431268659 : _aux->prepare(tid);
1769 :
1770 444551421 : if (_displaced_problem &&
1771 : // _reinit_displaced_neighbor applies to interface type objects which will do computations
1772 : // based on both elem and neighbor. Consequently, despite what you might think by its name, we
1773 : // must make sure we prepare the displaced elem
1774 444551421 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1775 : {
1776 8536374 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), tid);
1777 8536346 : if (_has_nonlocal_coupling)
1778 0 : _displaced_problem->prepareNonlocal(tid);
1779 : }
1780 431268631 : }
1781 :
1782 : void
1783 29479 : FEProblemBase::prepareFace(const Elem * elem, const THREAD_ID tid)
1784 : {
1785 58958 : for (auto & nl : _nl)
1786 29479 : nl->prepareFace(tid, true);
1787 29479 : _aux->prepareFace(tid, false);
1788 :
1789 29479 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1790 0 : _displaced_problem->prepareFace(_displaced_mesh->elemPtr(elem->id()), tid);
1791 29479 : }
1792 :
1793 : void
1794 0 : FEProblemBase::prepare(const Elem * elem,
1795 : unsigned int ivar,
1796 : unsigned int jvar,
1797 : const std::vector<dof_id_type> & dof_indices,
1798 : const THREAD_ID tid)
1799 : {
1800 0 : for (const auto i : index_range(_nl))
1801 : {
1802 0 : _assembly[tid][i]->reinit(elem);
1803 0 : _nl[i]->prepare(tid);
1804 : }
1805 :
1806 0 : _aux->prepare(tid);
1807 0 : const auto current_nl_sys_num = _current_nl_sys->number();
1808 0 : _assembly[tid][current_nl_sys_num]->prepareBlock(ivar, jvar, dof_indices);
1809 0 : if (_has_nonlocal_coupling)
1810 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1811 : {
1812 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1813 0 : _assembly[tid][current_nl_sys_num]->prepareBlockNonlocal(
1814 : ivar, jvar, dof_indices, jv.allDofIndices());
1815 : }
1816 :
1817 0 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1818 : {
1819 0 : _displaced_problem->prepare(_displaced_mesh->elemPtr(elem->id()), ivar, jvar, dof_indices, tid);
1820 0 : if (_has_nonlocal_coupling)
1821 0 : if (_nonlocal_cm[current_nl_sys_num](ivar, jvar) != 0)
1822 : {
1823 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
1824 0 : _displaced_problem->prepareBlockNonlocal(ivar, jvar, dof_indices, jv.allDofIndices(), tid);
1825 : }
1826 : }
1827 0 : }
1828 :
1829 : void
1830 472098943 : FEProblemBase::setCurrentSubdomainID(const Elem * elem, const THREAD_ID tid)
1831 : {
1832 472098943 : SubdomainID did = elem->subdomain_id();
1833 948135545 : for (const auto i : index_range(_solver_systems))
1834 : {
1835 476036602 : _assembly[tid][i]->setCurrentSubdomainID(did);
1836 490246206 : if (_displaced_problem &&
1837 490246206 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1838 8817056 : _displaced_problem->assembly(tid, i).setCurrentSubdomainID(did);
1839 : }
1840 472098943 : }
1841 :
1842 : void
1843 1664505707 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, unsigned int side, const THREAD_ID tid)
1844 : {
1845 1664505707 : SubdomainID did = elem->neighbor_ptr(side)->subdomain_id();
1846 3334291814 : for (const auto i : index_range(_nl))
1847 : {
1848 1669786107 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1849 1719470210 : if (_displaced_problem &&
1850 1719470210 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1851 29111734 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1852 : }
1853 1664505707 : }
1854 :
1855 : void
1856 23572031 : FEProblemBase::setNeighborSubdomainID(const Elem * elem, const THREAD_ID tid)
1857 : {
1858 23572031 : SubdomainID did = elem->subdomain_id();
1859 48214288 : for (const auto i : index_range(_nl))
1860 : {
1861 24642257 : _assembly[tid][i]->setCurrentNeighborSubdomainID(did);
1862 24699961 : if (_displaced_problem &&
1863 24699961 : (_reinit_displaced_elem || _reinit_displaced_face || _reinit_displaced_neighbor))
1864 57704 : _displaced_problem->assembly(tid, i).setCurrentNeighborSubdomainID(did);
1865 : }
1866 23572031 : }
1867 :
1868 : void
1869 153242 : FEProblemBase::prepareAssembly(const THREAD_ID tid)
1870 : {
1871 153242 : _assembly[tid][_current_nl_sys->number()]->prepare();
1872 153242 : if (_has_nonlocal_coupling)
1873 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
1874 :
1875 153242 : if (_displaced_problem && (_reinit_displaced_elem || _reinit_displaced_face))
1876 : {
1877 57704 : _displaced_problem->prepareAssembly(tid);
1878 57704 : if (_has_nonlocal_coupling)
1879 0 : _displaced_problem->prepareNonlocal(tid);
1880 : }
1881 153242 : }
1882 :
1883 : void
1884 300146 : FEProblemBase::addResidual(const THREAD_ID tid)
1885 : {
1886 600292 : _assembly[tid][_current_nl_sys->number()]->addResidual(Assembly::GlobalDataKey{},
1887 300146 : currentResidualVectorTags());
1888 :
1889 300146 : if (_displaced_problem)
1890 5288 : _displaced_problem->addResidual(tid);
1891 300146 : }
1892 :
1893 : void
1894 2337539 : FEProblemBase::addResidualNeighbor(const THREAD_ID tid)
1895 : {
1896 4675078 : _assembly[tid][_current_nl_sys->number()]->addResidualNeighbor(Assembly::GlobalDataKey{},
1897 2337539 : currentResidualVectorTags());
1898 :
1899 2337539 : if (_displaced_problem)
1900 68460 : _displaced_problem->addResidualNeighbor(tid);
1901 2337539 : }
1902 :
1903 : void
1904 2312671 : FEProblemBase::addResidualLower(const THREAD_ID tid)
1905 : {
1906 4625342 : _assembly[tid][_current_nl_sys->number()]->addResidualLower(Assembly::GlobalDataKey{},
1907 2312671 : currentResidualVectorTags());
1908 :
1909 2312671 : if (_displaced_problem)
1910 68688 : _displaced_problem->addResidualLower(tid);
1911 2312671 : }
1912 :
1913 : void
1914 64576 : FEProblemBase::addResidualScalar(const THREAD_ID tid /* = 0*/)
1915 : {
1916 129152 : _assembly[tid][_current_nl_sys->number()]->addResidualScalar(Assembly::GlobalDataKey{},
1917 64576 : currentResidualVectorTags());
1918 64576 : }
1919 :
1920 : void
1921 329589464 : FEProblemBase::cacheResidual(const THREAD_ID tid)
1922 : {
1923 329589464 : SubProblem::cacheResidual(tid);
1924 329589464 : if (_displaced_problem)
1925 8645975 : _displaced_problem->cacheResidual(tid);
1926 329589464 : }
1927 :
1928 : void
1929 55594 : FEProblemBase::cacheResidualNeighbor(const THREAD_ID tid)
1930 : {
1931 55594 : SubProblem::cacheResidualNeighbor(tid);
1932 55594 : if (_displaced_problem)
1933 36 : _displaced_problem->cacheResidualNeighbor(tid);
1934 55594 : }
1935 :
1936 : void
1937 19094341 : FEProblemBase::addCachedResidual(const THREAD_ID tid)
1938 : {
1939 19094341 : SubProblem::addCachedResidual(tid);
1940 19094341 : if (_displaced_problem)
1941 555761 : _displaced_problem->addCachedResidual(tid);
1942 19094341 : }
1943 :
1944 : void
1945 11826 : FEProblemBase::addCachedResidualDirectly(NumericVector<Number> & residual, const THREAD_ID tid)
1946 : {
1947 11826 : if (_current_nl_sys->hasVector(_current_nl_sys->timeVectorTag()))
1948 32553 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
1949 32553 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->timeVectorTag()));
1950 :
1951 11826 : if (_current_nl_sys->hasVector(_current_nl_sys->nonTimeVectorTag()))
1952 35478 : _assembly[tid][_current_nl_sys->number()]->addCachedResidualDirectly(
1953 35478 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->nonTimeVectorTag()));
1954 :
1955 : // We do this because by adding the cached residual directly, we cannot ensure that all of the
1956 : // cached residuals are emptied after only the two add calls above
1957 11826 : _assembly[tid][_current_nl_sys->number()]->clearCachedResiduals(Assembly::GlobalDataKey{});
1958 :
1959 11826 : if (_displaced_problem)
1960 48 : _displaced_problem->addCachedResidualDirectly(residual, tid);
1961 11826 : }
1962 :
1963 : void
1964 0 : FEProblemBase::setResidual(NumericVector<Number> & residual, const THREAD_ID tid)
1965 : {
1966 0 : _assembly[tid][_current_nl_sys->number()]->setResidual(
1967 : residual,
1968 0 : Assembly::GlobalDataKey{},
1969 0 : getVectorTag(_nl[_current_nl_sys->number()]->residualVectorTag()));
1970 0 : if (_displaced_problem)
1971 0 : _displaced_problem->setResidual(residual, tid);
1972 0 : }
1973 :
1974 : void
1975 0 : FEProblemBase::setResidualNeighbor(NumericVector<Number> & residual, const THREAD_ID tid)
1976 : {
1977 0 : _assembly[tid][_current_nl_sys->number()]->setResidualNeighbor(
1978 0 : residual, Assembly::GlobalDataKey{}, getVectorTag(_current_nl_sys->residualVectorTag()));
1979 0 : if (_displaced_problem)
1980 0 : _displaced_problem->setResidualNeighbor(residual, tid);
1981 0 : }
1982 :
1983 : void
1984 42097 : FEProblemBase::addJacobian(const THREAD_ID tid)
1985 : {
1986 42097 : _assembly[tid][_current_nl_sys->number()]->addJacobian(Assembly::GlobalDataKey{});
1987 42097 : if (_has_nonlocal_coupling)
1988 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianNonlocal(Assembly::GlobalDataKey{});
1989 42097 : if (_displaced_problem)
1990 : {
1991 224 : _displaced_problem->addJacobian(tid);
1992 224 : if (_has_nonlocal_coupling)
1993 0 : _displaced_problem->addJacobianNonlocal(tid);
1994 : }
1995 42097 : }
1996 :
1997 : void
1998 9105 : FEProblemBase::addJacobianNeighbor(const THREAD_ID tid)
1999 : {
2000 9105 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighbor(Assembly::GlobalDataKey{});
2001 9105 : if (_displaced_problem)
2002 54 : _displaced_problem->addJacobianNeighbor(tid);
2003 9105 : }
2004 :
2005 : void
2006 127506 : FEProblemBase::addJacobianNeighborLowerD(const THREAD_ID tid)
2007 : {
2008 127506 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborLowerD(Assembly::GlobalDataKey{});
2009 127506 : if (_displaced_problem)
2010 3456 : _displaced_problem->addJacobianNeighborLowerD(tid);
2011 127506 : }
2012 :
2013 : void
2014 5220 : FEProblemBase::addJacobianLowerD(const THREAD_ID tid)
2015 : {
2016 5220 : _assembly[tid][_current_nl_sys->number()]->addJacobianLowerD(Assembly::GlobalDataKey{});
2017 5220 : if (_displaced_problem)
2018 216 : _displaced_problem->addJacobianLowerD(tid);
2019 5220 : }
2020 :
2021 : void
2022 15622 : FEProblemBase::addJacobianScalar(const THREAD_ID tid /* = 0*/)
2023 : {
2024 15622 : _assembly[tid][_current_nl_sys->number()]->addJacobianScalar(Assembly::GlobalDataKey{});
2025 15622 : }
2026 :
2027 : void
2028 39233 : FEProblemBase::addJacobianOffDiagScalar(unsigned int ivar, const THREAD_ID tid /* = 0*/)
2029 : {
2030 78466 : _assembly[tid][_current_nl_sys->number()]->addJacobianOffDiagScalar(ivar,
2031 39233 : Assembly::GlobalDataKey{});
2032 39233 : }
2033 :
2034 : void
2035 55537885 : FEProblemBase::cacheJacobian(const THREAD_ID tid)
2036 : {
2037 55537885 : SubProblem::cacheJacobian(tid);
2038 55537885 : if (_displaced_problem)
2039 1767232 : _displaced_problem->cacheJacobian(tid);
2040 55537885 : }
2041 :
2042 : void
2043 10576 : FEProblemBase::cacheJacobianNeighbor(const THREAD_ID tid)
2044 : {
2045 10576 : SubProblem::cacheJacobianNeighbor(tid);
2046 10576 : if (_displaced_problem)
2047 0 : _displaced_problem->cacheJacobianNeighbor(tid);
2048 10576 : }
2049 :
2050 : void
2051 3471251 : FEProblemBase::addCachedJacobian(const THREAD_ID tid)
2052 : {
2053 3471251 : SubProblem::addCachedJacobian(tid);
2054 3471247 : if (_displaced_problem)
2055 108971 : _displaced_problem->addCachedJacobian(tid);
2056 3471247 : }
2057 :
2058 : void
2059 81172 : FEProblemBase::addJacobianBlockTags(SparseMatrix<Number> & jacobian,
2060 : unsigned int ivar,
2061 : unsigned int jvar,
2062 : const DofMap & dof_map,
2063 : std::vector<dof_id_type> & dof_indices,
2064 : const std::set<TagID> & tags,
2065 : const THREAD_ID tid)
2066 : {
2067 162344 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockTags(
2068 81172 : jacobian, ivar, jvar, dof_map, dof_indices, Assembly::GlobalDataKey{}, tags);
2069 :
2070 81172 : if (_has_nonlocal_coupling)
2071 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2072 : {
2073 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2074 0 : _assembly[tid][_current_nl_sys->number()]->addJacobianBlockNonlocalTags(
2075 : jacobian,
2076 : ivar,
2077 : jvar,
2078 : dof_map,
2079 : dof_indices,
2080 : jv.allDofIndices(),
2081 0 : Assembly::GlobalDataKey{},
2082 : tags);
2083 : }
2084 :
2085 81172 : if (_displaced_problem)
2086 : {
2087 0 : _displaced_problem->addJacobianBlockTags(jacobian, ivar, jvar, dof_map, dof_indices, tags, tid);
2088 0 : if (_has_nonlocal_coupling)
2089 0 : if (_nonlocal_cm[_current_nl_sys->number()](ivar, jvar) != 0)
2090 : {
2091 0 : MooseVariableFEBase & jv = _current_nl_sys->getVariable(tid, jvar);
2092 0 : _displaced_problem->addJacobianBlockNonlocal(
2093 : jacobian, ivar, jvar, dof_map, dof_indices, jv.allDofIndices(), tags, tid);
2094 : }
2095 : }
2096 81172 : }
2097 :
2098 : void
2099 1344 : FEProblemBase::addJacobianNeighbor(SparseMatrix<Number> & jacobian,
2100 : unsigned int ivar,
2101 : unsigned int jvar,
2102 : const DofMap & dof_map,
2103 : std::vector<dof_id_type> & dof_indices,
2104 : std::vector<dof_id_type> & neighbor_dof_indices,
2105 : const std::set<TagID> & tags,
2106 : const THREAD_ID tid)
2107 : {
2108 2688 : _assembly[tid][_current_nl_sys->number()]->addJacobianNeighborTags(jacobian,
2109 : ivar,
2110 : jvar,
2111 : dof_map,
2112 : dof_indices,
2113 : neighbor_dof_indices,
2114 1344 : Assembly::GlobalDataKey{},
2115 : tags);
2116 1344 : if (_displaced_problem)
2117 0 : _displaced_problem->addJacobianNeighbor(
2118 : jacobian, ivar, jvar, dof_map, dof_indices, neighbor_dof_indices, tags, tid);
2119 1344 : }
2120 :
2121 : void
2122 146074809 : FEProblemBase::prepareShapes(unsigned int var, const THREAD_ID tid)
2123 : {
2124 146074809 : _assembly[tid][_current_nl_sys->number()]->copyShapes(var);
2125 146074809 : }
2126 :
2127 : void
2128 722841 : FEProblemBase::prepareFaceShapes(unsigned int var, const THREAD_ID tid)
2129 : {
2130 722841 : _assembly[tid][_current_nl_sys->number()]->copyFaceShapes(var);
2131 722841 : }
2132 :
2133 : void
2134 217901 : FEProblemBase::prepareNeighborShapes(unsigned int var, const THREAD_ID tid)
2135 : {
2136 217901 : _assembly[tid][_current_nl_sys->number()]->copyNeighborShapes(var);
2137 217901 : }
2138 :
2139 : void
2140 965942 : FEProblemBase::addGhostedElem(dof_id_type elem_id)
2141 : {
2142 965942 : if (_mesh.elemPtr(elem_id)->processor_id() != processor_id())
2143 205714 : _ghosted_elems.insert(elem_id);
2144 965942 : }
2145 :
2146 : void
2147 31700 : FEProblemBase::addGhostedBoundary(BoundaryID boundary_id)
2148 : {
2149 31700 : _mesh.addGhostedBoundary(boundary_id);
2150 31700 : if (_displaced_problem)
2151 28798 : _displaced_mesh->addGhostedBoundary(boundary_id);
2152 31700 : }
2153 :
2154 : void
2155 70384 : FEProblemBase::ghostGhostedBoundaries()
2156 : {
2157 351920 : TIME_SECTION("ghostGhostedBoundaries", 3, "Ghosting Ghosted Boundaries");
2158 :
2159 70384 : _mesh.ghostGhostedBoundaries();
2160 :
2161 70384 : if (_displaced_problem)
2162 2879 : _displaced_mesh->ghostGhostedBoundaries();
2163 70384 : }
2164 :
2165 : void
2166 0 : FEProblemBase::sizeZeroes(unsigned int /*size*/, const THREAD_ID /*tid*/)
2167 : {
2168 0 : mooseDoOnce(mooseWarning(
2169 : "This function is deprecated and no longer performs any function. Please do not call it."));
2170 0 : }
2171 :
2172 : bool
2173 342243 : FEProblemBase::reinitDirac(const Elem * elem, const THREAD_ID tid)
2174 : {
2175 342243 : std::vector<Point> & points = _dirac_kernel_info.getPoints()[elem].first;
2176 :
2177 342243 : unsigned int n_points = points.size();
2178 :
2179 342243 : if (n_points)
2180 : {
2181 336731 : if (n_points > _max_qps)
2182 : {
2183 0 : _max_qps = n_points;
2184 :
2185 : /**
2186 : * The maximum number of qps can rise if several Dirac points are added to a single element.
2187 : * In that case we need to resize the zeros to compensate.
2188 : */
2189 0 : unsigned int max_qpts = getMaxQps();
2190 0 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
2191 : {
2192 : // the highest available order in libMesh is 43
2193 0 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
2194 0 : _zero[tid].resize(max_qpts, 0);
2195 0 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
2196 0 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
2197 0 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
2198 0 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
2199 : }
2200 : }
2201 :
2202 673462 : for (const auto i : index_range(_nl))
2203 : {
2204 336731 : _assembly[tid][i]->reinitAtPhysical(elem, points);
2205 336731 : _nl[i]->prepare(tid);
2206 : }
2207 336731 : _aux->prepare(tid);
2208 :
2209 336731 : reinitElem(elem, tid);
2210 : }
2211 :
2212 342243 : _assembly[tid][_current_nl_sys->number()]->prepare();
2213 342243 : if (_has_nonlocal_coupling)
2214 0 : _assembly[tid][_current_nl_sys->number()]->prepareNonlocal();
2215 :
2216 342243 : bool have_points = n_points > 0;
2217 342243 : if (_displaced_problem && (_reinit_displaced_elem))
2218 : {
2219 5512 : have_points |= _displaced_problem->reinitDirac(_displaced_mesh->elemPtr(elem->id()), tid);
2220 5512 : if (_has_nonlocal_coupling)
2221 0 : _displaced_problem->prepareNonlocal(tid);
2222 : }
2223 :
2224 342243 : return have_points;
2225 : }
2226 :
2227 : void
2228 431533452 : FEProblemBase::reinitElem(const Elem * elem, const THREAD_ID tid)
2229 : {
2230 865865864 : for (auto & sys : _solver_systems)
2231 434332412 : sys->reinitElem(elem, tid);
2232 431533452 : _aux->reinitElem(elem, tid);
2233 :
2234 431533452 : if (_displaced_problem && _reinit_displaced_elem)
2235 4961598 : _displaced_problem->reinitElem(_displaced_mesh->elemPtr(elem->id()), tid);
2236 431533452 : }
2237 :
2238 : void
2239 53607 : FEProblemBase::reinitElemPhys(const Elem * const elem,
2240 : const std::vector<Point> & phys_points_in_elem,
2241 : const THREAD_ID tid)
2242 : {
2243 : mooseAssert(_mesh.queryElemPtr(elem->id()) == elem,
2244 : "Are you calling this method with a displaced mesh element?");
2245 :
2246 107214 : for (const auto i : index_range(_solver_systems))
2247 : {
2248 53607 : _assembly[tid][i]->reinitAtPhysical(elem, phys_points_in_elem);
2249 53607 : _solver_systems[i]->prepare(tid);
2250 53607 : _assembly[tid][i]->prepare();
2251 53607 : if (_has_nonlocal_coupling)
2252 0 : _assembly[tid][i]->prepareNonlocal();
2253 : }
2254 53607 : _aux->prepare(tid);
2255 :
2256 53607 : reinitElem(elem, tid);
2257 53607 : }
2258 :
2259 : void
2260 0 : FEProblemBase::reinitElemFace(const Elem * const elem,
2261 : const unsigned int side,
2262 : const BoundaryID,
2263 : const THREAD_ID tid)
2264 : {
2265 0 : mooseDeprecated(
2266 : "reinitElemFace with a BoundaryID argument is deprecated because the boundary id was never "
2267 : "used. Please call reinitElemFace without the BoundaryID argument instead");
2268 :
2269 0 : reinitElemFace(elem, side, tid);
2270 0 : }
2271 :
2272 : void
2273 8556469 : FEProblemBase::reinitElemFace(const Elem * const elem, const unsigned int side, const THREAD_ID tid)
2274 : {
2275 17113343 : for (const auto i : index_range(_solver_systems))
2276 : {
2277 8556874 : _assembly[tid][i]->reinit(elem, side);
2278 8556874 : _solver_systems[i]->reinitElemFace(elem, side, tid);
2279 : }
2280 8556469 : _aux->reinitElemFace(elem, side, tid);
2281 :
2282 8556469 : if (_displaced_problem && _reinit_displaced_face)
2283 97294 : _displaced_problem->reinitElemFace(_displaced_mesh->elemPtr(elem->id()), side, tid);
2284 8556469 : }
2285 :
2286 : void
2287 300210 : FEProblemBase::reinitLowerDElem(const Elem * lower_d_elem,
2288 : const THREAD_ID tid,
2289 : const std::vector<Point> * const pts,
2290 : const std::vector<Real> * const weights)
2291 : {
2292 300210 : SubProblem::reinitLowerDElem(lower_d_elem, tid, pts, weights);
2293 :
2294 300210 : if (_displaced_problem && _displaced_mesh)
2295 1080 : _displaced_problem->reinitLowerDElem(
2296 1080 : _displaced_mesh->elemPtr(lower_d_elem->id()), tid, pts, weights);
2297 300210 : }
2298 :
2299 : void
2300 29242669 : FEProblemBase::reinitNode(const Node * node, const THREAD_ID tid)
2301 : {
2302 29242669 : if (_displaced_problem && _reinit_displaced_elem)
2303 1122418 : _displaced_problem->reinitNode(&_displaced_mesh->nodeRef(node->id()), tid);
2304 :
2305 58488614 : for (const auto i : index_range(_nl))
2306 : {
2307 29245945 : _assembly[tid][i]->reinit(node);
2308 29245945 : _nl[i]->reinitNode(node, tid);
2309 : }
2310 29242669 : _aux->reinitNode(node, tid);
2311 29242669 : }
2312 :
2313 : void
2314 71962227 : FEProblemBase::reinitNodeFace(const Node * node, BoundaryID bnd_id, const THREAD_ID tid)
2315 : {
2316 71962227 : if (_displaced_problem && _reinit_displaced_face)
2317 3818746 : _displaced_problem->reinitNodeFace(&_displaced_mesh->nodeRef(node->id()), bnd_id, tid);
2318 :
2319 144801862 : for (const auto i : index_range(_nl))
2320 : {
2321 72839635 : _assembly[tid][i]->reinit(node);
2322 72839635 : _nl[i]->reinitNodeFace(node, bnd_id, tid);
2323 : }
2324 71962227 : _aux->reinitNodeFace(node, bnd_id, tid);
2325 71962227 : }
2326 :
2327 : void
2328 6813 : FEProblemBase::reinitNodes(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2329 : {
2330 6813 : if (_displaced_problem && _reinit_displaced_elem)
2331 0 : _displaced_problem->reinitNodes(nodes, tid);
2332 :
2333 13626 : for (auto & nl : _nl)
2334 6813 : nl->reinitNodes(nodes, tid);
2335 6813 : _aux->reinitNodes(nodes, tid);
2336 6813 : }
2337 :
2338 : void
2339 723 : FEProblemBase::reinitNodesNeighbor(const std::vector<dof_id_type> & nodes, const THREAD_ID tid)
2340 : {
2341 723 : if (_displaced_problem && _reinit_displaced_elem)
2342 0 : _displaced_problem->reinitNodesNeighbor(nodes, tid);
2343 :
2344 1446 : for (auto & nl : _nl)
2345 723 : nl->reinitNodesNeighbor(nodes, tid);
2346 723 : _aux->reinitNodesNeighbor(nodes, tid);
2347 723 : }
2348 :
2349 : void
2350 9162916 : FEProblemBase::reinitScalars(const THREAD_ID tid, bool reinit_for_derivative_reordering /*=false*/)
2351 : {
2352 45814580 : TIME_SECTION("reinitScalars", 3, "Reinitializing Scalar Variables");
2353 :
2354 9162916 : if (_displaced_problem && _reinit_displaced_elem)
2355 124637 : _displaced_problem->reinitScalars(tid, reinit_for_derivative_reordering);
2356 :
2357 18570571 : for (auto & nl : _nl)
2358 9407655 : nl->reinitScalars(tid, reinit_for_derivative_reordering);
2359 9162916 : _aux->reinitScalars(tid, reinit_for_derivative_reordering);
2360 :
2361 : // This is called outside of residual/Jacobian call-backs
2362 18577375 : for (auto & assembly : _assembly[tid])
2363 9414459 : assembly->prepareScalar();
2364 9162916 : }
2365 :
2366 : void
2367 210604 : FEProblemBase::reinitOffDiagScalars(const THREAD_ID tid)
2368 : {
2369 210604 : _assembly[tid][_current_nl_sys->number()]->prepareOffDiagScalar();
2370 210604 : if (_displaced_problem)
2371 90 : _displaced_problem->reinitOffDiagScalars(tid);
2372 210604 : }
2373 :
2374 : void
2375 4228377 : FEProblemBase::reinitNeighbor(const Elem * elem, unsigned int side, const THREAD_ID tid)
2376 : {
2377 4228377 : setNeighborSubdomainID(elem, side, tid);
2378 :
2379 4228377 : const Elem * neighbor = elem->neighbor_ptr(side);
2380 4228377 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(elem);
2381 :
2382 8456784 : for (const auto i : index_range(_nl))
2383 : {
2384 4228407 : _assembly[tid][i]->reinitElemAndNeighbor(elem, side, neighbor, neighbor_side);
2385 4228407 : _nl[i]->prepareNeighbor(tid);
2386 : // Called during stateful material property evaluation outside of solve
2387 4228407 : _assembly[tid][i]->prepareNeighbor();
2388 : }
2389 4228377 : _aux->prepareNeighbor(tid);
2390 :
2391 8456784 : for (auto & nl : _nl)
2392 : {
2393 4228407 : nl->reinitElemFace(elem, side, tid);
2394 4228407 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2395 : }
2396 4228377 : _aux->reinitElemFace(elem, side, tid);
2397 4228377 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2398 :
2399 4228377 : if (_displaced_problem && _reinit_displaced_neighbor)
2400 : {
2401 : // There are cases like for cohesive zone modeling without significant sliding where we cannot
2402 : // use FEInterface::inverse_map in Assembly::reinitElemAndNeighbor in the displaced problem
2403 : // because the physical points coming from the element don't actually lie on the neighbor.
2404 : // Moreover, what's the point of doing another physical point inversion in other cases? We only
2405 : // care about the reference points which we can just take from the undisplaced computation
2406 71886 : const auto & displaced_ref_pts = _assembly[tid][0]->qRuleNeighbor()->get_points();
2407 :
2408 71886 : _displaced_problem->reinitNeighbor(
2409 71886 : _displaced_mesh->elemPtr(elem->id()), side, tid, &displaced_ref_pts);
2410 : }
2411 4228377 : }
2412 :
2413 : void
2414 2433898 : FEProblemBase::reinitElemNeighborAndLowerD(const Elem * elem,
2415 : unsigned int side,
2416 : const THREAD_ID tid)
2417 : {
2418 2433898 : reinitNeighbor(elem, side, tid);
2419 :
2420 2433898 : const Elem * lower_d_elem = _mesh.getLowerDElem(elem, side);
2421 2433898 : if (lower_d_elem && _mesh.interiorLowerDBlocks().count(lower_d_elem->subdomain_id()) > 0)
2422 11610 : reinitLowerDElem(lower_d_elem, tid);
2423 : else
2424 : {
2425 : // with mesh refinement, lower-dimensional element might be defined on neighbor side
2426 2422288 : auto & neighbor = _assembly[tid][0]->neighbor();
2427 2422288 : auto & neighbor_side = _assembly[tid][0]->neighborSide();
2428 2422288 : const Elem * lower_d_elem_neighbor = _mesh.getLowerDElem(neighbor, neighbor_side);
2429 2422288 : if (lower_d_elem_neighbor &&
2430 2422288 : _mesh.interiorLowerDBlocks().count(lower_d_elem_neighbor->subdomain_id()) > 0)
2431 : {
2432 0 : auto qps = _assembly[tid][0]->qPointsFaceNeighbor().stdVector();
2433 0 : std::vector<Point> reference_points;
2434 0 : FEMap::inverse_map(
2435 0 : lower_d_elem_neighbor->dim(), lower_d_elem_neighbor, qps, reference_points);
2436 0 : reinitLowerDElem(lower_d_elem_neighbor, tid, &reference_points);
2437 0 : }
2438 : }
2439 :
2440 2433898 : if (_displaced_problem && (_reinit_displaced_face || _reinit_displaced_neighbor))
2441 71820 : _displaced_problem->reinitElemNeighborAndLowerD(
2442 71820 : _displaced_mesh->elemPtr(elem->id()), side, tid);
2443 2433898 : }
2444 :
2445 : void
2446 108178 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2447 : unsigned int neighbor_side,
2448 : const std::vector<Point> & physical_points,
2449 : const THREAD_ID tid)
2450 : {
2451 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2452 : "Are you calling this method with a displaced mesh element?");
2453 :
2454 216356 : for (const auto i : index_range(_nl))
2455 : {
2456 : // Reinits shape the functions at the physical points
2457 108178 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, neighbor_side, physical_points);
2458 :
2459 : // Sets the neighbor dof indices
2460 108178 : _nl[i]->prepareNeighbor(tid);
2461 : }
2462 108178 : _aux->prepareNeighbor(tid);
2463 :
2464 : // Resizes Re and Ke
2465 108178 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2466 :
2467 : // Compute the values of each variable at the points
2468 216356 : for (auto & nl : _nl)
2469 108178 : nl->reinitNeighborFace(neighbor, neighbor_side, tid);
2470 108178 : _aux->reinitNeighborFace(neighbor, neighbor_side, tid);
2471 108178 : }
2472 :
2473 : void
2474 22720 : FEProblemBase::reinitNeighborPhys(const Elem * neighbor,
2475 : const std::vector<Point> & physical_points,
2476 : const THREAD_ID tid)
2477 : {
2478 : mooseAssert(_mesh.queryElemPtr(neighbor->id()) == neighbor,
2479 : "Are you calling this method with a displaced mesh element?");
2480 :
2481 45440 : for (const auto i : index_range(_nl))
2482 : {
2483 : // Reinits shape the functions at the physical points
2484 22720 : _assembly[tid][i]->reinitNeighborAtPhysical(neighbor, physical_points);
2485 :
2486 : // Sets the neighbor dof indices
2487 22720 : _nl[i]->prepareNeighbor(tid);
2488 : }
2489 22720 : _aux->prepareNeighbor(tid);
2490 :
2491 : // Resizes Re and Ke
2492 22720 : _assembly[tid][_current_nl_sys->number()]->prepareNeighbor();
2493 :
2494 : // Compute the values of each variable at the points
2495 45440 : for (auto & nl : _nl)
2496 22720 : nl->reinitNeighbor(neighbor, tid);
2497 22720 : _aux->reinitNeighbor(neighbor, tid);
2498 22720 : }
2499 :
2500 : void
2501 38812 : FEProblemBase::getDiracElements(std::set<const Elem *> & elems)
2502 : {
2503 : // First add in the undisplaced elements
2504 38812 : elems = _dirac_kernel_info.getElements();
2505 :
2506 38812 : if (_displaced_problem)
2507 : {
2508 2539 : std::set<const Elem *> displaced_elements;
2509 2539 : _displaced_problem->getDiracElements(displaced_elements);
2510 :
2511 : { // Use the ids from the displaced elements to get the undisplaced elements
2512 : // and add them to the list
2513 8051 : for (const auto & elem : displaced_elements)
2514 5512 : elems.insert(_mesh.elemPtr(elem->id()));
2515 : }
2516 2539 : }
2517 38812 : }
2518 :
2519 : void
2520 3875597 : FEProblemBase::clearDiracInfo()
2521 : {
2522 3875597 : _dirac_kernel_info.clearPoints();
2523 :
2524 3875597 : if (_displaced_problem)
2525 159253 : _displaced_problem->clearDiracInfo();
2526 3875597 : }
2527 :
2528 : void
2529 6065760 : FEProblemBase::subdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2530 : {
2531 6065760 : _all_materials.subdomainSetup(subdomain, tid);
2532 : // Call the subdomain methods of the output system, these are not threaded so only call it once
2533 6065760 : if (tid == 0)
2534 5616311 : _app.getOutputWarehouse().subdomainSetup();
2535 :
2536 12283050 : for (auto & nl : _nl)
2537 6217290 : nl->subdomainSetup(subdomain, tid);
2538 :
2539 : // FIXME: call displaced_problem->subdomainSetup() ?
2540 : // When adding possibility with materials being evaluated on displaced mesh
2541 6065760 : }
2542 :
2543 : void
2544 18307735 : FEProblemBase::neighborSubdomainSetup(SubdomainID subdomain, const THREAD_ID tid)
2545 : {
2546 18307735 : _all_materials.neighborSubdomainSetup(subdomain, tid);
2547 18307735 : }
2548 :
2549 : void
2550 53316 : FEProblemBase::addFunction(const std::string & type,
2551 : const std::string & name,
2552 : InputParameters & parameters)
2553 : {
2554 : parallel_object_only();
2555 :
2556 106632 : parameters.set<SubProblem *>("_subproblem") = this;
2557 :
2558 110579 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2559 : {
2560 57400 : std::shared_ptr<Function> func = _factory.create<Function>(type, name, parameters, tid);
2561 57263 : logAdd("Function", name, type, parameters);
2562 57263 : _functions.addObject(func, tid);
2563 :
2564 57263 : if (auto * const functor = dynamic_cast<Moose::FunctorBase<Real> *>(func.get()))
2565 : {
2566 57263 : this->addFunctor(name, *functor, tid);
2567 57263 : if (_displaced_problem)
2568 1953 : _displaced_problem->addFunctor(name, *functor, tid);
2569 : }
2570 : else
2571 0 : mooseError("Unrecognized function functor type");
2572 57263 : }
2573 53179 : }
2574 :
2575 : void
2576 160256 : FEProblemBase::addConvergence(const std::string & type,
2577 : const std::string & name,
2578 : InputParameters & parameters)
2579 : {
2580 : parallel_object_only();
2581 :
2582 336091 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2583 : {
2584 175871 : std::shared_ptr<Convergence> conv = _factory.create<Convergence>(type, name, parameters, tid);
2585 175835 : _convergences.addObject(conv, tid);
2586 175835 : }
2587 160220 : }
2588 :
2589 : void
2590 63288 : FEProblemBase::addDefaultNonlinearConvergence(const InputParameters & params_to_apply)
2591 : {
2592 63288 : const std::string class_name = "DefaultNonlinearConvergence";
2593 63288 : InputParameters params = _factory.getValidParams(class_name);
2594 63288 : params.applyParameters(params_to_apply);
2595 63288 : params.applyParameters(parameters());
2596 63288 : params.set<bool>("added_as_default") = true;
2597 125515 : for (const auto & conv_name : getNonlinearConvergenceNames())
2598 62227 : addConvergence(class_name, conv_name, params);
2599 63288 : }
2600 :
2601 : void
2602 64123 : FEProblemBase::addDefaultMultiAppFixedPointConvergence(const InputParameters & params_to_apply)
2603 : {
2604 64123 : const std::string class_name = "DefaultMultiAppFixedPointConvergence";
2605 64123 : InputParameters params = _factory.getValidParams(class_name);
2606 64123 : params.applyParameters(params_to_apply);
2607 64123 : params.applyParameters(parameters());
2608 64123 : params.set<bool>("added_as_default") = true;
2609 64123 : addConvergence(class_name, getMultiAppFixedPointConvergenceName(), params);
2610 64111 : }
2611 :
2612 : void
2613 31966 : FEProblemBase::addDefaultSteadyStateConvergence(const InputParameters & params_to_apply)
2614 : {
2615 31966 : const std::string class_name = "DefaultSteadyStateConvergence";
2616 31966 : InputParameters params = _factory.getValidParams(class_name);
2617 31966 : params.applyParameters(params_to_apply);
2618 31966 : params.applyParameters(parameters());
2619 31966 : params.set<bool>("added_as_default") = true;
2620 31966 : addConvergence(class_name, getSteadyStateConvergenceName(), params);
2621 31966 : }
2622 :
2623 : bool
2624 89078 : FEProblemBase::hasFunction(const std::string & name, const THREAD_ID tid)
2625 : {
2626 89078 : return _functions.hasActiveObject(name, tid);
2627 : }
2628 :
2629 : Function &
2630 64968 : FEProblemBase::getFunction(const std::string & name, const THREAD_ID tid)
2631 : {
2632 : // This thread lock is necessary since this method will create functions
2633 : // for all threads if one is missing.
2634 64968 : Threads::spin_mutex::scoped_lock lock(get_function_mutex);
2635 :
2636 64968 : if (!hasFunction(name, tid))
2637 : {
2638 : // If we didn't find a function, it might be a default function, attempt to construct one now
2639 20250 : std::istringstream ss(name);
2640 : Real real_value;
2641 :
2642 : // First see if it's just a constant. If it is, build a ConstantFunction
2643 20250 : if (ss >> real_value && ss.eof())
2644 : {
2645 12984 : InputParameters params = _factory.getValidParams("ConstantFunction");
2646 12984 : params.set<Real>("value") = real_value;
2647 19476 : addFunction("ConstantFunction", ss.str(), params);
2648 6492 : }
2649 : else
2650 : {
2651 13758 : FunctionParserBase<Real> fp;
2652 13758 : std::string vars = "x,y,z,t,NaN,pi,e";
2653 13758 : if (fp.Parse(name, vars) == -1) // -1 for success
2654 : {
2655 : // It parsed ok, so build a MooseParsedFunction
2656 41262 : InputParameters params = _factory.getValidParams("ParsedFunction");
2657 13754 : params.set<std::string>("expression") = name;
2658 27508 : addFunction("ParsedFunction", name, params);
2659 13754 : }
2660 13758 : }
2661 :
2662 : // Try once more
2663 20250 : if (!hasFunction(name, tid))
2664 4 : mooseError("Unable to find function " + name);
2665 20246 : }
2666 :
2667 64964 : auto * const ret = dynamic_cast<Function *>(_functions.getActiveObject(name, tid).get());
2668 64964 : if (!ret)
2669 0 : mooseError("No function named ", name, " of appropriate type");
2670 :
2671 64964 : return *ret;
2672 64964 : }
2673 :
2674 : bool
2675 222040 : FEProblemBase::hasConvergence(const std::string & name, const THREAD_ID tid) const
2676 : {
2677 222040 : return _convergences.hasActiveObject(name, tid);
2678 : }
2679 :
2680 : Convergence &
2681 1234002 : FEProblemBase::getConvergence(const std::string & name, const THREAD_ID tid) const
2682 : {
2683 1234002 : auto * const ret = dynamic_cast<Convergence *>(_convergences.getActiveObject(name, tid).get());
2684 1234002 : if (!ret)
2685 0 : mooseError("The Convergence object '", name, "' does not exist.");
2686 :
2687 1234002 : return *ret;
2688 : }
2689 :
2690 : const std::vector<std::shared_ptr<Convergence>> &
2691 210740 : FEProblemBase::getConvergenceObjects(const THREAD_ID tid) const
2692 : {
2693 210740 : return _convergences.getActiveObjects(tid);
2694 : }
2695 :
2696 : void
2697 770 : FEProblemBase::addMeshDivision(const std::string & type,
2698 : const std::string & name,
2699 : InputParameters & parameters)
2700 : {
2701 : parallel_object_only();
2702 1540 : parameters.set<FEProblemBase *>("_fe_problem_base") = this;
2703 1540 : parameters.set<SubProblem *>("_subproblem") = this;
2704 1623 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
2705 : {
2706 865 : std::shared_ptr<MeshDivision> func = _factory.create<MeshDivision>(type, name, parameters, tid);
2707 853 : _mesh_divisions.addObject(func, tid);
2708 853 : }
2709 758 : }
2710 :
2711 : MeshDivision &
2712 1665 : FEProblemBase::getMeshDivision(const std::string & name, const THREAD_ID tid) const
2713 : {
2714 1665 : auto * const ret = dynamic_cast<MeshDivision *>(_mesh_divisions.getActiveObject(name, tid).get());
2715 1665 : if (!ret)
2716 0 : mooseError("No MeshDivision object named ", name, " of appropriate type");
2717 1665 : return *ret;
2718 : }
2719 :
2720 : void
2721 0 : FEProblemBase::lineSearch()
2722 : {
2723 0 : _line_search->lineSearch();
2724 0 : }
2725 :
2726 : NonlinearSystem &
2727 0 : FEProblemBase::getNonlinearSystem(const unsigned int sys_num)
2728 : {
2729 0 : mooseDeprecated("FEProblemBase::getNonlinearSystem() is deprecated, please use "
2730 : "FEProblemBase::getNonlinearSystemBase() \n");
2731 :
2732 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
2733 0 : auto nl_sys = std::dynamic_pointer_cast<NonlinearSystem>(_nl[sys_num]);
2734 :
2735 0 : if (!nl_sys)
2736 0 : mooseError("This is not a NonlinearSystem");
2737 :
2738 0 : return *nl_sys;
2739 0 : }
2740 :
2741 : void
2742 0 : FEProblemBase::addDistribution(const std::string & type,
2743 : const std::string & name,
2744 : InputParameters & parameters)
2745 : {
2746 0 : parameters.set<std::string>("type") = type;
2747 0 : addObject<Distribution>(type, name, parameters, /* threaded = */ false);
2748 0 : }
2749 :
2750 : Distribution &
2751 0 : FEProblemBase::getDistribution(const std::string & name)
2752 : {
2753 0 : std::vector<Distribution *> objs;
2754 0 : theWarehouse()
2755 0 : .query()
2756 0 : .condition<AttribSystem>("Distribution")
2757 0 : .condition<AttribName>(name)
2758 0 : .queryInto(objs);
2759 0 : if (objs.empty())
2760 0 : mooseError("Unable to find Distribution with name '" + name + "'");
2761 0 : return *(objs[0]);
2762 0 : }
2763 :
2764 : void
2765 356 : FEProblemBase::addSampler(const std::string & type,
2766 : const std::string & name,
2767 : InputParameters & parameters)
2768 : {
2769 356 : const auto samplers = addObject<Sampler>(type, name, parameters);
2770 726 : for (auto & sampler : samplers)
2771 382 : sampler->init();
2772 344 : }
2773 :
2774 : Sampler &
2775 344 : FEProblemBase::getSampler(const std::string & name, const THREAD_ID tid)
2776 : {
2777 344 : std::vector<Sampler *> objs;
2778 344 : theWarehouse()
2779 688 : .query()
2780 344 : .condition<AttribSystem>("Sampler")
2781 344 : .condition<AttribThread>(tid)
2782 344 : .condition<AttribName>(name)
2783 344 : .queryInto(objs);
2784 344 : if (objs.empty())
2785 0 : mooseError(
2786 0 : "Unable to find Sampler with name '" + name +
2787 : "', if you are attempting to access this object in the constructor of another object then "
2788 : "the object being retrieved must occur prior to the caller within the input file.");
2789 688 : return *(objs[0]);
2790 344 : }
2791 :
2792 : bool
2793 161801 : FEProblemBase::duplicateVariableCheck(const std::string & var_name,
2794 : const FEType & type,
2795 : bool is_aux,
2796 : const std::set<SubdomainID> * const active_subdomains)
2797 : {
2798 161801 : std::set<SubdomainID> subdomainIDs;
2799 161801 : if (active_subdomains->size() == 0)
2800 : {
2801 153031 : const auto subdomains = _mesh.meshSubdomains();
2802 153031 : subdomainIDs.insert(subdomains.begin(), subdomains.end());
2803 153031 : }
2804 : else
2805 8770 : subdomainIDs.insert(active_subdomains->begin(), active_subdomains->end());
2806 :
2807 324343 : for (auto & sys : _solver_systems)
2808 : {
2809 162558 : SystemBase * curr_sys_ptr = sys.get();
2810 162558 : SystemBase * other_sys_ptr = _aux.get();
2811 162558 : std::string error_prefix = "";
2812 162558 : if (is_aux)
2813 : {
2814 96720 : curr_sys_ptr = _aux.get();
2815 96720 : other_sys_ptr = sys.get();
2816 96720 : error_prefix = "aux";
2817 : }
2818 :
2819 162558 : if (other_sys_ptr->hasVariable(var_name))
2820 4 : mooseError("Cannot have an auxiliary variable and a solver variable with the same name: ",
2821 : var_name);
2822 :
2823 162554 : if (curr_sys_ptr->hasVariable(var_name))
2824 : {
2825 : const Variable & var =
2826 12 : curr_sys_ptr->system().variable(curr_sys_ptr->system().variable_number(var_name));
2827 :
2828 : // variable type
2829 12 : if (var.type() != type)
2830 : {
2831 16 : const auto stringifyType = [](FEType t)
2832 16 : { return Moose::stringify(t.family) + " of order " + Moose::stringify(t.order); };
2833 :
2834 8 : mooseError("Mismatching types are specified for ",
2835 : error_prefix,
2836 : "variable with name '",
2837 : var_name,
2838 : "': '",
2839 8 : stringifyType(var.type()),
2840 : "' and '",
2841 8 : stringifyType(type),
2842 : "'");
2843 : }
2844 :
2845 : // block-restriction
2846 4 : if (!(active_subdomains->size() == 0 && var.active_subdomains().size() == 0))
2847 : {
2848 4 : const auto varActiveSubdomains = var.active_subdomains();
2849 4 : std::set<SubdomainID> varSubdomainIDs;
2850 4 : if (varActiveSubdomains.size() == 0)
2851 : {
2852 0 : const auto subdomains = _mesh.meshSubdomains();
2853 0 : varSubdomainIDs.insert(subdomains.begin(), subdomains.end());
2854 0 : }
2855 : else
2856 4 : varSubdomainIDs.insert(varActiveSubdomains.begin(), varActiveSubdomains.end());
2857 :
2858 : // Is subdomainIDs a subset of varSubdomainIDs? With this we allow the case that the newly
2859 : // requested block restriction is only a subset of the existing one.
2860 4 : const auto isSubset = std::includes(varSubdomainIDs.begin(),
2861 : varSubdomainIDs.end(),
2862 : subdomainIDs.begin(),
2863 : subdomainIDs.end());
2864 :
2865 4 : if (!isSubset)
2866 : {
2867 : // helper function: make a string from a set of subdomain ids
2868 8 : const auto stringifySubdomains = [this](std::set<SubdomainID> subdomainIDs)
2869 : {
2870 8 : std::stringstream s;
2871 20 : for (auto const i : subdomainIDs)
2872 : {
2873 : // do we need to insert a comma?
2874 12 : if (s.tellp() != 0)
2875 4 : s << ", ";
2876 :
2877 : // insert subdomain name and id -or- only the id (if no name is given)
2878 12 : const auto subdomainName = _mesh.getSubdomainName(i);
2879 12 : if (subdomainName.empty())
2880 12 : s << i;
2881 : else
2882 0 : s << subdomainName << " (" << i << ")";
2883 12 : }
2884 16 : return s.str();
2885 8 : };
2886 :
2887 8 : const std::string msg = "Mismatching block-restrictions are specified for " +
2888 8 : error_prefix + "variable with name '" + var_name + "': {" +
2889 16 : stringifySubdomains(varSubdomainIDs) + "} and {" +
2890 12 : stringifySubdomains(subdomainIDs) + "}";
2891 :
2892 4 : mooseError(msg);
2893 0 : }
2894 0 : }
2895 :
2896 0 : return true;
2897 : }
2898 162542 : }
2899 :
2900 161785 : return false;
2901 161785 : }
2902 :
2903 : void
2904 65237 : FEProblemBase::addVariable(const std::string & var_type,
2905 : const std::string & var_name,
2906 : InputParameters & params)
2907 : {
2908 : parallel_object_only();
2909 :
2910 65237 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
2911 65237 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
2912 65237 : const auto fe_type = FEType(order, family);
2913 :
2914 : const auto active_subdomains_vector =
2915 65237 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
2916 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
2917 65237 : active_subdomains_vector.end());
2918 :
2919 65237 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ false, &active_subdomains))
2920 0 : return;
2921 :
2922 195699 : params.set<FEProblemBase *>("_fe_problem_base") = this;
2923 65233 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_SOLVER;
2924 65233 : SolverSystemName sys_name = params.get<SolverSystemName>("solver_sys");
2925 :
2926 65233 : const auto solver_system_number = solverSysNum(sys_name);
2927 65233 : logAdd("Variable", var_name, var_type, params);
2928 65233 : _solver_systems[solver_system_number]->addVariable(var_type, var_name, params);
2929 65217 : if (_displaced_problem)
2930 : // MooseObjects need to be unique so change the name here
2931 3839 : _displaced_problem->addVariable(var_type, var_name, params, solver_system_number);
2932 :
2933 65217 : _solver_var_to_sys_num[var_name] = solver_system_number;
2934 :
2935 65217 : markFamilyPRefinement(params);
2936 65217 : }
2937 :
2938 : std::pair<bool, unsigned int>
2939 5056598 : FEProblemBase::determineSolverSystem(const std::string & var_name,
2940 : const bool error_if_not_found) const
2941 : {
2942 5056598 : auto map_it = _solver_var_to_sys_num.find(var_name);
2943 5056598 : const bool var_in_sys = map_it != _solver_var_to_sys_num.end();
2944 5056598 : if (var_in_sys)
2945 : mooseAssert(_solver_systems[map_it->second]->hasVariable(var_name) ||
2946 : _solver_systems[map_it->second]->hasScalarVariable(var_name),
2947 : "If the variable is in our FEProblem solver system map, then it must be in the "
2948 : "solver system we expect");
2949 3414015 : else if (error_if_not_found)
2950 : {
2951 42 : if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
2952 28 : mooseError("No solver variable named ",
2953 : var_name,
2954 : " found. Did you specify an auxiliary variable when you meant to specify a "
2955 : "solver variable?");
2956 : else
2957 14 : mooseError("Unknown variable '",
2958 : var_name,
2959 : "'. It does not exist in the solver system(s) or auxiliary system");
2960 : }
2961 :
2962 10113112 : return std::make_pair(var_in_sys, var_in_sys ? map_it->second : libMesh::invalid_uint);
2963 : }
2964 :
2965 : void
2966 170776 : FEProblemBase::setResidualObjectParamsAndLog(const std::string & ro_name,
2967 : const std::string & name,
2968 : InputParameters & parameters,
2969 : const unsigned int nl_sys_num,
2970 : const std::string & base_name,
2971 : bool & reinit_displaced)
2972 : {
2973 170776 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
2974 : {
2975 2160 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
2976 2160 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
2977 1080 : reinit_displaced = true;
2978 : }
2979 : else
2980 : {
2981 169696 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
2982 : {
2983 : // We allow Kernels to request that they use_displaced_mesh,
2984 : // but then be overridden when no displacements variables are
2985 : // provided in the Mesh block. If that happened, update the value
2986 : // of use_displaced_mesh appropriately for this Kernel.
2987 136 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
2988 272 : parameters.set<bool>("use_displaced_mesh") = false;
2989 : }
2990 :
2991 339392 : parameters.set<SubProblem *>("_subproblem") = this;
2992 509088 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
2993 : }
2994 :
2995 170776 : logAdd(base_name, name, ro_name, parameters);
2996 170776 : }
2997 :
2998 : void
2999 68745 : FEProblemBase::setAuxKernelParamsAndLog(const std::string & ak_name,
3000 : const std::string & name,
3001 : InputParameters & parameters,
3002 : const std::string & base_name)
3003 : {
3004 68745 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3005 : {
3006 24808 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3007 24808 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3008 24808 : parameters.set<SystemBase *>("_nl_sys") = &_displaced_problem->solverSys(0);
3009 12404 : if (!parameters.get<std::vector<BoundaryName>>("boundary").empty())
3010 11916 : _reinit_displaced_face = true;
3011 : else
3012 488 : _reinit_displaced_elem = true;
3013 : }
3014 : else
3015 : {
3016 56341 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3017 : {
3018 : // We allow AuxKernels to request that they use_displaced_mesh,
3019 : // but then be overridden when no displacements variables are
3020 : // provided in the Mesh block. If that happened, update the value
3021 : // of use_displaced_mesh appropriately for this AuxKernel.
3022 863 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3023 1726 : parameters.set<bool>("use_displaced_mesh") = false;
3024 : }
3025 :
3026 112682 : parameters.set<SubProblem *>("_subproblem") = this;
3027 112682 : parameters.set<SystemBase *>("_sys") = _aux.get();
3028 169023 : parameters.set<SystemBase *>("_nl_sys") = _solver_systems[0].get();
3029 : }
3030 :
3031 68745 : logAdd(base_name, name, ak_name, parameters);
3032 68745 : }
3033 :
3034 : void
3035 83751 : FEProblemBase::addKernel(const std::string & kernel_name,
3036 : const std::string & name,
3037 : InputParameters & parameters)
3038 : {
3039 : parallel_object_only();
3040 167502 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3041 83735 : if (!isSolverSystemNonlinear(nl_sys_num))
3042 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
3043 : "supported at the moment!");
3044 83735 : setResidualObjectParamsAndLog(
3045 83735 : kernel_name, name, parameters, nl_sys_num, "Kernel", _reinit_displaced_elem);
3046 :
3047 83735 : _nl[nl_sys_num]->addKernel(kernel_name, name, parameters);
3048 83531 : }
3049 :
3050 : void
3051 808 : FEProblemBase::addHDGKernel(const std::string & kernel_name,
3052 : const std::string & name,
3053 : InputParameters & parameters)
3054 : {
3055 : parallel_object_only();
3056 1616 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3057 808 : if (!isSolverSystemNonlinear(nl_sys_num))
3058 0 : mooseError("You are trying to add a HDGKernel to a linear variable/system, which is not "
3059 : "supported at the moment!");
3060 808 : setResidualObjectParamsAndLog(
3061 808 : kernel_name, name, parameters, nl_sys_num, "HDGKernel", _reinit_displaced_elem);
3062 :
3063 808 : _nl[nl_sys_num]->addHDGKernel(kernel_name, name, parameters);
3064 808 : }
3065 :
3066 : void
3067 620 : FEProblemBase::addNodalKernel(const std::string & kernel_name,
3068 : const std::string & name,
3069 : InputParameters & parameters)
3070 : {
3071 : parallel_object_only();
3072 :
3073 1240 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3074 616 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3075 : {
3076 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3077 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3078 0 : _reinit_displaced_elem = true;
3079 : }
3080 : else
3081 : {
3082 616 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3083 : {
3084 : // We allow NodalKernels to request that they use_displaced_mesh,
3085 : // but then be overridden when no displacements variables are
3086 : // provided in the Mesh block. If that happened, update the value
3087 : // of use_displaced_mesh appropriately for this NodalKernel.
3088 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3089 0 : parameters.set<bool>("use_displaced_mesh") = false;
3090 : }
3091 :
3092 1232 : parameters.set<SubProblem *>("_subproblem") = this;
3093 1848 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3094 : }
3095 616 : logAdd("NodalKernel", name, kernel_name, parameters);
3096 616 : _nl[nl_sys_num]->addNodalKernel(kernel_name, name, parameters);
3097 616 : }
3098 :
3099 : void
3100 1551 : FEProblemBase::addScalarKernel(const std::string & kernel_name,
3101 : const std::string & name,
3102 : InputParameters & parameters)
3103 : {
3104 : parallel_object_only();
3105 :
3106 3102 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3107 1551 : if (!isSolverSystemNonlinear(nl_sys_num))
3108 0 : mooseError("You are trying to add a ScalarKernel to a linear variable/system, which is not "
3109 : "supported at the moment!");
3110 :
3111 1551 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3112 : {
3113 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3114 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3115 : }
3116 : else
3117 : {
3118 1551 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3119 : {
3120 : // We allow ScalarKernels to request that they use_displaced_mesh,
3121 : // but then be overridden when no displacements variables are
3122 : // provided in the Mesh block. If that happened, update the value
3123 : // of use_displaced_mesh appropriately for this ScalarKernel.
3124 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3125 0 : parameters.set<bool>("use_displaced_mesh") = false;
3126 : }
3127 :
3128 3102 : parameters.set<SubProblem *>("_subproblem") = this;
3129 4653 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3130 : }
3131 :
3132 1551 : logAdd("ScalarKernel", name, kernel_name, parameters);
3133 1551 : _nl[nl_sys_num]->addScalarKernel(kernel_name, name, parameters);
3134 1543 : }
3135 :
3136 : void
3137 82940 : FEProblemBase::addBoundaryCondition(const std::string & bc_name,
3138 : const std::string & name,
3139 : InputParameters & parameters)
3140 : {
3141 : parallel_object_only();
3142 :
3143 165884 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3144 82930 : if (!isSolverSystemNonlinear(nl_sys_num))
3145 0 : mooseError(
3146 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
3147 : "supported at the moment!");
3148 :
3149 82930 : setResidualObjectParamsAndLog(
3150 82930 : bc_name, name, parameters, nl_sys_num, "BoundaryCondition", _reinit_displaced_face);
3151 82930 : _nl[nl_sys_num]->addBoundaryCondition(bc_name, name, parameters);
3152 82872 : }
3153 :
3154 : void
3155 1731 : FEProblemBase::addConstraint(const std::string & c_name,
3156 : const std::string & name,
3157 : InputParameters & parameters)
3158 : {
3159 : parallel_object_only();
3160 :
3161 1731 : _has_constraints = true;
3162 :
3163 1731 : auto determine_var_param_name = [¶meters, this]()
3164 : {
3165 5193 : if (parameters.isParamValid("variable"))
3166 1125 : return "variable";
3167 : else
3168 : {
3169 : // must be a mortar constraint
3170 1212 : const bool has_secondary_var = parameters.isParamValid("secondary_variable");
3171 1212 : const bool has_primary_var = parameters.isParamValid("primary_variable");
3172 606 : if (!has_secondary_var && !has_primary_var)
3173 0 : mooseError(
3174 : "Either a 'secondary_variable' or 'primary_variable' parameter must be supplied for '",
3175 0 : parameters.getObjectName(),
3176 : "'");
3177 606 : return has_secondary_var ? "secondary_variable" : "primary_variable";
3178 : }
3179 1731 : };
3180 :
3181 : const auto nl_sys_num =
3182 3462 : determineSolverSystem(parameters.varName(determine_var_param_name(), name), true).second;
3183 1727 : if (!isSolverSystemNonlinear(nl_sys_num))
3184 0 : mooseError("You are trying to add a Constraint to a linear variable/system, which is not "
3185 : "supported at the moment!");
3186 :
3187 1727 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3188 : {
3189 294 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3190 294 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3191 147 : _reinit_displaced_face = true;
3192 : }
3193 : else
3194 : {
3195 : // It might _want_ to use a displaced mesh... but we're not so set it to false
3196 1580 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3197 3160 : parameters.set<bool>("use_displaced_mesh") = false;
3198 :
3199 3160 : parameters.set<SubProblem *>("_subproblem") = this;
3200 4740 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3201 : }
3202 :
3203 1727 : logAdd("Constraint", name, c_name, parameters);
3204 1727 : _nl[nl_sys_num]->addConstraint(c_name, name, parameters);
3205 1719 : }
3206 :
3207 : void
3208 96564 : FEProblemBase::addAuxVariable(const std::string & var_type,
3209 : const std::string & var_name,
3210 : InputParameters & params)
3211 : {
3212 : parallel_object_only();
3213 :
3214 96564 : const auto order = Utility::string_to_enum<Order>(params.get<MooseEnum>("order"));
3215 96564 : const auto family = Utility::string_to_enum<FEFamily>(params.get<MooseEnum>("family"));
3216 96564 : const auto fe_type = FEType(order, family);
3217 :
3218 : const auto active_subdomains_vector =
3219 96564 : _mesh.getSubdomainIDs(params.get<std::vector<SubdomainName>>("block"));
3220 : const std::set<SubdomainID> active_subdomains(active_subdomains_vector.begin(),
3221 96564 : active_subdomains_vector.end());
3222 :
3223 96564 : if (duplicateVariableCheck(var_name, fe_type, /* is_aux = */ true, &active_subdomains))
3224 0 : return;
3225 :
3226 289656 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3227 193104 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3228 :
3229 96552 : logAdd("AuxVariable", var_name, var_type, params);
3230 96552 : _aux->addVariable(var_type, var_name, params);
3231 96552 : if (_displaced_problem)
3232 : // MooseObjects need to be unique so change the name here
3233 11112 : _displaced_problem->addAuxVariable(var_type, var_name, params);
3234 :
3235 96552 : markFamilyPRefinement(params);
3236 96552 : }
3237 :
3238 : void
3239 0 : FEProblemBase::addAuxVariable(const std::string & var_name,
3240 : const FEType & type,
3241 : const std::set<SubdomainID> * const active_subdomains)
3242 : {
3243 : parallel_object_only();
3244 :
3245 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3246 :
3247 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3248 0 : return;
3249 :
3250 0 : std::string var_type;
3251 0 : if (type == FEType(0, MONOMIAL))
3252 0 : var_type = "MooseVariableConstMonomial";
3253 0 : else if (type.family == SCALAR)
3254 0 : var_type = "MooseVariableScalar";
3255 0 : else if (FEInterface::field_type(type) == TYPE_VECTOR)
3256 0 : var_type = "VectorMooseVariable";
3257 : else
3258 0 : var_type = "MooseVariable";
3259 :
3260 0 : InputParameters params = _factory.getValidParams(var_type);
3261 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3262 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3263 0 : params.set<MooseEnum>("order") = type.order.get_order();
3264 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3265 :
3266 0 : if (active_subdomains)
3267 0 : for (const SubdomainID & id : *active_subdomains)
3268 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3269 :
3270 0 : logAdd("AuxVariable", var_name, var_type, params);
3271 0 : _aux->addVariable(var_type, var_name, params);
3272 0 : if (_displaced_problem)
3273 0 : _displaced_problem->addAuxVariable("MooseVariable", var_name, params);
3274 :
3275 0 : markFamilyPRefinement(params);
3276 0 : }
3277 :
3278 : void
3279 0 : FEProblemBase::addAuxArrayVariable(const std::string & var_name,
3280 : const FEType & type,
3281 : unsigned int components,
3282 : const std::set<SubdomainID> * const active_subdomains)
3283 : {
3284 : parallel_object_only();
3285 :
3286 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3287 :
3288 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3289 0 : return;
3290 :
3291 0 : InputParameters params = _factory.getValidParams("ArrayMooseVariable");
3292 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3293 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3294 0 : params.set<MooseEnum>("order") = type.order.get_order();
3295 0 : params.set<MooseEnum>("family") = Moose::stringify(type.family);
3296 0 : params.set<unsigned int>("components") = components;
3297 :
3298 0 : if (active_subdomains)
3299 0 : for (const SubdomainID & id : *active_subdomains)
3300 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3301 :
3302 0 : logAdd("Variable", var_name, "ArrayMooseVariable", params);
3303 0 : _aux->addVariable("ArrayMooseVariable", var_name, params);
3304 0 : if (_displaced_problem)
3305 0 : _displaced_problem->addAuxVariable("ArrayMooseVariable", var_name, params);
3306 :
3307 0 : markFamilyPRefinement(params);
3308 0 : }
3309 :
3310 : void
3311 0 : FEProblemBase::addAuxScalarVariable(const std::string & var_name,
3312 : Order order,
3313 : Real /*scale_factor*/,
3314 : const std::set<SubdomainID> * const active_subdomains)
3315 : {
3316 : parallel_object_only();
3317 :
3318 0 : mooseDeprecated("Please use the addAuxVariable(var_type, var_name, params) API instead");
3319 :
3320 0 : if (order > _max_scalar_order)
3321 0 : _max_scalar_order = order;
3322 :
3323 0 : FEType type(order, SCALAR);
3324 0 : if (duplicateVariableCheck(var_name, type, /* is_aux = */ true, active_subdomains))
3325 0 : return;
3326 :
3327 0 : InputParameters params = _factory.getValidParams("MooseVariableScalar");
3328 0 : params.set<FEProblemBase *>("_fe_problem_base") = this;
3329 0 : params.set<Moose::VarKindType>("_var_kind") = Moose::VarKindType::VAR_AUXILIARY;
3330 :
3331 0 : params.set<MooseEnum>("order") = type.order.get_order();
3332 0 : params.set<MooseEnum>("family") = "SCALAR";
3333 0 : params.set<std::vector<Real>>("scaling") = {1};
3334 0 : if (active_subdomains)
3335 0 : for (const SubdomainID & id : *active_subdomains)
3336 0 : params.set<std::vector<SubdomainName>>("block").push_back(Moose::stringify(id));
3337 :
3338 0 : logAdd("ScalarVariable", var_name, "MooseVariableScalar", params);
3339 0 : _aux->addVariable("MooseVariableScalar", var_name, params);
3340 0 : if (_displaced_problem)
3341 0 : _displaced_problem->addAuxVariable("MooseVariableScalar", var_name, params);
3342 0 : }
3343 :
3344 : void
3345 68429 : FEProblemBase::addAuxKernel(const std::string & kernel_name,
3346 : const std::string & name,
3347 : InputParameters & parameters)
3348 : {
3349 : parallel_object_only();
3350 :
3351 68429 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "AuxKernel");
3352 :
3353 68429 : _aux->addKernel(kernel_name, name, parameters);
3354 68277 : }
3355 :
3356 : void
3357 537 : FEProblemBase::addAuxScalarKernel(const std::string & kernel_name,
3358 : const std::string & name,
3359 : InputParameters & parameters)
3360 : {
3361 : parallel_object_only();
3362 :
3363 537 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3364 : {
3365 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3366 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
3367 : }
3368 : else
3369 : {
3370 537 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3371 : {
3372 : // We allow AuxScalarKernels to request that they use_displaced_mesh,
3373 : // but then be overridden when no displacements variables are
3374 : // provided in the Mesh block. If that happened, update the value
3375 : // of use_displaced_mesh appropriately for this AuxScalarKernel.
3376 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3377 0 : parameters.set<bool>("use_displaced_mesh") = false;
3378 : }
3379 :
3380 1074 : parameters.set<SubProblem *>("_subproblem") = this;
3381 1611 : parameters.set<SystemBase *>("_sys") = _aux.get();
3382 : }
3383 :
3384 537 : logAdd("AuxScalarKernel", name, kernel_name, parameters);
3385 537 : _aux->addScalarKernel(kernel_name, name, parameters);
3386 533 : }
3387 :
3388 : void
3389 942 : FEProblemBase::addDiracKernel(const std::string & kernel_name,
3390 : const std::string & name,
3391 : InputParameters & parameters)
3392 : {
3393 : parallel_object_only();
3394 :
3395 1884 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3396 938 : if (!isSolverSystemNonlinear(nl_sys_num))
3397 0 : mooseError("You are trying to add a DiracKernel to a linear variable/system, which is not "
3398 : "supported at the moment!");
3399 :
3400 938 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3401 : {
3402 26 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3403 26 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3404 13 : _reinit_displaced_elem = true;
3405 : }
3406 : else
3407 : {
3408 925 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3409 : {
3410 : // We allow DiracKernels to request that they use_displaced_mesh,
3411 : // but then be overridden when no displacements variables are
3412 : // provided in the Mesh block. If that happened, update the value
3413 : // of use_displaced_mesh appropriately for this DiracKernel.
3414 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3415 0 : parameters.set<bool>("use_displaced_mesh") = false;
3416 : }
3417 :
3418 1850 : parameters.set<SubProblem *>("_subproblem") = this;
3419 2775 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3420 : }
3421 :
3422 938 : logAdd("DiracKernel", name, kernel_name, parameters);
3423 938 : _nl[nl_sys_num]->addDiracKernel(kernel_name, name, parameters);
3424 930 : }
3425 :
3426 : // DGKernels ////
3427 :
3428 : void
3429 1453 : FEProblemBase::addDGKernel(const std::string & dg_kernel_name,
3430 : const std::string & name,
3431 : InputParameters & parameters)
3432 : {
3433 : parallel_object_only();
3434 :
3435 2906 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3436 1449 : if (!isSolverSystemNonlinear(nl_sys_num))
3437 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
3438 : "supported at the moment!");
3439 :
3440 1449 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3441 : {
3442 52 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3443 52 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3444 26 : _reinit_displaced_neighbor = true;
3445 : }
3446 : else
3447 : {
3448 1423 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3449 : {
3450 : // We allow DGKernels to request that they use_displaced_mesh,
3451 : // but then be overridden when no displacements variables are
3452 : // provided in the Mesh block. If that happened, update the value
3453 : // of use_displaced_mesh appropriately for this DGKernel.
3454 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3455 0 : parameters.set<bool>("use_displaced_mesh") = false;
3456 : }
3457 :
3458 2846 : parameters.set<SubProblem *>("_subproblem") = this;
3459 4269 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3460 : }
3461 :
3462 1449 : logAdd("DGKernel", name, dg_kernel_name, parameters);
3463 1449 : _nl[nl_sys_num]->addDGKernel(dg_kernel_name, name, parameters);
3464 :
3465 1449 : _has_internal_edge_residual_objects = true;
3466 1449 : }
3467 :
3468 : void
3469 9122 : FEProblemBase::addFVKernel(const std::string & fv_kernel_name,
3470 : const std::string & name,
3471 : InputParameters & parameters)
3472 : {
3473 9122 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3474 : // FVElementalKernels are computed in the historically finite element threaded loops. They rely
3475 : // on Assembly data like _current_elem. When we call reinit on the FEProblemBase we will only
3476 : // reinit the DisplacedProblem and its associated Assembly objects if we mark this boolean as
3477 : // true
3478 0 : _reinit_displaced_elem = true;
3479 9122 : addObject<FVKernel>(fv_kernel_name, name, parameters);
3480 9114 : }
3481 :
3482 : void
3483 7779 : FEProblemBase::addFVBC(const std::string & fv_bc_name,
3484 : const std::string & name,
3485 : InputParameters & parameters)
3486 : {
3487 7779 : addObject<FVBoundaryCondition>(fv_bc_name, name, parameters);
3488 7779 : }
3489 :
3490 : void
3491 327 : FEProblemBase::addFVInterfaceKernel(const std::string & fv_ik_name,
3492 : const std::string & name,
3493 : InputParameters & parameters)
3494 : {
3495 : /// We assume that variable1 and variable2 can live on different systems, in this case
3496 : /// the user needs to create two interface kernels with flipped variables and parameters
3497 327 : addObject<FVInterfaceKernel>(
3498 : fv_ik_name, name, parameters, /*threaded=*/true, /*variable_param_name=*/"variable1");
3499 315 : }
3500 :
3501 : void
3502 3295 : FEProblemBase::addLinearFVKernel(const std::string & kernel_name,
3503 : const std::string & name,
3504 : InputParameters & parameters)
3505 : {
3506 3295 : addObject<LinearFVKernel>(kernel_name, name, parameters);
3507 3295 : }
3508 :
3509 : void
3510 2382 : FEProblemBase::addLinearFVBC(const std::string & bc_name,
3511 : const std::string & name,
3512 : InputParameters & parameters)
3513 : {
3514 2382 : addObject<LinearFVBoundaryCondition>(bc_name, name, parameters);
3515 2382 : }
3516 :
3517 : // InterfaceKernels ////
3518 :
3519 : void
3520 919 : FEProblemBase::addInterfaceKernel(const std::string & interface_kernel_name,
3521 : const std::string & name,
3522 : InputParameters & parameters)
3523 : {
3524 : parallel_object_only();
3525 :
3526 1838 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
3527 915 : if (!isSolverSystemNonlinear(nl_sys_num))
3528 0 : mooseError("You are trying to add a InterfaceKernel to a linear variable/system, which is not "
3529 : "supported at the moment!");
3530 :
3531 915 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
3532 : {
3533 26 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
3534 26 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(nl_sys_num);
3535 13 : _reinit_displaced_neighbor = true;
3536 : }
3537 : else
3538 : {
3539 902 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
3540 : {
3541 : // We allow InterfaceKernels to request that they use_displaced_mesh,
3542 : // but then be overridden when no displacements variables are
3543 : // provided in the Mesh block. If that happened, update the value
3544 : // of use_displaced_mesh appropriately for this InterfaceKernel.
3545 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
3546 0 : parameters.set<bool>("use_displaced_mesh") = false;
3547 : }
3548 :
3549 1804 : parameters.set<SubProblem *>("_subproblem") = this;
3550 2706 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
3551 : }
3552 :
3553 915 : logAdd("InterfaceKernel", name, interface_kernel_name, parameters);
3554 915 : _nl[nl_sys_num]->addInterfaceKernel(interface_kernel_name, name, parameters);
3555 :
3556 915 : _has_internal_edge_residual_objects = true;
3557 915 : }
3558 :
3559 : void
3560 34004 : FEProblemBase::checkICRestartError(const std::string & ic_name,
3561 : const std::string & name,
3562 : const VariableName & var_name)
3563 : {
3564 34004 : if (!_allow_ics_during_restart)
3565 : {
3566 33908 : std::string restart_method = "";
3567 33908 : if (_app.isRestarting())
3568 : restart_method =
3569 0 : "a checkpoint restart, by IC object '" + ic_name + "' for variable '" + name + "'";
3570 33908 : else if (_app.getExReaderForRestart())
3571 : {
3572 4 : std::vector<std::string> restarted_vars = _app.getExReaderForRestart()->get_elem_var_names();
3573 4 : const auto nodal_vars = _app.getExReaderForRestart()->get_nodal_var_names();
3574 4 : const auto global_vars = _app.getExReaderForRestart()->get_global_var_names();
3575 4 : restarted_vars.insert(restarted_vars.end(), nodal_vars.begin(), nodal_vars.end());
3576 4 : restarted_vars.insert(restarted_vars.end(), global_vars.begin(), global_vars.end());
3577 :
3578 4 : if (std::find(restarted_vars.begin(), restarted_vars.end(), var_name) != restarted_vars.end())
3579 8 : restart_method = "an Exodus restart, by IC object '" + ic_name + "' for variable '" + name +
3580 4 : "' that is also being restarted";
3581 4 : }
3582 33908 : if (!restart_method.empty())
3583 4 : mooseError(
3584 : "Initial conditions have been specified during ",
3585 : restart_method,
3586 : ".\nThis is only allowed if you specify 'allow_initial_conditions_with_restart' to "
3587 : "the [Problem], as initial conditions can override restarted fields");
3588 33904 : }
3589 34000 : }
3590 :
3591 : void
3592 31148 : FEProblemBase::addInitialCondition(const std::string & ic_name,
3593 : const std::string & name,
3594 : InputParameters & parameters)
3595 : {
3596 : parallel_object_only();
3597 :
3598 : // before we start to mess with the initial condition, we need to check parameters for errors.
3599 31148 : parameters.checkParams(name);
3600 31144 : const std::string & var_name = parameters.get<VariableName>("variable");
3601 :
3602 : // Forbid initial conditions on a restarted problem, as they would override the restart
3603 31144 : checkICRestartError(ic_name, name, var_name);
3604 :
3605 62280 : parameters.set<SubProblem *>("_subproblem") = this;
3606 :
3607 : // field IC
3608 31140 : if (hasVariable(var_name))
3609 : {
3610 61706 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3611 : {
3612 32360 : MooseVariableFEBase & var = getVariable(
3613 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3614 64720 : parameters.set<SystemBase *>("_sys") = &var.sys();
3615 32360 : std::shared_ptr<InitialConditionBase> ic;
3616 32360 : if (dynamic_cast<MooseVariable *>(&var))
3617 29752 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3618 2608 : else if (dynamic_cast<VectorMooseVariable *>(&var))
3619 272 : ic = _factory.create<VectorInitialCondition>(ic_name, name, parameters, tid);
3620 2336 : else if (dynamic_cast<ArrayMooseVariable *>(&var))
3621 1314 : ic = _factory.create<ArrayInitialCondition>(ic_name, name, parameters, tid);
3622 1022 : else if (dynamic_cast<MooseVariableFVReal *>(&var))
3623 1022 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3624 0 : else if (dynamic_cast<MooseLinearVariableFVReal *>(&var))
3625 0 : ic = _factory.create<InitialCondition>(ic_name, name, parameters, tid);
3626 : else
3627 0 : mooseError("Your FE variable in initial condition ",
3628 : name,
3629 : " must be either of scalar or vector type");
3630 32332 : logAdd("IC", name, ic_name, parameters);
3631 32332 : _ics.addObject(ic, tid);
3632 32312 : }
3633 : }
3634 :
3635 : // scalar IC
3636 1746 : else if (hasScalarVariable(var_name))
3637 : {
3638 1746 : MooseVariableScalar & var = getScalarVariable(0, var_name);
3639 3492 : parameters.set<SystemBase *>("_sys") = &var.sys();
3640 : std::shared_ptr<ScalarInitialCondition> ic =
3641 1746 : _factory.create<ScalarInitialCondition>(ic_name, name, parameters);
3642 1746 : logAdd("ScalarIC", name, ic_name, parameters);
3643 1746 : _scalar_ics.addObject(ic);
3644 1746 : }
3645 :
3646 : else
3647 0 : mooseError(
3648 : "Variable '", var_name, "' requested in initial condition '", name, "' does not exist.");
3649 31092 : }
3650 :
3651 : void
3652 2860 : FEProblemBase::addFVInitialCondition(const std::string & ic_name,
3653 : const std::string & name,
3654 : InputParameters & parameters)
3655 : {
3656 : parallel_object_only();
3657 :
3658 : // before we start to mess with the initial condition, we need to check parameters for errors.
3659 2860 : parameters.checkParams(name);
3660 2860 : const std::string & var_name = parameters.get<VariableName>("variable");
3661 :
3662 : // Forbid initial conditions on a restarted problem, as they would override the restart
3663 2860 : checkICRestartError(ic_name, name, var_name);
3664 :
3665 5720 : parameters.set<SubProblem *>("_subproblem") = this;
3666 :
3667 : // field IC
3668 2860 : if (hasVariable(var_name))
3669 : {
3670 5802 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
3671 : {
3672 2942 : auto & var = getVariable(
3673 : tid, var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_ANY);
3674 5884 : parameters.set<SystemBase *>("_sys") = &var.sys();
3675 2942 : std::shared_ptr<FVInitialConditionBase> ic;
3676 2942 : if (var.isFV())
3677 2942 : ic = _factory.create<FVInitialCondition>(ic_name, name, parameters, tid);
3678 : else
3679 0 : mooseError(
3680 : "Your variable for an FVInitialCondition needs to be an a finite volume variable!");
3681 2942 : _fv_ics.addObject(ic, tid);
3682 2942 : }
3683 : }
3684 : else
3685 0 : mooseError("Variable '",
3686 : var_name,
3687 : "' requested in finite volume initial condition '",
3688 : name,
3689 : "' does not exist.");
3690 2860 : }
3691 :
3692 : void
3693 58859 : FEProblemBase::projectSolution()
3694 : {
3695 294295 : TIME_SECTION("projectSolution", 2, "Projecting Initial Solutions")
3696 :
3697 58859 : FloatingPointExceptionGuard fpe_guard(_app);
3698 :
3699 58859 : ConstElemRange & elem_range = *_mesh.getActiveLocalElementRange();
3700 58859 : ComputeInitialConditionThread cic(*this);
3701 58859 : Threads::parallel_reduce(elem_range, cic);
3702 :
3703 58851 : if (haveFV())
3704 : {
3705 : using ElemInfoRange = StoredRange<MooseMesh::const_elem_info_iterator, const ElemInfo *>;
3706 5226 : ElemInfoRange elem_info_range(_mesh.ownedElemInfoBegin(), _mesh.ownedElemInfoEnd());
3707 :
3708 5226 : ComputeFVInitialConditionThread cfvic(*this);
3709 5226 : Threads::parallel_reduce(elem_info_range, cfvic);
3710 5226 : }
3711 :
3712 : // Need to close the solution vector here so that boundary ICs take precendence
3713 116633 : for (auto & nl : _nl)
3714 57782 : nl->solution().close();
3715 58851 : _aux->solution().close();
3716 :
3717 : // now run boundary-restricted initial conditions
3718 58851 : ConstBndNodeRange & bnd_nodes = *_mesh.getBoundaryNodeRange();
3719 58851 : ComputeBoundaryInitialConditionThread cbic(*this);
3720 58851 : Threads::parallel_reduce(bnd_nodes, cbic);
3721 :
3722 116633 : for (auto & nl : _nl)
3723 57782 : nl->solution().close();
3724 58851 : _aux->solution().close();
3725 :
3726 : // Also, load values into the SCALAR dofs
3727 : // Note: We assume that all SCALAR dofs are on the
3728 : // processor with highest ID
3729 58851 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3730 : {
3731 678 : const auto & ics = _scalar_ics.getActiveObjects();
3732 1864 : for (const auto & ic : ics)
3733 : {
3734 1186 : MooseVariableScalar & var = ic->variable();
3735 1186 : var.reinit();
3736 :
3737 1186 : DenseVector<Number> vals(var.order());
3738 1186 : ic->compute(vals);
3739 :
3740 1186 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3741 2757 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3742 : {
3743 1571 : const auto global_index = var.dofIndices()[i];
3744 1571 : var.sys().solution().set(global_index, vals(i));
3745 1571 : var.setValue(i, vals(i));
3746 : }
3747 1186 : }
3748 : }
3749 :
3750 117959 : for (auto & sys : _solver_systems)
3751 : {
3752 59108 : sys->solution().close();
3753 59108 : sys->solution().localize(*sys->system().current_local_solution, sys->dofMap().get_send_list());
3754 : }
3755 :
3756 58851 : _aux->solution().close();
3757 58851 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3758 58851 : }
3759 :
3760 : void
3761 2191 : FEProblemBase::projectInitialConditionOnCustomRange(
3762 : ConstElemRange & elem_range,
3763 : ConstBndNodeRange & bnd_nodes,
3764 : const std::optional<std::set<VariableName>> & target_vars)
3765 : {
3766 2191 : if (target_vars)
3767 : {
3768 2191 : ComputeInitialConditionThread cic(*this, &(*target_vars));
3769 2191 : Threads::parallel_reduce(elem_range, cic);
3770 : }
3771 : else
3772 : {
3773 0 : ComputeInitialConditionThread cic(*this);
3774 0 : Threads::parallel_reduce(elem_range, cic);
3775 : }
3776 :
3777 : // Need to close the solution vector here so that boundary ICs take precendence
3778 4382 : for (auto & nl : _nl)
3779 2191 : nl->solution().close();
3780 2191 : _aux->solution().close();
3781 :
3782 2191 : if (target_vars)
3783 : {
3784 2191 : ComputeBoundaryInitialConditionThread cbic(*this, &(*target_vars));
3785 2191 : Threads::parallel_reduce(bnd_nodes, cbic);
3786 2191 : }
3787 : else
3788 : {
3789 0 : ComputeBoundaryInitialConditionThread cbic(*this);
3790 0 : Threads::parallel_reduce(bnd_nodes, cbic);
3791 0 : }
3792 :
3793 4382 : for (auto & nl : _nl)
3794 2191 : nl->solution().close();
3795 2191 : _aux->solution().close();
3796 :
3797 : // Also, load values into the SCALAR dofs
3798 : // Note: We assume that all SCALAR dofs are on the
3799 : // processor with highest ID
3800 2191 : if (processor_id() == (n_processors() - 1) && _scalar_ics.hasActiveObjects())
3801 : {
3802 0 : const auto & ics = _scalar_ics.getActiveObjects();
3803 0 : for (const auto & ic : ics)
3804 : {
3805 0 : MooseVariableScalar & var = ic->variable();
3806 :
3807 0 : if (target_vars && !target_vars->count(var.name()))
3808 0 : continue;
3809 :
3810 0 : var.reinit();
3811 :
3812 0 : DenseVector<Number> vals(var.order());
3813 0 : ic->compute(vals);
3814 :
3815 0 : const unsigned int n_scalar_dofs = var.dofIndices().size();
3816 0 : for (unsigned int i = 0; i < n_scalar_dofs; i++)
3817 : {
3818 0 : const auto global_index = var.dofIndices()[i];
3819 0 : var.sys().solution().set(global_index, vals(i));
3820 0 : var.setValue(i, vals(i));
3821 : }
3822 0 : }
3823 : }
3824 :
3825 4382 : for (auto & nl : _nl)
3826 : {
3827 2191 : nl->solution().close();
3828 2191 : nl->solution().localize(*nl->system().current_local_solution, nl->dofMap().get_send_list());
3829 : }
3830 :
3831 2191 : _aux->solution().close();
3832 2191 : _aux->solution().localize(*_aux->sys().current_local_solution, _aux->dofMap().get_send_list());
3833 2191 : }
3834 :
3835 : void
3836 765 : FEProblemBase::projectFunctionOnCustomRange(ConstElemRange & elem_range,
3837 : Number (*func)(const Point &,
3838 : const libMesh::Parameters &,
3839 : const std::string &,
3840 : const std::string &),
3841 : Gradient (*func_grad)(const Point &,
3842 : const libMesh::Parameters &,
3843 : const std::string &,
3844 : const std::string &),
3845 : const libMesh::Parameters & params,
3846 : const VariableName & target_var)
3847 : {
3848 : mooseAssert(!Threads::in_threads,
3849 : "We're performing a projection based on data from just the thread 0 variable, so any "
3850 : "modifications to the variable solution must have been thread joined already");
3851 :
3852 765 : const auto & var = getStandardVariable(0, target_var);
3853 765 : const auto var_num = var.number();
3854 765 : const auto sn = systemNumForVariable(target_var);
3855 765 : auto & sys = getSystemBase(sn);
3856 :
3857 : // Let libmesh handle the projection
3858 765 : System & libmesh_sys = getSystem(target_var);
3859 765 : auto temp_vec = libmesh_sys.current_local_solution->zero_clone();
3860 765 : libmesh_sys.project_vector(func, func_grad, params, *temp_vec);
3861 765 : temp_vec->close();
3862 :
3863 : // Get the dof indices to copy
3864 765 : DofMap & dof_map = sys.dofMap();
3865 765 : std::set<dof_id_type> dof_indices;
3866 765 : std::vector<dof_id_type> elem_dof_indices;
3867 :
3868 4853 : for (const auto & elem : elem_range)
3869 : {
3870 4088 : dof_map.dof_indices(elem, elem_dof_indices, var_num);
3871 4088 : dof_indices.insert(elem_dof_indices.begin(), elem_dof_indices.end());
3872 : }
3873 765 : std::vector<dof_id_type> dof_indices_v(dof_indices.begin(), dof_indices.end());
3874 :
3875 : // Copy the projected values into the solution vector
3876 765 : std::vector<Real> dof_vals;
3877 765 : temp_vec->get(dof_indices_v, dof_vals);
3878 : mooseAssert(sys.solution().closed(),
3879 : "The solution should be closed before mapping our projection");
3880 765 : sys.solution().insert(dof_vals, dof_indices_v);
3881 765 : sys.solution().close();
3882 765 : sys.solution().localize(*libmesh_sys.current_local_solution, sys.dofMap().get_send_list());
3883 765 : }
3884 :
3885 : std::shared_ptr<MaterialBase>
3886 276 : FEProblemBase::getMaterial(std::string name,
3887 : Moose::MaterialDataType type,
3888 : const THREAD_ID tid,
3889 : bool no_warn)
3890 : {
3891 276 : switch (type)
3892 : {
3893 70 : case Moose::NEIGHBOR_MATERIAL_DATA:
3894 70 : name += "_neighbor";
3895 70 : break;
3896 70 : case Moose::FACE_MATERIAL_DATA:
3897 70 : name += "_face";
3898 70 : break;
3899 136 : default:
3900 136 : break;
3901 : }
3902 :
3903 276 : std::shared_ptr<MaterialBase> material = _all_materials[type].getActiveObject(name, tid);
3904 716 : if (!no_warn && material->getParam<bool>("compute") && type == Moose::BLOCK_MATERIAL_DATA)
3905 4 : mooseWarning("You are retrieving a Material object (",
3906 4 : material->name(),
3907 : "), but its compute flag is set to true. This indicates that MOOSE is "
3908 : "computing this property which may not be desired and produce un-expected "
3909 : "results.");
3910 :
3911 268 : return material;
3912 : }
3913 :
3914 : MaterialData &
3915 50273886 : FEProblemBase::getMaterialData(Moose::MaterialDataType type,
3916 : const THREAD_ID tid,
3917 : const MooseObject * object) const
3918 : {
3919 50273886 : switch (type)
3920 : {
3921 908307 : case Moose::BLOCK_MATERIAL_DATA:
3922 908307 : if (object)
3923 282733 : _material_props.addConsumer(type, object);
3924 908307 : return _material_props.getMaterialData(tid);
3925 23467686 : case Moose::NEIGHBOR_MATERIAL_DATA:
3926 23467686 : if (object)
3927 26553 : _neighbor_material_props.addConsumer(type, object);
3928 23467686 : return _neighbor_material_props.getMaterialData(tid);
3929 25897893 : case Moose::BOUNDARY_MATERIAL_DATA:
3930 : case Moose::FACE_MATERIAL_DATA:
3931 : case Moose::INTERFACE_MATERIAL_DATA:
3932 25897893 : if (object)
3933 59030 : _bnd_material_props.addConsumer(type, object);
3934 25897893 : return _bnd_material_props.getMaterialData(tid);
3935 : }
3936 :
3937 0 : mooseError("FEProblemBase::getMaterialData(): Invalid MaterialDataType ", type);
3938 : }
3939 :
3940 : const std::set<const MooseObject *> &
3941 0 : FEProblemBase::getMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
3942 : {
3943 0 : switch (type)
3944 : {
3945 0 : case Moose::BLOCK_MATERIAL_DATA:
3946 0 : return _material_props.getConsumers(type);
3947 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
3948 0 : return _neighbor_material_props.getConsumers(type);
3949 0 : case Moose::BOUNDARY_MATERIAL_DATA:
3950 : case Moose::FACE_MATERIAL_DATA:
3951 : case Moose::INTERFACE_MATERIAL_DATA:
3952 0 : return _bnd_material_props.getConsumers(type);
3953 : }
3954 :
3955 0 : mooseError("FEProblemBase::getMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
3956 : type);
3957 : }
3958 :
3959 : void
3960 0 : FEProblemBase::setPreserveMatrixSparsityPattern(bool preserve)
3961 : {
3962 0 : if (_ignore_zeros_in_jacobian && preserve)
3963 0 : paramWarning(
3964 : "ignore_zeros_in_jacobian",
3965 : "We likely cannot preserve the sparsity pattern if ignoring zeros in the Jacobian, which "
3966 : "leads to removing those entries from the Jacobian sparsity pattern");
3967 0 : _preserve_matrix_sparsity_pattern = preserve;
3968 0 : }
3969 :
3970 : bool
3971 340893 : FEProblemBase::acceptInvalidSolution() const
3972 : {
3973 681692 : return allowInvalidSolution() || // invalid solutions are always allowed
3974 681692 : !_app.solutionInvalidity().hasInvalidSolutionError(); // if not allowed, check for errors
3975 : }
3976 :
3977 : void
3978 1177 : FEProblemBase::addFunctorMaterial(const std::string & functor_material_name,
3979 : const std::string & name,
3980 : InputParameters & parameters)
3981 : {
3982 : parallel_object_only();
3983 :
3984 1177 : auto add_functor_materials = [&](const auto & parameters, const auto & name)
3985 : {
3986 2437 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
3987 : {
3988 : // Create the general Block/Boundary MaterialBase object
3989 1260 : std::shared_ptr<MaterialBase> material =
3990 1260 : _factory.create<MaterialBase>(functor_material_name, name, parameters, tid);
3991 2520 : logAdd("FunctorMaterial", name, functor_material_name, parameters);
3992 1260 : _all_materials.addObject(material, tid);
3993 1260 : _materials.addObject(material, tid);
3994 : }
3995 1177 : };
3996 :
3997 2354 : parameters.set<SubProblem *>("_subproblem") = this;
3998 1177 : add_functor_materials(parameters, name);
3999 1177 : if (_displaced_problem)
4000 : {
4001 0 : auto disp_params = parameters;
4002 0 : disp_params.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4003 0 : add_functor_materials(disp_params, name + "_displaced");
4004 0 : }
4005 1177 : }
4006 :
4007 : void
4008 13522 : FEProblemBase::addMaterial(const std::string & mat_name,
4009 : const std::string & name,
4010 : InputParameters & parameters)
4011 : {
4012 27044 : addMaterialHelper({&_materials}, mat_name, name, parameters);
4013 13392 : }
4014 :
4015 : void
4016 348 : FEProblemBase::addInterfaceMaterial(const std::string & mat_name,
4017 : const std::string & name,
4018 : InputParameters & parameters)
4019 : {
4020 696 : addMaterialHelper({&_interface_materials}, mat_name, name, parameters);
4021 348 : }
4022 :
4023 : void
4024 14164 : FEProblemBase::addMaterialHelper(std::vector<MaterialWarehouse *> warehouses,
4025 : const std::string & mat_name,
4026 : const std::string & name,
4027 : InputParameters & parameters)
4028 : {
4029 : parallel_object_only();
4030 :
4031 14164 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4032 : {
4033 274 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4034 137 : _reinit_displaced_elem = _reinit_displaced_face = _reinit_displaced_neighbor = true;
4035 : }
4036 : else
4037 : {
4038 14027 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
4039 : {
4040 : // We allow Materials to request that they use_displaced_mesh,
4041 : // but then be overridden when no displacements variables are
4042 : // provided in the Mesh block. If that happened, update the value
4043 : // of use_displaced_mesh appropriately for this Material.
4044 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
4045 0 : parameters.set<bool>("use_displaced_mesh") = false;
4046 : }
4047 :
4048 42081 : parameters.set<SubProblem *>("_subproblem") = this;
4049 : }
4050 :
4051 29393 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
4052 : {
4053 : // Create the general Block/Boundary MaterialBase object
4054 : std::shared_ptr<MaterialBase> material =
4055 15365 : _factory.create<MaterialBase>(mat_name, name, parameters, tid);
4056 15229 : logAdd("Material", name, mat_name, parameters);
4057 30458 : bool discrete = !material->getParam<bool>("compute");
4058 :
4059 : // If the object is boundary restricted or if it is a functor material we do not create the
4060 : // neighbor and face objects
4061 15229 : if (material->boundaryRestricted() || dynamic_cast<FunctorMaterial *>(material.get()))
4062 : {
4063 3429 : _all_materials.addObject(material, tid);
4064 3429 : if (discrete)
4065 5 : _discrete_materials.addObject(material, tid);
4066 : else
4067 6848 : for (auto && warehouse : warehouses)
4068 3424 : warehouse->addObject(material, tid);
4069 : }
4070 :
4071 : // Non-boundary restricted require face and neighbor objects
4072 : else
4073 : {
4074 : // TODO: we only need to do this if we have needs for face materials (e.g.
4075 : // FV, DG, etc.) - but currently we always do it. Figure out how to fix
4076 : // this.
4077 :
4078 : // The name of the object being created, this is changed multiple times as objects are
4079 : // created below
4080 11800 : std::string object_name;
4081 :
4082 : // Create a copy of the supplied parameters to the setting for "_material_data_type" isn't
4083 : // used from a previous tid loop
4084 11800 : InputParameters current_parameters = parameters;
4085 :
4086 : // face material
4087 11800 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4088 : Moose::FACE_MATERIAL_DATA;
4089 11800 : object_name = name + "_face";
4090 : std::shared_ptr<MaterialBase> face_material =
4091 11800 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4092 :
4093 : // neighbor material
4094 23600 : current_parameters.set<Moose::MaterialDataType>("_material_data_type") =
4095 : Moose::NEIGHBOR_MATERIAL_DATA;
4096 11800 : current_parameters.set<bool>("_neighbor") = true;
4097 11800 : object_name = name + "_neighbor";
4098 : std::shared_ptr<MaterialBase> neighbor_material =
4099 11800 : _factory.create<MaterialBase>(mat_name, object_name, current_parameters, tid);
4100 :
4101 : // Store the material objects
4102 11800 : _all_materials.addObjects(material, neighbor_material, face_material, tid);
4103 :
4104 11800 : if (discrete)
4105 80 : _discrete_materials.addObjects(material, neighbor_material, face_material, tid);
4106 : else
4107 23440 : for (auto && warehouse : warehouses)
4108 11720 : warehouse->addObjects(material, neighbor_material, face_material, tid);
4109 :
4110 : // Names of all controllable parameters for this Material object
4111 11800 : const std::string & base = parameters.getBase();
4112 35400 : MooseObjectParameterName name(MooseObjectName(base, material->name()), "*");
4113 : const auto param_names =
4114 11800 : _app.getInputParameterWarehouse().getControllableParameterNames(name);
4115 :
4116 : // Connect parameters of the primary Material object to those on the face and neighbor
4117 : // objects
4118 29957 : for (const auto & p_name : param_names)
4119 : {
4120 36314 : MooseObjectParameterName primary_name(MooseObjectName(base, material->name()),
4121 36314 : p_name.parameter());
4122 36314 : MooseObjectParameterName face_name(MooseObjectName(base, face_material->name()),
4123 36314 : p_name.parameter());
4124 36314 : MooseObjectParameterName neighbor_name(MooseObjectName(base, neighbor_material->name()),
4125 36314 : p_name.parameter());
4126 18157 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4127 : primary_name, face_name, false);
4128 18157 : _app.getInputParameterWarehouse().addControllableParameterConnection(
4129 : primary_name, neighbor_name, false);
4130 18157 : }
4131 11800 : }
4132 15229 : }
4133 14028 : }
4134 :
4135 : void
4136 5605423 : FEProblemBase::prepareMaterials(const std::unordered_set<unsigned int> & consumer_needed_mat_props,
4137 : const SubdomainID blk_id,
4138 : const THREAD_ID tid)
4139 : {
4140 5605423 : std::set<MooseVariableFEBase *> needed_moose_vars;
4141 5605423 : std::unordered_set<unsigned int> needed_mat_props;
4142 :
4143 5605423 : if (_all_materials.hasActiveBlockObjects(blk_id, tid))
4144 : {
4145 678273 : _all_materials.updateVariableDependency(needed_moose_vars, tid);
4146 678273 : _all_materials.updateBlockMatPropDependency(blk_id, needed_mat_props, tid);
4147 : }
4148 :
4149 5605423 : const auto & ids = _mesh.getSubdomainBoundaryIds(blk_id);
4150 25805974 : for (const auto id : ids)
4151 : {
4152 20200551 : _materials.updateBoundaryVariableDependency(id, needed_moose_vars, tid);
4153 20200551 : _materials.updateBoundaryMatPropDependency(id, needed_mat_props, tid);
4154 : }
4155 :
4156 5605423 : const auto & current_active_elemental_moose_variables = getActiveElementalMooseVariables(tid);
4157 5605423 : needed_moose_vars.insert(current_active_elemental_moose_variables.begin(),
4158 : current_active_elemental_moose_variables.end());
4159 :
4160 5605423 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
4161 :
4162 5605423 : setActiveElementalMooseVariables(needed_moose_vars, tid);
4163 5605423 : setActiveMaterialProperties(needed_mat_props, tid);
4164 5605423 : }
4165 :
4166 : void
4167 418947166 : FEProblemBase::reinitMaterials(SubdomainID blk_id, const THREAD_ID tid, bool swap_stateful)
4168 : {
4169 418947166 : if (hasActiveMaterialProperties(tid))
4170 : {
4171 20009316 : auto && elem = _assembly[tid][0]->elem();
4172 20009316 : unsigned int n_points = _assembly[tid][0]->qRule()->n_points();
4173 :
4174 20009316 : auto & material_data = _material_props.getMaterialData(tid);
4175 20009316 : material_data.resize(n_points);
4176 :
4177 : // Only swap if requested
4178 20009316 : if (swap_stateful)
4179 20001977 : material_data.swap(*elem);
4180 :
4181 20009316 : if (_discrete_materials.hasActiveBlockObjects(blk_id, tid))
4182 3089 : material_data.reset(_discrete_materials.getActiveBlockObjects(blk_id, tid));
4183 :
4184 20009312 : if (_materials.hasActiveBlockObjects(blk_id, tid))
4185 19979208 : material_data.reinit(_materials.getActiveBlockObjects(blk_id, tid));
4186 : }
4187 418947084 : }
4188 :
4189 : void
4190 8596481 : FEProblemBase::reinitMaterialsFace(const SubdomainID blk_id,
4191 : const THREAD_ID tid,
4192 : const bool swap_stateful,
4193 : const std::deque<MaterialBase *> * const reinit_mats)
4194 : {
4195 : // we reinit more often than needed here because we dont have a way to check whether
4196 : // we need to compute the face materials on a particular (possibly external) face
4197 8596481 : if (hasActiveMaterialProperties(tid))
4198 : {
4199 2896935 : auto && elem = _assembly[tid][0]->elem();
4200 2896935 : unsigned int side = _assembly[tid][0]->side();
4201 2896935 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4202 :
4203 2896935 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4204 2896935 : bnd_material_data.resize(n_points);
4205 :
4206 2896935 : if (swap_stateful && !bnd_material_data.isSwapped())
4207 2872583 : bnd_material_data.swap(*elem, side);
4208 :
4209 2896935 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4210 0 : bnd_material_data.reset(
4211 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4212 :
4213 2896935 : if (reinit_mats)
4214 24352 : bnd_material_data.reinit(*reinit_mats);
4215 2872583 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4216 2872583 : bnd_material_data.reinit(
4217 2872583 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4218 : }
4219 8596481 : }
4220 :
4221 : void
4222 4346725 : FEProblemBase::reinitMaterialsFaceOnBoundary(const BoundaryID boundary_id,
4223 : const SubdomainID blk_id,
4224 : const THREAD_ID tid,
4225 : const bool swap_stateful,
4226 : const std::deque<MaterialBase *> * const reinit_mats)
4227 : {
4228 4737879 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4229 391154 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4230 391154 : needInternalNeighborSideMaterial(blk_id, tid)))
4231 : {
4232 519715 : const auto * const elem = _assembly[tid][0]->elem();
4233 519715 : unsigned int side = _assembly[tid][0]->side();
4234 519715 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4235 :
4236 519715 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4237 519715 : bnd_material_data.resize(n_points);
4238 :
4239 519715 : if (swap_stateful && !bnd_material_data.isSwapped())
4240 519715 : bnd_material_data.swap(*elem, side);
4241 :
4242 519715 : if (_discrete_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4243 0 : bnd_material_data.reset(
4244 0 : _discrete_materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4245 :
4246 519715 : if (reinit_mats)
4247 0 : bnd_material_data.reinit(*reinit_mats);
4248 519715 : else if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4249 503690 : bnd_material_data.reinit(
4250 503690 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4251 : }
4252 4346725 : }
4253 :
4254 : void
4255 54438 : FEProblemBase::reinitMaterialsNeighborOnBoundary(
4256 : const BoundaryID boundary_id,
4257 : const SubdomainID blk_id,
4258 : const THREAD_ID tid,
4259 : const bool swap_stateful,
4260 : const std::deque<MaterialBase *> * const reinit_mats)
4261 : {
4262 : // Since objects don't declare whether they need the face or neighbor (side) material properties,
4263 : // we use the same criteria for skipping material property computations as for face material
4264 : // properties This could be a future optimization.
4265 59223 : if (hasActiveMaterialProperties(tid) && (needBoundaryMaterialOnSide(boundary_id, tid) ||
4266 4785 : needInterfaceMaterialOnSide(boundary_id, tid) ||
4267 4785 : needInternalNeighborSideMaterial(blk_id, tid)))
4268 45110 : reinitMaterialsNeighbor(blk_id, tid, swap_stateful, reinit_mats);
4269 54438 : }
4270 :
4271 : void
4272 4596908 : FEProblemBase::reinitMaterialsNeighbor(const SubdomainID blk_id,
4273 : const THREAD_ID tid,
4274 : const bool swap_stateful,
4275 : const std::deque<MaterialBase *> * const reinit_mats)
4276 : {
4277 4596908 : if (hasActiveMaterialProperties(tid))
4278 : {
4279 : // NOTE: this will not work with h-adaptivity
4280 : // lindsayad: why not?
4281 :
4282 1092527 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4283 1092527 : unsigned int neighbor_side = neighbor->which_neighbor_am_i(_assembly[tid][0]->elem());
4284 :
4285 : mooseAssert(neighbor, "neighbor should be non-null");
4286 : mooseAssert(blk_id == neighbor->subdomain_id(),
4287 : "The provided blk_id " << blk_id << " and neighbor subdomain ID "
4288 : << neighbor->subdomain_id() << " do not match.");
4289 :
4290 1092527 : unsigned int n_points = _assembly[tid][0]->qRuleNeighbor()->n_points();
4291 :
4292 1092527 : auto & neighbor_material_data = _neighbor_material_props.getMaterialData(tid);
4293 1092527 : neighbor_material_data.resize(n_points);
4294 :
4295 : // Only swap if requested
4296 1092527 : if (swap_stateful)
4297 1068175 : neighbor_material_data.swap(*neighbor, neighbor_side);
4298 :
4299 1092527 : if (_discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4300 0 : neighbor_material_data.reset(
4301 0 : _discrete_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4302 :
4303 1092527 : if (reinit_mats)
4304 24352 : neighbor_material_data.reinit(*reinit_mats);
4305 1068175 : else if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
4306 1067876 : neighbor_material_data.reinit(
4307 1067876 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid));
4308 : }
4309 4596908 : }
4310 :
4311 : void
4312 4631154 : FEProblemBase::reinitMaterialsBoundary(const BoundaryID boundary_id,
4313 : const THREAD_ID tid,
4314 : const bool swap_stateful,
4315 : const std::deque<MaterialBase *> * const reinit_mats)
4316 : {
4317 4631154 : if (hasActiveMaterialProperties(tid) && needBoundaryMaterialOnSide(boundary_id, tid))
4318 : {
4319 365687 : auto && elem = _assembly[tid][0]->elem();
4320 365687 : unsigned int side = _assembly[tid][0]->side();
4321 365687 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4322 :
4323 365687 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4324 365687 : bnd_material_data.resize(n_points);
4325 :
4326 365687 : if (swap_stateful && !bnd_material_data.isSwapped())
4327 310985 : bnd_material_data.swap(*elem, side);
4328 :
4329 365687 : if (_discrete_materials.hasActiveBoundaryObjects(boundary_id, tid))
4330 0 : bnd_material_data.reset(_discrete_materials.getActiveBoundaryObjects(boundary_id, tid));
4331 :
4332 365687 : if (reinit_mats)
4333 24352 : bnd_material_data.reinit(*reinit_mats);
4334 341335 : else if (_materials.hasActiveBoundaryObjects(boundary_id, tid))
4335 25963 : bnd_material_data.reinit(_materials.getActiveBoundaryObjects(boundary_id, tid));
4336 : }
4337 4631154 : }
4338 :
4339 : void
4340 57823 : FEProblemBase::reinitMaterialsInterface(BoundaryID boundary_id,
4341 : const THREAD_ID tid,
4342 : bool swap_stateful)
4343 : {
4344 57823 : if (hasActiveMaterialProperties(tid) && needInterfaceMaterialOnSide(boundary_id, tid))
4345 : {
4346 46343 : const Elem * const & elem = _assembly[tid][0]->elem();
4347 46343 : unsigned int side = _assembly[tid][0]->side();
4348 46343 : unsigned int n_points = _assembly[tid][0]->qRuleFace()->n_points();
4349 :
4350 46343 : auto & bnd_material_data = _bnd_material_props.getMaterialData(tid);
4351 46343 : bnd_material_data.resize(n_points);
4352 :
4353 46343 : if (swap_stateful && !bnd_material_data.isSwapped())
4354 44568 : bnd_material_data.swap(*elem, side);
4355 :
4356 46343 : if (_interface_materials.hasActiveBoundaryObjects(boundary_id, tid))
4357 1317 : bnd_material_data.reinit(_interface_materials.getActiveBoundaryObjects(boundary_id, tid));
4358 : }
4359 57823 : }
4360 :
4361 : void
4362 417117350 : FEProblemBase::swapBackMaterials(const THREAD_ID tid)
4363 : {
4364 417117350 : auto && elem = _assembly[tid][0]->elem();
4365 417117350 : _material_props.getMaterialData(tid).swapBack(*elem);
4366 417117350 : }
4367 :
4368 : void
4369 12658758 : FEProblemBase::swapBackMaterialsFace(const THREAD_ID tid)
4370 : {
4371 12658758 : auto && elem = _assembly[tid][0]->elem();
4372 12658758 : unsigned int side = _assembly[tid][0]->side();
4373 12658758 : _bnd_material_props.getMaterialData(tid).swapBack(*elem, side);
4374 12658758 : }
4375 :
4376 : void
4377 4213629 : FEProblemBase::swapBackMaterialsNeighbor(const THREAD_ID tid)
4378 : {
4379 : // NOTE: this will not work with h-adaptivity
4380 4213629 : const Elem * neighbor = _assembly[tid][0]->neighbor();
4381 : unsigned int neighbor_side =
4382 4213629 : neighbor ? neighbor->which_neighbor_am_i(_assembly[tid][0]->elem()) : libMesh::invalid_uint;
4383 :
4384 4213629 : if (!neighbor)
4385 : {
4386 0 : if (haveFV())
4387 : {
4388 : // If neighbor is null, then we're on the neighbor side of a mesh boundary, e.g. we're off
4389 : // the mesh in ghost-land. If we're using the finite volume method, then variable values and
4390 : // consequently material properties have well-defined values in this ghost region outside of
4391 : // the mesh and we really do want to reinit our neighbor materials in this case. Since we're
4392 : // off in ghost land it's safe to do swaps with `MaterialPropertyStorage` using the elem and
4393 : // elem_side keys
4394 0 : neighbor = _assembly[tid][0]->elem();
4395 0 : neighbor_side = _assembly[tid][0]->side();
4396 : mooseAssert(neighbor, "We should have an appropriate value for elem coming from Assembly");
4397 : }
4398 : else
4399 0 : mooseError("neighbor is null in Assembly!");
4400 : }
4401 :
4402 4213629 : _neighbor_material_props.getMaterialData(tid).swapBack(*neighbor, neighbor_side);
4403 4213629 : }
4404 :
4405 : void
4406 1010203 : FEProblemBase::logAdd(const std::string & system,
4407 : const std::string & name,
4408 : const std::string & type,
4409 : const InputParameters & params) const
4410 : {
4411 1010203 : if (_verbose_setup != "false")
4412 19260 : _console << "[DBG] Adding " << system << " '" << name << "' of type " << type << std::endl;
4413 1010203 : if (_verbose_setup == "extra")
4414 0 : _console << params << std::endl;
4415 1010203 : }
4416 :
4417 : void
4418 136083 : FEProblemBase::addObjectParamsHelper(InputParameters & parameters,
4419 : const std::string & object_name,
4420 : const std::string & var_param_name)
4421 : {
4422 : // Due to objects like SolutionUserObject which manipulate libmesh objects
4423 : // and variables directly at the back end, we need a default option here
4424 : // which is going to be the pointer to the first solver system within this
4425 : // problem
4426 136083 : unsigned int sys_num = 0;
4427 136083 : if (parameters.isParamValid(var_param_name))
4428 : {
4429 59518 : const auto variable_name = parameters.varName(var_param_name, object_name);
4430 59518 : if (this->hasVariable(variable_name) || this->hasScalarVariable(variable_name))
4431 57861 : sys_num = getSystem(variable_name).number();
4432 59518 : }
4433 272166 : if (parameters.isParamValid("solver_sys"))
4434 : {
4435 929 : const auto var_sys_num = sys_num;
4436 929 : sys_num = getSystemBase(parameters.get<SolverSystemName>("solver_sys")).number();
4437 929 : if (sys_num != var_sys_num && parameters.isParamValid(var_param_name))
4438 0 : mooseError("We dont support setting 'variable' to a variable that is not set to the same "
4439 : "system as the 'solver_sys' parameter");
4440 : }
4441 :
4442 137366 : if (_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4443 137366 : parameters.get<bool>("use_displaced_mesh"))
4444 : {
4445 1260 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
4446 630 : if (sys_num == _aux->number())
4447 1197 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->systemBaseAuxiliary();
4448 : else
4449 693 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->solverSys(sys_num);
4450 : }
4451 : else
4452 : {
4453 : // The object requested use_displaced_mesh, but it was overridden
4454 : // due to there being no displacements variables in the [Mesh] block.
4455 : // If that happened, update the value of use_displaced_mesh appropriately.
4456 235106 : if (!_displaced_problem && parameters.have_parameter<bool>("use_displaced_mesh") &&
4457 235106 : parameters.get<bool>("use_displaced_mesh"))
4458 146 : parameters.set<bool>("use_displaced_mesh") = false;
4459 :
4460 270906 : parameters.set<SubProblem *>("_subproblem") = this;
4461 :
4462 135453 : if (sys_num == _aux->number())
4463 38646 : parameters.set<SystemBase *>("_sys") = _aux.get();
4464 : else
4465 367713 : parameters.set<SystemBase *>("_sys") = _solver_systems[sys_num].get();
4466 : }
4467 136083 : }
4468 :
4469 : void
4470 55314 : FEProblemBase::addPostprocessor(const std::string & pp_name,
4471 : const std::string & name,
4472 : InputParameters & parameters)
4473 : {
4474 : // Check for name collision
4475 55314 : if (hasUserObject(name))
4476 4 : mooseError("A ",
4477 4 : getUserObjectBase(name).typeAndName(),
4478 : " already exists. You may not add a Postprocessor by the same name.");
4479 :
4480 55310 : addUserObject(pp_name, name, parameters);
4481 55274 : }
4482 :
4483 : void
4484 5806 : FEProblemBase::addVectorPostprocessor(const std::string & pp_name,
4485 : const std::string & name,
4486 : InputParameters & parameters)
4487 : {
4488 : // Check for name collision
4489 5806 : if (hasUserObject(name))
4490 4 : mooseError("A ",
4491 4 : getUserObjectBase(name).typeAndName(),
4492 : " already exists. You may not add a VectorPostprocessor by the same name.");
4493 :
4494 5802 : addUserObject(pp_name, name, parameters);
4495 5762 : }
4496 :
4497 : void
4498 4916 : FEProblemBase::addReporter(const std::string & type,
4499 : const std::string & name,
4500 : InputParameters & parameters)
4501 : {
4502 : // Check for name collision
4503 4916 : if (hasUserObject(name))
4504 4 : mooseError("A ",
4505 4 : getUserObjectBase(name).typeAndName(),
4506 : " already exists. You may not add a Reporter by the same name.");
4507 :
4508 4912 : addUserObject(type, name, parameters);
4509 4868 : }
4510 :
4511 : std::vector<std::shared_ptr<UserObject>>
4512 80155 : FEProblemBase::addUserObject(const std::string & user_object_name,
4513 : const std::string & name,
4514 : InputParameters & parameters)
4515 : {
4516 : parallel_object_only();
4517 :
4518 80155 : std::vector<std::shared_ptr<UserObject>> uos;
4519 :
4520 : // Add the _subproblem and _sys parameters depending on use_displaced_mesh
4521 80155 : addObjectParamsHelper(parameters, name);
4522 :
4523 127575 : for (const auto tid : make_range(libMesh::n_threads()))
4524 : {
4525 : // Create the UserObject
4526 : std::shared_ptr<UserObject> user_object =
4527 83750 : _factory.create<UserObject>(user_object_name, name, parameters, tid);
4528 83370 : logAdd("UserObject", name, user_object_name, parameters);
4529 83370 : uos.push_back(user_object);
4530 :
4531 83370 : if (tid != 0)
4532 3595 : user_object->setPrimaryThreadCopy(uos[0].get());
4533 :
4534 : // TODO: delete this line after apps have been updated to not call getUserObjects
4535 83370 : _all_user_objects.addObject(user_object, tid);
4536 :
4537 83362 : theWarehouse().add(user_object);
4538 :
4539 : // Attempt to create all the possible UserObject types
4540 83362 : auto euo = std::dynamic_pointer_cast<ElementUserObject>(user_object);
4541 83362 : auto suo = std::dynamic_pointer_cast<SideUserObject>(user_object);
4542 83362 : auto isuo = std::dynamic_pointer_cast<InternalSideUserObject>(user_object);
4543 83362 : auto iuo = std::dynamic_pointer_cast<InterfaceUserObjectBase>(user_object);
4544 83362 : auto nuo = std::dynamic_pointer_cast<NodalUserObject>(user_object);
4545 83362 : auto duo = std::dynamic_pointer_cast<DomainUserObject>(user_object);
4546 83362 : auto guo = std::dynamic_pointer_cast<GeneralUserObject>(user_object);
4547 83362 : auto tguo = std::dynamic_pointer_cast<ThreadedGeneralUserObject>(user_object);
4548 83362 : auto muo = std::dynamic_pointer_cast<MortarUserObject>(user_object);
4549 :
4550 : // Account for displaced mesh use
4551 83362 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
4552 : {
4553 : // Whether to re-init or not depends on the attributes of the base classes.
4554 : // For example, InterfaceUOBase has "_current_side_elem" and "_neighbor_elem"
4555 : // so it needs to reinit on displaced neighbors and faces
4556 : // _reinit_displaced_elem -> _current_elem will be reinited
4557 : // _reinit_displaced_face -> _current_elem, lowerD if any and _current_side_elem to be
4558 : // reinited _reinit_displaced_neighbor -> _current_elem, lowerD if any and _current_neighbor
4559 : // to be reinited Note that as soon as you use materials on the displaced mesh, all three get
4560 : // turned on.
4561 680 : if (euo || nuo || duo)
4562 606 : _reinit_displaced_elem = true;
4563 680 : if (suo || duo || isuo || iuo)
4564 35 : _reinit_displaced_face = true;
4565 680 : if (iuo || duo || isuo)
4566 0 : _reinit_displaced_neighbor = true;
4567 : }
4568 :
4569 : // These objects only require one thread
4570 83362 : if ((guo && !tguo) || muo)
4571 35942 : break;
4572 406840 : }
4573 :
4574 : // Add as a Functor if it is one. We usually need to add the user object from thread 0 as the
4575 : // registered functor for all threads because when user objects are thread joined, generally only
4576 : // the primary thread copy ends up with all the data
4577 166663 : for (const auto tid : make_range(libMesh::n_threads()))
4578 : {
4579 86896 : const decltype(uos)::size_type uo_index = uos.front()->needThreadedCopy() ? tid : 0;
4580 86896 : if (const auto functor = dynamic_cast<Moose::FunctorBase<Real> *>(uos[uo_index].get()))
4581 : {
4582 65301 : this->addFunctor(name, *functor, tid);
4583 65301 : if (_displaced_problem)
4584 804 : _displaced_problem->addFunctor(name, *functor, tid);
4585 : }
4586 : }
4587 :
4588 79767 : return uos;
4589 16 : }
4590 :
4591 : const UserObject &
4592 330457 : FEProblemBase::getUserObjectBase(const std::string & name, const THREAD_ID tid /* = 0 */) const
4593 : {
4594 330457 : std::vector<UserObject *> objs;
4595 330457 : theWarehouse()
4596 660914 : .query()
4597 330457 : .condition<AttribSystem>("UserObject")
4598 330457 : .condition<AttribThread>(tid)
4599 330457 : .condition<AttribName>(name)
4600 330457 : .queryInto(objs);
4601 330457 : if (objs.empty())
4602 0 : mooseError("Unable to find user object with name '" + name + "'");
4603 : mooseAssert(objs.size() == 1, "Should only find one UO");
4604 660914 : return *(objs[0]);
4605 330457 : }
4606 :
4607 : const Positions &
4608 1154 : FEProblemBase::getPositionsObject(const std::string & name) const
4609 : {
4610 1154 : std::vector<Positions *> objs;
4611 1154 : theWarehouse()
4612 2308 : .query()
4613 1154 : .condition<AttribSystem>("UserObject")
4614 1154 : .condition<AttribName>(name)
4615 1154 : .queryInto(objs);
4616 1154 : if (objs.empty())
4617 0 : mooseError("Unable to find Positions object with name '" + name + "'");
4618 : mooseAssert(objs.size() == 1, "Should only find one Positions");
4619 2308 : return *(objs[0]);
4620 1154 : }
4621 :
4622 : bool
4623 93818 : FEProblemBase::hasUserObject(const std::string & name) const
4624 : {
4625 93818 : std::vector<UserObject *> objs;
4626 93818 : theWarehouse()
4627 93818 : .query()
4628 93818 : .condition<AttribSystem>("UserObject")
4629 187636 : .condition<AttribThread>(0)
4630 93818 : .condition<AttribName>(name)
4631 93818 : .queryInto(objs);
4632 187636 : return !objs.empty();
4633 93818 : }
4634 :
4635 : bool
4636 477 : FEProblemBase::hasPostprocessorValueByName(const PostprocessorName & name) const
4637 : {
4638 477 : return _reporter_data.hasReporterValue<PostprocessorValue>(PostprocessorReporterName(name));
4639 : }
4640 :
4641 : const PostprocessorValue &
4642 911895 : FEProblemBase::getPostprocessorValueByName(const PostprocessorName & name,
4643 : std::size_t t_index) const
4644 : {
4645 1823790 : return _reporter_data.getReporterValue<PostprocessorValue>(PostprocessorReporterName(name),
4646 1823790 : t_index);
4647 : }
4648 :
4649 : void
4650 602836 : FEProblemBase::setPostprocessorValueByName(const PostprocessorName & name,
4651 : const PostprocessorValue & value,
4652 : std::size_t t_index)
4653 : {
4654 602836 : _reporter_data.setReporterValue<PostprocessorValue>(
4655 1205672 : PostprocessorReporterName(name), value, t_index);
4656 602836 : }
4657 :
4658 : bool
4659 56 : FEProblemBase::hasPostprocessor(const std::string & name) const
4660 : {
4661 56 : mooseDeprecated("FEProblemBase::hasPostprocssor is being removed; use "
4662 : "hasPostprocessorValueByName instead.");
4663 56 : return hasPostprocessorValueByName(name);
4664 : }
4665 :
4666 : const VectorPostprocessorValue &
4667 57 : FEProblemBase::getVectorPostprocessorValueByName(const std::string & object_name,
4668 : const std::string & vector_name,
4669 : std::size_t t_index) const
4670 : {
4671 57 : return _reporter_data.getReporterValue<VectorPostprocessorValue>(
4672 114 : VectorPostprocessorReporterName(object_name, vector_name), t_index);
4673 : }
4674 :
4675 : void
4676 20 : FEProblemBase::setVectorPostprocessorValueByName(const std::string & object_name,
4677 : const std::string & vector_name,
4678 : const VectorPostprocessorValue & value,
4679 : std::size_t t_index)
4680 : {
4681 20 : _reporter_data.setReporterValue<VectorPostprocessorValue>(
4682 40 : VectorPostprocessorReporterName(object_name, vector_name), value, t_index);
4683 20 : }
4684 :
4685 : const VectorPostprocessor &
4686 9268 : FEProblemBase::getVectorPostprocessorObjectByName(const std::string & object_name,
4687 : const THREAD_ID tid) const
4688 : {
4689 9268 : return getUserObject<VectorPostprocessor>(object_name, tid);
4690 : }
4691 :
4692 : void
4693 77 : FEProblemBase::parentOutputPositionChanged()
4694 : {
4695 2156 : for (const auto & it : _multi_apps)
4696 : {
4697 2079 : const auto & objects = it.second.getActiveObjects();
4698 2100 : for (const auto & obj : objects)
4699 21 : obj->parentOutputPositionChanged();
4700 : }
4701 77 : }
4702 :
4703 : void
4704 0 : FEProblemBase::computeIndicatorsAndMarkers()
4705 : {
4706 0 : computeIndicators();
4707 0 : computeMarkers();
4708 0 : }
4709 :
4710 : void
4711 240856 : FEProblemBase::computeIndicators()
4712 : {
4713 : // Initialize indicator aux variable fields
4714 240856 : if (_indicators.hasActiveObjects() || _internal_side_indicators.hasActiveObjects())
4715 : {
4716 12880 : TIME_SECTION("computeIndicators", 1, "Computing Indicators");
4717 :
4718 : // Internal side indicators may lead to creating a much larger sparsity pattern than dictated by
4719 : // the actual finite element scheme (e.g. CFEM)
4720 2576 : const auto old_do_derivatives = ADReal::do_derivatives;
4721 2576 : ADReal::do_derivatives = false;
4722 :
4723 2576 : std::vector<std::string> fields;
4724 :
4725 : // Indicator Fields
4726 2576 : const auto & indicators = _indicators.getActiveObjects();
4727 2736 : for (const auto & indicator : indicators)
4728 160 : fields.push_back(indicator->name());
4729 :
4730 : // InternalSideIndicator Fields
4731 2576 : const auto & internal_indicators = _internal_side_indicators.getActiveObjects();
4732 5056 : for (const auto & internal_indicator : internal_indicators)
4733 2480 : fields.push_back(internal_indicator->name());
4734 :
4735 2576 : _aux->zeroVariables(fields);
4736 :
4737 : // compute Indicators
4738 2576 : ComputeIndicatorThread cit(*this);
4739 2576 : Threads::parallel_reduce(*_mesh.getActiveLocalElementRange(), cit);
4740 2576 : _aux->solution().close();
4741 2576 : _aux->update();
4742 :
4743 2576 : ComputeIndicatorThread finalize_cit(*this, true);
4744 2576 : Threads::parallel_reduce(*_mesh.getActiveLocalElementRange(), finalize_cit);
4745 2576 : _aux->solution().close();
4746 2576 : _aux->update();
4747 :
4748 2576 : ADReal::do_derivatives = old_do_derivatives;
4749 2576 : }
4750 240856 : }
4751 :
4752 : void
4753 240880 : FEProblemBase::computeMarkers()
4754 : {
4755 240880 : if (_markers.hasActiveObjects())
4756 : {
4757 35360 : TIME_SECTION("computeMarkers", 1, "Computing Markers");
4758 :
4759 7072 : std::vector<std::string> fields;
4760 :
4761 : // Marker Fields
4762 7072 : const auto & markers = _markers.getActiveObjects();
4763 15122 : for (const auto & marker : markers)
4764 8050 : fields.push_back(marker->name());
4765 :
4766 7072 : _aux->zeroVariables(fields);
4767 :
4768 7072 : _adaptivity.updateErrorVectors();
4769 :
4770 14764 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
4771 : {
4772 7696 : const auto & markers = _markers.getActiveObjects(tid);
4773 16453 : for (const auto & marker : markers)
4774 8761 : marker->markerSetup();
4775 : }
4776 :
4777 7068 : ComputeMarkerThread cmt(*this);
4778 7068 : Threads::parallel_reduce(*_mesh.getActiveLocalElementRange(), cmt);
4779 :
4780 7068 : _aux->solution().close();
4781 7068 : _aux->update();
4782 7068 : }
4783 240876 : }
4784 :
4785 : const ExecFlagType &
4786 2747308 : FEProblemBase::getCurrentExecuteOnFlag() const
4787 : {
4788 2747308 : return _current_execute_on_flag;
4789 : }
4790 :
4791 : void
4792 7504338 : FEProblemBase::setCurrentExecuteOnFlag(const ExecFlagType & flag)
4793 : {
4794 7504338 : _current_execute_on_flag = flag;
4795 7504338 : }
4796 :
4797 : void
4798 80 : FEProblemBase::executeAllObjects(const ExecFlagType & /*exec_type*/)
4799 : {
4800 80 : }
4801 :
4802 : void
4803 1924747 : FEProblemBase::customSetup(const ExecFlagType & exec_type)
4804 : {
4805 1924747 : SubProblem::customSetup(exec_type);
4806 :
4807 1924747 : if (_line_search)
4808 0 : _line_search->customSetup(exec_type);
4809 :
4810 1924747 : unsigned int n_threads = libMesh::n_threads();
4811 4021625 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4812 : {
4813 2096878 : _all_materials.customSetup(exec_type, tid);
4814 2096878 : _functions.customSetup(exec_type, tid);
4815 : }
4816 :
4817 : #ifdef MOOSE_KOKKOS_ENABLED
4818 1268185 : _kokkos_functions.customSetup(exec_type);
4819 : #endif
4820 :
4821 1924747 : _aux->customSetup(exec_type);
4822 3880100 : for (auto & nl : _nl)
4823 1955353 : nl->customSetup(exec_type);
4824 :
4825 1924747 : if (_displaced_problem)
4826 158920 : _displaced_problem->customSetup(exec_type);
4827 :
4828 4021625 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
4829 : {
4830 2096878 : _internal_side_indicators.customSetup(exec_type, tid);
4831 2096878 : _indicators.customSetup(exec_type, tid);
4832 2096878 : _markers.customSetup(exec_type, tid);
4833 : }
4834 :
4835 1924747 : std::vector<UserObject *> userobjs;
4836 1924747 : theWarehouse().query().condition<AttribSystem>("UserObject").queryIntoUnsorted(userobjs);
4837 4778021 : for (auto obj : userobjs)
4838 2853274 : obj->customSetup(exec_type);
4839 :
4840 1924747 : _app.getOutputWarehouse().customSetup(exec_type);
4841 1924747 : }
4842 :
4843 : void
4844 2324422 : FEProblemBase::execute(const ExecFlagType & exec_type)
4845 : {
4846 : // Set the current flag
4847 2324422 : setCurrentExecuteOnFlag(exec_type);
4848 :
4849 2324422 : if (exec_type != EXEC_INITIAL)
4850 2266591 : executeControls(exec_type);
4851 :
4852 : // intentially call this after executing controls because the setups may rely on the controls
4853 : // FIXME: we skip the following flags because they have dedicated setup functions in
4854 : // SetupInterface and it may not be appropriate to call them here.
4855 4251865 : if (!(exec_type == EXEC_INITIAL || exec_type == EXEC_TIMESTEP_BEGIN ||
4856 1927467 : exec_type == EXEC_SUBDOMAIN || exec_type == EXEC_NONLINEAR || exec_type == EXEC_LINEAR))
4857 1924747 : customSetup(exec_type);
4858 :
4859 : // Samplers; EXEC_INITIAL is not called because the Sampler::init() method that is called after
4860 : // construction makes the first Sampler::execute() call. This ensures that the random number
4861 : // generator object is the correct state prior to any other object (e.g., Transfers) attempts to
4862 : // extract data from the Sampler. That is, if the Sampler::execute() call is delayed to here
4863 : // then it is not in the correct state for other objects.
4864 2324398 : if (exec_type != EXEC_INITIAL)
4865 2266567 : executeSamplers(exec_type);
4866 :
4867 : // Pre-aux UserObjects
4868 2324364 : computeUserObjects(exec_type, Moose::PRE_AUX);
4869 :
4870 : // Systems (includes system time derivative and aux kernel calculations)
4871 2324364 : computeSystems(exec_type);
4872 : // With the auxiliary system solution computed, sync the displaced problem auxiliary solution
4873 : // before computation of post-aux user objects. The undisplaced auxiliary system current local
4874 : // solution is updated (via System::update) within the AuxiliarySystem class's variable
4875 : // computation methods (e.g. computeElementalVarsHelper, computeNodalVarsHelper), so it is safe to
4876 : // use it here
4877 2324319 : if (_displaced_problem)
4878 196607 : _displaced_problem->syncAuxSolution(*getAuxiliarySystem().currentSolution());
4879 :
4880 : // Post-aux UserObjects
4881 2324319 : computeUserObjects(exec_type, Moose::POST_AUX);
4882 :
4883 : // Return the current flag to None
4884 2324202 : setCurrentExecuteOnFlag(EXEC_NONE);
4885 :
4886 2324202 : if (_uo_aux_state_check && !_checking_uo_aux_state)
4887 : {
4888 : // we will only check aux variables and postprocessors
4889 : // checking more reporter data can be added in the future if needed
4890 624 : std::unique_ptr<NumericVector<Number>> x = _aux->currentSolution()->clone();
4891 624 : DenseVector<Real> pp_values = getReporterData().getAllRealReporterValues();
4892 :
4893 : // call THIS execute one more time for checking the possible states
4894 624 : _checking_uo_aux_state = true;
4895 624 : FEProblemBase::execute(exec_type);
4896 624 : _checking_uo_aux_state = false;
4897 :
4898 624 : const Real check_tol = 1e-8;
4899 :
4900 624 : const Real xnorm = x->l2_norm();
4901 624 : *x -= *_aux->currentSolution();
4902 624 : if (x->l2_norm() > check_tol * xnorm)
4903 : {
4904 4 : const auto & sys = _aux->system();
4905 4 : const unsigned int n_vars = sys.n_vars();
4906 4 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
4907 20 : for (const auto i : make_range(n_vars))
4908 : {
4909 16 : const Real vnorm = sys.calculate_norm(*x, i, DISCRETE_L2);
4910 16 : ordered_map.emplace(vnorm, sys.variable_name(i));
4911 : }
4912 :
4913 4 : std::ostringstream oss;
4914 20 : for (const auto & [error_norm, var_name] : ordered_map)
4915 16 : oss << " {" << var_name << ", " << error_norm << "},\n";
4916 :
4917 4 : mooseError("Aux kernels, user objects appear to have states for aux variables on ",
4918 : exec_type,
4919 : ".\nVariable error norms in descending order:\n",
4920 4 : oss.str());
4921 0 : }
4922 :
4923 620 : const DenseVector<Real> new_pp_values = getReporterData().getAllRealReporterValues();
4924 620 : if (pp_values.size() != new_pp_values.size())
4925 0 : mooseError("Second execution for uo/aux state check should not change the number of "
4926 : "real reporter values");
4927 :
4928 620 : const Real ppnorm = pp_values.l2_norm();
4929 620 : pp_values -= new_pp_values;
4930 620 : if (pp_values.l2_norm() > check_tol * ppnorm)
4931 : {
4932 4 : const auto pp_names = getReporterData().getAllRealReporterFullNames();
4933 4 : std::multimap<Real, std::string, std::greater<Real>> ordered_map;
4934 16 : for (const auto i : index_range(pp_names))
4935 12 : ordered_map.emplace(std::abs(pp_values(i)), pp_names[i]);
4936 :
4937 4 : std::ostringstream oss;
4938 16 : for (const auto & [error_norm, pp_name] : ordered_map)
4939 12 : oss << " {" << pp_name << ", " << error_norm << "},\n";
4940 :
4941 4 : mooseError("Aux kernels, user objects appear to have states for real reporter values on ",
4942 : exec_type,
4943 : ".\nErrors of real reporter values in descending order:\n",
4944 4 : oss.str());
4945 0 : }
4946 616 : }
4947 2324194 : }
4948 :
4949 : // Finalize, threadJoin, and update PP values of Elemental/Nodal/Side/InternalSideUserObjects
4950 : void
4951 1428812 : FEProblemBase::joinAndFinalize(TheWarehouse::Query query, bool isgen)
4952 : {
4953 1428812 : std::vector<UserObject *> objs;
4954 1428812 : query.queryInto(objs);
4955 1428812 : if (!isgen)
4956 : {
4957 : // join all threaded user objects (i.e. not regular general user objects) to the primary
4958 : // thread
4959 1511977 : for (auto obj : objs)
4960 392484 : if (obj->primaryThreadCopy())
4961 29195 : obj->primaryThreadCopy()->threadJoin(*obj);
4962 : }
4963 :
4964 1428812 : query.condition<AttribThread>(0).queryInto(objs);
4965 :
4966 : // finalize objects and retrieve/store any postprocessor values
4967 2036466 : for (auto obj : objs)
4968 : {
4969 607763 : if (isgen && dynamic_cast<ThreadedGeneralUserObject *>(obj))
4970 116 : continue;
4971 607647 : if (isgen)
4972 : {
4973 : // general user objects are not run in their own threaded loop object - so run them here
4974 244358 : if (shouldPrintExecution(0))
4975 798 : _console << "[DBG] Initializing, executing & finalizing general UO '" << obj->name()
4976 798 : << "' on " << _current_execute_on_flag.name() << std::endl;
4977 244358 : obj->initialize();
4978 244358 : obj->execute();
4979 : }
4980 :
4981 607578 : obj->finalize();
4982 :
4983 : // These have to be stored piecemeal (with every call to this function) because general
4984 : // postprocessors (which run last after other userobjects have been completed) might depend on
4985 : // them being stored. This wouldn't be a problem if all userobjects satisfied the dependency
4986 : // resolver interface and could be sorted appropriately with the general userobjects, but they
4987 : // don't.
4988 607558 : auto pp = dynamic_cast<const Postprocessor *>(obj);
4989 607558 : if (pp)
4990 : {
4991 529069 : _reporter_data.finalize(obj->name());
4992 529069 : setPostprocessorValueByName(obj->name(), pp->getValue());
4993 : }
4994 :
4995 607546 : auto vpp = dynamic_cast<VectorPostprocessor *>(obj);
4996 607546 : if (vpp)
4997 14358 : _reporter_data.finalize(obj->name());
4998 :
4999 : // Update Reporter data
5000 607546 : auto reporter = dynamic_cast<Reporter *>(obj);
5001 607546 : if (reporter)
5002 5391 : _reporter_data.finalize(obj->name());
5003 : }
5004 1428703 : }
5005 :
5006 : void
5007 57176 : FEProblemBase::computeUserObjectByName(const ExecFlagType & type,
5008 : const Moose::AuxGroup & group,
5009 : const std::string & name)
5010 : {
5011 57176 : const auto old_exec_flag = _current_execute_on_flag;
5012 57176 : _current_execute_on_flag = type;
5013 57176 : TheWarehouse::Query query = theWarehouse()
5014 114352 : .query()
5015 57176 : .condition<AttribSystem>("UserObject")
5016 57176 : .condition<AttribExecOns>(type)
5017 57176 : .condition<AttribName>(name);
5018 57176 : computeUserObjectsInternal(type, group, query);
5019 57176 : _current_execute_on_flag = old_exec_flag;
5020 57176 : }
5021 :
5022 : void
5023 12477598 : FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGroup & group)
5024 : {
5025 : TheWarehouse::Query query =
5026 12477598 : theWarehouse().query().condition<AttribSystem>("UserObject").condition<AttribExecOns>(type);
5027 12477598 : computeUserObjectsInternal(type, group, query);
5028 12477481 : }
5029 :
5030 : void
5031 12534774 : FEProblemBase::computeUserObjectsInternal(const ExecFlagType & type,
5032 : const Moose::AuxGroup & group,
5033 : TheWarehouse::Query & primary_query)
5034 : {
5035 : try
5036 : {
5037 62673870 : TIME_SECTION("computeUserObjects", 1, "Computing User Objects");
5038 :
5039 : // Add group to query
5040 12534774 : if (group == Moose::PRE_IC)
5041 57891 : primary_query.condition<AttribPreIC>(true);
5042 12476883 : else if (group == Moose::PRE_AUX)
5043 6238206 : primary_query.condition<AttribPreAux>(type);
5044 6238677 : else if (group == Moose::POST_AUX)
5045 6238633 : primary_query.condition<AttribPostAux>(type);
5046 :
5047 : // query everything first to obtain a list of execution groups
5048 12534774 : std::vector<UserObject *> uos;
5049 12534774 : primary_query.clone().queryIntoUnsorted(uos);
5050 12534774 : std::set<int> execution_groups;
5051 13171752 : for (const auto & uo : uos)
5052 1910934 : execution_groups.insert(uo->getParam<int>("execution_order_group"));
5053 :
5054 : // iterate over execution order groups
5055 12843992 : for (const auto execution_group : execution_groups)
5056 : {
5057 309335 : auto query = primary_query.clone().condition<AttribExecutionOrderGroup>(execution_group);
5058 :
5059 309335 : std::vector<GeneralUserObject *> genobjs;
5060 309335 : query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject).queryInto(genobjs);
5061 :
5062 309335 : std::vector<UserObject *> userobjs;
5063 309335 : query.clone()
5064 618670 : .condition<AttribInterfaces>(Interfaces::ElementUserObject | Interfaces::SideUserObject |
5065 : Interfaces::InternalSideUserObject |
5066 618670 : Interfaces::InterfaceUserObject |
5067 : Interfaces::DomainUserObject)
5068 309335 : .queryInto(userobjs);
5069 :
5070 309335 : std::vector<UserObject *> tgobjs;
5071 309335 : query.clone()
5072 618670 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5073 309335 : .queryInto(tgobjs);
5074 :
5075 309335 : std::vector<UserObject *> nodal;
5076 309335 : query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).queryInto(nodal);
5077 :
5078 309335 : std::vector<MortarUserObject *> mortar;
5079 309335 : query.clone().condition<AttribInterfaces>(Interfaces::MortarUserObject).queryInto(mortar);
5080 :
5081 309335 : if (userobjs.empty() && genobjs.empty() && tgobjs.empty() && nodal.empty() && mortar.empty())
5082 0 : continue;
5083 :
5084 : // Start the timer here since we have at least one active user object
5085 309335 : std::string compute_uo_tag = "computeUserObjects(" + Moose::stringify(type) + ")";
5086 :
5087 : // Perform Residual/Jacobian setups
5088 309335 : if (type == EXEC_LINEAR)
5089 : {
5090 132237 : for (auto obj : userobjs)
5091 74488 : obj->residualSetup();
5092 62541 : for (auto obj : nodal)
5093 4792 : obj->residualSetup();
5094 57749 : for (auto obj : mortar)
5095 0 : obj->residualSetup();
5096 57749 : for (auto obj : tgobjs)
5097 0 : obj->residualSetup();
5098 69324 : for (auto obj : genobjs)
5099 11575 : obj->residualSetup();
5100 : }
5101 251586 : else if (type == EXEC_NONLINEAR)
5102 : {
5103 11622 : for (auto obj : userobjs)
5104 2809 : obj->jacobianSetup();
5105 9237 : for (auto obj : nodal)
5106 424 : obj->jacobianSetup();
5107 8813 : for (auto obj : mortar)
5108 0 : obj->jacobianSetup();
5109 8813 : for (auto obj : tgobjs)
5110 0 : obj->jacobianSetup();
5111 24077 : for (auto obj : genobjs)
5112 15264 : obj->jacobianSetup();
5113 : }
5114 :
5115 678947 : for (auto obj : userobjs)
5116 369612 : obj->initialize();
5117 :
5118 : // Execute Side/InternalSide/Interface/Elemental/DomainUserObjects
5119 309335 : if (!userobjs.empty())
5120 : {
5121 : // non-nodal user objects have to be run separately before the nodal user objects run
5122 : // because some nodal user objects (NodalNormal related) depend on elemental user objects
5123 : // :-(
5124 220311 : ComputeUserObjectsThread cppt(*this, query);
5125 220311 : Threads::parallel_reduce(*_mesh.getActiveLocalElementRange(), cppt);
5126 :
5127 : // There is one instance in rattlesnake where an elemental user object's finalize depends
5128 : // on a side user object having been finalized first :-(
5129 220303 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::SideUserObject));
5130 220299 : joinAndFinalize(
5131 440598 : query.clone().condition<AttribInterfaces>(Interfaces::InternalSideUserObject));
5132 220299 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::InterfaceUserObject));
5133 220299 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::ElementUserObject));
5134 220295 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::DomainUserObject));
5135 220295 : }
5136 :
5137 : // if any userobject may have written to variables we need to close the aux solution
5138 678829 : for (const auto & uo : userobjs)
5139 369546 : if (auto euo = dynamic_cast<const ElementUserObject *>(uo);
5140 369546 : euo && euo->hasWritableCoupledVariables())
5141 : {
5142 36 : _aux->solution().close();
5143 36 : _aux->system().update();
5144 36 : break;
5145 : }
5146 :
5147 : // Execute NodalUserObjects
5148 : // BISON has an axial reloc elemental user object that has a finalize func that depends on a
5149 : // nodal user object's prev value. So we can't initialize this until after elemental objects
5150 : // have been finalized :-(
5151 331779 : for (auto obj : nodal)
5152 22460 : obj->initialize();
5153 309319 : if (query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject).count() > 0)
5154 : {
5155 17882 : ComputeNodalUserObjectsThread cnppt(*this, query);
5156 17882 : Threads::parallel_reduce(*_mesh.getLocalNodeRange(), cnppt);
5157 17882 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::NodalUserObject));
5158 17882 : }
5159 :
5160 : // if any userobject may have written to variables we need to close the aux solution
5161 331740 : for (const auto & uo : nodal)
5162 22445 : if (auto nuo = dynamic_cast<const NodalUserObject *>(uo);
5163 22445 : nuo && nuo->hasWritableCoupledVariables())
5164 : {
5165 24 : _aux->solution().close();
5166 24 : _aux->system().update();
5167 24 : break;
5168 : }
5169 :
5170 : // Execute MortarUserObjects
5171 : {
5172 309343 : for (auto obj : mortar)
5173 24 : obj->initialize();
5174 309319 : if (!mortar.empty())
5175 : {
5176 36 : auto create_and_run_mortar_functors = [this, type, &mortar](const bool displaced)
5177 : {
5178 : // go over mortar interfaces and construct functors
5179 36 : const auto & mortar_interfaces = getMortarInterfaces(displaced);
5180 60 : for (const auto & [primary_secondary_boundary_pair, mortar_generation_ptr] :
5181 96 : mortar_interfaces)
5182 : {
5183 : auto mortar_uos_to_execute =
5184 24 : getMortarUserObjects(primary_secondary_boundary_pair.first,
5185 24 : primary_secondary_boundary_pair.second,
5186 : displaced,
5187 24 : mortar);
5188 :
5189 : auto * const subproblem = displaced
5190 24 : ? static_cast<SubProblem *>(_displaced_problem.get())
5191 24 : : static_cast<SubProblem *>(this);
5192 : MortarUserObjectThread muot(mortar_uos_to_execute,
5193 24 : *mortar_generation_ptr,
5194 : *subproblem,
5195 : *this,
5196 : displaced,
5197 24 : subproblem->assembly(0, 0));
5198 :
5199 24 : muot();
5200 24 : }
5201 60 : };
5202 :
5203 24 : create_and_run_mortar_functors(false);
5204 24 : if (_displaced_problem)
5205 12 : create_and_run_mortar_functors(true);
5206 24 : }
5207 309343 : for (auto obj : mortar)
5208 24 : obj->finalize();
5209 : }
5210 :
5211 : // Execute threaded general user objects
5212 309771 : for (auto obj : tgobjs)
5213 452 : obj->initialize();
5214 309319 : std::vector<GeneralUserObject *> tguos_zero;
5215 309319 : query.clone()
5216 309319 : .condition<AttribThread>(0)
5217 618638 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject)
5218 309319 : .queryInto(tguos_zero);
5219 309435 : for (auto obj : tguos_zero)
5220 : {
5221 116 : std::vector<GeneralUserObject *> tguos;
5222 116 : auto q = query.clone()
5223 116 : .condition<AttribName>(obj->name())
5224 116 : .condition<AttribInterfaces>(Interfaces::ThreadedGeneralUserObject);
5225 116 : q.queryInto(tguos);
5226 :
5227 116 : ComputeThreadedGeneralUserObjectsThread ctguot(*this);
5228 116 : Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(), tguos.end()), ctguot);
5229 116 : joinAndFinalize(q);
5230 116 : }
5231 :
5232 : // Execute general user objects
5233 309319 : joinAndFinalize(query.clone().condition<AttribInterfaces>(Interfaces::GeneralUserObject),
5234 : true);
5235 309218 : }
5236 12534657 : }
5237 0 : catch (...)
5238 : {
5239 0 : handleException("computeUserObjectsInternal");
5240 0 : }
5241 12534657 : }
5242 :
5243 : void
5244 6213689 : FEProblemBase::executeControls(const ExecFlagType & exec_type)
5245 : {
5246 6213689 : if (_control_warehouse[exec_type].hasActiveObjects())
5247 : {
5248 41685 : TIME_SECTION("executeControls", 1, "Executing Controls");
5249 :
5250 8337 : DependencyResolver<std::shared_ptr<Control>> resolver;
5251 :
5252 8337 : auto controls_wh = _control_warehouse[exec_type];
5253 : // Add all of the dependencies into the resolver and sort them
5254 20920 : for (const auto & it : controls_wh.getActiveObjects())
5255 : {
5256 : // Make sure an item with no dependencies comes out too!
5257 12587 : resolver.addItem(it);
5258 :
5259 12587 : std::vector<std::string> & dependent_controls = it->getDependencies();
5260 15712 : for (const auto & depend_name : dependent_controls)
5261 : {
5262 3129 : if (controls_wh.hasActiveObject(depend_name))
5263 : {
5264 3125 : auto dep_control = controls_wh.getActiveObject(depend_name);
5265 3125 : resolver.addEdge(dep_control, it);
5266 3125 : }
5267 : else
5268 4 : mooseError("The Control \"",
5269 : depend_name,
5270 : "\" was not created, did you make a "
5271 : "spelling mistake or forget to include it "
5272 : "in your input file?");
5273 : }
5274 : }
5275 :
5276 8333 : const auto & ordered_controls = resolver.getSortedValues();
5277 :
5278 8333 : if (!ordered_controls.empty())
5279 : {
5280 8333 : _control_warehouse.setup(exec_type);
5281 : // Run the controls in the proper order
5282 20860 : for (const auto & control : ordered_controls)
5283 12583 : control->execute();
5284 : }
5285 8277 : }
5286 6213629 : }
5287 :
5288 : void
5289 2266567 : FEProblemBase::executeSamplers(const ExecFlagType & exec_type)
5290 : {
5291 : // TODO: This should be done in a threaded loop, but this should be super quick so for now
5292 : // do a serial loop.
5293 4735268 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5294 : {
5295 2468735 : std::vector<Sampler *> objects;
5296 2468735 : theWarehouse()
5297 4937470 : .query()
5298 2468735 : .condition<AttribSystem>("Sampler")
5299 2468735 : .condition<AttribThread>(tid)
5300 2468735 : .condition<AttribExecOns>(exec_type)
5301 2468735 : .queryInto(objects);
5302 :
5303 2468735 : if (!objects.empty())
5304 : {
5305 285 : TIME_SECTION("executeSamplers", 1, "Executing Samplers");
5306 57 : FEProblemBase::objectSetupHelper<Sampler>(objects, exec_type);
5307 57 : FEProblemBase::objectExecuteHelper<Sampler>(objects);
5308 23 : }
5309 2468701 : }
5310 2266533 : }
5311 :
5312 : void
5313 339083 : FEProblemBase::updateActiveObjects()
5314 : {
5315 1695415 : TIME_SECTION("updateActiveObjects", 5, "Updating Active Objects");
5316 :
5317 707978 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
5318 : {
5319 740314 : for (auto & nl : _nl)
5320 371419 : nl->updateActive(tid);
5321 368895 : _aux->updateActive(tid);
5322 368895 : _indicators.updateActive(tid);
5323 368895 : _internal_side_indicators.updateActive(tid);
5324 368895 : _markers.updateActive(tid);
5325 368895 : _all_materials.updateActive(tid);
5326 368895 : _materials.updateActive(tid);
5327 368895 : _discrete_materials.updateActive(tid);
5328 : }
5329 :
5330 339083 : _control_warehouse.updateActive();
5331 339083 : _multi_apps.updateActive();
5332 339083 : _transient_multi_apps.updateActive();
5333 339083 : _transfers.updateActive();
5334 339083 : _to_multi_app_transfers.updateActive();
5335 339083 : _from_multi_app_transfers.updateActive();
5336 339083 : _between_multi_app_transfers.updateActive();
5337 :
5338 : #ifdef MOOSE_KOKKOS_ENABLED
5339 223632 : _kokkos_materials.updateActive();
5340 : #endif
5341 339083 : }
5342 :
5343 : void
5344 0 : FEProblemBase::reportMooseObjectDependency(MooseObject * /*a*/, MooseObject * /*b*/)
5345 : {
5346 : //<< "Object " << a->name() << " -> " << b->name() << std::endl;
5347 0 : }
5348 :
5349 : void
5350 70319 : FEProblemBase::reinitBecauseOfGhostingOrNewGeomObjects(const bool mortar_changed)
5351 : {
5352 351595 : TIME_SECTION("reinitBecauseOfGhostingOrNewGeomObjects",
5353 : 3,
5354 : "Reinitializing Because of Geometric Search Objects");
5355 :
5356 : // Need to see if _any_ processor has ghosted elems or geometry objects.
5357 70319 : bool needs_reinit = !_ghosted_elems.empty();
5358 140228 : needs_reinit = needs_reinit || !_geometric_search_data._nearest_node_locators.empty() ||
5359 69909 : (_mortar_data->hasObjects() && mortar_changed);
5360 70319 : needs_reinit =
5361 142181 : needs_reinit || (_displaced_problem &&
5362 5002 : (!_displaced_problem->geomSearchData()._nearest_node_locators.empty() ||
5363 71225 : (_mortar_data->hasDisplacedObjects() && mortar_changed)));
5364 70319 : _communicator.max(needs_reinit);
5365 :
5366 70319 : if (needs_reinit)
5367 : {
5368 : // Call reinit to get the ghosted vectors correct now that some geometric search has been done
5369 2354 : es().reinit();
5370 :
5371 2354 : if (_displaced_mesh)
5372 1748 : _displaced_problem->es().reinit();
5373 : }
5374 70319 : }
5375 :
5376 : void
5377 181 : FEProblemBase::addDamper(const std::string & damper_name,
5378 : const std::string & name,
5379 : InputParameters & parameters)
5380 : {
5381 : parallel_object_only();
5382 :
5383 : const auto nl_sys_num =
5384 181 : parameters.isParamValid("variable")
5385 582 : ? determineSolverSystem(parameters.varName("variable", name), true).second
5386 177 : : (unsigned int)0;
5387 :
5388 177 : if (!isSolverSystemNonlinear(nl_sys_num))
5389 0 : mooseError("You are trying to add a DGKernel to a linear variable/system, which is not "
5390 : "supported at the moment!");
5391 :
5392 354 : parameters.set<SubProblem *>("_subproblem") = this;
5393 354 : parameters.set<SystemBase *>("_sys") = _nl[nl_sys_num].get();
5394 :
5395 177 : _has_dampers = true;
5396 177 : logAdd("Damper", name, damper_name, parameters);
5397 177 : _nl[nl_sys_num]->addDamper(damper_name, name, parameters);
5398 177 : }
5399 :
5400 : void
5401 163 : FEProblemBase::setupDampers()
5402 : {
5403 326 : for (auto & nl : _nl)
5404 163 : nl->setupDampers();
5405 163 : }
5406 :
5407 : void
5408 697 : FEProblemBase::addIndicator(const std::string & indicator_name,
5409 : const std::string & name,
5410 : InputParameters & parameters)
5411 : {
5412 : parallel_object_only();
5413 :
5414 697 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5415 : {
5416 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5417 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5418 0 : _reinit_displaced_elem = true;
5419 : }
5420 : else
5421 : {
5422 697 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5423 : {
5424 : // We allow Indicators to request that they use_displaced_mesh,
5425 : // but then be overridden when no displacements variables are
5426 : // provided in the Mesh block. If that happened, update the value
5427 : // of use_displaced_mesh appropriately for this Indicator.
5428 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5429 0 : parameters.set<bool>("use_displaced_mesh") = false;
5430 : }
5431 :
5432 1394 : parameters.set<SubProblem *>("_subproblem") = this;
5433 2091 : parameters.set<SystemBase *>("_sys") = _aux.get();
5434 : }
5435 :
5436 1456 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5437 : {
5438 : std::shared_ptr<Indicator> indicator =
5439 759 : _factory.create<Indicator>(indicator_name, name, parameters, tid);
5440 759 : logAdd("Indicator", name, indicator_name, parameters);
5441 : std::shared_ptr<InternalSideIndicatorBase> isi =
5442 759 : std::dynamic_pointer_cast<InternalSideIndicatorBase>(indicator);
5443 759 : if (isi)
5444 650 : _internal_side_indicators.addObject(isi, tid);
5445 : else
5446 109 : _indicators.addObject(indicator, tid);
5447 759 : }
5448 697 : }
5449 :
5450 : void
5451 2095 : FEProblemBase::addMarker(const std::string & marker_name,
5452 : const std::string & name,
5453 : InputParameters & parameters)
5454 : {
5455 : parallel_object_only();
5456 :
5457 2095 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5458 : {
5459 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5460 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5461 0 : _reinit_displaced_elem = true;
5462 : }
5463 : else
5464 : {
5465 2095 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5466 : {
5467 : // We allow Markers to request that they use_displaced_mesh,
5468 : // but then be overridden when no displacements variables are
5469 : // provided in the Mesh block. If that happened, update the value
5470 : // of use_displaced_mesh appropriately for this Marker.
5471 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5472 0 : parameters.set<bool>("use_displaced_mesh") = false;
5473 : }
5474 :
5475 4190 : parameters.set<SubProblem *>("_subproblem") = this;
5476 6285 : parameters.set<SystemBase *>("_sys") = _aux.get();
5477 : }
5478 :
5479 4360 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
5480 : {
5481 2269 : std::shared_ptr<Marker> marker = _factory.create<Marker>(marker_name, name, parameters, tid);
5482 2265 : logAdd("Marker", name, marker_name, parameters);
5483 2265 : _markers.addObject(marker, tid);
5484 2265 : }
5485 2091 : }
5486 :
5487 : void
5488 7672 : FEProblemBase::addMultiApp(const std::string & multi_app_name,
5489 : const std::string & name,
5490 : InputParameters & parameters)
5491 : {
5492 : parallel_object_only();
5493 :
5494 15344 : parameters.set<MPI_Comm>("_mpi_comm") = _communicator.get();
5495 :
5496 7672 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5497 : {
5498 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5499 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5500 0 : _reinit_displaced_elem = true;
5501 : }
5502 : else
5503 : {
5504 7672 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5505 : {
5506 : // We allow MultiApps to request that they use_displaced_mesh,
5507 : // but then be overridden when no displacements variables are
5508 : // provided in the Mesh block. If that happened, update the value
5509 : // of use_displaced_mesh appropriately for this MultiApp.
5510 49 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5511 98 : parameters.set<bool>("use_displaced_mesh") = false;
5512 : }
5513 :
5514 15344 : parameters.set<SubProblem *>("_subproblem") = this;
5515 23016 : parameters.set<SystemBase *>("_sys") = _aux.get();
5516 : }
5517 :
5518 7672 : std::shared_ptr<MultiApp> multi_app = _factory.create<MultiApp>(multi_app_name, name, parameters);
5519 7656 : logAdd("MultiApp", name, multi_app_name, parameters);
5520 7656 : multi_app->setupPositions();
5521 :
5522 7541 : _multi_apps.addObject(multi_app);
5523 :
5524 : // Store TransientMultiApp objects in another container, this is needed for calling computeDT
5525 : std::shared_ptr<TransientMultiApp> trans_multi_app =
5526 7541 : std::dynamic_pointer_cast<TransientMultiApp>(multi_app);
5527 7541 : if (trans_multi_app)
5528 5401 : _transient_multi_apps.addObject(trans_multi_app);
5529 7541 : }
5530 :
5531 : bool
5532 120680 : FEProblemBase::hasMultiApps(ExecFlagType type) const
5533 : {
5534 120680 : return _multi_apps[type].hasActiveObjects();
5535 : }
5536 :
5537 : bool
5538 0 : FEProblemBase::hasMultiApp(const std::string & multi_app_name) const
5539 : {
5540 0 : return _multi_apps.hasActiveObject(multi_app_name);
5541 : }
5542 :
5543 : std::shared_ptr<MultiApp>
5544 24983 : FEProblemBase::getMultiApp(const std::string & multi_app_name) const
5545 : {
5546 24983 : return _multi_apps.getObject(multi_app_name);
5547 : }
5548 :
5549 : void
5550 15741113 : FEProblemBase::execMultiAppTransfers(ExecFlagType type, Transfer::DIRECTION direction)
5551 : {
5552 15741113 : bool to_multiapp = direction == MultiAppTransfer::TO_MULTIAPP;
5553 15741113 : bool from_multiapp = direction == MultiAppTransfer::FROM_MULTIAPP;
5554 15741113 : std::string string_direction;
5555 15741113 : if (to_multiapp)
5556 5247110 : string_direction = " To ";
5557 10494003 : else if (from_multiapp)
5558 5246941 : string_direction = " From ";
5559 : else
5560 5247062 : string_direction = " Between ";
5561 :
5562 15741330 : const MooseObjectWarehouse<Transfer> & wh = to_multiapp ? _to_multi_app_transfers[type]
5563 26234995 : : from_multiapp ? _from_multi_app_transfers[type]
5564 36729215 : : _between_multi_app_transfers[type];
5565 :
5566 15741113 : if (wh.hasActiveObjects())
5567 : {
5568 570085 : TIME_SECTION("execMultiAppTransfers", 1, "Executing Transfers");
5569 :
5570 114017 : const auto & transfers = wh.getActiveObjects();
5571 :
5572 114017 : if (_verbose_multiapps)
5573 : {
5574 2486 : _console << COLOR_CYAN << "\nTransfers on " << Moose::stringify(type) << string_direction
5575 2486 : << "MultiApps" << COLOR_DEFAULT << ":" << std::endl;
5576 :
5577 : VariadicTable<std::string, std::string, std::string, std::string> table(
5578 4972 : {"Name", "Type", "From", "To"});
5579 :
5580 : // Build Table of Transfer Info
5581 5932 : for (const auto & transfer : transfers)
5582 : {
5583 3446 : auto multiapp_transfer = dynamic_cast<MultiAppTransfer *>(transfer.get());
5584 :
5585 3446 : table.addRow(multiapp_transfer->name(),
5586 3446 : multiapp_transfer->type(),
5587 6892 : multiapp_transfer->getFromName(),
5588 6892 : multiapp_transfer->getToName());
5589 : }
5590 :
5591 : // Print it
5592 2486 : table.print(_console);
5593 2486 : }
5594 :
5595 233099 : for (const auto & transfer : transfers)
5596 : {
5597 119205 : transfer->setCurrentDirection(direction);
5598 119205 : transfer->execute();
5599 : }
5600 :
5601 113894 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5602 :
5603 113894 : if (_verbose_multiapps)
5604 4956 : _console << COLOR_CYAN << "Transfers on " << Moose::stringify(type) << " Are Finished\n"
5605 2478 : << COLOR_DEFAULT << std::endl;
5606 113894 : }
5607 15627096 : else if (_multi_apps[type].getActiveObjects().size())
5608 : {
5609 116089 : if (_verbose_multiapps)
5610 7988 : _console << COLOR_CYAN << "\nNo Transfers on " << Moose::stringify(type) << string_direction
5611 3994 : << "MultiApps\n"
5612 3994 : << COLOR_DEFAULT << std::endl;
5613 : }
5614 15740990 : }
5615 :
5616 : std::vector<std::shared_ptr<Transfer>>
5617 13 : FEProblemBase::getTransfers(ExecFlagType type, Transfer::DIRECTION direction) const
5618 : {
5619 13 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5620 13 : return _to_multi_app_transfers[type].getActiveObjects();
5621 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5622 0 : return _from_multi_app_transfers[type].getActiveObjects();
5623 : else
5624 0 : return _between_multi_app_transfers[type].getActiveObjects();
5625 : }
5626 :
5627 : std::vector<std::shared_ptr<Transfer>>
5628 0 : FEProblemBase::getTransfers(Transfer::DIRECTION direction) const
5629 : {
5630 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5631 0 : return _to_multi_app_transfers.getActiveObjects();
5632 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5633 0 : return _from_multi_app_transfers.getActiveObjects();
5634 : else
5635 0 : return _between_multi_app_transfers.getActiveObjects();
5636 : }
5637 :
5638 : const ExecuteMooseObjectWarehouse<Transfer> &
5639 0 : FEProblemBase::getMultiAppTransferWarehouse(Transfer::DIRECTION direction) const
5640 : {
5641 0 : if (direction == MultiAppTransfer::TO_MULTIAPP)
5642 0 : return _to_multi_app_transfers;
5643 0 : else if (direction == MultiAppTransfer::FROM_MULTIAPP)
5644 0 : return _from_multi_app_transfers;
5645 : else
5646 0 : return _between_multi_app_transfers;
5647 : }
5648 :
5649 : bool
5650 5247114 : FEProblemBase::execMultiApps(ExecFlagType type, bool auto_advance)
5651 : {
5652 : // Active MultiApps
5653 : const std::vector<MooseSharedPointer<MultiApp>> & multi_apps =
5654 5247114 : _multi_apps[type].getActiveObjects();
5655 :
5656 : // Do anything that needs to be done to Apps before transfers
5657 5315060 : for (const auto & multi_app : multi_apps)
5658 67950 : multi_app->preTransfer(_dt, _time);
5659 :
5660 : // Execute Transfers _to_ MultiApps
5661 5247110 : execMultiAppTransfers(type, MultiAppTransfer::TO_MULTIAPP);
5662 :
5663 : // Execute Transfers _between_ Multiapps
5664 5247062 : execMultiAppTransfers(type, MultiAppTransfer::BETWEEN_MULTIAPP);
5665 :
5666 : // Execute MultiApps
5667 5247062 : if (multi_apps.size())
5668 : {
5669 333535 : TIME_SECTION("execMultiApps", 1, "Executing MultiApps", false);
5670 :
5671 66707 : if (_verbose_multiapps)
5672 4192 : _console << COLOR_CYAN << "\nExecuting MultiApps on " << Moose::stringify(type)
5673 2096 : << COLOR_DEFAULT << std::endl;
5674 :
5675 66707 : bool success = true;
5676 :
5677 134481 : for (const auto & multi_app : multi_apps)
5678 : {
5679 67887 : success = multi_app->solveStep(_dt, _time, auto_advance);
5680 : // no need to finish executing the subapps if one fails
5681 67875 : if (!success)
5682 101 : break;
5683 : }
5684 :
5685 66695 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5686 :
5687 66695 : _communicator.min(success);
5688 :
5689 66695 : if (!success)
5690 109 : return false;
5691 :
5692 66586 : if (_verbose_multiapps)
5693 4192 : _console << COLOR_CYAN << "Finished Executing MultiApps on " << Moose::stringify(type) << "\n"
5694 2096 : << COLOR_DEFAULT << std::endl;
5695 66695 : }
5696 :
5697 : // Execute Transfers _from_ MultiApps
5698 5246941 : execMultiAppTransfers(type, MultiAppTransfer::FROM_MULTIAPP);
5699 :
5700 : // If we made it here then everything passed
5701 5246866 : return true;
5702 : }
5703 :
5704 : void
5705 50232 : FEProblemBase::finalizeMultiApps()
5706 : {
5707 50232 : const auto & multi_apps = _multi_apps.getActiveObjects();
5708 :
5709 56591 : for (const auto & multi_app : multi_apps)
5710 6359 : multi_app->finalize();
5711 50232 : }
5712 :
5713 : void
5714 51840 : FEProblemBase::postExecute()
5715 : {
5716 51840 : const auto & multi_apps = _multi_apps.getActiveObjects();
5717 :
5718 58556 : for (const auto & multi_app : multi_apps)
5719 6716 : multi_app->postExecute();
5720 51840 : }
5721 :
5722 : void
5723 727404 : FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
5724 : {
5725 727404 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
5726 :
5727 727404 : if (multi_apps.size())
5728 28639 : for (const auto & multi_app : multi_apps)
5729 14497 : multi_app->incrementTStep(_time);
5730 727404 : }
5731 :
5732 : void
5733 39872 : FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
5734 : {
5735 39872 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
5736 :
5737 39872 : if (multi_apps.size())
5738 : {
5739 9407 : if (_verbose_multiapps)
5740 416 : _console << COLOR_CYAN << "\nAdvancing MultiApps on " << type.name() << COLOR_DEFAULT
5741 416 : << std::endl;
5742 :
5743 18828 : for (const auto & multi_app : multi_apps)
5744 9421 : multi_app->finishStep(recurse_through_multiapp_levels);
5745 :
5746 9407 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5747 :
5748 9407 : if (_verbose_multiapps)
5749 416 : _console << COLOR_CYAN << "Finished Advancing MultiApps on " << type.name() << "\n"
5750 416 : << COLOR_DEFAULT << std::endl;
5751 : }
5752 39872 : }
5753 :
5754 : void
5755 1217665 : FEProblemBase::backupMultiApps(ExecFlagType type)
5756 : {
5757 1217665 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
5758 :
5759 1217665 : if (multi_apps.size())
5760 : {
5761 106985 : TIME_SECTION("backupMultiApps", 5, "Backing Up MultiApp");
5762 :
5763 21397 : if (_verbose_multiapps)
5764 1007 : _console << COLOR_CYAN << "\nBacking Up MultiApps on " << type.name() << COLOR_DEFAULT
5765 1007 : << std::endl;
5766 :
5767 43915 : for (const auto & multi_app : multi_apps)
5768 22518 : multi_app->backup();
5769 :
5770 21397 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5771 :
5772 21397 : if (_verbose_multiapps)
5773 1007 : _console << COLOR_CYAN << "Finished Backing Up MultiApps on " << type.name() << "\n"
5774 1007 : << COLOR_DEFAULT << std::endl;
5775 21397 : }
5776 1217665 : }
5777 :
5778 : void
5779 118944 : FEProblemBase::restoreMultiApps(ExecFlagType type, bool force)
5780 : {
5781 118944 : const auto & multi_apps = _multi_apps[type].getActiveObjects();
5782 :
5783 118944 : if (multi_apps.size())
5784 : {
5785 45872 : if (_verbose_multiapps)
5786 : {
5787 1092 : if (force)
5788 3 : _console << COLOR_CYAN << "\nRestoring Multiapps on " << type.name()
5789 3 : << " because of solve failure!" << COLOR_DEFAULT << std::endl;
5790 : else
5791 1089 : _console << COLOR_CYAN << "\nRestoring MultiApps on " << type.name() << COLOR_DEFAULT
5792 1089 : << std::endl;
5793 : }
5794 :
5795 91829 : for (const auto & multi_app : multi_apps)
5796 45961 : multi_app->restore(force);
5797 :
5798 45868 : MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);
5799 :
5800 45868 : if (_verbose_multiapps)
5801 1092 : _console << COLOR_CYAN << "Finished Restoring MultiApps on " << type.name() << "\n"
5802 1092 : << COLOR_DEFAULT << std::endl;
5803 : }
5804 118940 : }
5805 :
5806 : Real
5807 756296 : FEProblemBase::computeMultiAppsDT(ExecFlagType type)
5808 : {
5809 756296 : const auto & multi_apps = _transient_multi_apps[type].getActiveObjects();
5810 :
5811 756296 : Real smallest_dt = std::numeric_limits<Real>::max();
5812 :
5813 774822 : for (const auto & multi_app : multi_apps)
5814 18526 : smallest_dt = std::min(smallest_dt, multi_app->computeDT());
5815 :
5816 756296 : return smallest_dt;
5817 : }
5818 :
5819 : void
5820 5191831 : FEProblemBase::execTransfers(ExecFlagType type)
5821 : {
5822 5191831 : if (_transfers[type].hasActiveObjects())
5823 : {
5824 0 : TIME_SECTION("execTransfers", 3, "Executing Transfers");
5825 :
5826 0 : const auto & transfers = _transfers[type].getActiveObjects();
5827 :
5828 0 : for (const auto & transfer : transfers)
5829 0 : transfer->execute();
5830 0 : }
5831 5191831 : }
5832 :
5833 : void
5834 12603 : FEProblemBase::addTransfer(const std::string & transfer_name,
5835 : const std::string & name,
5836 : InputParameters & parameters)
5837 : {
5838 : parallel_object_only();
5839 :
5840 12603 : if (_displaced_problem && parameters.get<bool>("use_displaced_mesh"))
5841 : {
5842 0 : parameters.set<SubProblem *>("_subproblem") = _displaced_problem.get();
5843 0 : parameters.set<SystemBase *>("_sys") = &_displaced_problem->auxSys();
5844 0 : _reinit_displaced_elem = true;
5845 : }
5846 : else
5847 : {
5848 12603 : if (_displaced_problem == nullptr && parameters.get<bool>("use_displaced_mesh"))
5849 : {
5850 : // We allow Transfers to request that they use_displaced_mesh,
5851 : // but then be overridden when no displacements variables are
5852 : // provided in the Mesh block. If that happened, update the value
5853 : // of use_displaced_mesh appropriately for this Transfer.
5854 0 : if (parameters.have_parameter<bool>("use_displaced_mesh"))
5855 0 : parameters.set<bool>("use_displaced_mesh") = false;
5856 : }
5857 :
5858 25206 : parameters.set<SubProblem *>("_subproblem") = this;
5859 37809 : parameters.set<SystemBase *>("_sys") = _aux.get();
5860 : }
5861 :
5862 : // Handle the "SAME_AS_MULTIAPP" execute option. The get method is used to test for the
5863 : // flag so the set by user flag is not reset, calling set with the true flag causes the set
5864 : // by user status to be reset, which should only be done if the EXEC_SAME_AS_MULTIAPP is
5865 : // being applied to the object.
5866 12603 : if (parameters.get<ExecFlagEnum>("execute_on").isValueSet(EXEC_SAME_AS_MULTIAPP))
5867 : {
5868 10280 : ExecFlagEnum & exec_enum = parameters.set<ExecFlagEnum>("execute_on", true);
5869 10280 : std::shared_ptr<MultiApp> multiapp;
5870 20560 : if (parameters.isParamValid("multi_app"))
5871 56 : multiapp = getMultiApp(parameters.get<MultiAppName>("multi_app"));
5872 : // This catches the sibling transfer case, where we want to be executing only as often as the
5873 : // receiving application. A transfer 'to' a multiapp is executed before that multiapp
5874 20448 : else if (parameters.isParamValid("to_multi_app"))
5875 5465 : multiapp = getMultiApp(parameters.get<MultiAppName>("to_multi_app"));
5876 9518 : else if (parameters.isParamValid("from_multi_app"))
5877 4751 : multiapp = getMultiApp(parameters.get<MultiAppName>("from_multi_app"));
5878 : // else do nothing because the user has provided invalid input. They should get a nice error
5879 : // about this during transfer construction. This necessitates checking for null in this next
5880 : // line, however
5881 10280 : if (multiapp)
5882 30816 : exec_enum = multiapp->getParam<ExecFlagEnum>("execute_on");
5883 10280 : }
5884 :
5885 : // Create the Transfer objects
5886 12603 : std::shared_ptr<Transfer> transfer = _factory.create<Transfer>(transfer_name, name, parameters);
5887 12527 : logAdd("Transfer", name, transfer_name, parameters);
5888 :
5889 : // Add MultiAppTransfer object
5890 : std::shared_ptr<MultiAppTransfer> multi_app_transfer =
5891 12527 : std::dynamic_pointer_cast<MultiAppTransfer>(transfer);
5892 12527 : if (multi_app_transfer)
5893 : {
5894 12527 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::TO_MULTIAPP))
5895 5352 : _to_multi_app_transfers.addObject(multi_app_transfer);
5896 12527 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::FROM_MULTIAPP))
5897 5979 : _from_multi_app_transfers.addObject(multi_app_transfer);
5898 12527 : if (multi_app_transfer->directions().isValueSet(MultiAppTransfer::BETWEEN_MULTIAPP))
5899 1196 : _between_multi_app_transfers.addObject(multi_app_transfer);
5900 : }
5901 : else
5902 0 : _transfers.addObject(transfer);
5903 12527 : }
5904 :
5905 : bool
5906 1512826 : FEProblemBase::hasVariable(const std::string & var_name) const
5907 : {
5908 2450847 : for (auto & sys : _solver_systems)
5909 1518737 : if (sys->hasVariable(var_name))
5910 580716 : return true;
5911 932110 : if (_aux->hasVariable(var_name))
5912 859520 : return true;
5913 :
5914 72590 : return false;
5915 : }
5916 :
5917 : bool
5918 0 : FEProblemBase::hasSolverVariable(const std::string & var_name) const
5919 : {
5920 0 : for (auto & sys : _solver_systems)
5921 0 : if (sys->hasVariable(var_name))
5922 0 : return true;
5923 :
5924 0 : return false;
5925 : }
5926 :
5927 : const MooseVariableFieldBase &
5928 4776378 : FEProblemBase::getVariable(const THREAD_ID tid,
5929 : const std::string & var_name,
5930 : Moose::VarKindType expected_var_type,
5931 : Moose::VarFieldType expected_var_field_type) const
5932 : {
5933 14329126 : return getVariableHelper(
5934 4776378 : tid, var_name, expected_var_type, expected_var_field_type, _solver_systems, *_aux);
5935 : }
5936 :
5937 : MooseVariable &
5938 8001 : FEProblemBase::getStandardVariable(const THREAD_ID tid, const std::string & var_name)
5939 : {
5940 12111 : for (auto & sys : _solver_systems)
5941 8001 : if (sys->hasVariable(var_name))
5942 3891 : return sys->getFieldVariable<Real>(tid, var_name);
5943 4110 : if (_aux->hasVariable(var_name))
5944 4106 : return _aux->getFieldVariable<Real>(tid, var_name);
5945 :
5946 4 : mooseError("Unknown variable " + var_name);
5947 : }
5948 :
5949 : MooseVariableFieldBase &
5950 661 : FEProblemBase::getActualFieldVariable(const THREAD_ID tid, const std::string & var_name)
5951 : {
5952 877 : for (auto & sys : _solver_systems)
5953 661 : if (sys->hasVariable(var_name))
5954 445 : return sys->getActualFieldVariable<Real>(tid, var_name);
5955 216 : if (_aux->hasVariable(var_name))
5956 216 : return _aux->getActualFieldVariable<Real>(tid, var_name);
5957 :
5958 0 : mooseError("Unknown variable " + var_name);
5959 : }
5960 :
5961 : VectorMooseVariable &
5962 0 : FEProblemBase::getVectorVariable(const THREAD_ID tid, const std::string & var_name)
5963 : {
5964 0 : for (auto & sys : _solver_systems)
5965 0 : if (sys->hasVariable(var_name))
5966 0 : return sys->getFieldVariable<RealVectorValue>(tid, var_name);
5967 0 : if (_aux->hasVariable(var_name))
5968 0 : return _aux->getFieldVariable<RealVectorValue>(tid, var_name);
5969 :
5970 0 : mooseError("Unknown variable " + var_name);
5971 : }
5972 :
5973 : ArrayMooseVariable &
5974 378 : FEProblemBase::getArrayVariable(const THREAD_ID tid, const std::string & var_name)
5975 : {
5976 657 : for (auto & sys : _solver_systems)
5977 378 : if (sys->hasVariable(var_name))
5978 99 : return sys->getFieldVariable<RealEigenVector>(tid, var_name);
5979 279 : if (_aux->hasVariable(var_name))
5980 279 : return _aux->getFieldVariable<RealEigenVector>(tid, var_name);
5981 :
5982 0 : mooseError("Unknown variable " + var_name);
5983 : }
5984 :
5985 : bool
5986 190665 : FEProblemBase::hasScalarVariable(const std::string & var_name) const
5987 : {
5988 355923 : for (auto & sys : _solver_systems)
5989 191288 : if (sys->hasScalarVariable(var_name))
5990 26030 : return true;
5991 164635 : if (_aux->hasScalarVariable(var_name))
5992 12603 : return true;
5993 :
5994 152032 : return false;
5995 : }
5996 :
5997 : MooseVariableScalar &
5998 56126 : FEProblemBase::getScalarVariable(const THREAD_ID tid, const std::string & var_name)
5999 : {
6000 71989 : for (auto & sys : _solver_systems)
6001 56126 : if (sys->hasScalarVariable(var_name))
6002 40263 : return sys->getScalarVariable(tid, var_name);
6003 15863 : if (_aux->hasScalarVariable(var_name))
6004 15863 : return _aux->getScalarVariable(tid, var_name);
6005 :
6006 0 : mooseError("Unknown variable " + var_name);
6007 : }
6008 :
6009 : System &
6010 66522 : FEProblemBase::getSystem(const std::string & var_name)
6011 : {
6012 66522 : const auto [var_in_sys, sys_num] = determineSolverSystem(var_name);
6013 66522 : if (var_in_sys)
6014 47676 : return _solver_systems[sys_num]->system();
6015 18846 : else if (_aux->hasVariable(var_name) || _aux->hasScalarVariable(var_name))
6016 18846 : return _aux->system();
6017 : else
6018 0 : mooseError("Unable to find a system containing the variable " + var_name);
6019 : }
6020 :
6021 : const RestartableEquationSystems &
6022 0 : FEProblemBase::getRestartableEquationSystems() const
6023 : {
6024 0 : return _req.get();
6025 : }
6026 :
6027 : void
6028 541903 : FEProblemBase::setActiveFEVariableCoupleableMatrixTags(std::set<TagID> & mtags, const THREAD_ID tid)
6029 : {
6030 541903 : SubProblem::setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6031 :
6032 541903 : if (_displaced_problem)
6033 114396 : _displaced_problem->setActiveFEVariableCoupleableMatrixTags(mtags, tid);
6034 541903 : }
6035 :
6036 : void
6037 7269292 : FEProblemBase::setActiveFEVariableCoupleableVectorTags(std::set<TagID> & vtags, const THREAD_ID tid)
6038 : {
6039 7269292 : SubProblem::setActiveFEVariableCoupleableVectorTags(vtags, tid);
6040 :
6041 7269292 : if (_displaced_problem)
6042 400132 : _displaced_problem->setActiveFEVariableCoupleableVectorTags(vtags, tid);
6043 7269292 : }
6044 :
6045 : void
6046 56031 : FEProblemBase::setActiveScalarVariableCoupleableMatrixTags(std::set<TagID> & mtags,
6047 : const THREAD_ID tid)
6048 : {
6049 56031 : SubProblem::setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6050 :
6051 56031 : if (_displaced_problem)
6052 0 : _displaced_problem->setActiveScalarVariableCoupleableMatrixTags(mtags, tid);
6053 56031 : }
6054 :
6055 : void
6056 56031 : FEProblemBase::setActiveScalarVariableCoupleableVectorTags(std::set<TagID> & vtags,
6057 : const THREAD_ID tid)
6058 : {
6059 56031 : SubProblem::setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6060 :
6061 56031 : if (_displaced_problem)
6062 0 : _displaced_problem->setActiveScalarVariableCoupleableVectorTags(vtags, tid);
6063 56031 : }
6064 :
6065 : void
6066 11294381 : FEProblemBase::setActiveElementalMooseVariables(const std::set<MooseVariableFEBase *> & moose_vars,
6067 : const THREAD_ID tid)
6068 : {
6069 11294381 : SubProblem::setActiveElementalMooseVariables(moose_vars, tid);
6070 :
6071 11294381 : if (_displaced_problem)
6072 567951 : _displaced_problem->setActiveElementalMooseVariables(moose_vars, tid);
6073 11294381 : }
6074 :
6075 : void
6076 4698140 : FEProblemBase::clearActiveElementalMooseVariables(const THREAD_ID tid)
6077 : {
6078 4698140 : SubProblem::clearActiveElementalMooseVariables(tid);
6079 :
6080 4698140 : if (_displaced_problem)
6081 204061 : _displaced_problem->clearActiveElementalMooseVariables(tid);
6082 4698140 : }
6083 :
6084 : void
6085 249318 : FEProblemBase::clearActiveFEVariableCoupleableMatrixTags(const THREAD_ID tid)
6086 : {
6087 249318 : SubProblem::clearActiveFEVariableCoupleableMatrixTags(tid);
6088 :
6089 249318 : if (_displaced_problem)
6090 50039 : _displaced_problem->clearActiveFEVariableCoupleableMatrixTags(tid);
6091 249318 : }
6092 :
6093 : void
6094 249318 : FEProblemBase::clearActiveFEVariableCoupleableVectorTags(const THREAD_ID tid)
6095 : {
6096 249318 : SubProblem::clearActiveFEVariableCoupleableVectorTags(tid);
6097 :
6098 249318 : if (_displaced_problem)
6099 50039 : _displaced_problem->clearActiveFEVariableCoupleableVectorTags(tid);
6100 249318 : }
6101 :
6102 : void
6103 56031 : FEProblemBase::clearActiveScalarVariableCoupleableMatrixTags(const THREAD_ID tid)
6104 : {
6105 56031 : SubProblem::clearActiveScalarVariableCoupleableMatrixTags(tid);
6106 :
6107 56031 : if (_displaced_problem)
6108 0 : _displaced_problem->clearActiveScalarVariableCoupleableMatrixTags(tid);
6109 56031 : }
6110 :
6111 : void
6112 56031 : FEProblemBase::clearActiveScalarVariableCoupleableVectorTags(const THREAD_ID tid)
6113 : {
6114 56031 : SubProblem::clearActiveScalarVariableCoupleableVectorTags(tid);
6115 :
6116 56031 : if (_displaced_problem)
6117 0 : _displaced_problem->clearActiveScalarVariableCoupleableVectorTags(tid);
6118 56031 : }
6119 :
6120 : void
6121 5834056 : FEProblemBase::setActiveMaterialProperties(const std::unordered_set<unsigned int> & mat_prop_ids,
6122 : const THREAD_ID tid)
6123 : {
6124 : // mark active properties in every material
6125 7073952 : for (auto & mat : _all_materials.getObjects(tid))
6126 1239896 : mat->setActiveProperties(mat_prop_ids);
6127 6817376 : for (auto & mat : _all_materials[Moose::FACE_MATERIAL_DATA].getObjects(tid))
6128 983320 : mat->setActiveProperties(mat_prop_ids);
6129 6817376 : for (auto & mat : _all_materials[Moose::NEIGHBOR_MATERIAL_DATA].getObjects(tid))
6130 983320 : mat->setActiveProperties(mat_prop_ids);
6131 :
6132 5834056 : _has_active_material_properties[tid] = !mat_prop_ids.empty();
6133 5834056 : }
6134 :
6135 : bool
6136 443161248 : FEProblemBase::hasActiveMaterialProperties(const THREAD_ID tid) const
6137 : {
6138 443161248 : return _has_active_material_properties[tid];
6139 : }
6140 :
6141 : void
6142 4805563 : FEProblemBase::clearActiveMaterialProperties(const THREAD_ID tid)
6143 : {
6144 4805563 : _has_active_material_properties[tid] = 0;
6145 4805563 : }
6146 :
6147 : void
6148 62192 : FEProblemBase::addAnyRedistributers()
6149 : {
6150 : #ifdef LIBMESH_ENABLE_AMR
6151 64612 : if ((_adaptivity.isOn() || _num_grid_steps) &&
6152 2420 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
6153 2349 : _neighbor_material_props.hasStatefulProperties()))
6154 : {
6155 : // Even on a serialized Mesh, we don't keep our material
6156 : // properties serialized, so we'll rely on the callback to
6157 : // redistribute() to redistribute properties at the same time
6158 : // libMesh is redistributing elements.
6159 71 : auto add_redistributer = [this](MooseMesh & mesh,
6160 : const std::string & redistributer_name,
6161 : const bool use_displaced_mesh)
6162 : {
6163 71 : InputParameters redistribute_params = RedistributeProperties::validParams();
6164 71 : redistribute_params.set<MooseApp *>(MooseBase::app_param) = &_app;
6165 142 : redistribute_params.set<std::string>("for_whom") = this->name();
6166 213 : redistribute_params.set<MooseMesh *>("mesh") = &mesh;
6167 71 : redistribute_params.set<Moose::RelationshipManagerType>("rm_type") =
6168 : Moose::RelationshipManagerType::GEOMETRIC;
6169 142 : redistribute_params.set<bool>("use_displaced_mesh") = use_displaced_mesh;
6170 71 : redistribute_params.setHitNode(*parameters().getHitNode(), {});
6171 :
6172 : std::shared_ptr<RedistributeProperties> redistributer =
6173 71 : _factory.create<RedistributeProperties>(
6174 142 : "RedistributeProperties", redistributer_name, redistribute_params);
6175 :
6176 71 : if (_material_props.hasStatefulProperties())
6177 71 : redistributer->addMaterialPropertyStorage(_material_props);
6178 :
6179 71 : if (_bnd_material_props.hasStatefulProperties())
6180 71 : redistributer->addMaterialPropertyStorage(_bnd_material_props);
6181 :
6182 71 : if (_neighbor_material_props.hasStatefulProperties())
6183 71 : redistributer->addMaterialPropertyStorage(_neighbor_material_props);
6184 :
6185 71 : mesh.getMesh().add_ghosting_functor(redistributer);
6186 142 : };
6187 :
6188 71 : add_redistributer(_mesh, "mesh_property_redistributer", false);
6189 71 : if (_displaced_problem)
6190 0 : add_redistributer(_displaced_problem->mesh(), "displaced_mesh_property_redistributer", true);
6191 : }
6192 : #endif // LIBMESH_ENABLE_AMR
6193 62192 : }
6194 :
6195 : void
6196 64441 : FEProblemBase::updateMaxQps()
6197 : {
6198 : // Find the maximum number of quadrature points
6199 : {
6200 64441 : MaxQpsThread mqt(*this);
6201 64441 : Threads::parallel_reduce(*_mesh.getActiveLocalElementRange(), mqt);
6202 64441 : _max_qps = mqt.max();
6203 :
6204 : // If we have more shape functions or more quadrature points on
6205 : // another processor, then we may need to handle those elements
6206 : // ourselves later after repartitioning.
6207 64441 : _communicator.max(_max_qps);
6208 : }
6209 :
6210 64441 : unsigned int max_qpts = getMaxQps();
6211 64441 : if (max_qpts > Moose::constMaxQpsPerElem)
6212 0 : mooseError("Max quadrature points per element assumptions made in some code (e.g. Coupleable ",
6213 : "and MaterialPropertyInterface classes) have been violated.\n",
6214 : "Complain to Moose developers to have constMaxQpsPerElem increased from ",
6215 : Moose::constMaxQpsPerElem,
6216 : " to ",
6217 : max_qpts);
6218 135142 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6219 : {
6220 : // the highest available order in libMesh is 43
6221 70701 : _scalar_zero[tid].resize(FORTYTHIRD, 0);
6222 70701 : _zero[tid].resize(max_qpts, 0);
6223 70701 : _ad_zero[tid].resize(max_qpts, 0);
6224 70701 : _grad_zero[tid].resize(max_qpts, RealGradient(0.));
6225 70701 : _ad_grad_zero[tid].resize(max_qpts, ADRealGradient(0));
6226 70701 : _second_zero[tid].resize(max_qpts, RealTensor(0.));
6227 70701 : _ad_second_zero[tid].resize(max_qpts, ADRealTensorValue(0));
6228 70701 : _vector_zero[tid].resize(max_qpts, RealGradient(0.));
6229 70701 : _vector_curl_zero[tid].resize(max_qpts, RealGradient(0.));
6230 : }
6231 64441 : }
6232 :
6233 : void
6234 84 : FEProblemBase::bumpVolumeQRuleOrder(Order order, SubdomainID block)
6235 : {
6236 180 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6237 192 : for (const auto i : index_range(_nl))
6238 96 : _assembly[tid][i]->bumpVolumeQRuleOrder(order, block);
6239 :
6240 84 : if (_displaced_problem)
6241 0 : _displaced_problem->bumpVolumeQRuleOrder(order, block);
6242 :
6243 84 : updateMaxQps();
6244 84 : }
6245 :
6246 : void
6247 14 : FEProblemBase::bumpAllQRuleOrder(Order order, SubdomainID block)
6248 : {
6249 30 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6250 32 : for (const auto i : index_range(_nl))
6251 16 : _assembly[tid][i]->bumpAllQRuleOrder(order, block);
6252 :
6253 14 : if (_displaced_problem)
6254 0 : _displaced_problem->bumpAllQRuleOrder(order, block);
6255 :
6256 14 : updateMaxQps();
6257 14 : }
6258 :
6259 : void
6260 64343 : FEProblemBase::createQRules(QuadratureType type,
6261 : Order order,
6262 : Order volume_order,
6263 : Order face_order,
6264 : SubdomainID block,
6265 : const bool allow_negative_qweights)
6266 : {
6267 64343 : if (order == INVALID_ORDER)
6268 : {
6269 : // automatically determine the integration order
6270 63836 : order = _solver_systems[0]->getMinQuadratureOrder();
6271 64115 : for (const auto i : make_range(std::size_t(1), _solver_systems.size()))
6272 279 : if (order < _solver_systems[i]->getMinQuadratureOrder())
6273 0 : order = _solver_systems[i]->getMinQuadratureOrder();
6274 63836 : if (order < _aux->getMinQuadratureOrder())
6275 5616 : order = _aux->getMinQuadratureOrder();
6276 : }
6277 :
6278 64343 : if (volume_order == INVALID_ORDER)
6279 64248 : volume_order = order;
6280 :
6281 64343 : if (face_order == INVALID_ORDER)
6282 64248 : face_order = order;
6283 :
6284 134932 : for (unsigned int tid = 0; tid < libMesh::n_threads(); ++tid)
6285 141475 : for (const auto i : index_range(_solver_systems))
6286 70886 : _assembly[tid][i]->createQRules(
6287 : type, order, volume_order, face_order, block, allow_negative_qweights);
6288 :
6289 64343 : if (_displaced_problem)
6290 2246 : _displaced_problem->createQRules(
6291 : type, order, volume_order, face_order, block, allow_negative_qweights);
6292 :
6293 64343 : updateMaxQps();
6294 64343 : }
6295 :
6296 : void
6297 22768 : FEProblemBase::setCoupling(Moose::CouplingType type)
6298 : {
6299 22768 : if (_trust_user_coupling_matrix)
6300 : {
6301 4 : if (_coupling != Moose::COUPLING_CUSTOM)
6302 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6303 : "haven't been provided a coupling matrix!");
6304 :
6305 : // We've been told to trust the user coupling matrix, so we're going to leave things alone
6306 4 : return;
6307 : }
6308 :
6309 22764 : _coupling = type;
6310 : }
6311 :
6312 : void
6313 0 : FEProblemBase::setCouplingMatrix(CouplingMatrix * cm, const unsigned int i)
6314 : {
6315 : // TODO: Deprecate method
6316 0 : setCoupling(Moose::COUPLING_CUSTOM);
6317 0 : _cm[i].reset(cm);
6318 0 : }
6319 :
6320 : void
6321 15104 : FEProblemBase::setCouplingMatrix(std::unique_ptr<CouplingMatrix> cm, const unsigned int i)
6322 : {
6323 15104 : setCoupling(Moose::COUPLING_CUSTOM);
6324 15104 : _cm[i] = std::move(cm);
6325 15104 : }
6326 :
6327 : void
6328 4 : FEProblemBase::trustUserCouplingMatrix()
6329 : {
6330 4 : if (_coupling != Moose::COUPLING_CUSTOM)
6331 0 : mooseError("Someone told us (the FEProblemBase) to trust the user coupling matrix, but we "
6332 : "haven't been provided a coupling matrix!");
6333 :
6334 4 : _trust_user_coupling_matrix = true;
6335 4 : }
6336 :
6337 : void
6338 81 : FEProblemBase::setNonlocalCouplingMatrix()
6339 : {
6340 405 : TIME_SECTION("setNonlocalCouplingMatrix", 5, "Setting Nonlocal Coupling Matrix");
6341 :
6342 81 : if (_nl.size() > 1)
6343 0 : mooseError("Nonlocal kernels are weirdly stored on the FEProblem so we don't currently support "
6344 : "multiple nonlinear systems with nonlocal kernels.");
6345 :
6346 162 : for (const auto nl_sys_num : index_range(_nl))
6347 : {
6348 81 : auto & nl = _nl[nl_sys_num];
6349 81 : auto & nonlocal_cm = _nonlocal_cm[nl_sys_num];
6350 81 : unsigned int n_vars = nl->nVariables();
6351 81 : nonlocal_cm.resize(n_vars);
6352 81 : const auto & vars = nl->getVariables(0);
6353 81 : const auto & nonlocal_kernel = _nonlocal_kernels.getObjects();
6354 81 : const auto & nonlocal_integrated_bc = _nonlocal_integrated_bcs.getObjects();
6355 243 : for (const auto & ivar : vars)
6356 : {
6357 252 : for (const auto & kernel : nonlocal_kernel)
6358 : {
6359 180 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6360 90 : if (i == kernel->variable().number())
6361 135 : for (const auto & jvar : vars)
6362 : {
6363 90 : const auto it = _var_dof_map.find(jvar->name());
6364 90 : if (it != _var_dof_map.end())
6365 : {
6366 80 : unsigned int j = jvar->number();
6367 80 : nonlocal_cm(i, j) = 1;
6368 : }
6369 : }
6370 : }
6371 234 : for (const auto & integrated_bc : nonlocal_integrated_bc)
6372 : {
6373 144 : for (unsigned int i = ivar->number(); i < ivar->number() + ivar->count(); ++i)
6374 72 : if (i == integrated_bc->variable().number())
6375 108 : for (const auto & jvar : vars)
6376 : {
6377 72 : const auto it = _var_dof_map.find(jvar->name());
6378 72 : if (it != _var_dof_map.end())
6379 : {
6380 36 : unsigned int j = jvar->number();
6381 36 : nonlocal_cm(i, j) = 1;
6382 : }
6383 : }
6384 : }
6385 : }
6386 : }
6387 81 : }
6388 :
6389 : bool
6390 712 : FEProblemBase::areCoupled(const unsigned int ivar,
6391 : const unsigned int jvar,
6392 : const unsigned int nl_sys) const
6393 : {
6394 712 : return (*_cm[nl_sys])(ivar, jvar);
6395 : }
6396 :
6397 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6398 19040629 : FEProblemBase::couplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6399 : {
6400 19040629 : return _assembly[tid][nl_sys]->couplingEntries();
6401 : }
6402 :
6403 : std::vector<std::pair<MooseVariableFEBase *, MooseVariableFEBase *>> &
6404 5558 : FEProblemBase::nonlocalCouplingEntries(const THREAD_ID tid, const unsigned int nl_sys)
6405 : {
6406 5558 : return _assembly[tid][nl_sys]->nonlocalCouplingEntries();
6407 : }
6408 :
6409 : void
6410 62797 : FEProblemBase::init()
6411 : {
6412 62797 : if (_initialized)
6413 0 : return;
6414 :
6415 313985 : TIME_SECTION("init", 2, "Initializing");
6416 :
6417 : // call executioner's preProblemInit so that it can do some setups before problem init
6418 62797 : _app.getExecutioner()->preProblemInit();
6419 :
6420 : // If we have AD and we are doing global AD indexing, then we should by default set the matrix
6421 : // coupling to full. If the user has told us to trust their coupling matrix, then this call will
6422 : // not do anything
6423 62797 : if (haveADObjects() && Moose::globalADIndexing())
6424 7562 : setCoupling(Moose::COUPLING_FULL);
6425 :
6426 124529 : for (const auto i : index_range(_nl))
6427 : {
6428 61732 : auto & nl = _nl[i];
6429 61732 : auto & cm = _cm[i];
6430 :
6431 61732 : unsigned int n_vars = nl->nVariables();
6432 : {
6433 308660 : TIME_SECTION("fillCouplingMatrix", 3, "Filling Coupling Matrix");
6434 :
6435 61732 : switch (_coupling)
6436 : {
6437 44725 : case Moose::COUPLING_DIAG:
6438 44725 : cm = std::make_unique<CouplingMatrix>(n_vars);
6439 83173 : for (unsigned int i = 0; i < n_vars; i++)
6440 38448 : (*cm)(i, i) = 1;
6441 44725 : break;
6442 :
6443 : // for full jacobian
6444 7705 : case Moose::COUPLING_FULL:
6445 7705 : cm = std::make_unique<CouplingMatrix>(n_vars);
6446 19555 : for (unsigned int i = 0; i < n_vars; i++)
6447 34034 : for (unsigned int j = 0; j < n_vars; j++)
6448 22184 : (*cm)(i, j) = 1;
6449 7705 : break;
6450 :
6451 9302 : case Moose::COUPLING_CUSTOM:
6452 : // do nothing, _cm was already set through couplingMatrix() call
6453 9302 : break;
6454 : }
6455 61732 : }
6456 :
6457 61732 : nl->dofMap()._dof_coupling = cm.get();
6458 :
6459 : // If there are no variables, make sure to pass a nullptr coupling
6460 : // matrix, to avoid warnings about non-nullptr yet empty
6461 : // CouplingMatrices.
6462 61732 : if (n_vars == 0)
6463 12266 : nl->dofMap()._dof_coupling = nullptr;
6464 :
6465 61732 : nl->dofMap().attach_extra_sparsity_function(&extraSparsity, nl.get());
6466 61732 : nl->dofMap().attach_extra_send_list_function(&extraSendList, nl.get());
6467 61732 : _aux->dofMap().attach_extra_send_list_function(&extraSendList, _aux.get());
6468 :
6469 61732 : if (!_skip_nl_system_check && _solve && n_vars == 0)
6470 0 : mooseError("No variables specified in nonlinear system '", nl->name(), "'.");
6471 : }
6472 :
6473 62797 : ghostGhostedBoundaries(); // We do this again right here in case new boundaries have been added
6474 :
6475 : // We may have added element/nodes to the mesh in ghostGhostedBoundaries so we need to update
6476 : // all of our mesh information. We need to make sure that mesh information is up-to-date before
6477 : // EquationSystems::init because that will call through to updateGeomSearch (for sparsity
6478 : // augmentation) and if we haven't added back boundary node information before that latter call,
6479 : // then we're screwed. We'll get things like "Unable to find closest node!"
6480 62797 : _mesh.meshChanged();
6481 62797 : if (_displaced_problem)
6482 2246 : _displaced_mesh->meshChanged();
6483 :
6484 62797 : if (_mesh.doingPRefinement())
6485 : {
6486 234 : preparePRefinement();
6487 234 : if (_displaced_problem)
6488 0 : _displaced_problem->preparePRefinement();
6489 : }
6490 :
6491 : // do not assemble system matrix for JFNK solve
6492 124529 : for (auto & nl : _nl)
6493 61732 : if (solverParams(nl->number())._type == Moose::ST_JFNK)
6494 139 : nl->turnOffJacobian();
6495 :
6496 125869 : for (auto & sys : _solver_systems)
6497 63072 : sys->preInit();
6498 62797 : _aux->preInit();
6499 :
6500 : // Build the mortar segment meshes, if they haven't been already, for a couple reasons:
6501 : // 1) Get the ghosting correct for both static and dynamic meshes
6502 : // 2) Make sure the mortar mesh is built for mortar constraints that live on the static mesh
6503 : //
6504 : // It is worth-while to note that mortar meshes that live on a dynamic mesh will be built
6505 : // during residual and Jacobian evaluation because when displacements are solution variables
6506 : // the mortar mesh will move and change during the course of a non-linear solve. We DO NOT
6507 : // redo ghosting during non-linear solve, so for purpose 1) the below call has to be made
6508 62797 : if (!_mortar_data->initialized())
6509 46545 : updateMortarMesh();
6510 :
6511 : {
6512 313985 : TIME_SECTION("EquationSystems::Init", 2, "Initializing Equation Systems");
6513 62797 : es().init();
6514 62797 : }
6515 :
6516 125869 : for (auto & sys : _solver_systems)
6517 63072 : sys->postInit();
6518 62797 : _aux->postInit();
6519 :
6520 : // Now that the equation system and the dof distribution is done, we can generate the
6521 : // finite volume-related parts if needed.
6522 62797 : if (haveFV())
6523 5407 : _mesh.setupFiniteVolumeMeshData();
6524 :
6525 125869 : for (auto & sys : _solver_systems)
6526 63072 : sys->update();
6527 62797 : _aux->update();
6528 :
6529 131473 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
6530 136304 : for (const auto i : index_range(_nl))
6531 : {
6532 : mooseAssert(
6533 : _cm[i],
6534 : "Coupling matrix not set for system "
6535 : << i
6536 : << ". This should only happen if a preconditioner was not setup for this system");
6537 67628 : _assembly[tid][i]->init(_cm[i].get());
6538 : }
6539 :
6540 62797 : if (_displaced_problem)
6541 2246 : _displaced_problem->init();
6542 :
6543 : #ifdef MOOSE_KOKKOS_ENABLED
6544 42062 : if (_has_kokkos_objects)
6545 807 : initKokkos();
6546 : #endif
6547 :
6548 62797 : _initialized = true;
6549 62797 : }
6550 :
6551 : unsigned int
6552 13677 : FEProblemBase::nlSysNum(const NonlinearSystemName & nl_sys_name) const
6553 : {
6554 13677 : std::istringstream ss(nl_sys_name);
6555 : unsigned int nl_sys_num;
6556 13677 : if (!(ss >> nl_sys_num) || !ss.eof())
6557 13677 : nl_sys_num = libmesh_map_find(_nl_sys_name_to_num, nl_sys_name);
6558 :
6559 13677 : return nl_sys_num;
6560 13677 : }
6561 :
6562 : unsigned int
6563 39884 : FEProblemBase::linearSysNum(const LinearSystemName & linear_sys_name) const
6564 : {
6565 39884 : std::istringstream ss(linear_sys_name);
6566 : unsigned int linear_sys_num;
6567 39884 : if (!(ss >> linear_sys_num) || !ss.eof())
6568 39884 : linear_sys_num = libmesh_map_find(_linear_sys_name_to_num, linear_sys_name);
6569 :
6570 39884 : return linear_sys_num;
6571 39884 : }
6572 :
6573 : unsigned int
6574 130931 : FEProblemBase::solverSysNum(const SolverSystemName & solver_sys_name) const
6575 : {
6576 130931 : std::istringstream ss(solver_sys_name);
6577 : unsigned int solver_sys_num;
6578 130931 : if (!(ss >> solver_sys_num) || !ss.eof())
6579 : {
6580 130931 : const auto & search = _solver_sys_name_to_num.find(solver_sys_name);
6581 130931 : if (search == _solver_sys_name_to_num.end())
6582 0 : mooseError("The solver system number was requested for system '" + solver_sys_name,
6583 : "' but this system does not exist in the Problem. Systems can be added to the "
6584 : "problem using the 'nl_sys_names'/'linear_sys_names' parameter.\nSystems in the "
6585 0 : "Problem: " +
6586 0 : Moose::stringify(_solver_sys_names));
6587 130931 : solver_sys_num = search->second;
6588 : }
6589 :
6590 130931 : return solver_sys_num;
6591 130931 : }
6592 :
6593 : unsigned int
6594 1641 : FEProblemBase::systemNumForVariable(const VariableName & variable_name) const
6595 : {
6596 1752 : for (const auto & solver_sys : _solver_systems)
6597 1641 : if (solver_sys->hasVariable(variable_name))
6598 1530 : return solver_sys->number();
6599 : mooseAssert(_aux, "Should have an auxiliary system");
6600 111 : if (_aux->hasVariable(variable_name))
6601 111 : return _aux->number();
6602 :
6603 0 : mooseError("Variable '",
6604 : variable_name,
6605 : "' was not found in any solver (nonlinear/linear) or auxiliary system");
6606 : }
6607 :
6608 : void
6609 348712 : FEProblemBase::solve(const unsigned int nl_sys_num)
6610 : {
6611 1743560 : TIME_SECTION("solve", 1, "Solving", false);
6612 :
6613 348712 : setCurrentNonlinearSystem(nl_sys_num);
6614 :
6615 : // This prevents stale dof indices from lingering around and possibly leading to invalid reads
6616 : // and writes. Dof indices may be made stale through operations like mesh adaptivity
6617 348712 : clearAllDofIndices();
6618 348712 : if (_displaced_problem)
6619 35869 : _displaced_problem->clearAllDofIndices();
6620 :
6621 : // Setup the output system for printing linear/nonlinear iteration information and some solver
6622 : // settings, including setting matrix prefixes. This must occur before petscSetOptions
6623 348712 : initPetscOutputAndSomeSolverSettings();
6624 :
6625 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6626 : Moose::PetscSupport::petscSetOptions(
6627 : _petsc_options, _solver_params); // Make sure the PETSc options are setup for this app
6628 : #else
6629 : // Now this database will be the default
6630 : // Each app should have only one database
6631 348712 : if (!_app.isUltimateMaster())
6632 89156 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6633 : // We did not add PETSc options to database yet
6634 348712 : if (!_is_petsc_options_inserted)
6635 : {
6636 : // Insert options for all systems all at once
6637 42272 : Moose::PetscSupport::petscSetOptions(_petsc_options, _solver_params, this);
6638 42272 : _is_petsc_options_inserted = true;
6639 : }
6640 : #endif
6641 :
6642 : // set up DM which is required if use a field split preconditioner
6643 : // We need to setup DM every "solve()" because libMesh destroy SNES after solve()
6644 : // Do not worry, DM setup is very cheap
6645 348712 : _current_nl_sys->setupDM();
6646 :
6647 348712 : possiblyRebuildGeomSearchPatches();
6648 :
6649 : // reset flag so that residual evaluation does not get skipped
6650 : // and the next non-linear iteration does not automatically fail with
6651 : // "DIVERGED_NANORINF", when we throw an exception and stop solve
6652 348712 : _fail_next_system_convergence_check = false;
6653 :
6654 348712 : if (_solve)
6655 : {
6656 317244 : _current_nl_sys->solve();
6657 317164 : _current_nl_sys->update();
6658 : }
6659 :
6660 : // sync solutions in displaced problem
6661 348632 : if (_displaced_problem)
6662 35865 : _displaced_problem->syncSolutions();
6663 :
6664 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6665 348632 : if (!_app.isUltimateMaster())
6666 89156 : LibmeshPetscCall(PetscOptionsPop());
6667 : #endif
6668 348632 : }
6669 :
6670 : void
6671 213 : FEProblemBase::setException(const std::string & message)
6672 : {
6673 213 : _has_exception = true;
6674 213 : _exception_message = message;
6675 213 : }
6676 :
6677 : void
6678 21841140 : FEProblemBase::checkExceptionAndStopSolve(bool print_message)
6679 : {
6680 21841140 : if (_skip_exception_check)
6681 240 : return;
6682 :
6683 65522700 : TIME_SECTION("checkExceptionAndStopSolve", 5);
6684 :
6685 : // See if any processor had an exception. If it did, get back the
6686 : // processor that the exception occurred on.
6687 : unsigned int processor_id;
6688 :
6689 21840900 : _communicator.maxloc(_has_exception, processor_id);
6690 :
6691 21840900 : if (_has_exception)
6692 : {
6693 337 : _communicator.broadcast(_exception_message, processor_id);
6694 :
6695 451 : if (_current_execute_on_flag == EXEC_LINEAR || _current_execute_on_flag == EXEC_NONLINEAR ||
6696 114 : _current_execute_on_flag == EXEC_POSTCHECK)
6697 : {
6698 : // Print the message
6699 337 : if (_communicator.rank() == 0 && print_message)
6700 : {
6701 197 : _console << "\n" << _exception_message << "\n";
6702 197 : if (isTransient())
6703 : _console
6704 : << "To recover, the solution will fail and then be re-attempted with a reduced time "
6705 180 : "step.\n"
6706 180 : << std::endl;
6707 : }
6708 :
6709 : // Stop the solve -- this entails setting
6710 : // SNESSetFunctionDomainError() or directly inserting NaNs in the
6711 : // residual vector to let PETSc >= 3.6 return DIVERGED_NANORINF.
6712 337 : if (_current_nl_sys)
6713 337 : _current_nl_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
6714 :
6715 337 : if (_current_linear_sys)
6716 0 : _current_linear_sys->stopSolve(_current_execute_on_flag, _fe_vector_tags);
6717 :
6718 : // and close Aux system (we MUST do this here; see #11525)
6719 337 : _aux->solution().close();
6720 :
6721 : // We've handled this exception, so we no longer have one.
6722 337 : _has_exception = false;
6723 :
6724 : // Force the next non-linear convergence check to fail (and all further residual evaluation
6725 : // to be skipped).
6726 337 : _fail_next_system_convergence_check = true;
6727 :
6728 : // Repropagate the exception, so it can be caught at a higher level, typically
6729 : // this is NonlinearSystem::computeResidual().
6730 337 : throw MooseException(_exception_message);
6731 : }
6732 : else
6733 0 : mooseError("The following parallel-communicated exception was detected during " +
6734 0 : Moose::stringify(_current_execute_on_flag) + " evaluation:\n" +
6735 0 : _exception_message +
6736 : "\nBecause this did not occur during residual evaluation, there"
6737 : " is no way to handle this, so the solution is aborting.\n");
6738 : }
6739 21840900 : }
6740 :
6741 : void
6742 3886843 : FEProblemBase::resetState()
6743 : {
6744 : // Our default state is to allow computing derivatives
6745 3886843 : ADReal::do_derivatives = true;
6746 3886843 : _current_execute_on_flag = EXEC_NONE;
6747 :
6748 : // Clear the VectorTags and MatrixTags
6749 3886843 : clearCurrentResidualVectorTags();
6750 3886843 : clearCurrentJacobianMatrixTags();
6751 :
6752 3886843 : _safe_access_tagged_vectors = true;
6753 3886843 : _safe_access_tagged_matrices = true;
6754 :
6755 3886843 : setCurrentlyComputingResidual(false);
6756 3886843 : setCurrentlyComputingJacobian(false);
6757 3886843 : setCurrentlyComputingResidualAndJacobian(false);
6758 3886843 : if (_displaced_problem)
6759 : {
6760 160528 : _displaced_problem->setCurrentlyComputingResidual(false);
6761 160528 : _displaced_problem->setCurrentlyComputingJacobian(false);
6762 160528 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(false);
6763 : }
6764 3886843 : }
6765 :
6766 : void
6767 5648 : FEProblemBase::solveLinearSystem(const unsigned int linear_sys_num,
6768 : const Moose::PetscSupport::PetscOptions * po)
6769 : {
6770 28240 : TIME_SECTION("solve", 1, "Solving", false);
6771 :
6772 5648 : setCurrentLinearSystem(linear_sys_num);
6773 :
6774 5648 : const Moose::PetscSupport::PetscOptions & options = po ? *po : _petsc_options;
6775 5648 : auto & solver_params = _solver_params[numNonlinearSystems() + linear_sys_num];
6776 :
6777 : // Set custom convergence criteria
6778 5648 : Moose::PetscSupport::petscSetDefaults(*this);
6779 :
6780 : #if PETSC_RELEASE_LESS_THAN(3, 12, 0)
6781 : LibmeshPetscCall(Moose::PetscSupport::petscSetOptions(
6782 : options, solver_params)); // Make sure the PETSc options are setup for this app
6783 : #else
6784 : // Now this database will be the default
6785 : // Each app should have only one database
6786 5648 : if (!_app.isUltimateMaster())
6787 235 : LibmeshPetscCall(PetscOptionsPush(_petsc_option_data_base));
6788 :
6789 : // We did not add PETSc options to database yet
6790 5648 : if (!_is_petsc_options_inserted)
6791 : {
6792 1257 : Moose::PetscSupport::petscSetOptions(options, solver_params, this);
6793 1257 : _is_petsc_options_inserted = true;
6794 : }
6795 : #endif
6796 :
6797 5648 : if (_solve)
6798 5636 : _current_linear_sys->solve();
6799 :
6800 : #if !PETSC_RELEASE_LESS_THAN(3, 12, 0)
6801 5648 : if (!_app.isUltimateMaster())
6802 235 : LibmeshPetscCall(PetscOptionsPop());
6803 : #endif
6804 5648 : }
6805 :
6806 : bool
6807 346085 : FEProblemBase::solverSystemConverged(const unsigned int sys_num)
6808 : {
6809 346085 : if (_solve)
6810 346061 : return _solver_systems[sys_num]->converged();
6811 : else
6812 24 : return true;
6813 : }
6814 :
6815 : unsigned int
6816 6328 : FEProblemBase::nNonlinearIterations(const unsigned int nl_sys_num) const
6817 : {
6818 6328 : return _nl[nl_sys_num]->nNonlinearIterations();
6819 : }
6820 :
6821 : unsigned int
6822 5425 : FEProblemBase::nLinearIterations(const unsigned int nl_sys_num) const
6823 : {
6824 5425 : return _nl[nl_sys_num]->nLinearIterations();
6825 : }
6826 :
6827 : Real
6828 258 : FEProblemBase::finalNonlinearResidual(const unsigned int nl_sys_num) const
6829 : {
6830 258 : return _nl[nl_sys_num]->finalNonlinearResidual();
6831 : }
6832 :
6833 : bool
6834 845824 : FEProblemBase::computingPreSMOResidual(const unsigned int nl_sys_num) const
6835 : {
6836 845824 : return _nl[nl_sys_num]->computingPreSMOResidual();
6837 : }
6838 :
6839 : void
6840 57313 : FEProblemBase::copySolutionsBackwards()
6841 : {
6842 286565 : TIME_SECTION("copySolutionsBackwards", 3, "Copying Solutions Backward");
6843 :
6844 114883 : for (auto & sys : _solver_systems)
6845 57570 : sys->copySolutionsBackwards();
6846 57313 : _aux->copySolutionsBackwards();
6847 57313 : }
6848 :
6849 : void
6850 250324 : FEProblemBase::advanceState()
6851 : {
6852 1251620 : TIME_SECTION("advanceState", 5, "Advancing State");
6853 :
6854 504307 : for (auto & sys : _solver_systems)
6855 253983 : sys->copyOldSolutions();
6856 250324 : _aux->copyOldSolutions();
6857 :
6858 250324 : if (_displaced_problem)
6859 : {
6860 67642 : for (const auto i : index_range(_solver_systems))
6861 33821 : _displaced_problem->solverSys(i).copyOldSolutions();
6862 33821 : _displaced_problem->auxSys().copyOldSolutions();
6863 : }
6864 :
6865 250324 : _reporter_data.copyValuesBack();
6866 :
6867 250324 : getMooseApp().getChainControlDataSystem().copyValuesBack();
6868 :
6869 250324 : if (_material_props.hasStatefulProperties())
6870 1970 : _material_props.shift();
6871 :
6872 250324 : if (_bnd_material_props.hasStatefulProperties())
6873 1816 : _bnd_material_props.shift();
6874 :
6875 250324 : if (_neighbor_material_props.hasStatefulProperties())
6876 1682 : _neighbor_material_props.shift();
6877 :
6878 : #ifdef MOOSE_KOKKOS_ENABLED
6879 164704 : if (_kokkos_material_props.hasStatefulProperties())
6880 376 : _kokkos_material_props.shift();
6881 :
6882 164704 : if (_kokkos_bnd_material_props.hasStatefulProperties())
6883 426 : _kokkos_bnd_material_props.shift();
6884 :
6885 164704 : if (_kokkos_neighbor_material_props.hasStatefulProperties())
6886 376 : _kokkos_neighbor_material_props.shift();
6887 : #endif
6888 250324 : }
6889 :
6890 : void
6891 5374 : FEProblemBase::restoreSolutions()
6892 : {
6893 26870 : TIME_SECTION("restoreSolutions", 5, "Restoring Solutions");
6894 :
6895 5374 : if (!_not_zeroed_tagged_vectors.empty())
6896 0 : paramError("not_zeroed_tag_vectors",
6897 : "There is currently no way to restore not-zeroed vectors.");
6898 :
6899 10750 : for (auto & sys : _solver_systems)
6900 : {
6901 5376 : if (_verbose_restore)
6902 395 : _console << "Restoring solutions on system " << sys->name() << "..." << std::endl;
6903 5376 : sys->restoreSolutions();
6904 : }
6905 :
6906 5374 : if (_verbose_restore)
6907 394 : _console << "Restoring solutions on Auxiliary system..." << std::endl;
6908 5374 : _aux->restoreSolutions();
6909 :
6910 5374 : if (_verbose_restore)
6911 394 : _console << "Restoring postprocessor, vector-postprocessor, and reporter data..." << std::endl;
6912 5374 : _reporter_data.restoreState(_verbose_restore);
6913 :
6914 5374 : if (_displaced_problem)
6915 136 : _displaced_problem->updateMesh();
6916 5374 : }
6917 :
6918 : void
6919 115 : FEProblemBase::saveOldSolutions()
6920 : {
6921 575 : TIME_SECTION("saveOldSolutions", 5, "Saving Old Solutions");
6922 :
6923 230 : for (auto & sys : _solver_systems)
6924 115 : sys->saveOldSolutions();
6925 115 : _aux->saveOldSolutions();
6926 115 : }
6927 :
6928 : void
6929 115 : FEProblemBase::restoreOldSolutions()
6930 : {
6931 575 : TIME_SECTION("restoreOldSolutions", 5, "Restoring Old Solutions");
6932 :
6933 230 : for (auto & sys : _solver_systems)
6934 115 : sys->restoreOldSolutions();
6935 115 : _aux->restoreOldSolutions();
6936 115 : }
6937 :
6938 : void
6939 1337186 : FEProblemBase::outputStep(ExecFlagType type)
6940 : {
6941 6685930 : TIME_SECTION("outputStep", 1, "Outputting");
6942 :
6943 1337186 : setCurrentExecuteOnFlag(type);
6944 :
6945 2689816 : for (auto & sys : _solver_systems)
6946 1352630 : sys->update();
6947 1337186 : _aux->update();
6948 :
6949 1337186 : if (_displaced_problem)
6950 138047 : _displaced_problem->syncSolutions();
6951 1337186 : _app.getOutputWarehouse().outputStep(type);
6952 :
6953 1337174 : setCurrentExecuteOnFlag(EXEC_NONE);
6954 1337174 : }
6955 :
6956 : void
6957 89528 : FEProblemBase::allowOutput(bool state)
6958 : {
6959 89528 : _app.getOutputWarehouse().allowOutput(state);
6960 89528 : }
6961 :
6962 : void
6963 25 : FEProblemBase::forceOutput()
6964 : {
6965 25 : _app.getOutputWarehouse().forceOutput();
6966 25 : }
6967 :
6968 : void
6969 369716 : FEProblemBase::initPetscOutputAndSomeSolverSettings()
6970 : {
6971 369716 : _app.getOutputWarehouse().solveSetup();
6972 369716 : Moose::PetscSupport::petscSetDefaults(*this);
6973 369716 : }
6974 :
6975 : void
6976 259865 : FEProblemBase::onTimestepBegin()
6977 : {
6978 779595 : TIME_SECTION("onTimestepBegin", 2);
6979 :
6980 523102 : for (auto & nl : _nl)
6981 263237 : nl->onTimestepBegin();
6982 259865 : }
6983 :
6984 : void
6985 335187 : FEProblemBase::onTimestepEnd()
6986 : {
6987 335187 : }
6988 :
6989 : Real
6990 8145249 : FEProblemBase::getTimeFromStateArg(const Moose::StateArg & state) const
6991 : {
6992 8145249 : if (state.iteration_type != Moose::SolutionIterationType::Time)
6993 : // If we are any iteration type other than time (e.g. nonlinear), then temporally we are still
6994 : // in the present time
6995 0 : return time();
6996 :
6997 8145249 : switch (state.state)
6998 : {
6999 8145249 : case 0:
7000 8145249 : return time();
7001 :
7002 0 : case 1:
7003 0 : return timeOld();
7004 :
7005 0 : default:
7006 0 : mooseError("Unhandled state ", state.state, " in FEProblemBase::getTimeFromStateArg");
7007 : }
7008 : }
7009 :
7010 : void
7011 32102 : FEProblemBase::addTimeIntegrator(const std::string & type,
7012 : const std::string & name,
7013 : InputParameters & parameters)
7014 : {
7015 : parallel_object_only();
7016 :
7017 96306 : parameters.set<SubProblem *>("_subproblem") = this;
7018 32102 : logAdd("TimeIntegrator", name, type, parameters);
7019 32102 : _aux->addTimeIntegrator(type, name + ":aux", parameters);
7020 64238 : for (auto & sys : _solver_systems)
7021 32136 : sys->addTimeIntegrator(type, name + ":" + sys->name(), parameters);
7022 32102 : _has_time_integrator = true;
7023 :
7024 : // add vectors to store u_dot, u_dotdot, udot_old, u_dotdot_old and
7025 : // solution vectors older than 2 time steps, if requested by the time
7026 : // integrator
7027 32102 : _aux->addDotVectors();
7028 64212 : for (auto & nl : _nl)
7029 : {
7030 32110 : nl->addDotVectors();
7031 :
7032 32110 : auto tag_udot = nl->getTimeIntegrators()[0]->uDotFactorTag();
7033 32110 : if (!nl->hasVector(tag_udot))
7034 32052 : nl->associateVectorToTag(*nl->solutionUDot(), tag_udot);
7035 32110 : auto tag_udotdot = nl->getTimeIntegrators()[0]->uDotDotFactorTag();
7036 32110 : if (!nl->hasVector(tag_udotdot) && uDotDotRequested())
7037 164 : nl->associateVectorToTag(*nl->solutionUDotDot(), tag_udotdot);
7038 : }
7039 :
7040 32102 : if (_displaced_problem)
7041 : // Time integrator does not exist when displaced problem is created.
7042 1908 : _displaced_problem->addTimeIntegrator();
7043 32102 : }
7044 :
7045 : void
7046 24 : FEProblemBase::addPredictor(const std::string & type,
7047 : const std::string & name,
7048 : InputParameters & parameters)
7049 : {
7050 : parallel_object_only();
7051 :
7052 24 : if (!numNonlinearSystems() && numLinearSystems())
7053 0 : mooseError("Vector bounds cannot be used with LinearSystems!");
7054 :
7055 48 : parameters.set<SubProblem *>("_subproblem") = this;
7056 24 : std::shared_ptr<Predictor> predictor = _factory.create<Predictor>(type, name, parameters);
7057 24 : logAdd("Predictor", name, type, parameters);
7058 :
7059 48 : for (auto & nl : _nl)
7060 24 : nl->setPredictor(predictor);
7061 24 : }
7062 :
7063 : Real
7064 67294 : FEProblemBase::computeResidualL2Norm(NonlinearSystemBase & sys)
7065 : {
7066 67294 : _current_nl_sys = &sys;
7067 67294 : computeResidual(*sys.currentSolution(), sys.RHS(), sys.number());
7068 67294 : return sys.RHS().l2_norm();
7069 : }
7070 :
7071 : Real
7072 102 : FEProblemBase::computeResidualL2Norm(LinearSystem & sys)
7073 : {
7074 102 : _current_linear_sys = &sys;
7075 :
7076 : // We assemble the current system to check the current residual
7077 102 : computeLinearSystemSys(sys.linearImplicitSystem(),
7078 102 : *sys.linearImplicitSystem().matrix,
7079 102 : *sys.linearImplicitSystem().rhs,
7080 : /*compute fresh gradients*/ true);
7081 :
7082 : // Unfortunate, but we have to allocate a new vector for the residual
7083 102 : auto residual = sys.linearImplicitSystem().rhs->clone();
7084 102 : residual->scale(-1.0);
7085 102 : residual->add_vector(*sys.currentSolution(), *sys.linearImplicitSystem().matrix);
7086 204 : return residual->l2_norm();
7087 102 : }
7088 :
7089 : Real
7090 67360 : FEProblemBase::computeResidualL2Norm()
7091 : {
7092 336800 : TIME_SECTION("computeResidualL2Norm", 2, "Computing L2 Norm of Residual");
7093 :
7094 : // We use sum the squared norms of the individual systems and then take the square root of it
7095 67360 : Real l2_norm = 0.0;
7096 134654 : for (auto sys : _nl)
7097 : {
7098 67294 : const auto norm = computeResidualL2Norm(*sys);
7099 67294 : l2_norm += norm * norm;
7100 67294 : }
7101 :
7102 67462 : for (auto sys : _linear_systems)
7103 : {
7104 102 : const auto norm = computeResidualL2Norm(*sys);
7105 102 : l2_norm += norm * norm;
7106 102 : }
7107 :
7108 134720 : return std::sqrt(l2_norm);
7109 67360 : }
7110 :
7111 : void
7112 3242340 : FEProblemBase::computeResidualSys(NonlinearImplicitSystem & sys,
7113 : const NumericVector<Number> & soln,
7114 : NumericVector<Number> & residual)
7115 : {
7116 : parallel_object_only();
7117 :
7118 9727020 : TIME_SECTION("computeResidualSys", 5);
7119 :
7120 3242340 : computeResidual(soln, residual, sys.number());
7121 3242296 : }
7122 :
7123 : void
7124 0 : FEProblemBase::computeResidual(NonlinearImplicitSystem & sys,
7125 : const NumericVector<Number> & soln,
7126 : NumericVector<Number> & residual)
7127 : {
7128 0 : mooseDeprecated("Please use computeResidualSys");
7129 :
7130 0 : computeResidualSys(sys, soln, residual);
7131 0 : }
7132 :
7133 : void
7134 3305521 : FEProblemBase::computeResidual(const NumericVector<Number> & soln,
7135 : NumericVector<Number> & residual,
7136 : const unsigned int nl_sys_num)
7137 : {
7138 3305521 : setCurrentNonlinearSystem(nl_sys_num);
7139 :
7140 : // We associate the residual tag with the given residual vector to make sure we
7141 : // don't filter it out below
7142 3305521 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7143 3305521 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7144 :
7145 : mooseAssert(_fe_vector_tags.empty(), "This should be empty indicating a clean starting state");
7146 : // We filter out tags which do not have associated vectors in the current nonlinear
7147 : // system. This is essential to be able to use system-dependent residual tags.
7148 3305521 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7149 :
7150 3305521 : computeResidualInternal(soln, residual, _fe_vector_tags);
7151 3305477 : _fe_vector_tags.clear();
7152 3305477 : }
7153 :
7154 : void
7155 3467 : FEProblemBase::computeResidualAndJacobian(const NumericVector<Number> & soln,
7156 : NumericVector<Number> & residual,
7157 : SparseMatrix<Number> & jacobian)
7158 : {
7159 : try
7160 : {
7161 : try
7162 : {
7163 : // vector tags
7164 3467 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7165 3467 : const auto & residual_vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7166 :
7167 : mooseAssert(_fe_vector_tags.empty(),
7168 : "This should be empty indicating a clean starting state");
7169 : // We filter out tags which do not have associated vectors in the current nonlinear
7170 : // system. This is essential to be able to use system-dependent residual tags.
7171 3467 : selectVectorTagsFromSystem(*_current_nl_sys, residual_vector_tags, _fe_vector_tags);
7172 :
7173 3467 : setCurrentResidualVectorTags(_fe_vector_tags);
7174 :
7175 : // matrix tags
7176 : {
7177 3467 : _fe_matrix_tags.clear();
7178 :
7179 3467 : auto & tags = getMatrixTags();
7180 10401 : for (auto & tag : tags)
7181 6934 : _fe_matrix_tags.insert(tag.second);
7182 : }
7183 :
7184 3467 : _current_nl_sys->setSolution(soln);
7185 :
7186 3467 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7187 3467 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7188 :
7189 10401 : for (const auto tag : _fe_matrix_tags)
7190 6934 : if (_current_nl_sys->hasMatrix(tag))
7191 : {
7192 3467 : auto & matrix = _current_nl_sys->getMatrix(tag);
7193 3467 : matrix.zero();
7194 3467 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7195 : // PETSc algorithms require diagonal allocations regardless of whether there is non-zero
7196 : // diagonal dependence. With global AD indexing we only add non-zero
7197 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7198 139927 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7199 136580 : matrix.add(index, index, 0);
7200 : }
7201 :
7202 3467 : _aux->zeroVariablesForResidual();
7203 :
7204 3467 : unsigned int n_threads = libMesh::n_threads();
7205 :
7206 3467 : _current_execute_on_flag = EXEC_LINEAR;
7207 :
7208 : // Random interface objects
7209 3467 : for (const auto & it : _random_data_objects)
7210 0 : it.second->updateSeeds(EXEC_LINEAR);
7211 :
7212 3467 : setCurrentlyComputingResidual(true);
7213 3467 : setCurrentlyComputingJacobian(true);
7214 3467 : setCurrentlyComputingResidualAndJacobian(true);
7215 3467 : if (_displaced_problem)
7216 : {
7217 1172 : _displaced_problem->setCurrentlyComputingResidual(true);
7218 1172 : _displaced_problem->setCurrentlyComputingJacobian(true);
7219 1172 : _displaced_problem->setCurrentlyComputingResidualAndJacobian(true);
7220 : }
7221 :
7222 3467 : execTransfers(EXEC_LINEAR);
7223 :
7224 3467 : execMultiApps(EXEC_LINEAR);
7225 :
7226 7273 : for (unsigned int tid = 0; tid < n_threads; tid++)
7227 3806 : reinitScalars(tid);
7228 :
7229 3467 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7230 :
7231 3467 : _aux->residualSetup();
7232 :
7233 3467 : if (_displaced_problem)
7234 : {
7235 1172 : computeSystems(EXEC_PRE_DISPLACE);
7236 1172 : _displaced_problem->updateMesh();
7237 1172 : if (_mortar_data->hasDisplacedObjects())
7238 1172 : updateMortarMesh();
7239 : }
7240 :
7241 7273 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7242 : {
7243 3806 : _all_materials.residualSetup(tid);
7244 3806 : _functions.residualSetup(tid);
7245 : }
7246 :
7247 : #ifdef MOOSE_KOKKOS_ENABLED
7248 2165 : _kokkos_functions.residualSetup();
7249 : #endif
7250 :
7251 3467 : computeSystems(EXEC_LINEAR);
7252 :
7253 3467 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7254 :
7255 3467 : executeControls(EXEC_LINEAR);
7256 :
7257 3467 : _app.getOutputWarehouse().residualSetup();
7258 :
7259 3467 : _safe_access_tagged_vectors = false;
7260 3467 : _safe_access_tagged_matrices = false;
7261 :
7262 3467 : _current_nl_sys->computeResidualAndJacobianTags(_fe_vector_tags, _fe_matrix_tags);
7263 :
7264 3467 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7265 3467 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7266 : }
7267 0 : catch (...)
7268 : {
7269 0 : handleException("computeResidualAndJacobian");
7270 0 : }
7271 : }
7272 0 : catch (const MooseException &)
7273 : {
7274 : // The buck stops here, we have already handled the exception by
7275 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7276 : // "diverged" reason during the next solve.
7277 0 : }
7278 0 : catch (...)
7279 : {
7280 0 : mooseError("Unexpected exception type");
7281 0 : }
7282 :
7283 3467 : resetState();
7284 3467 : _fe_vector_tags.clear();
7285 3467 : _fe_matrix_tags.clear();
7286 3467 : }
7287 :
7288 : void
7289 0 : FEProblemBase::computeResidualTag(const NumericVector<Number> & soln,
7290 : NumericVector<Number> & residual,
7291 : TagID tag)
7292 : {
7293 : try
7294 : {
7295 0 : _current_nl_sys->setSolution(soln);
7296 :
7297 0 : _current_nl_sys->associateVectorToTag(residual, tag);
7298 :
7299 0 : computeResidualTags({tag});
7300 :
7301 0 : _current_nl_sys->disassociateVectorFromTag(residual, tag);
7302 : }
7303 0 : catch (MooseException & e)
7304 : {
7305 : // If a MooseException propagates all the way to here, it means
7306 : // that it was thrown from a MOOSE system where we do not
7307 : // (currently) properly support the throwing of exceptions, and
7308 : // therefore we have no choice but to error out. It may be
7309 : // *possible* to handle exceptions from other systems, but in the
7310 : // meantime, we don't want to silently swallow any unhandled
7311 : // exceptions here.
7312 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7313 : "contact the MOOSE team for assistance.");
7314 0 : }
7315 0 : }
7316 :
7317 : void
7318 3311125 : FEProblemBase::computeResidualInternal(const NumericVector<Number> & soln,
7319 : NumericVector<Number> & residual,
7320 : const std::set<TagID> & tags)
7321 : {
7322 : parallel_object_only();
7323 :
7324 9933375 : TIME_SECTION("computeResidualInternal", 1);
7325 :
7326 : try
7327 : {
7328 3311125 : _current_nl_sys->setSolution(soln);
7329 :
7330 3311125 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7331 :
7332 3311125 : computeResidualTags(tags);
7333 :
7334 3311081 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7335 : }
7336 0 : catch (MooseException & e)
7337 : {
7338 : // If a MooseException propagates all the way to here, it means
7339 : // that it was thrown from a MOOSE system where we do not
7340 : // (currently) properly support the throwing of exceptions, and
7341 : // therefore we have no choice but to error out. It may be
7342 : // *possible* to handle exceptions from other systems, but in the
7343 : // meantime, we don't want to silently swallow any unhandled
7344 : // exceptions here.
7345 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7346 : "contact the MOOSE team for assistance.");
7347 0 : }
7348 3311081 : }
7349 :
7350 : void
7351 0 : FEProblemBase::computeResidualType(const NumericVector<Number> & soln,
7352 : NumericVector<Number> & residual,
7353 : TagID tag)
7354 : {
7355 0 : TIME_SECTION("computeResidualType", 5);
7356 :
7357 : try
7358 : {
7359 0 : _current_nl_sys->setSolution(soln);
7360 :
7361 0 : _current_nl_sys->associateVectorToTag(residual, _current_nl_sys->residualVectorTag());
7362 :
7363 0 : computeResidualTags({tag, _current_nl_sys->residualVectorTag()});
7364 :
7365 0 : _current_nl_sys->disassociateVectorFromTag(residual, _current_nl_sys->residualVectorTag());
7366 : }
7367 0 : catch (MooseException & e)
7368 : {
7369 : // If a MooseException propagates all the way to here, it means
7370 : // that it was thrown from a MOOSE system where we do not
7371 : // (currently) properly support the throwing of exceptions, and
7372 : // therefore we have no choice but to error out. It may be
7373 : // *possible* to handle exceptions from other systems, but in the
7374 : // meantime, we don't want to silently swallow any unhandled
7375 : // exceptions here.
7376 0 : mooseError("An unhandled MooseException was raised during residual computation. Please "
7377 : "contact the MOOSE team for assistance.");
7378 0 : }
7379 0 : }
7380 :
7381 : void
7382 4 : FEProblemBase::handleException(const std::string & calling_method)
7383 : {
7384 : auto create_exception_message =
7385 4 : [&calling_method](const std::string & exception_type, const auto & exception)
7386 : {
7387 : return std::string("A " + exception_type + " was raised during FEProblemBase::" +
7388 8 : calling_method + "\n" + std::string(exception.what()));
7389 4 : };
7390 :
7391 : try
7392 : {
7393 4 : throw;
7394 : }
7395 4 : catch (const libMesh::LogicError & e)
7396 : {
7397 0 : setException(create_exception_message("libMesh::LogicError", e));
7398 0 : }
7399 0 : catch (const MooseException & e)
7400 : {
7401 0 : setException(create_exception_message("MooseException", e));
7402 0 : }
7403 0 : catch (const MetaPhysicL::LogicError & e)
7404 : {
7405 0 : moose::translateMetaPhysicLError(e);
7406 0 : }
7407 4 : catch (const libMesh::PetscSolverException & e)
7408 : {
7409 : // One PETSc solver exception that we cannot currently recover from are new nonzero errors. In
7410 : // particular I have observed the following scenario in a parallel test:
7411 : // - Both processes throw because of a new nonzero during MOOSE's computeJacobianTags
7412 : // - We potentially handle the exceptions nicely here
7413 : // - When the matrix is closed in libMesh's libmesh_petsc_snes_solver, there is a new nonzero
7414 : // throw which we do not catch here in MOOSE and the simulation terminates. This only appears
7415 : // in parallel (and not all the time; a test I was examining threw with distributed mesh, but
7416 : // not with replicated). In serial there are no new throws from libmesh_petsc_snes_solver.
7417 : // So for uniformity of behavior across serial/parallel, we will choose to abort here and always
7418 : // produce a non-zero exit code
7419 8 : mooseError(create_exception_message("libMesh::PetscSolverException", e));
7420 0 : }
7421 0 : catch (const std::exception & e)
7422 : {
7423 0 : const auto message = create_exception_message("std::exception", e);
7424 0 : if (_regard_general_exceptions_as_errors)
7425 0 : mooseError(message);
7426 : else
7427 0 : setException(message);
7428 0 : }
7429 :
7430 0 : checkExceptionAndStopSolve();
7431 0 : }
7432 :
7433 : void
7434 3347146 : FEProblemBase::computeResidualTags(const std::set<TagID> & tags)
7435 : {
7436 : parallel_object_only();
7437 :
7438 : try
7439 : {
7440 : try
7441 : {
7442 16735730 : TIME_SECTION("computeResidualTags", 5, "Computing Residual");
7443 :
7444 3347146 : ADReal::do_derivatives = false;
7445 :
7446 3347146 : setCurrentResidualVectorTags(tags);
7447 :
7448 3347146 : _aux->zeroVariablesForResidual();
7449 :
7450 3347146 : unsigned int n_threads = libMesh::n_threads();
7451 :
7452 3347146 : _current_execute_on_flag = EXEC_LINEAR;
7453 :
7454 : // Random interface objects
7455 3362929 : for (const auto & it : _random_data_objects)
7456 15783 : it.second->updateSeeds(EXEC_LINEAR);
7457 :
7458 3347146 : execTransfers(EXEC_LINEAR);
7459 :
7460 3347146 : execMultiApps(EXEC_LINEAR);
7461 :
7462 6994932 : for (unsigned int tid = 0; tid < n_threads; tid++)
7463 3647786 : reinitScalars(tid);
7464 :
7465 3347146 : computeUserObjects(EXEC_LINEAR, Moose::PRE_AUX);
7466 :
7467 3347146 : _aux->residualSetup();
7468 :
7469 3347146 : if (_displaced_problem)
7470 : {
7471 136012 : computeSystems(EXEC_PRE_DISPLACE);
7472 136012 : _displaced_problem->updateMesh();
7473 136012 : if (_mortar_data->hasDisplacedObjects())
7474 2429 : updateMortarMesh();
7475 : }
7476 :
7477 6994932 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7478 : {
7479 3647786 : _all_materials.residualSetup(tid);
7480 3647786 : _functions.residualSetup(tid);
7481 : }
7482 :
7483 : #ifdef MOOSE_KOKKOS_ENABLED
7484 2199382 : _kokkos_functions.residualSetup();
7485 : #endif
7486 :
7487 3347146 : computeSystems(EXEC_LINEAR);
7488 :
7489 3347146 : computeUserObjects(EXEC_LINEAR, Moose::POST_AUX);
7490 :
7491 3347146 : executeControls(EXEC_LINEAR);
7492 :
7493 3347146 : _app.getOutputWarehouse().residualSetup();
7494 :
7495 3347146 : _safe_access_tagged_vectors = false;
7496 3347146 : _current_nl_sys->computeResidualTags(tags);
7497 3347102 : }
7498 0 : catch (...)
7499 : {
7500 0 : handleException("computeResidualTags");
7501 0 : }
7502 : }
7503 0 : catch (const MooseException &)
7504 : {
7505 : // The buck stops here, we have already handled the exception by
7506 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7507 : // "diverged" reason during the next solve.
7508 0 : }
7509 0 : catch (...)
7510 : {
7511 0 : mooseError("Unexpected exception type");
7512 0 : }
7513 :
7514 3347102 : resetState();
7515 3347102 : }
7516 :
7517 : void
7518 527300 : FEProblemBase::computeJacobianSys(NonlinearImplicitSystem & sys,
7519 : const NumericVector<Number> & soln,
7520 : SparseMatrix<Number> & jacobian)
7521 : {
7522 527300 : computeJacobian(soln, jacobian, sys.number());
7523 527280 : }
7524 :
7525 : void
7526 4894 : FEProblemBase::computeJacobianTag(const NumericVector<Number> & soln,
7527 : SparseMatrix<Number> & jacobian,
7528 : TagID tag)
7529 : {
7530 4894 : _current_nl_sys->setSolution(soln);
7531 :
7532 4894 : _current_nl_sys->associateMatrixToTag(jacobian, tag);
7533 :
7534 9788 : computeJacobianTags({tag});
7535 :
7536 4894 : _current_nl_sys->disassociateMatrixFromTag(jacobian, tag);
7537 4894 : }
7538 :
7539 : void
7540 526470 : FEProblemBase::computeJacobian(const NumericVector<Number> & soln,
7541 : SparseMatrix<Number> & jacobian,
7542 : const unsigned int nl_sys_num)
7543 : {
7544 526470 : setCurrentNonlinearSystem(nl_sys_num);
7545 :
7546 526470 : _fe_matrix_tags.clear();
7547 :
7548 526470 : auto & tags = getMatrixTags();
7549 1579682 : for (auto & tag : tags)
7550 1053212 : _fe_matrix_tags.insert(tag.second);
7551 :
7552 526470 : computeJacobianInternal(soln, jacobian, _fe_matrix_tags);
7553 526450 : }
7554 :
7555 : void
7556 526470 : FEProblemBase::computeJacobianInternal(const NumericVector<Number> & soln,
7557 : SparseMatrix<Number> & jacobian,
7558 : const std::set<TagID> & tags)
7559 : {
7560 1579410 : TIME_SECTION("computeJacobianInternal", 1);
7561 :
7562 526470 : _current_nl_sys->setSolution(soln);
7563 :
7564 526470 : _current_nl_sys->associateMatrixToTag(jacobian, _current_nl_sys->systemMatrixTag());
7565 :
7566 526470 : computeJacobianTags(tags);
7567 :
7568 526450 : _current_nl_sys->disassociateMatrixFromTag(jacobian, _current_nl_sys->systemMatrixTag());
7569 526450 : }
7570 :
7571 : void
7572 536294 : FEProblemBase::computeJacobianTags(const std::set<TagID> & tags)
7573 : {
7574 : try
7575 : {
7576 : try
7577 : {
7578 536294 : if (!_has_jacobian || !_const_jacobian)
7579 : {
7580 2645705 : TIME_SECTION("computeJacobianTags", 5, "Computing Jacobian");
7581 :
7582 1579585 : for (auto tag : tags)
7583 1050444 : if (_current_nl_sys->hasMatrix(tag))
7584 : {
7585 531127 : auto & matrix = _current_nl_sys->getMatrix(tag);
7586 531127 : if (_restore_original_nonzero_pattern)
7587 8195 : matrix.restore_original_nonzero_pattern();
7588 : else
7589 522932 : matrix.zero();
7590 531127 : if (haveADObjects() && !_current_nl_sys->system().has_static_condensation())
7591 : // PETSc algorithms require diagonal allocations regardless of whether there is
7592 : // non-zero diagonal dependence. With global AD indexing we only add non-zero
7593 : // dependence, so PETSc will scream at us unless we artificially add the diagonals.
7594 8316520 : for (auto index : make_range(matrix.row_start(), matrix.row_stop()))
7595 8258014 : matrix.add(index, index, 0);
7596 : }
7597 :
7598 529141 : _aux->zeroVariablesForJacobian();
7599 :
7600 529141 : unsigned int n_threads = libMesh::n_threads();
7601 :
7602 : // Random interface objects
7603 531649 : for (const auto & it : _random_data_objects)
7604 2508 : it.second->updateSeeds(EXEC_NONLINEAR);
7605 :
7606 529141 : _current_execute_on_flag = EXEC_NONLINEAR;
7607 529141 : _currently_computing_jacobian = true;
7608 529141 : if (_displaced_problem)
7609 23348 : _displaced_problem->setCurrentlyComputingJacobian(true);
7610 :
7611 529141 : execTransfers(EXEC_NONLINEAR);
7612 529141 : execMultiApps(EXEC_NONLINEAR);
7613 :
7614 1107239 : for (unsigned int tid = 0; tid < n_threads; tid++)
7615 578098 : reinitScalars(tid);
7616 :
7617 529141 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7618 :
7619 529141 : _aux->jacobianSetup();
7620 :
7621 529141 : if (_displaced_problem)
7622 : {
7623 23348 : computeSystems(EXEC_PRE_DISPLACE);
7624 23348 : _displaced_problem->updateMesh();
7625 : }
7626 :
7627 1107230 : for (unsigned int tid = 0; tid < n_threads; tid++)
7628 : {
7629 578093 : _all_materials.jacobianSetup(tid);
7630 578093 : _functions.jacobianSetup(tid);
7631 : }
7632 :
7633 : #ifdef MOOSE_KOKKOS_ENABLED
7634 346418 : _kokkos_functions.jacobianSetup();
7635 : #endif
7636 :
7637 529137 : computeSystems(EXEC_NONLINEAR);
7638 :
7639 529137 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7640 :
7641 529137 : executeControls(EXEC_NONLINEAR);
7642 :
7643 529137 : _app.getOutputWarehouse().jacobianSetup();
7644 :
7645 529137 : _safe_access_tagged_matrices = false;
7646 :
7647 529137 : _current_nl_sys->computeJacobianTags(tags);
7648 :
7649 : // For explicit Euler calculations for example we often compute the Jacobian one time and
7650 : // then re-use it over and over. If we're performing automatic scaling, we don't want to
7651 : // use that kernel, diagonal-block only Jacobian for our actual matrix when performing
7652 : // solves!
7653 529121 : if (!_current_nl_sys->computingScalingJacobian())
7654 528519 : _has_jacobian = true;
7655 529125 : }
7656 : }
7657 4 : catch (...)
7658 : {
7659 4 : handleException("computeJacobianTags");
7660 0 : }
7661 : }
7662 0 : catch (const MooseException &)
7663 : {
7664 : // The buck stops here, we have already handled the exception by
7665 : // calling the system's stopSolve() method, it is now up to PETSc to return a
7666 : // "diverged" reason during the next solve.
7667 0 : }
7668 0 : catch (...)
7669 : {
7670 0 : mooseError("Unexpected exception type");
7671 0 : }
7672 :
7673 536274 : resetState();
7674 536274 : }
7675 :
7676 : void
7677 302 : FEProblemBase::computeJacobianBlocks(std::vector<JacobianBlock *> & blocks,
7678 : const unsigned int nl_sys_num)
7679 : {
7680 906 : TIME_SECTION("computeTransientImplicitJacobian", 2);
7681 302 : setCurrentNonlinearSystem(nl_sys_num);
7682 :
7683 302 : if (_displaced_problem)
7684 : {
7685 0 : computeSystems(EXEC_PRE_DISPLACE);
7686 0 : _displaced_problem->updateMesh();
7687 : }
7688 :
7689 302 : computeSystems(EXEC_NONLINEAR);
7690 :
7691 302 : _currently_computing_jacobian = true;
7692 302 : _current_nl_sys->computeJacobianBlocks(blocks);
7693 302 : _currently_computing_jacobian = false;
7694 302 : }
7695 :
7696 : void
7697 0 : FEProblemBase::computeJacobianBlock(SparseMatrix<Number> & jacobian,
7698 : libMesh::System & precond_system,
7699 : unsigned int ivar,
7700 : unsigned int jvar)
7701 : {
7702 0 : JacobianBlock jac_block(precond_system, jacobian, ivar, jvar);
7703 0 : std::vector<JacobianBlock *> blocks = {&jac_block};
7704 : mooseAssert(_current_nl_sys, "This should be non-null");
7705 0 : computeJacobianBlocks(blocks, _current_nl_sys->number());
7706 0 : }
7707 :
7708 : void
7709 778 : FEProblemBase::computeBounds(NonlinearImplicitSystem & libmesh_dbg_var(sys),
7710 : NumericVector<Number> & lower,
7711 : NumericVector<Number> & upper)
7712 : {
7713 : try
7714 : {
7715 : try
7716 : {
7717 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
7718 : "I expect these system numbers to be the same");
7719 :
7720 3890 : if (!_current_nl_sys->hasVector("lower_bound") || !_current_nl_sys->hasVector("upper_bound"))
7721 0 : return;
7722 :
7723 3890 : TIME_SECTION("computeBounds", 1, "Computing Bounds");
7724 :
7725 1556 : NumericVector<Number> & _lower = _current_nl_sys->getVector("lower_bound");
7726 1556 : NumericVector<Number> & _upper = _current_nl_sys->getVector("upper_bound");
7727 778 : _lower.swap(lower);
7728 778 : _upper.swap(upper);
7729 1620 : for (THREAD_ID tid = 0; tid < libMesh::n_threads(); tid++)
7730 842 : _all_materials.residualSetup(tid);
7731 :
7732 778 : _aux->residualSetup();
7733 778 : computeSystems(EXEC_LINEAR);
7734 778 : _lower.swap(lower);
7735 778 : _upper.swap(upper);
7736 778 : }
7737 0 : catch (...)
7738 : {
7739 0 : handleException("computeBounds");
7740 0 : }
7741 : }
7742 0 : catch (MooseException & e)
7743 : {
7744 0 : mooseError("Irrecoverable exception: " + std::string(e.what()));
7745 0 : }
7746 0 : catch (...)
7747 : {
7748 0 : mooseError("Unexpected exception type");
7749 0 : }
7750 : }
7751 :
7752 : void
7753 5738 : FEProblemBase::computeLinearSystemSys(LinearImplicitSystem & sys,
7754 : SparseMatrix<Number> & system_matrix,
7755 : NumericVector<Number> & rhs,
7756 : const bool compute_gradients)
7757 : {
7758 17214 : TIME_SECTION("computeLinearSystemSys", 5);
7759 :
7760 5738 : setCurrentLinearSystem(linearSysNum(sys.name()));
7761 :
7762 5738 : _current_linear_sys->associateVectorToTag(rhs, _current_linear_sys->rightHandSideVectorTag());
7763 5738 : _current_linear_sys->associateMatrixToTag(system_matrix, _current_linear_sys->systemMatrixTag());
7764 :
7765 : // We are using the residual tag system for right hand sides so we fetch everything
7766 5738 : const auto & vector_tags = getVectorTags(Moose::VECTOR_TAG_RESIDUAL);
7767 :
7768 : // We filter out tags which do not have associated vectors in the current
7769 : // system. This is essential to be able to use system-dependent vector tags.
7770 5738 : selectVectorTagsFromSystem(*_current_linear_sys, vector_tags, _linear_vector_tags);
7771 5738 : selectMatrixTagsFromSystem(*_current_linear_sys, getMatrixTags(), _linear_matrix_tags);
7772 :
7773 5738 : computeLinearSystemTags(*(_current_linear_sys->currentSolution()),
7774 5738 : _linear_vector_tags,
7775 5738 : _linear_matrix_tags,
7776 : compute_gradients);
7777 :
7778 5738 : _current_linear_sys->disassociateMatrixFromTag(system_matrix,
7779 5738 : _current_linear_sys->systemMatrixTag());
7780 5738 : _current_linear_sys->disassociateVectorFromTag(rhs,
7781 5738 : _current_linear_sys->rightHandSideVectorTag());
7782 : // We reset the tags to the default containers for further operations
7783 5738 : _current_linear_sys->associateVectorToTag(_current_linear_sys->getRightHandSideVector(),
7784 5738 : _current_linear_sys->rightHandSideVectorTag());
7785 5738 : _current_linear_sys->associateMatrixToTag(_current_linear_sys->getSystemMatrix(),
7786 5738 : _current_linear_sys->systemMatrixTag());
7787 5738 : }
7788 :
7789 : void
7790 5738 : FEProblemBase::computeLinearSystemTags(const NumericVector<Number> & soln,
7791 : const std::set<TagID> & vector_tags,
7792 : const std::set<TagID> & matrix_tags,
7793 : const bool compute_gradients)
7794 : {
7795 28690 : TIME_SECTION("computeLinearSystemTags", 5, "Computing Linear System");
7796 :
7797 5738 : _current_linear_sys->setSolution(soln);
7798 :
7799 11500 : for (auto tag : matrix_tags)
7800 : {
7801 5762 : auto & matrix = _current_linear_sys->getMatrix(tag);
7802 5762 : matrix.zero();
7803 : }
7804 :
7805 5738 : unsigned int n_threads = libMesh::n_threads();
7806 :
7807 5738 : _current_execute_on_flag = EXEC_NONLINEAR;
7808 :
7809 : // Random interface objects
7810 5738 : for (const auto & it : _random_data_objects)
7811 0 : it.second->updateSeeds(EXEC_NONLINEAR);
7812 :
7813 5738 : execTransfers(EXEC_NONLINEAR);
7814 5738 : execMultiApps(EXEC_NONLINEAR);
7815 :
7816 5738 : computeUserObjects(EXEC_NONLINEAR, Moose::PRE_AUX);
7817 :
7818 5738 : _aux->jacobianSetup();
7819 :
7820 11476 : for (THREAD_ID tid = 0; tid < n_threads; tid++)
7821 : {
7822 5738 : _functions.jacobianSetup(tid);
7823 : }
7824 :
7825 : #ifdef MOOSE_KOKKOS_ENABLED
7826 3357 : _kokkos_functions.jacobianSetup();
7827 : #endif
7828 :
7829 : try
7830 : {
7831 5738 : computeSystems(EXEC_NONLINEAR);
7832 : }
7833 0 : catch (MooseException & e)
7834 : {
7835 0 : _console << "\nA MooseException was raised during Auxiliary variable computation.\n"
7836 0 : << "The next solve will fail, the timestep will be reduced, and we will try again.\n"
7837 0 : << std::endl;
7838 :
7839 : // We know the next solve is going to fail, so there's no point in
7840 : // computing anything else after this. Plus, using incompletely
7841 : // computed AuxVariables in subsequent calculations could lead to
7842 : // other errors or unhandled exceptions being thrown.
7843 0 : return;
7844 0 : }
7845 :
7846 5738 : computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX);
7847 5738 : executeControls(EXEC_NONLINEAR);
7848 :
7849 5738 : _app.getOutputWarehouse().jacobianSetup();
7850 :
7851 5738 : _current_linear_sys->computeLinearSystemTags(vector_tags, matrix_tags, compute_gradients);
7852 :
7853 : // Reset execution flag as after this point we are no longer on LINEAR
7854 5738 : _current_execute_on_flag = EXEC_NONE;
7855 :
7856 : // These are the relevant parts of resetState()
7857 5738 : _safe_access_tagged_vectors = true;
7858 5738 : _safe_access_tagged_matrices = true;
7859 5738 : }
7860 :
7861 : void
7862 325140 : FEProblemBase::computeNearNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
7863 : std::vector<NumericVector<Number> *> & sp)
7864 : {
7865 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
7866 : "I expect these system numbers to be the same");
7867 :
7868 325140 : sp.clear();
7869 975420 : for (unsigned int i = 0; i < subspaceDim("NearNullSpace"); ++i)
7870 : {
7871 0 : std::stringstream postfix;
7872 0 : postfix << "_" << i;
7873 0 : std::string modename = "NearNullSpace" + postfix.str();
7874 0 : sp.push_back(&_current_nl_sys->getVector(modename));
7875 0 : }
7876 325140 : }
7877 :
7878 : void
7879 325140 : FEProblemBase::computeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
7880 : std::vector<NumericVector<Number> *> & sp)
7881 : {
7882 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
7883 : "I expect these system numbers to be the same");
7884 325140 : sp.clear();
7885 975492 : for (unsigned int i = 0; i < subspaceDim("NullSpace"); ++i)
7886 : {
7887 24 : std::stringstream postfix;
7888 24 : postfix << "_" << i;
7889 24 : sp.push_back(&_current_nl_sys->getVector("NullSpace" + postfix.str()));
7890 24 : }
7891 325140 : }
7892 :
7893 : void
7894 325140 : FEProblemBase::computeTransposeNullSpace(NonlinearImplicitSystem & libmesh_dbg_var(sys),
7895 : std::vector<NumericVector<Number> *> & sp)
7896 : {
7897 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
7898 : "I expect these system numbers to be the same");
7899 325140 : sp.clear();
7900 975456 : for (unsigned int i = 0; i < subspaceDim("TransposeNullSpace"); ++i)
7901 : {
7902 12 : std::stringstream postfix;
7903 12 : postfix << "_" << i;
7904 12 : sp.push_back(&_current_nl_sys->getVector("TransposeNullSpace" + postfix.str()));
7905 12 : }
7906 325140 : }
7907 :
7908 : void
7909 1976 : FEProblemBase::computePostCheck(NonlinearImplicitSystem & sys,
7910 : const NumericVector<Number> & old_soln,
7911 : NumericVector<Number> & search_direction,
7912 : NumericVector<Number> & new_soln,
7913 : bool & changed_search_direction,
7914 : bool & changed_new_soln)
7915 : {
7916 : mooseAssert(_current_nl_sys && (sys.number() == _current_nl_sys->number()),
7917 : "I expect these system numbers to be the same");
7918 :
7919 : // This function replaces the old PetscSupport::dampedCheck() function.
7920 : //
7921 : // 1.) Recreate code in PetscSupport::dampedCheck() for constructing
7922 : // ghosted "soln" and "update" vectors.
7923 : // 2.) Call FEProblemBase::computeDamping() with these ghost vectors.
7924 : // 3.) Recreate the code in PetscSupport::dampedCheck() to actually update
7925 : // the solution vector based on the damping, and set the "changed" flags
7926 : // appropriately.
7927 :
7928 9880 : TIME_SECTION("computePostCheck", 2, "Computing Post Check");
7929 :
7930 1976 : _current_execute_on_flag = EXEC_POSTCHECK;
7931 :
7932 : // MOOSE's FEProblemBase doesn't update the solution during the
7933 : // postcheck, but FEProblemBase-derived classes might.
7934 1976 : if (_has_dampers || shouldUpdateSolution())
7935 : {
7936 : // We need ghosted versions of new_soln and search_direction (the
7937 : // ones we get from libmesh/PETSc are PARALLEL vectors. To make
7938 : // our lives simpler, we use the same ghosting pattern as the
7939 : // system's current_local_solution to create new ghosted vectors.
7940 :
7941 : // Construct zeroed-out clones with the same ghosted dofs as the
7942 : // System's current_local_solution.
7943 : std::unique_ptr<NumericVector<Number>> ghosted_solution =
7944 1420 : sys.current_local_solution->zero_clone(),
7945 : ghosted_search_direction =
7946 1420 : sys.current_local_solution->zero_clone();
7947 :
7948 : // Copy values from input vectors into clones with ghosted values.
7949 1420 : *ghosted_solution = new_soln;
7950 1420 : *ghosted_search_direction = search_direction;
7951 :
7952 1420 : if (_has_dampers)
7953 : {
7954 : // Compute the damping coefficient using the ghosted vectors
7955 1420 : Real damping = computeDamping(*ghosted_solution, *ghosted_search_direction);
7956 :
7957 : // If some non-trivial damping was computed, update the new_soln
7958 : // vector accordingly.
7959 1420 : if (damping < 1.0)
7960 : {
7961 995 : new_soln = old_soln;
7962 995 : new_soln.add(-damping, search_direction);
7963 995 : changed_new_soln = true;
7964 : }
7965 : }
7966 :
7967 1420 : if (shouldUpdateSolution())
7968 : {
7969 : // Update the ghosted copy of the new solution, if necessary.
7970 0 : if (changed_new_soln)
7971 0 : *ghosted_solution = new_soln;
7972 :
7973 0 : bool updated_solution = updateSolution(new_soln, *ghosted_solution);
7974 0 : if (updated_solution)
7975 0 : changed_new_soln = true;
7976 : }
7977 1420 : }
7978 :
7979 1976 : if (vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
7980 : {
7981 556 : _current_nl_sys->setPreviousNewtonSolution(old_soln);
7982 556 : _aux->copyCurrentIntoPreviousNL();
7983 : }
7984 :
7985 : // MOOSE doesn't change the search_direction
7986 1976 : changed_search_direction = false;
7987 :
7988 1976 : _current_execute_on_flag = EXEC_NONE;
7989 1976 : }
7990 :
7991 : Real
7992 1420 : FEProblemBase::computeDamping(const NumericVector<Number> & soln,
7993 : const NumericVector<Number> & update)
7994 : {
7995 : // Default to no damping
7996 1420 : Real damping = 1.0;
7997 :
7998 1420 : if (_has_dampers)
7999 : {
8000 7100 : TIME_SECTION("computeDamping", 1, "Computing Damping");
8001 :
8002 : // Save pointer to the current solution
8003 1420 : const NumericVector<Number> * _saved_current_solution = _current_nl_sys->currentSolution();
8004 :
8005 1420 : _current_nl_sys->setSolution(soln);
8006 : // For now, do not re-compute auxiliary variables. Doing so allows a wild solution increment
8007 : // to get to the material models, which may not be able to cope with drastically different
8008 : // values. Once more complete dependency checking is in place, auxiliary variables (and
8009 : // material properties) will be computed as needed by dampers.
8010 : // _aux.compute();
8011 1420 : damping = _current_nl_sys->computeDamping(soln, update);
8012 :
8013 : // restore saved solution
8014 1420 : _current_nl_sys->setSolution(*_saved_current_solution);
8015 1420 : }
8016 :
8017 1420 : return damping;
8018 : }
8019 :
8020 : bool
8021 318819 : FEProblemBase::shouldUpdateSolution()
8022 : {
8023 318819 : return false;
8024 : }
8025 :
8026 : bool
8027 0 : FEProblemBase::updateSolution(NumericVector<Number> & /*vec_solution*/,
8028 : NumericVector<Number> & /*ghosted_solution*/)
8029 : {
8030 0 : return false;
8031 : }
8032 :
8033 : void
8034 223 : FEProblemBase::predictorCleanup(NumericVector<Number> & /*ghosted_solution*/)
8035 : {
8036 223 : }
8037 :
8038 : void
8039 2246 : FEProblemBase::addDisplacedProblem(std::shared_ptr<DisplacedProblem> displaced_problem)
8040 : {
8041 : parallel_object_only();
8042 :
8043 2246 : _displaced_mesh = &displaced_problem->mesh();
8044 2246 : _displaced_problem = displaced_problem;
8045 2246 : }
8046 :
8047 : void
8048 124957 : FEProblemBase::updateGeomSearch(GeometricSearchData::GeometricSearchType type)
8049 : {
8050 624785 : TIME_SECTION("updateGeometricSearch", 3, "Updating Geometric Search");
8051 :
8052 124957 : _geometric_search_data.update(type);
8053 :
8054 124957 : if (_displaced_problem)
8055 4669 : _displaced_problem->updateGeomSearch(type);
8056 124957 : }
8057 :
8058 : void
8059 74109 : FEProblemBase::updateMortarMesh()
8060 : {
8061 370545 : TIME_SECTION("updateMortarMesh", 5, "Updating Mortar Mesh");
8062 :
8063 74109 : FloatingPointExceptionGuard fpe_guard(_app);
8064 :
8065 74109 : _mortar_data->update();
8066 74109 : }
8067 :
8068 : void
8069 1448 : FEProblemBase::createMortarInterface(
8070 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8071 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8072 : bool on_displaced,
8073 : bool periodic,
8074 : const bool debug,
8075 : const bool correct_edge_dropping,
8076 : const Real minimum_projection_angle)
8077 : {
8078 1448 : _has_mortar = true;
8079 :
8080 1448 : if (on_displaced)
8081 276 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8082 : primary_secondary_subdomain_pair,
8083 138 : *_displaced_problem,
8084 : on_displaced,
8085 : periodic,
8086 : debug,
8087 : correct_edge_dropping,
8088 138 : minimum_projection_angle);
8089 : else
8090 1310 : return _mortar_data->createMortarInterface(primary_secondary_boundary_pair,
8091 : primary_secondary_subdomain_pair,
8092 : *this,
8093 : on_displaced,
8094 : periodic,
8095 : debug,
8096 : correct_edge_dropping,
8097 1310 : minimum_projection_angle);
8098 : }
8099 :
8100 : const AutomaticMortarGeneration &
8101 0 : FEProblemBase::getMortarInterface(
8102 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8103 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8104 : bool on_displaced) const
8105 : {
8106 0 : return _mortar_data->getMortarInterface(
8107 0 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8108 : }
8109 :
8110 : AutomaticMortarGeneration &
8111 258234 : FEProblemBase::getMortarInterface(
8112 : const std::pair<BoundaryID, BoundaryID> & primary_secondary_boundary_pair,
8113 : const std::pair<SubdomainID, SubdomainID> & primary_secondary_subdomain_pair,
8114 : bool on_displaced)
8115 : {
8116 258234 : return _mortar_data->getMortarInterface(
8117 258234 : primary_secondary_boundary_pair, primary_secondary_subdomain_pair, on_displaced);
8118 : }
8119 :
8120 : void
8121 348712 : FEProblemBase::possiblyRebuildGeomSearchPatches()
8122 : {
8123 348712 : if (_displaced_problem) // Only need to do this if things are moving...
8124 : {
8125 179345 : TIME_SECTION("possiblyRebuildGeomSearchPatches", 5, "Rebuilding Geometric Search Patches");
8126 :
8127 35869 : switch (_mesh.getPatchUpdateStrategy())
8128 : {
8129 34675 : case Moose::Never:
8130 34675 : break;
8131 398 : case Moose::Iteration:
8132 : // Update the list of ghosted elements at the start of the time step
8133 398 : _geometric_search_data.updateGhostedElems();
8134 398 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8135 :
8136 398 : _displaced_problem->geomSearchData().updateGhostedElems();
8137 398 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8138 :
8139 : // The commands below ensure that the sparsity of the Jacobian matrix is
8140 : // augmented at the start of the time step using neighbor nodes from the end
8141 : // of the previous time step.
8142 :
8143 398 : reinitBecauseOfGhostingOrNewGeomObjects();
8144 :
8145 : // This is needed to reinitialize PETSc output
8146 398 : initPetscOutputAndSomeSolverSettings();
8147 :
8148 398 : break;
8149 :
8150 361 : case Moose::Auto:
8151 : {
8152 361 : Real max = _displaced_problem->geomSearchData().maxPatchPercentage();
8153 361 : _communicator.max(max);
8154 :
8155 : // If we haven't moved very far through the patch
8156 361 : if (max < 0.4)
8157 325 : break;
8158 : }
8159 : libmesh_fallthrough();
8160 :
8161 : // Let this fall through if things do need to be updated...
8162 : case Moose::Always:
8163 : // Flush output here to see the message before the reinitialization, which could take a
8164 : // while
8165 471 : _console << "\n\nUpdating geometric search patches\n" << std::endl;
8166 :
8167 471 : _geometric_search_data.clearNearestNodeLocators();
8168 471 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8169 :
8170 471 : _displaced_problem->geomSearchData().clearNearestNodeLocators();
8171 471 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8172 :
8173 471 : reinitBecauseOfGhostingOrNewGeomObjects();
8174 :
8175 : // This is needed to reinitialize PETSc output
8176 471 : initPetscOutputAndSomeSolverSettings();
8177 : }
8178 35869 : }
8179 348712 : }
8180 :
8181 : #ifdef LIBMESH_ENABLE_AMR
8182 : void
8183 57849 : FEProblemBase::initialAdaptMesh()
8184 : {
8185 57849 : unsigned int n = adaptivity().getInitialSteps();
8186 57849 : _cycles_completed = 0;
8187 57849 : if (n)
8188 : {
8189 695 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8190 4 : mooseError("HFEM does not support mesh adaptivity currently.");
8191 :
8192 3455 : TIME_SECTION("initialAdaptMesh", 2, "Performing Initial Adaptivity");
8193 :
8194 1667 : for (unsigned int i = 0; i < n; i++)
8195 : {
8196 1169 : computeIndicators();
8197 1169 : computeMarkers();
8198 :
8199 1169 : if (_adaptivity.initialAdaptMesh())
8200 : {
8201 976 : meshChanged(
8202 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8203 :
8204 : // reproject the initial condition
8205 976 : projectSolution();
8206 :
8207 976 : _cycles_completed++;
8208 : }
8209 : else
8210 : {
8211 193 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8212 193 : return;
8213 : }
8214 : }
8215 691 : }
8216 : }
8217 :
8218 : bool
8219 182790 : FEProblemBase::adaptMesh()
8220 : {
8221 : // reset cycle counter
8222 182790 : _cycles_completed = 0;
8223 :
8224 182790 : if (!_adaptivity.isAdaptivityDue())
8225 177854 : return false;
8226 :
8227 24680 : TIME_SECTION("adaptMesh", 3, "Adapting Mesh");
8228 :
8229 4936 : unsigned int cycles_per_step = _adaptivity.getCyclesPerStep();
8230 :
8231 4936 : bool mesh_changed = false;
8232 :
8233 8674 : for (unsigned int i = 0; i < cycles_per_step; ++i)
8234 : {
8235 5102 : if (!_mesh.interiorLowerDBlocks().empty() || !_mesh.boundaryLowerDBlocks().empty())
8236 0 : mooseError("HFEM does not support mesh adaptivity currently.");
8237 :
8238 : // Markers were already computed once by Executioner
8239 5102 : if (_adaptivity.getRecomputeMarkersFlag() && i > 0)
8240 24 : computeMarkers();
8241 :
8242 : bool mesh_changed_this_step;
8243 5102 : mesh_changed_this_step = _adaptivity.adaptMesh();
8244 :
8245 5102 : if (mesh_changed_this_step)
8246 : {
8247 3738 : mesh_changed = true;
8248 :
8249 3738 : meshChanged(
8250 : /*intermediate_change=*/true, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
8251 3738 : _cycles_completed++;
8252 : }
8253 : else
8254 : {
8255 : // If the mesh didn't change, we still need to update the displaced mesh
8256 : // to undo the undisplacement performed in Adaptivity::adaptMesh
8257 1364 : if (_displaced_problem)
8258 48 : _displaced_problem->updateMesh();
8259 :
8260 1364 : _console << "Mesh unchanged, skipping remaining steps..." << std::endl;
8261 1364 : break;
8262 : }
8263 :
8264 : // Show adaptivity progress
8265 3738 : _console << std::flush;
8266 : }
8267 :
8268 : // We're done with all intermediate changes; now get systems ready
8269 : // for real if necessary.
8270 4936 : if (mesh_changed)
8271 3584 : es().reinit_systems();
8272 :
8273 : // Execute multi-apps that need to run after adaptivity, but before the next timestep.
8274 4936 : execMultiApps(EXEC_POST_ADAPTIVITY);
8275 :
8276 4936 : return mesh_changed;
8277 4936 : }
8278 : #endif // LIBMESH_ENABLE_AMR
8279 :
8280 : void
8281 0 : FEProblemBase::initXFEM(std::shared_ptr<XFEMInterface> xfem)
8282 : {
8283 0 : _xfem = xfem;
8284 0 : _xfem->setMesh(&_mesh);
8285 0 : if (_displaced_mesh)
8286 0 : _xfem->setDisplacedMesh(_displaced_mesh);
8287 :
8288 0 : auto fill_data = [](auto & storage)
8289 : {
8290 0 : std::vector<MaterialData *> data(libMesh::n_threads());
8291 0 : for (const auto tid : make_range(libMesh::n_threads()))
8292 0 : data[tid] = &storage.getMaterialData(tid);
8293 0 : return data;
8294 0 : };
8295 0 : _xfem->setMaterialData(fill_data(_material_props));
8296 0 : _xfem->setBoundaryMaterialData(fill_data(_bnd_material_props));
8297 :
8298 0 : unsigned int n_threads = libMesh::n_threads();
8299 0 : for (unsigned int i = 0; i < n_threads; ++i)
8300 0 : for (const auto nl_sys_num : index_range(_nl))
8301 : {
8302 0 : _assembly[i][nl_sys_num]->setXFEM(_xfem);
8303 0 : if (_displaced_problem)
8304 0 : _displaced_problem->assembly(i, nl_sys_num).setXFEM(_xfem);
8305 : }
8306 0 : }
8307 :
8308 : bool
8309 0 : FEProblemBase::updateMeshXFEM()
8310 : {
8311 0 : TIME_SECTION("updateMeshXFEM", 5, "Updating XFEM");
8312 :
8313 0 : bool updated = false;
8314 0 : if (haveXFEM())
8315 : {
8316 0 : if (_xfem->updateHeal())
8317 : // XFEM exodiff tests rely on a given numbering because they cannot use map = true due to
8318 : // having coincident elements. While conceptually speaking we do not need to contract the
8319 : // mesh, we need its call to renumber_nodes_and_elements in order to preserve these tests
8320 0 : meshChanged(
8321 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8322 :
8323 0 : updated = _xfem->update(_time, _nl, *_aux);
8324 0 : if (updated)
8325 : {
8326 0 : meshChanged(
8327 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/false);
8328 0 : _xfem->initSolution(_nl, *_aux);
8329 0 : restoreSolutions();
8330 0 : _console << "\nXFEM update complete: Mesh modified" << std::endl;
8331 : }
8332 : else
8333 0 : _console << "\nXFEM update complete: Mesh not modified" << std::endl;
8334 : }
8335 0 : return updated;
8336 0 : }
8337 :
8338 : void
8339 7587 : FEProblemBase::meshChanged(const bool intermediate_change,
8340 : const bool contract_mesh,
8341 : const bool clean_refinement_flags)
8342 : {
8343 37935 : TIME_SECTION("meshChanged", 3, "Handling Mesh Changes");
8344 :
8345 14942 : if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8346 7355 : _neighbor_material_props.hasStatefulProperties())
8347 232 : _mesh.cacheChangedLists(); // Currently only used with adaptivity and stateful material
8348 : // properties
8349 :
8350 : // Clear these out because they corresponded to the old mesh
8351 7587 : _ghosted_elems.clear();
8352 7587 : ghostGhostedBoundaries();
8353 :
8354 : // The mesh changed. We notify the MooseMesh first, because
8355 : // callbacks (e.g. for sparsity calculations) triggered by the
8356 : // EquationSystems reinit may require up-to-date MooseMesh caches.
8357 7587 : _mesh.meshChanged();
8358 :
8359 : // If we're just going to alter the mesh again, all we need to
8360 : // handle here is AMR and projections, not full system reinit
8361 7587 : if (intermediate_change)
8362 3773 : es().reinit_solutions();
8363 : else
8364 3814 : es().reinit();
8365 :
8366 7587 : if (contract_mesh)
8367 : // Once vectors are restricted, we can delete children of coarsened elements
8368 4817 : _mesh.getMesh().contract();
8369 7587 : if (clean_refinement_flags)
8370 : {
8371 : // Finally clear refinement flags so that if someone tries to project vectors again without
8372 : // an intervening mesh refinement to clear flags they won't run into trouble
8373 4817 : MeshRefinement refinement(_mesh.getMesh());
8374 4817 : refinement.clean_refinement_flags();
8375 4817 : }
8376 :
8377 7587 : if (!intermediate_change)
8378 : {
8379 : // Since the mesh has changed, we need to make sure that we update any of our
8380 : // MOOSE-system specific data.
8381 7628 : for (auto & sys : _solver_systems)
8382 3814 : sys->reinit();
8383 3814 : _aux->reinit();
8384 : }
8385 :
8386 : // Updating MooseMesh first breaks other adaptivity code, unless we
8387 : // then *again* update the MooseMesh caches. E.g. the definition of
8388 : // "active" and "local" may have been *changed* by refinement and
8389 : // repartitioning done in EquationSystems::reinit().
8390 7587 : _mesh.meshChanged();
8391 :
8392 : // If we have finite volume variables, we will need to recompute additional elemental/face
8393 : // quantities
8394 7587 : if (haveFV() && _mesh.isFiniteVolumeInfoDirty())
8395 384 : _mesh.setupFiniteVolumeMeshData();
8396 :
8397 : // Let the meshChangedInterface notify the mesh changed event before we update the active
8398 : // semilocal nodes, because the set of ghosted elements may potentially be updated during a mesh
8399 : // changed event.
8400 133674 : for (const auto & mci : _notify_when_mesh_changes)
8401 126087 : mci->meshChanged();
8402 :
8403 : // Since the Mesh changed, update the PointLocator object used by DiracKernels.
8404 7587 : _dirac_kernel_info.updatePointLocator(_mesh);
8405 :
8406 : // Need to redo ghosting
8407 7587 : _geometric_search_data.reinit();
8408 :
8409 7587 : if (_displaced_problem)
8410 : {
8411 633 : _displaced_problem->meshChanged(contract_mesh, clean_refinement_flags);
8412 633 : _displaced_mesh->updateActiveSemiLocalNodeRange(_ghosted_elems);
8413 : }
8414 :
8415 7587 : _mesh.updateActiveSemiLocalNodeRange(_ghosted_elems);
8416 :
8417 7587 : _evaluable_local_elem_range.reset();
8418 7587 : _nl_evaluable_local_elem_range.reset();
8419 :
8420 : // Just like we reinitialized our geometric search objects, we also need to reinitialize our
8421 : // mortar meshes. Note that this needs to happen after DisplacedProblem::meshChanged because the
8422 : // mortar mesh discretization will depend necessarily on the displaced mesh being re-displaced
8423 7587 : updateMortarMesh();
8424 :
8425 : // Nonlinear systems hold the mortar mesh functors. The domains of definition of the mortar
8426 : // functors might have changed when the mesh changed.
8427 15162 : for (auto & nl_sys : _nl)
8428 7575 : nl_sys->reinitMortarFunctors();
8429 :
8430 7587 : reinitBecauseOfGhostingOrNewGeomObjects(/*mortar_changed=*/true);
8431 :
8432 : // We need to create new storage for newly active elements, and copy
8433 : // stateful properties from the old elements.
8434 7819 : if (_has_initialized_stateful &&
8435 232 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties()))
8436 : {
8437 232 : if (havePRefinement())
8438 72 : _mesh.buildPRefinementAndCoarseningMaps(_assembly[0][0].get());
8439 :
8440 : // Prolong properties onto newly refined elements' children
8441 : {
8442 : ProjectMaterialProperties pmp(
8443 232 : /* refine = */ true, *this, _material_props, _bnd_material_props, _assembly);
8444 232 : const auto & range = *_mesh.refinedElementRange();
8445 232 : Threads::parallel_reduce(range, pmp);
8446 :
8447 : // Concurrent erasure from the shared hash map is not safe while we are reading from it in
8448 : // ProjectMaterialProperties, so we handle erasure here. Moreover, erasure based on key is
8449 : // not thread safe in and of itself because it is a read-write operation. Note that we do not
8450 : // do the erasure for p-refinement because the coarse level element is the same as our active
8451 : // refined level element
8452 232 : if (!doingPRefinement())
8453 3758 : for (const auto & elem : range)
8454 : {
8455 3598 : _material_props.eraseProperty(elem);
8456 3598 : _bnd_material_props.eraseProperty(elem);
8457 3598 : _neighbor_material_props.eraseProperty(elem);
8458 : }
8459 232 : }
8460 :
8461 : // Restrict properties onto newly coarsened elements
8462 : {
8463 : ProjectMaterialProperties pmp(
8464 232 : /* refine = */ false, *this, _material_props, _bnd_material_props, _assembly);
8465 232 : const auto & range = *_mesh.coarsenedElementRange();
8466 232 : Threads::parallel_reduce(range, pmp);
8467 : // Note that we do not do the erasure for p-refinement because the coarse level element is the
8468 : // same as our active refined level element
8469 232 : if (!doingPRefinement())
8470 1525 : for (const auto & elem : range)
8471 : {
8472 1365 : auto && coarsened_children = _mesh.coarsenedElementChildren(elem);
8473 8861 : for (auto && child : coarsened_children)
8474 : {
8475 7496 : _material_props.eraseProperty(child);
8476 7496 : _bnd_material_props.eraseProperty(child);
8477 7496 : _neighbor_material_props.eraseProperty(child);
8478 : }
8479 : }
8480 232 : }
8481 : }
8482 :
8483 7587 : if (_calculate_jacobian_in_uo)
8484 0 : setVariableAllDoFMap(_uo_jacobian_moose_vars[0]);
8485 :
8486 7587 : _has_jacobian = false; // we have to recompute jacobian when mesh changed
8487 :
8488 : // Now for backwards compatibility with user code that overrode the old no-arg meshChanged we must
8489 : // call it here
8490 7587 : meshChanged();
8491 7587 : }
8492 :
8493 : void
8494 975335 : FEProblemBase::notifyWhenMeshChanges(MeshChangedInterface * mci)
8495 : {
8496 975335 : _notify_when_mesh_changes.push_back(mci);
8497 975335 : }
8498 :
8499 : void
8500 85611 : FEProblemBase::notifyWhenMeshDisplaces(MeshDisplacedInterface * mdi)
8501 : {
8502 85611 : _notify_when_mesh_displaces.push_back(mdi);
8503 85611 : }
8504 :
8505 : void
8506 70072 : FEProblemBase::meshDisplaced()
8507 : {
8508 101233 : for (const auto & mdi : _notify_when_mesh_displaces)
8509 31161 : mdi->meshDisplaced();
8510 70072 : }
8511 :
8512 : void
8513 10246 : FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const bool threaded)
8514 : {
8515 : ComputeMaterialsObjectThread cmt(
8516 10246 : *this, _material_props, _bnd_material_props, _neighbor_material_props, _assembly);
8517 10246 : if (threaded)
8518 10246 : Threads::parallel_reduce(elem_range, cmt);
8519 : else
8520 0 : cmt(elem_range, true);
8521 :
8522 : #ifdef MOOSE_KOKKOS_ENABLED
8523 6918 : if (_has_kokkos_objects)
8524 310 : initKokkosStatefulProps();
8525 : #endif
8526 10246 : }
8527 :
8528 : void
8529 62728 : FEProblemBase::checkProblemIntegrity()
8530 : {
8531 188184 : TIME_SECTION("checkProblemIntegrity", 5);
8532 :
8533 : // Subdomains specified by the "Problem/block" parameter
8534 125456 : const auto & subdomain_names = getParam<std::vector<SubdomainName>>("block");
8535 62728 : auto mesh_subdomains_vec = MooseMeshUtils::getSubdomainIDs(_mesh, subdomain_names);
8536 62728 : std::set<SubdomainID> mesh_subdomains(mesh_subdomains_vec.begin(), mesh_subdomains_vec.end());
8537 :
8538 : // Check kernel coverage of subdomains (blocks) in the mesh
8539 62728 : if (!_skip_nl_system_check && _solve && _kernel_coverage_check != CoverageCheckMode::FALSE &&
8540 45160 : _kernel_coverage_check != CoverageCheckMode::OFF)
8541 : {
8542 45147 : std::set<SubdomainID> blocks;
8543 45147 : if (_kernel_coverage_check == CoverageCheckMode::TRUE ||
8544 237 : _kernel_coverage_check == CoverageCheckMode::ON)
8545 44910 : blocks = mesh_subdomains;
8546 237 : else if (_kernel_coverage_check == CoverageCheckMode::SKIP_LIST)
8547 : {
8548 13 : blocks = mesh_subdomains;
8549 26 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8550 : {
8551 13 : const auto id = _mesh.getSubdomainID(subdomain_name);
8552 13 : if (id == Moose::INVALID_BLOCK_ID)
8553 0 : paramError("kernel_coverage_block_list",
8554 : "Subdomain \"",
8555 : subdomain_name,
8556 : "\" not found in mesh.");
8557 13 : blocks.erase(id);
8558 : }
8559 : }
8560 224 : else if (_kernel_coverage_check == CoverageCheckMode::ONLY_LIST)
8561 448 : for (const auto & subdomain_name : _kernel_coverage_blocks)
8562 : {
8563 224 : const auto id = _mesh.getSubdomainID(subdomain_name);
8564 224 : if (id == Moose::INVALID_BLOCK_ID)
8565 0 : paramError("kernel_coverage_block_list",
8566 : "Subdomain \"",
8567 : subdomain_name,
8568 : "\" not found in mesh.");
8569 224 : blocks.insert(id);
8570 : }
8571 45147 : if (!blocks.empty())
8572 89200 : for (auto & nl : _nl)
8573 44069 : nl->checkKernelCoverage(blocks);
8574 45131 : }
8575 :
8576 : // Check materials
8577 : {
8578 : #ifdef LIBMESH_ENABLE_AMR
8579 65132 : if ((_adaptivity.isOn() || _num_grid_steps) &&
8580 2420 : (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
8581 2349 : _neighbor_material_props.hasStatefulProperties()))
8582 : {
8583 71 : _console << "Using EXPERIMENTAL Stateful Material Property projection with Adaptivity!\n"
8584 71 : << std::flush;
8585 : }
8586 : #endif
8587 :
8588 62712 : std::set<SubdomainID> local_mesh_subs(mesh_subdomains);
8589 :
8590 62712 : if (_material_coverage_check != CoverageCheckMode::FALSE &&
8591 62620 : _material_coverage_check != CoverageCheckMode::OFF)
8592 : {
8593 : /**
8594 : * If a material is specified for any block in the simulation, then all blocks must
8595 : * have a material specified.
8596 : */
8597 62620 : bool check_material_coverage = false;
8598 62620 : std::set<SubdomainID> ids = _all_materials.getActiveBlocks();
8599 75834 : for (const auto & id : ids)
8600 : {
8601 13214 : local_mesh_subs.erase(id);
8602 13214 : check_material_coverage = true;
8603 : }
8604 :
8605 : // did the user limit the subdomains to be checked?
8606 62620 : if (_material_coverage_check == CoverageCheckMode::SKIP_LIST)
8607 : {
8608 26 : for (const auto & subdomain_name : _material_coverage_blocks)
8609 : {
8610 13 : const auto id = _mesh.getSubdomainID(subdomain_name);
8611 13 : if (id == Moose::INVALID_BLOCK_ID)
8612 0 : paramError("material_coverage_block_list",
8613 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8614 13 : local_mesh_subs.erase(id);
8615 : }
8616 : }
8617 62607 : else if (_material_coverage_check == CoverageCheckMode::ONLY_LIST)
8618 : {
8619 224 : std::set<SubdomainID> blocks(local_mesh_subs);
8620 448 : for (const auto & subdomain_name : _material_coverage_blocks)
8621 : {
8622 224 : const auto id = _mesh.getSubdomainID(subdomain_name);
8623 224 : if (id == Moose::INVALID_BLOCK_ID)
8624 0 : paramError("material_coverage_block_list",
8625 0 : "Subdomain \"" + subdomain_name + "\" not found in mesh.");
8626 224 : blocks.erase(id);
8627 : }
8628 237 : for (const auto id : blocks)
8629 13 : local_mesh_subs.erase(id);
8630 224 : }
8631 :
8632 : // also exclude mortar spaces from the material check
8633 62620 : auto && mortar_subdomain_ids = _mortar_data->getMortarSubdomainIDs();
8634 64660 : for (auto subdomain_id : mortar_subdomain_ids)
8635 2040 : local_mesh_subs.erase(subdomain_id);
8636 :
8637 : // Check Material Coverage
8638 62620 : if (check_material_coverage && !local_mesh_subs.empty())
8639 : {
8640 8 : std::stringstream extra_subdomain_ids;
8641 : /// unsigned int is necessary to print SubdomainIDs in the statement below
8642 8 : std::copy(local_mesh_subs.begin(),
8643 : local_mesh_subs.end(),
8644 16 : std::ostream_iterator<unsigned int>(extra_subdomain_ids, " "));
8645 : /// vector is necessary to get the subdomain names
8646 : std::vector<SubdomainID> local_mesh_subs_vec(local_mesh_subs.begin(),
8647 8 : local_mesh_subs.end());
8648 :
8649 24 : mooseError("The following blocks from your input mesh do not contain an active material: " +
8650 16 : extra_subdomain_ids.str() +
8651 24 : "(names: " + Moose::stringify(_mesh.getSubdomainNames(local_mesh_subs_vec)) +
8652 : ")\nWhen ANY mesh block contains a Material object, "
8653 : "all blocks must contain a Material object.\n");
8654 0 : }
8655 62612 : }
8656 :
8657 : // Check material properties on blocks and boundaries
8658 62704 : checkBlockMatProps();
8659 62662 : checkBoundaryMatProps();
8660 :
8661 : // Check that material properties exist when requested by other properties on a given block
8662 62650 : const auto & materials = _all_materials.getActiveObjects();
8663 77440 : for (const auto & material : materials)
8664 14790 : material->checkStatefulSanity();
8665 :
8666 : // auto mats_to_check = _materials.getActiveBlockObjects();
8667 : // const auto & discrete_materials = _discrete_materials.getActiveBlockObjects();
8668 : // for (const auto & map_it : discrete_materials)
8669 : // for (const auto & container_element : map_it.second)
8670 : // mats_to_check[map_it.first].push_back(container_element);
8671 62650 : if (_material_dependency_check)
8672 62624 : checkDependMaterialsHelper(_all_materials.getActiveBlockObjects());
8673 62634 : }
8674 :
8675 62634 : checkUserObjects();
8676 :
8677 : // Verify that we don't have any Element type/Coordinate Type conflicts
8678 62634 : checkCoordinateSystems();
8679 :
8680 : // Coordinate transforms are only intended for use with MultiApps at this time. If you are not
8681 : // using multiapps but still require these, contact a moose developer
8682 62785 : if (_mesh.coordTransform().hasScalingOrRotationTransformation() && _app.isUltimateMaster() &&
8683 155 : !hasMultiApps())
8684 4 : mooseError("Coordinate transformation parameters, listed below, are only to be used in the "
8685 : "context of application to application field transfers at this time. The mesh is "
8686 : "not modified by these parameters within an application.\n"
8687 : "You should likely use a 'TransformGenerator' in the [Mesh] block to achieve the "
8688 : "desired mesh modification.\n\n",
8689 4 : Moose::stringify(MooseAppCoordTransform::validParams()));
8690 :
8691 : // If using displacements, verify that the order of the displacement
8692 : // variables matches the order of the elements in the displaced
8693 : // mesh.
8694 62626 : checkDisplacementOrders();
8695 :
8696 : // Check for postprocessor names with same name as a scalar variable
8697 62622 : checkDuplicatePostprocessorVariableNames();
8698 62622 : }
8699 :
8700 : void
8701 62626 : FEProblemBase::checkDisplacementOrders()
8702 : {
8703 62626 : if (_displaced_problem)
8704 : {
8705 2246 : bool mesh_has_second_order_elements = false;
8706 4492 : for (const auto & elem : as_range(_displaced_mesh->activeLocalElementsBegin(),
8707 535056 : _displaced_mesh->activeLocalElementsEnd()))
8708 : {
8709 264537 : if (elem->default_order() == SECOND)
8710 : {
8711 378 : mesh_has_second_order_elements = true;
8712 378 : break;
8713 : }
8714 2246 : }
8715 :
8716 : // We checked our local elements, so take the max over all processors.
8717 2246 : _displaced_mesh->comm().max(mesh_has_second_order_elements);
8718 :
8719 : // If the Mesh has second order elements, make sure the
8720 : // displacement variables are second-order.
8721 2246 : if (mesh_has_second_order_elements)
8722 : {
8723 : const std::vector<std::string> & displacement_variables =
8724 378 : _displaced_problem->getDisplacementVarNames();
8725 :
8726 1242 : for (const auto & var_name : displacement_variables)
8727 : {
8728 : MooseVariableFEBase & mv =
8729 868 : _displaced_problem->getVariable(/*tid=*/0,
8730 : var_name,
8731 : Moose::VarKindType::VAR_ANY,
8732 : Moose::VarFieldType::VAR_FIELD_STANDARD);
8733 868 : if (mv.order() != SECOND)
8734 4 : mooseError("Error: mesh has SECOND order elements, so all displacement variables must be "
8735 : "SECOND order.");
8736 : }
8737 : }
8738 : }
8739 62622 : }
8740 :
8741 : void
8742 62634 : FEProblemBase::checkUserObjects()
8743 : {
8744 : // Check user_objects block coverage
8745 62634 : std::set<SubdomainID> mesh_subdomains = _mesh.meshSubdomains();
8746 62634 : std::set<SubdomainID> user_objects_blocks;
8747 :
8748 : // gather names of all user_objects that were defined in the input file
8749 : // and the blocks that they are defined on
8750 62634 : std::set<std::string> names;
8751 :
8752 62634 : std::vector<UserObject *> objects;
8753 62634 : theWarehouse().query().condition<AttribInterfaces>(Interfaces::UserObject).queryInto(objects);
8754 :
8755 144657 : for (const auto & obj : objects)
8756 82023 : names.insert(obj->name());
8757 :
8758 : // See if all referenced blocks are covered
8759 62634 : std::set<SubdomainID> difference;
8760 62634 : std::set_difference(user_objects_blocks.begin(),
8761 : user_objects_blocks.end(),
8762 : mesh_subdomains.begin(),
8763 : mesh_subdomains.end(),
8764 : std::inserter(difference, difference.end()));
8765 :
8766 62634 : if (!difference.empty())
8767 : {
8768 0 : std::ostringstream oss;
8769 0 : oss << "One or more UserObjects is referencing a nonexistent block:\n";
8770 0 : for (const auto & id : difference)
8771 0 : oss << id << "\n";
8772 0 : mooseError(oss.str());
8773 0 : }
8774 62634 : }
8775 :
8776 : void
8777 62624 : FEProblemBase::checkDependMaterialsHelper(
8778 : const std::map<SubdomainID, std::vector<std::shared_ptr<MaterialBase>>> & materials_map)
8779 : {
8780 75765 : for (const auto & it : materials_map)
8781 : {
8782 : /// These two sets are used to make sure that all dependent props on a block are actually supplied
8783 13151 : std::set<std::string> block_depend_props, block_supplied_props;
8784 :
8785 32599 : for (const auto & mat1 : it.second)
8786 : {
8787 19448 : auto & alldeps = mat1->getMatPropDependencies(); // includes requested stateful props
8788 21900 : for (auto & dep : alldeps)
8789 2452 : block_depend_props.insert(_material_prop_registry.getName(dep));
8790 :
8791 : // See if any of the active materials supply this property
8792 60144 : for (const auto & mat2 : it.second)
8793 : {
8794 40696 : const std::set<std::string> & supplied_props = mat2->MaterialBase::getSuppliedItems();
8795 40696 : block_supplied_props.insert(supplied_props.begin(), supplied_props.end());
8796 : }
8797 : }
8798 :
8799 : // Add zero material properties specific to this block and unrestricted
8800 13151 : block_supplied_props.insert(_zero_block_material_props[it.first].begin(),
8801 13151 : _zero_block_material_props[it.first].end());
8802 :
8803 : // Error check to make sure all properties consumed by materials are supplied on this block
8804 13151 : std::set<std::string> difference;
8805 13151 : std::set_difference(block_depend_props.begin(),
8806 : block_depend_props.end(),
8807 : block_supplied_props.begin(),
8808 : block_supplied_props.end(),
8809 : std::inserter(difference, difference.end()));
8810 :
8811 13151 : if (!difference.empty())
8812 : {
8813 10 : std::ostringstream oss;
8814 10 : oss << "One or more Material Properties were not supplied on block ";
8815 10 : const std::string & subdomain_name = _mesh.getSubdomainName(it.first);
8816 10 : if (subdomain_name.length() > 0)
8817 0 : oss << subdomain_name << " (" << it.first << ")";
8818 : else
8819 10 : oss << it.first;
8820 10 : oss << ":\n";
8821 20 : for (const auto & name : difference)
8822 10 : oss << name << "\n";
8823 10 : mooseError(oss.str());
8824 0 : }
8825 13141 : }
8826 :
8827 : // This loop checks that materials are not supplied by multiple Material objects
8828 75749 : for (const auto & it : materials_map)
8829 : {
8830 13141 : const auto & materials = it.second;
8831 13141 : std::set<std::string> inner_supplied, outer_supplied;
8832 :
8833 32565 : for (const auto & outer_mat : materials)
8834 : {
8835 : // Storage for properties for this material (outer) and all other materials (inner)
8836 19430 : outer_supplied = outer_mat->getSuppliedItems();
8837 19430 : inner_supplied.clear();
8838 :
8839 : // Property to material map for error reporting
8840 19430 : std::map<std::string, std::set<std::string>> prop_to_mat;
8841 40782 : for (const auto & name : outer_supplied)
8842 21352 : prop_to_mat[name].insert(outer_mat->name());
8843 :
8844 60098 : for (const auto & inner_mat : materials)
8845 : {
8846 40668 : if (outer_mat == inner_mat)
8847 19430 : continue;
8848 :
8849 : // Check whether these materials are an AD pair
8850 21238 : auto outer_mat_type = outer_mat->type();
8851 21238 : auto inner_mat_type = inner_mat->type();
8852 42476 : removeSubstring(outer_mat_type, "<RESIDUAL>");
8853 42476 : removeSubstring(outer_mat_type, "<JACOBIAN>");
8854 42476 : removeSubstring(inner_mat_type, "<RESIDUAL>");
8855 21238 : removeSubstring(inner_mat_type, "<JACOBIAN>");
8856 21238 : if (outer_mat_type == inner_mat_type && outer_mat_type != outer_mat->type() &&
8857 0 : inner_mat_type != inner_mat->type())
8858 0 : continue;
8859 :
8860 21238 : inner_supplied.insert(inner_mat->getSuppliedItems().begin(),
8861 21238 : inner_mat->getSuppliedItems().end());
8862 :
8863 124032 : for (const auto & inner_supplied_name : inner_supplied)
8864 102794 : prop_to_mat[inner_supplied_name].insert(inner_mat->name());
8865 21238 : }
8866 :
8867 : // Test that a property isn't supplied on multiple blocks
8868 19430 : std::set<std::string> intersection;
8869 19430 : std::set_intersection(outer_supplied.begin(),
8870 : outer_supplied.end(),
8871 : inner_supplied.begin(),
8872 : inner_supplied.end(),
8873 : std::inserter(intersection, intersection.end()));
8874 :
8875 19430 : if (!intersection.empty())
8876 : {
8877 6 : std::ostringstream oss;
8878 6 : oss << "The following material properties are declared on block " << it.first
8879 6 : << " by multiple materials:\n";
8880 12 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << "Material Property"
8881 6 : << "Material Objects\n";
8882 24 : for (const auto & outer_name : intersection)
8883 : {
8884 18 : oss << ConsoleUtils::indent(2) << std::setw(30) << std::left << outer_name;
8885 54 : for (const auto & inner_name : prop_to_mat[outer_name])
8886 36 : oss << inner_name << " ";
8887 18 : oss << '\n';
8888 : }
8889 :
8890 6 : mooseError(oss.str());
8891 : break;
8892 0 : }
8893 19424 : }
8894 13135 : }
8895 62608 : }
8896 :
8897 : void
8898 62634 : FEProblemBase::checkCoordinateSystems()
8899 : {
8900 62634 : _mesh.checkCoordinateSystems();
8901 62630 : }
8902 :
8903 : void
8904 509 : FEProblemBase::setRestartFile(const std::string & file_name)
8905 : {
8906 509 : if (_app.isRecovering())
8907 : {
8908 23 : mooseInfo("Restart file ", file_name, " is NOT being used since we are performing recovery.");
8909 : }
8910 : else
8911 : {
8912 486 : _app.setRestart(true);
8913 486 : _app.setRestartRecoverFileBase(file_name);
8914 486 : mooseInfo("Using ", file_name, " for restart.");
8915 : }
8916 509 : }
8917 :
8918 : std::vector<VariableName>
8919 381595 : FEProblemBase::getVariableNames()
8920 : {
8921 381595 : std::vector<VariableName> names;
8922 :
8923 767698 : for (auto & sys : _solver_systems)
8924 : {
8925 386103 : const std::vector<VariableName> & var_names = sys->getVariableNames();
8926 386103 : names.insert(names.end(), var_names.begin(), var_names.end());
8927 : }
8928 :
8929 381595 : const std::vector<VariableName> & aux_var_names = _aux->getVariableNames();
8930 381595 : names.insert(names.end(), aux_var_names.begin(), aux_var_names.end());
8931 :
8932 381595 : return names;
8933 0 : }
8934 :
8935 : SolverParams &
8936 1708349 : FEProblemBase::solverParams(const unsigned int solver_sys_num)
8937 : {
8938 : mooseAssert(solver_sys_num < numSolverSystems(),
8939 : "Solver system number '" << solver_sys_num << "' is out of bounds. We have '"
8940 : << numSolverSystems() << "' solver systems");
8941 1708349 : return _solver_params[solver_sys_num];
8942 : }
8943 :
8944 : const SolverParams &
8945 12911 : FEProblemBase::solverParams(const unsigned int solver_sys_num) const
8946 : {
8947 12911 : return const_cast<FEProblemBase *>(this)->solverParams(solver_sys_num);
8948 : }
8949 :
8950 : void
8951 456 : FEProblemBase::registerRandomInterface(RandomInterface & random_interface, const std::string & name)
8952 : {
8953 456 : auto insert_pair = moose_try_emplace(
8954 456 : _random_data_objects, name, std::make_unique<RandomData>(*this, random_interface));
8955 :
8956 456 : auto random_data_ptr = insert_pair.first->second.get();
8957 456 : random_interface.setRandomDataPointer(random_data_ptr);
8958 456 : }
8959 :
8960 : bool
8961 1917205 : FEProblemBase::needBoundaryMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
8962 : {
8963 1917205 : if (_bnd_mat_side_cache[tid].find(bnd_id) == _bnd_mat_side_cache[tid].end())
8964 : {
8965 31243 : auto & bnd_mat_side_cache = _bnd_mat_side_cache[tid][bnd_id];
8966 31243 : bnd_mat_side_cache = false;
8967 :
8968 : // Check systems
8969 31243 : if (_aux->needMaterialOnSide(bnd_id))
8970 : {
8971 578 : bnd_mat_side_cache = true;
8972 578 : return true;
8973 : }
8974 58204 : for (auto & nl : _nl)
8975 30339 : if (nl->needBoundaryMaterialOnSide(bnd_id, tid))
8976 : {
8977 2800 : bnd_mat_side_cache = true;
8978 2800 : return true;
8979 : }
8980 :
8981 : // TODO: these objects should be checked for whether they actually consume materials
8982 : // NOTE: InterfaceUO can use use boundary properties too
8983 27865 : if (theWarehouse()
8984 55730 : .query()
8985 27865 : .condition<AttribThread>(tid)
8986 27865 : .condition<AttribInterfaces>(Interfaces::SideUserObject | Interfaces::DomainUserObject |
8987 : Interfaces::InterfaceUserObject)
8988 27865 : .condition<AttribBoundaries>(bnd_id)
8989 27865 : .count() > 0)
8990 : {
8991 634 : bnd_mat_side_cache = true;
8992 634 : return true;
8993 : }
8994 : }
8995 :
8996 1913193 : return _bnd_mat_side_cache[tid][bnd_id];
8997 : }
8998 :
8999 : bool
9000 458582 : FEProblemBase::needInterfaceMaterialOnSide(BoundaryID bnd_id, const THREAD_ID tid)
9001 : {
9002 458582 : if (_interface_mat_side_cache[tid].find(bnd_id) == _interface_mat_side_cache[tid].end())
9003 : {
9004 3238 : auto & interface_mat_side_cache = _interface_mat_side_cache[tid][bnd_id];
9005 3238 : interface_mat_side_cache = false;
9006 :
9007 : // Aux-system has not needed interface materials so far
9008 6206 : for (auto & nl : _nl)
9009 3238 : if (nl->needInterfaceMaterialOnSide(bnd_id, tid))
9010 : {
9011 270 : interface_mat_side_cache = true;
9012 270 : return true;
9013 : }
9014 :
9015 : // TODO: these objects should be checked for whether they actually consume materials
9016 2968 : if (theWarehouse()
9017 5936 : .query()
9018 2968 : .condition<AttribThread>(tid)
9019 2968 : .condition<AttribInterfaces>(Interfaces::InterfaceUserObject |
9020 : Interfaces::DomainUserObject)
9021 2968 : .condition<AttribBoundaries>(bnd_id)
9022 2968 : .count() > 0)
9023 : {
9024 90 : interface_mat_side_cache = true;
9025 90 : return true;
9026 : }
9027 2878 : else if (_interface_materials.hasActiveBoundaryObjects(bnd_id, tid))
9028 : {
9029 11 : interface_mat_side_cache = true;
9030 11 : return true;
9031 : }
9032 : }
9033 458211 : return _interface_mat_side_cache[tid][bnd_id];
9034 : }
9035 :
9036 : bool
9037 446428 : FEProblemBase::needInternalNeighborSideMaterial(SubdomainID subdomain_id, const THREAD_ID tid)
9038 : {
9039 446428 : if (_block_mat_side_cache[tid].find(subdomain_id) == _block_mat_side_cache[tid].end())
9040 : {
9041 13461 : _block_mat_side_cache[tid][subdomain_id] = false;
9042 :
9043 26344 : for (auto & nl : _nl)
9044 13378 : if (nl->needInternalNeighborSideMaterial(subdomain_id, tid))
9045 : {
9046 495 : _block_mat_side_cache[tid][subdomain_id] = true;
9047 495 : return true;
9048 : }
9049 :
9050 : // TODO: these objects should be checked for whether they actually consume materials
9051 12966 : if (theWarehouse()
9052 25932 : .query()
9053 12966 : .condition<AttribThread>(tid)
9054 12966 : .condition<AttribInterfaces>(Interfaces::InternalSideUserObject |
9055 : Interfaces::DomainUserObject)
9056 12966 : .condition<AttribSubdomains>(subdomain_id)
9057 12966 : .count() > 0)
9058 : {
9059 38 : _block_mat_side_cache[tid][subdomain_id] = true;
9060 38 : return true;
9061 : }
9062 : }
9063 :
9064 445895 : return _block_mat_side_cache[tid][subdomain_id];
9065 : }
9066 :
9067 : bool
9068 316843 : FEProblemBase::needsPreviousNewtonIteration() const
9069 : {
9070 316843 : return vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG);
9071 : }
9072 :
9073 : void
9074 82 : FEProblemBase::needsPreviousNewtonIteration(bool state)
9075 : {
9076 82 : if (state && !vectorTagExists(Moose::PREVIOUS_NL_SOLUTION_TAG))
9077 0 : mooseError("Previous nonlinear solution is required but not added through "
9078 : "Problem/previous_nl_solution_required=true");
9079 82 : }
9080 :
9081 : bool
9082 8536346 : FEProblemBase::hasJacobian() const
9083 : {
9084 8536346 : return _has_jacobian;
9085 : }
9086 :
9087 : bool
9088 7990258 : FEProblemBase::constJacobian() const
9089 : {
9090 7990258 : return _const_jacobian;
9091 : }
9092 :
9093 : void
9094 298364 : FEProblemBase::addOutput(const std::string & object_type,
9095 : const std::string & object_name,
9096 : InputParameters & parameters)
9097 : {
9098 : parallel_object_only();
9099 :
9100 : // Get a reference to the OutputWarehouse
9101 298364 : OutputWarehouse & output_warehouse = _app.getOutputWarehouse();
9102 :
9103 : // Reject the reserved names for objects not built by MOOSE
9104 298364 : if (!parameters.get<bool>("_built_by_moose") && output_warehouse.isReservedName(object_name))
9105 8 : mooseError("The name '", object_name, "' is a reserved name for output objects");
9106 :
9107 : // Check that an object by the same name does not already exist; this must be done before the
9108 : // object is created to avoid getting misleading errors from the Parser
9109 298356 : if (output_warehouse.hasOutput(object_name))
9110 4 : mooseError("An output object named '", object_name, "' already exists");
9111 :
9112 : // Add a pointer to the FEProblemBase class
9113 596704 : parameters.addPrivateParam<FEProblemBase *>("_fe_problem_base", this);
9114 :
9115 : // Create common parameter exclude list
9116 298352 : std::vector<std::string> exclude;
9117 298352 : if (object_type == "Console")
9118 : {
9119 63435 : exclude.push_back("execute_on");
9120 :
9121 : // --show-input should enable the display of the input file on the screen
9122 190305 : if (_app.getParam<bool>("show_input") && parameters.get<bool>("output_screen"))
9123 60 : parameters.set<ExecFlagEnum>("execute_input_on") = EXEC_INITIAL;
9124 : }
9125 : // Need this because Checkpoint::validParams changes the default value of
9126 : // execute_on
9127 234917 : else if (object_type == "Checkpoint")
9128 102138 : exclude.push_back("execute_on");
9129 :
9130 : // Apply the common parameters loaded with Outputs input syntax
9131 298352 : const InputParameters * common = output_warehouse.getCommonParameters();
9132 298352 : if (common)
9133 298352 : parameters.applyParameters(*common, exclude);
9134 298352 : if (common && std::find(exclude.begin(), exclude.end(), "execute_on") != exclude.end() &&
9135 825712 : common->isParamSetByUser("execute_on") && object_type != "Console")
9136 7082 : mooseInfoRepeated(
9137 14164 : "'execute_on' parameter specified in [Outputs] block is ignored for object '" +
9138 21246 : object_name +
9139 : "'.\nDefine this object in its own sub-block of [Outputs] to modify its "
9140 : "execution schedule.");
9141 :
9142 : // Set the correct value for the binary flag for XDA/XDR output
9143 298352 : if (object_type == "XDR")
9144 130 : parameters.set<bool>("_binary") = true;
9145 298287 : else if (object_type == "XDA")
9146 242 : parameters.set<bool>("_binary") = false;
9147 :
9148 : // Adjust the checkpoint suffix if auto recovery was enabled
9149 298352 : if (object_name == "auto_recovery_checkpoint")
9150 0 : parameters.set<std::string>("suffix") = "auto_recovery";
9151 :
9152 : // Create the object and add it to the warehouse
9153 298352 : std::shared_ptr<Output> output = _factory.create<Output>(object_type, object_name, parameters);
9154 298336 : logAdd("Output", object_name, object_type, parameters);
9155 298336 : output_warehouse.addOutput(output);
9156 298336 : }
9157 :
9158 : void
9159 28343 : FEProblemBase::haveADObjects(const bool have_ad_objects)
9160 : {
9161 28343 : _have_ad_objects = have_ad_objects;
9162 28343 : if (_displaced_problem)
9163 247 : _displaced_problem->SubProblem::haveADObjects(have_ad_objects);
9164 28343 : }
9165 :
9166 : const SystemBase &
9167 0 : FEProblemBase::getSystemBase(const unsigned int sys_num) const
9168 : {
9169 0 : if (sys_num < _solver_systems.size())
9170 0 : return *_solver_systems[sys_num];
9171 :
9172 0 : return *_aux;
9173 : }
9174 :
9175 : SystemBase &
9176 929 : FEProblemBase::getSystemBase(const std::string & sys_name)
9177 : {
9178 929 : if (std::find(_solver_sys_names.begin(), _solver_sys_names.end(), sys_name) !=
9179 1858 : _solver_sys_names.end())
9180 929 : return getSystemBase(solverSysNum(sys_name));
9181 0 : else if (sys_name == "aux0")
9182 0 : return *_aux;
9183 : else
9184 0 : mooseError("System '" + sys_name + "' was requested from problem but does not exist.");
9185 : }
9186 :
9187 : SystemBase &
9188 2570 : FEProblemBase::getSystemBase(const unsigned int sys_num)
9189 : {
9190 2570 : if (sys_num < _solver_systems.size())
9191 2459 : return *_solver_systems[sys_num];
9192 :
9193 111 : return *_aux;
9194 : }
9195 :
9196 : const SystemBase &
9197 14623 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num) const
9198 : {
9199 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9200 14623 : return *_nl[sys_num];
9201 : }
9202 :
9203 : SystemBase &
9204 1396939 : FEProblemBase::systemBaseNonlinear(const unsigned int sys_num)
9205 : {
9206 : mooseAssert(sys_num < _nl.size(), "System number greater than the number of nonlinear systems");
9207 1396939 : return *_nl[sys_num];
9208 : }
9209 :
9210 : const SystemBase &
9211 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num) const
9212 : {
9213 : mooseAssert(sys_num < _linear_systems.size(),
9214 : "System number greater than the number of linear systems");
9215 0 : return *_linear_systems[sys_num];
9216 : }
9217 :
9218 : SystemBase &
9219 0 : FEProblemBase::systemBaseLinear(const unsigned int sys_num)
9220 : {
9221 : mooseAssert(sys_num < _linear_systems.size(),
9222 : "System number greater than the number of linear systems");
9223 0 : return *_linear_systems[sys_num];
9224 : }
9225 :
9226 : const SystemBase &
9227 0 : FEProblemBase::systemBaseSolver(const unsigned int sys_num) const
9228 : {
9229 : mooseAssert(sys_num < _solver_systems.size(),
9230 : "System number greater than the number of solver systems");
9231 0 : return *_solver_systems[sys_num];
9232 : }
9233 :
9234 : SystemBase &
9235 7402348 : FEProblemBase::systemBaseSolver(const unsigned int sys_num)
9236 : {
9237 : mooseAssert(sys_num < _solver_systems.size(),
9238 : "System number greater than the number of solver systems");
9239 7402348 : return *_solver_systems[sys_num];
9240 : }
9241 :
9242 : const SystemBase &
9243 18 : FEProblemBase::systemBaseAuxiliary() const
9244 : {
9245 18 : return *_aux;
9246 : }
9247 :
9248 : SystemBase &
9249 9412171 : FEProblemBase::systemBaseAuxiliary()
9250 : {
9251 9412171 : return *_aux;
9252 : }
9253 :
9254 : void
9255 4336472 : FEProblemBase::computingNonlinearResid(bool computing_nonlinear_residual)
9256 : {
9257 : parallel_object_only();
9258 :
9259 4336472 : if (_displaced_problem)
9260 212324 : _displaced_problem->computingNonlinearResid(computing_nonlinear_residual);
9261 4336472 : _computing_nonlinear_residual = computing_nonlinear_residual;
9262 4336472 : }
9263 :
9264 : void
9265 10584770 : FEProblemBase::setCurrentlyComputingResidual(bool currently_computing_residual)
9266 : {
9267 10584770 : if (_displaced_problem)
9268 433724 : _displaced_problem->setCurrentlyComputingResidual(currently_computing_residual);
9269 10584770 : _currently_computing_residual = currently_computing_residual;
9270 10584770 : }
9271 :
9272 : void
9273 57 : FEProblemBase::uniformRefine()
9274 : {
9275 : // ResetDisplacedMeshThread::onNode looks up the reference mesh by ID, so we need to make sure
9276 : // we undisplace before adapting the reference mesh
9277 57 : if (_displaced_problem)
9278 37 : _displaced_problem->undisplaceMesh();
9279 :
9280 57 : Adaptivity::uniformRefine(&_mesh, 1);
9281 57 : if (_displaced_problem)
9282 37 : Adaptivity::uniformRefine(&_displaced_problem->mesh(), 1);
9283 :
9284 57 : meshChanged(
9285 : /*intermediate_change=*/false, /*contract_mesh=*/true, /*clean_refinement_flags=*/true);
9286 57 : }
9287 :
9288 : void
9289 63946 : FEProblemBase::automaticScaling(bool automatic_scaling)
9290 : {
9291 63946 : if (_displaced_problem)
9292 2246 : _displaced_problem->automaticScaling(automatic_scaling);
9293 :
9294 63946 : SubProblem::automaticScaling(automatic_scaling);
9295 63946 : }
9296 :
9297 : void
9298 284429 : FEProblemBase::reinitElemFaceRef(const Elem * elem,
9299 : unsigned int side,
9300 : Real tolerance,
9301 : const std::vector<Point> * const pts,
9302 : const std::vector<Real> * const weights,
9303 : const THREAD_ID tid)
9304 : {
9305 284429 : SubProblem::reinitElemFaceRef(elem, side, tolerance, pts, weights, tid);
9306 :
9307 284429 : if (_displaced_problem)
9308 22778 : _displaced_problem->reinitElemFaceRef(
9309 22778 : _displaced_mesh->elemPtr(elem->id()), side, tolerance, pts, weights, tid);
9310 284429 : }
9311 :
9312 : void
9313 284429 : FEProblemBase::reinitNeighborFaceRef(const Elem * neighbor_elem,
9314 : unsigned int neighbor_side,
9315 : Real tolerance,
9316 : const std::vector<Point> * const pts,
9317 : const std::vector<Real> * const weights,
9318 : const THREAD_ID tid)
9319 : {
9320 284429 : SubProblem::reinitNeighborFaceRef(neighbor_elem, neighbor_side, tolerance, pts, weights, tid);
9321 :
9322 284429 : if (_displaced_problem)
9323 22778 : _displaced_problem->reinitNeighborFaceRef(
9324 22778 : _displaced_mesh->elemPtr(neighbor_elem->id()), neighbor_side, tolerance, pts, weights, tid);
9325 284429 : }
9326 :
9327 : void
9328 3857812 : FEProblemBase::getFVMatsAndDependencies(
9329 : const SubdomainID blk_id,
9330 : std::vector<std::shared_ptr<MaterialBase>> & face_materials,
9331 : std::vector<std::shared_ptr<MaterialBase>> & neighbor_materials,
9332 : std::set<MooseVariableFieldBase *> & variables,
9333 : const THREAD_ID tid)
9334 : {
9335 3857812 : if (_materials[Moose::FACE_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9336 : {
9337 : auto & this_face_mats =
9338 3996 : _materials[Moose::FACE_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9339 8196 : for (std::shared_ptr<MaterialBase> face_mat : this_face_mats)
9340 4200 : if (face_mat->ghostable())
9341 : {
9342 4200 : face_materials.push_back(face_mat);
9343 4200 : auto & var_deps = face_mat->getMooseVariableDependencies();
9344 4608 : for (auto * var : var_deps)
9345 : {
9346 408 : if (!var->isFV())
9347 0 : mooseError(
9348 : "Ghostable materials should only have finite volume variables coupled into them.");
9349 408 : else if (face_mat->hasStatefulProperties())
9350 0 : mooseError("Finite volume materials do not currently support stateful properties.");
9351 408 : variables.insert(var);
9352 : }
9353 4200 : }
9354 : }
9355 :
9356 3857812 : if (_materials[Moose::NEIGHBOR_MATERIAL_DATA].hasActiveBlockObjects(blk_id, tid))
9357 : {
9358 : auto & this_neighbor_mats =
9359 3996 : _materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveBlockObjects(blk_id, tid);
9360 8196 : for (std::shared_ptr<MaterialBase> neighbor_mat : this_neighbor_mats)
9361 4200 : if (neighbor_mat->ghostable())
9362 : {
9363 4200 : neighbor_materials.push_back(neighbor_mat);
9364 : #ifndef NDEBUG
9365 : auto & var_deps = neighbor_mat->getMooseVariableDependencies();
9366 : for (auto * var : var_deps)
9367 : {
9368 : if (!var->isFV())
9369 : mooseError(
9370 : "Ghostable materials should only have finite volume variables coupled into them.");
9371 : else if (neighbor_mat->hasStatefulProperties())
9372 : mooseError("Finite volume materials do not currently support stateful properties.");
9373 : auto pr = variables.insert(var);
9374 : mooseAssert(!pr.second,
9375 : "We should not have inserted any new variables dependencies from our "
9376 : "neighbor materials that didn't exist for our face materials");
9377 : }
9378 : #endif
9379 4200 : }
9380 : }
9381 3857812 : }
9382 :
9383 : void
9384 48898539 : FEProblemBase::resizeMaterialData(const Moose::MaterialDataType data_type,
9385 : const unsigned int nqp,
9386 : const THREAD_ID tid)
9387 : {
9388 48898539 : getMaterialData(data_type, tid).resize(nqp);
9389 48898539 : }
9390 :
9391 : void
9392 63866 : FEProblemBase::setNonlinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9393 : {
9394 63866 : if (convergence_names.size() != numNonlinearSystems())
9395 0 : paramError("nonlinear_convergence",
9396 : "There must be one convergence object per nonlinear system");
9397 63866 : _nonlinear_convergence_names = convergence_names;
9398 63866 : }
9399 :
9400 : void
9401 64213 : FEProblemBase::setMultiAppFixedPointConvergenceName(const ConvergenceName & convergence_name)
9402 : {
9403 64213 : _multiapp_fixed_point_convergence_name = convergence_name;
9404 64213 : }
9405 :
9406 : void
9407 32004 : FEProblemBase::setSteadyStateConvergenceName(const ConvergenceName & convergence_name)
9408 : {
9409 32004 : _steady_state_convergence_name = convergence_name;
9410 32004 : }
9411 :
9412 : const std::vector<ConvergenceName> &
9413 1080563 : FEProblemBase::getNonlinearConvergenceNames() const
9414 : {
9415 1080563 : if (_nonlinear_convergence_names)
9416 1080563 : return *_nonlinear_convergence_names;
9417 0 : mooseError("The nonlinear system convergence name(s) have not been set.");
9418 : }
9419 :
9420 : bool
9421 6097 : FEProblemBase::hasLinearConvergenceObjects() const
9422 : {
9423 : // If false,this means we have not set one, not that we are querying this too early
9424 : // TODO: once there is a default linear CV object, error on the 'not set' case
9425 6097 : return _linear_convergence_names.has_value();
9426 : }
9427 :
9428 : void
9429 181 : FEProblemBase::setLinearConvergenceNames(const std::vector<ConvergenceName> & convergence_names)
9430 : {
9431 181 : if (convergence_names.size() != numLinearSystems())
9432 0 : paramError("linear_convergence", "There must be one convergence object per linear system");
9433 181 : _linear_convergence_names = convergence_names;
9434 181 : }
9435 :
9436 : const std::vector<ConvergenceName> &
9437 2634 : FEProblemBase::getLinearConvergenceNames() const
9438 : {
9439 2634 : if (_linear_convergence_names)
9440 2634 : return *_linear_convergence_names;
9441 0 : mooseError("The linear convergence name(s) have not been set.");
9442 : }
9443 :
9444 : const ConvergenceName &
9445 265045 : FEProblemBase::getMultiAppFixedPointConvergenceName() const
9446 : {
9447 265045 : if (_multiapp_fixed_point_convergence_name)
9448 265045 : return _multiapp_fixed_point_convergence_name.value();
9449 : else
9450 0 : mooseError("The fixed point convergence name has not been set.");
9451 : }
9452 :
9453 : const ConvergenceName &
9454 109052 : FEProblemBase::getSteadyStateConvergenceName() const
9455 : {
9456 109052 : if (_steady_state_convergence_name)
9457 109052 : return _steady_state_convergence_name.value();
9458 : else
9459 0 : mooseError("The steady convergence name has not been set.");
9460 : }
9461 :
9462 : void
9463 3350744 : FEProblemBase::residualSetup()
9464 : {
9465 3350744 : SubProblem::residualSetup();
9466 : // We need to setup all the nonlinear systems other than our current one which actually called
9467 : // this method (so we have to make sure we don't go in a circle)
9468 6794028 : for (const auto i : make_range(numNonlinearSystems()))
9469 3443284 : if (i != currentNlSysNum())
9470 92540 : _nl[i]->residualSetup();
9471 : // We don't setup the aux sys because that's been done elsewhere
9472 3350744 : if (_displaced_problem)
9473 137184 : _displaced_problem->residualSetup();
9474 3350744 : }
9475 :
9476 : void
9477 529137 : FEProblemBase::jacobianSetup()
9478 : {
9479 529137 : SubProblem::jacobianSetup();
9480 : // We need to setup all the nonlinear systems other than our current one which actually called
9481 : // this method (so we have to make sure we don't go in a circle)
9482 1074334 : for (const auto i : make_range(numNonlinearSystems()))
9483 545197 : if (i != currentNlSysNum())
9484 16060 : _nl[i]->jacobianSetup();
9485 : // We don't setup the aux sys because that's been done elsewhere
9486 529137 : if (_displaced_problem)
9487 23344 : _displaced_problem->jacobianSetup();
9488 529137 : }
9489 :
9490 : MooseAppCoordTransform &
9491 104848 : FEProblemBase::coordTransform()
9492 : {
9493 104848 : return mesh().coordTransform();
9494 : }
9495 :
9496 : unsigned int
9497 594857794 : FEProblemBase::currentNlSysNum() const
9498 : {
9499 : // If we don't have nonlinear systems this should be an invalid number
9500 594857794 : unsigned int current_nl_sys_num = libMesh::invalid_uint;
9501 594857794 : if (_nl.size())
9502 594853698 : current_nl_sys_num = currentNonlinearSystem().number();
9503 :
9504 594857794 : return current_nl_sys_num;
9505 : }
9506 :
9507 : unsigned int
9508 0 : FEProblemBase::currentLinearSysNum() const
9509 : {
9510 : // If we don't have linear systems this should be an invalid number
9511 0 : unsigned int current_linear_sys_num = libMesh::invalid_uint;
9512 0 : if (_linear_systems.size())
9513 0 : current_linear_sys_num = currentLinearSystem().number();
9514 :
9515 0 : return current_linear_sys_num;
9516 : }
9517 :
9518 : bool
9519 139536723 : FEProblemBase::shouldPrintExecution(const THREAD_ID tid) const
9520 : {
9521 : // For now, only support printing from thread 0
9522 139536723 : if (tid != 0)
9523 8146143 : return false;
9524 :
9525 262493898 : if (_print_execution_on.isValueSet(_current_execute_on_flag) ||
9526 131103318 : _print_execution_on.isValueSet(EXEC_ALWAYS))
9527 376597 : return true;
9528 : else
9529 131013983 : return false;
9530 : }
9531 :
9532 : std::vector<MortarUserObject *>
9533 283097 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9534 : const BoundaryID secondary_boundary_id,
9535 : const bool displaced,
9536 : const std::vector<MortarUserObject *> & mortar_uo_superset)
9537 : {
9538 283097 : std::vector<MortarUserObject *> mortar_uos;
9539 283097 : auto * const subproblem = displaced ? static_cast<SubProblem *>(_displaced_problem.get())
9540 283097 : : static_cast<SubProblem *>(this);
9541 283121 : for (auto * const obj : mortar_uo_superset)
9542 48 : if (obj->onInterface(primary_boundary_id, secondary_boundary_id) &&
9543 24 : (&obj->getSubProblem() == subproblem))
9544 24 : mortar_uos.push_back(obj);
9545 :
9546 283097 : return mortar_uos;
9547 0 : }
9548 :
9549 : std::vector<MortarUserObject *>
9550 283073 : FEProblemBase::getMortarUserObjects(const BoundaryID primary_boundary_id,
9551 : const BoundaryID secondary_boundary_id,
9552 : const bool displaced)
9553 : {
9554 283073 : std::vector<MortarUserObject *> mortar_uos;
9555 283073 : theWarehouse()
9556 283073 : .query()
9557 566146 : .condition<AttribInterfaces>(Interfaces::MortarUserObject)
9558 283073 : .queryInto(mortar_uos);
9559 566146 : return getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced, mortar_uos);
9560 283073 : }
9561 :
9562 : void
9563 283073 : FEProblemBase::reinitMortarUserObjects(const BoundaryID primary_boundary_id,
9564 : const BoundaryID secondary_boundary_id,
9565 : const bool displaced)
9566 : {
9567 : const auto mortar_uos =
9568 283073 : getMortarUserObjects(primary_boundary_id, secondary_boundary_id, displaced);
9569 283073 : for (auto * const mortar_uo : mortar_uos)
9570 : {
9571 0 : mortar_uo->setNormals();
9572 0 : mortar_uo->reinit();
9573 : }
9574 283073 : }
9575 :
9576 : void
9577 1154 : FEProblemBase::setVerboseProblem(bool verbose)
9578 : {
9579 1154 : _verbose_setup = verbose ? "true" : "false";
9580 1154 : _verbose_multiapps = verbose;
9581 1154 : _verbose_restore = verbose;
9582 1154 : }
9583 :
9584 : void
9585 126057 : FEProblemBase::setCurrentLowerDElem(const Elem * const lower_d_elem, const THREAD_ID tid)
9586 : {
9587 126057 : SubProblem::setCurrentLowerDElem(lower_d_elem, tid);
9588 126057 : if (_displaced_problem)
9589 31448 : _displaced_problem->setCurrentLowerDElem(
9590 0 : lower_d_elem ? _displaced_mesh->elemPtr(lower_d_elem->id()) : nullptr, tid);
9591 126057 : }
9592 :
9593 : void
9594 135889821 : FEProblemBase::setCurrentBoundaryID(BoundaryID bid, const THREAD_ID tid)
9595 : {
9596 135889821 : SubProblem::setCurrentBoundaryID(bid, tid);
9597 135889821 : if (_displaced_problem)
9598 10148584 : _displaced_problem->setCurrentBoundaryID(bid, tid);
9599 135889821 : }
9600 :
9601 : void
9602 8158396 : FEProblemBase::setCurrentNonlinearSystem(const unsigned int nl_sys_num)
9603 : {
9604 : mooseAssert(nl_sys_num < _nl.size(),
9605 : "System number greater than the number of nonlinear systems");
9606 8158396 : _current_nl_sys = _nl[nl_sys_num].get();
9607 8158396 : _current_solver_sys = _current_nl_sys;
9608 8158396 : }
9609 :
9610 : void
9611 17124 : FEProblemBase::setCurrentLinearSystem(const unsigned int sys_num)
9612 : {
9613 : mooseAssert(sys_num < _linear_systems.size(),
9614 : "System number greater than the number of linear systems");
9615 17124 : _current_linear_sys = _linear_systems[sys_num].get();
9616 17124 : _current_solver_sys = _current_linear_sys;
9617 17124 : }
9618 :
9619 : void
9620 6371824 : FEProblemBase::computeSystems(const ExecFlagType & type)
9621 : {
9622 : // When performing an adjoint solve in the optimization module, the current solver system is the
9623 : // adjoint. However, the adjoint solve requires having accurate time derivative calculations for
9624 : // the forward system. The cleanest way to handle such uses is just to compute the time
9625 : // derivatives for all solver systems instead of trying to guess which ones we need and don't need
9626 12899417 : for (auto & solver_sys : _solver_systems)
9627 6527593 : solver_sys->compute(type);
9628 :
9629 6371824 : _aux->compute(type);
9630 6371779 : }
9631 :
9632 : const ConstElemRange &
9633 3881685 : FEProblemBase::getCurrentAlgebraicElementRange()
9634 : {
9635 3881685 : if (!_current_algebraic_elem_range)
9636 3881685 : return *_mesh.getActiveLocalElementRange();
9637 :
9638 0 : return *_current_algebraic_elem_range;
9639 : }
9640 : const ConstNodeRange &
9641 20503 : FEProblemBase::getCurrentAlgebraicNodeRange()
9642 : {
9643 20503 : if (!_current_algebraic_node_range)
9644 20503 : return *_mesh.getLocalNodeRange();
9645 :
9646 0 : return *_current_algebraic_node_range;
9647 : }
9648 : const ConstBndNodeRange &
9649 3643612 : FEProblemBase::getCurrentAlgebraicBndNodeRange()
9650 : {
9651 3643612 : if (!_current_algebraic_bnd_node_range)
9652 3643612 : return *_mesh.getBoundaryNodeRange();
9653 :
9654 0 : return *_current_algebraic_bnd_node_range;
9655 : }
9656 :
9657 : void
9658 0 : FEProblemBase::setCurrentAlgebraicElementRange(ConstElemRange * range)
9659 : {
9660 0 : if (!range)
9661 : {
9662 0 : _current_algebraic_elem_range = nullptr;
9663 0 : return;
9664 : }
9665 :
9666 0 : _current_algebraic_elem_range = std::make_unique<ConstElemRange>(*range);
9667 : }
9668 : void
9669 0 : FEProblemBase::setCurrentAlgebraicNodeRange(ConstNodeRange * range)
9670 : {
9671 0 : if (!range)
9672 : {
9673 0 : _current_algebraic_node_range = nullptr;
9674 0 : return;
9675 : }
9676 :
9677 0 : _current_algebraic_node_range = std::make_unique<ConstNodeRange>(*range);
9678 : }
9679 : void
9680 0 : FEProblemBase::setCurrentAlgebraicBndNodeRange(ConstBndNodeRange * range)
9681 : {
9682 0 : if (!range)
9683 : {
9684 0 : _current_algebraic_bnd_node_range = nullptr;
9685 0 : return;
9686 : }
9687 :
9688 0 : _current_algebraic_bnd_node_range = std::make_unique<ConstBndNodeRange>(*range);
9689 : }
9690 :
9691 : unsigned short
9692 66032 : FEProblemBase::getCurrentICState()
9693 : {
9694 66032 : return _current_ic_state;
9695 : }
9696 :
9697 : std::string
9698 56172 : FEProblemBase::solverTypeString(const unsigned int solver_sys_num)
9699 : {
9700 56172 : return Moose::stringify(solverParams(solver_sys_num)._type);
9701 : }
9702 :
9703 : SolverParams
9704 1340 : FEProblemBase::makeLinearSolverParams()
9705 : {
9706 1340 : SolverParams solver_params;
9707 1340 : solver_params._type = Moose::SolveType::ST_LINEAR;
9708 1340 : solver_params._line_search = Moose::LineSearchType::LS_NONE;
9709 1340 : return solver_params;
9710 : }
9711 :
9712 : const libMesh::CouplingMatrix &
9713 73935 : FEProblemBase::nonlocalCouplingMatrix(const unsigned i) const
9714 : {
9715 73935 : return _nonlocal_cm[i];
9716 : }
9717 :
9718 : bool
9719 105852679 : FEProblemBase::checkNonlocalCouplingRequirement() const
9720 : {
9721 105852679 : return _requires_nonlocal_coupling;
9722 : }
9723 :
9724 : const std::unordered_map<std::pair<BoundaryID, BoundaryID>,
9725 : std::unique_ptr<AutomaticMortarGeneration>> &
9726 121640 : FEProblemBase::getMortarInterfaces(bool on_displaced) const
9727 : {
9728 121640 : return _mortar_data->getMortarInterfaces(on_displaced);
9729 : }
|
