Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://www.mooseframework.org
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "KokkosTypes.h"
11 :
12 : #include "FEProblemBase.h"
13 : #include "DisplacedProblem.h"
14 : #include "NonlinearSystemBase.h"
15 : #include "AuxiliarySystem.h"
16 : #include "MaterialBase.h"
17 :
18 : Moose::Kokkos::System &
19 0 : FEProblemBase::getKokkosSystem(const unsigned int sys_num)
20 : {
21 0 : return _kokkos_systems[sys_num];
22 : }
23 :
24 : const Moose::Kokkos::System &
25 0 : FEProblemBase::getKokkosSystem(const unsigned int sys_num) const
26 : {
27 0 : return _kokkos_systems[sys_num];
28 : }
29 :
30 : void
31 1335 : FEProblemBase::addKokkosKernel(const std::string & kernel_name,
32 : const std::string & name,
33 : InputParameters & parameters)
34 : {
35 : parallel_object_only();
36 :
37 2670 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
38 1335 : if (!isSolverSystemNonlinear(nl_sys_num))
39 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
40 : "supported at the moment!");
41 :
42 1649 : setResidualObjectParamsAndLog(
43 1021 : kernel_name, name, parameters, nl_sys_num, "KokkosKernel", _reinit_displaced_elem);
44 :
45 1335 : _nl[nl_sys_num]->addKokkosKernel(kernel_name, name, parameters);
46 :
47 1335 : _has_kokkos_objects = true;
48 1335 : _has_kokkos_residual_objects = true;
49 1335 : }
50 :
51 : void
52 272 : FEProblemBase::addKokkosNodalKernel(const std::string & kernel_name,
53 : const std::string & name,
54 : InputParameters & parameters)
55 : {
56 : parallel_object_only();
57 :
58 544 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
59 :
60 329 : setResidualObjectParamsAndLog(
61 215 : kernel_name, name, parameters, nl_sys_num, "KokkosNodalKernel", _reinit_displaced_elem);
62 :
63 272 : _nl[nl_sys_num]->addKokkosNodalKernel(kernel_name, name, parameters);
64 :
65 272 : _has_kokkos_objects = true;
66 272 : _has_kokkos_residual_objects = true;
67 272 : }
68 :
69 : void
70 1426 : FEProblemBase::addKokkosBoundaryCondition(const std::string & bc_name,
71 : const std::string & name,
72 : InputParameters & parameters)
73 : {
74 : parallel_object_only();
75 :
76 2852 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
77 1426 : if (!isSolverSystemNonlinear(nl_sys_num))
78 0 : mooseError(
79 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
80 : "supported at the moment!");
81 :
82 1765 : setResidualObjectParamsAndLog(
83 1087 : bc_name, name, parameters, nl_sys_num, "KokkosBoundaryCondition", _reinit_displaced_face);
84 :
85 1426 : _nl[nl_sys_num]->addKokkosBoundaryCondition(bc_name, name, parameters);
86 :
87 1426 : _has_kokkos_objects = true;
88 1426 : _has_kokkos_residual_objects = true;
89 1426 : }
90 :
91 : void
92 294 : FEProblemBase::addKokkosMaterial(const std::string & mat_name,
93 : const std::string & name,
94 : InputParameters & parameters)
95 : {
96 588 : addMaterialHelper({&_kokkos_materials}, mat_name, name, parameters);
97 :
98 288 : _has_kokkos_objects = true;
99 288 : }
100 :
101 : MaterialData &
102 8537 : FEProblemBase::getKokkosMaterialData(Moose::MaterialDataType type, const MooseObject * object) const
103 : {
104 8537 : switch (type)
105 : {
106 4759 : case Moose::BLOCK_MATERIAL_DATA:
107 4759 : if (object)
108 1920 : _kokkos_material_props.addConsumer(type, object);
109 4759 : return _kokkos_material_props.getMaterialData(0);
110 311 : case Moose::NEIGHBOR_MATERIAL_DATA:
111 311 : if (object)
112 311 : _kokkos_neighbor_material_props.addConsumer(type, object);
113 311 : return _kokkos_neighbor_material_props.getMaterialData(0);
114 3467 : case Moose::BOUNDARY_MATERIAL_DATA:
115 : case Moose::FACE_MATERIAL_DATA:
116 : case Moose::INTERFACE_MATERIAL_DATA:
117 3467 : if (object)
118 537 : _kokkos_bnd_material_props.addConsumer(type, object);
119 3467 : return _kokkos_bnd_material_props.getMaterialData(0);
120 : }
121 :
122 0 : mooseError("FEProblemBase::getKokkosMaterialData(): Invalid MaterialDataType ", type);
123 : }
124 :
125 : void
126 280 : FEProblemBase::addKokkosAuxKernel(const std::string & kernel_name,
127 : const std::string & name,
128 : InputParameters & parameters)
129 : {
130 : parallel_object_only();
131 :
132 280 : setAuxKernelParamsAndLog(kernel_name, name, parameters, "KokkosAuxKernel");
133 :
134 280 : _aux->addKokkosKernel(kernel_name, name, parameters);
135 :
136 280 : _has_kokkos_objects = true;
137 280 : }
138 :
139 : const std::set<const MooseObject *> &
140 741 : FEProblemBase::getKokkosMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
141 : {
142 741 : switch (type)
143 : {
144 0 : case Moose::BLOCK_MATERIAL_DATA:
145 0 : return _kokkos_material_props.getConsumers(type);
146 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
147 0 : return _kokkos_neighbor_material_props.getConsumers(type);
148 741 : case Moose::BOUNDARY_MATERIAL_DATA:
149 : case Moose::FACE_MATERIAL_DATA:
150 : case Moose::INTERFACE_MATERIAL_DATA:
151 741 : return _kokkos_bnd_material_props.getConsumers(type);
152 : }
153 :
154 0 : mooseError(
155 : "FEProblemBase::getKokkosMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
156 : type);
157 : }
158 :
159 : void
160 741 : FEProblemBase::initKokkos()
161 : {
162 : // Error on unsupported options
163 :
164 741 : if (haveDisplaced())
165 0 : mooseError("Kokkos does not support displaced mesh yet.");
166 :
167 741 : if (adaptivity().isOn())
168 0 : mooseError("Kokkos does not support adaptivity yet.");
169 :
170 : // Initialize Kokkos assembly
171 :
172 741 : _kokkos_assembly.init();
173 :
174 : // Initialize Kokkos systems
175 :
176 741 : unsigned int max_system_number = 0;
177 :
178 1482 : for (unsigned int s = 0; s < numNonlinearSystems(); ++s)
179 741 : max_system_number = std::max(max_system_number, getNonlinearSystemBase(s).number());
180 :
181 741 : max_system_number = std::max(max_system_number, getAuxiliarySystem().number());
182 :
183 : // Kokkos system does not have a default constructor, so this simply allocates buffer to which we
184 : // should do placement new later
185 741 : _kokkos_systems.create(max_system_number + 1);
186 :
187 1482 : for (unsigned int s = 0; s < numNonlinearSystems(); ++s)
188 : {
189 741 : auto & nl = getNonlinearSystemBase(s);
190 741 : new (&_kokkos_systems[nl.number()]) Moose::Kokkos::System(nl);
191 : }
192 :
193 741 : auto & aux = getAuxiliarySystem();
194 741 : new (&_kokkos_systems[aux.number()]) Moose::Kokkos::System(aux);
195 :
196 : // Initialize Kokkos material properties
197 :
198 741 : _kokkos_material_props.allocateKokkosProperties();
199 741 : _kokkos_bnd_material_props.allocateKokkosProperties();
200 741 : _kokkos_neighbor_material_props.allocateKokkosProperties();
201 741 : }
202 :
203 : void
204 310 : FEProblemBase::initKokkosStatefulProps()
205 : {
206 : // Resolve dependencies
207 :
208 310 : std::set<MooseVariableFieldBase *> needed_moose_vars;
209 310 : std::set<TagID> needed_fe_var_vector_tags;
210 :
211 632 : for (auto block : _mesh.meshSubdomains())
212 : {
213 322 : _kokkos_materials.updateBlockVariableDependency(block, needed_moose_vars);
214 322 : _kokkos_materials.updateBlockFEVariableCoupledVectorTagDependency(block,
215 : needed_fe_var_vector_tags);
216 : }
217 :
218 1484 : for (auto boundary : _mesh.meshBoundaryIds())
219 : {
220 1174 : _kokkos_materials.updateBoundaryVariableDependency(boundary, needed_moose_vars);
221 1174 : _kokkos_materials.updateBoundaryFEVariableCoupledVectorTagDependency(boundary,
222 : needed_fe_var_vector_tags);
223 : }
224 :
225 : // Copy data and preallocate quadature point solution vectors
226 :
227 930 : for (auto & system : _kokkos_systems)
228 : {
229 620 : system.setActiveVariables(needed_moose_vars);
230 620 : system.setActiveSolutionTags(needed_fe_var_vector_tags);
231 :
232 620 : system.sync(Moose::Kokkos::MemcpyKind::HOST_TO_DEVICE);
233 620 : system.reinit();
234 : }
235 :
236 310 : _kokkos_systems.copyToDevice();
237 :
238 : // Initialize stateful properties
239 :
240 720 : for (auto & material : _kokkos_materials.getActiveObjects())
241 410 : if (!material->hasRestoredProperties() && !material->boundaryRestricted())
242 356 : material->initStatefulProperties(0);
243 :
244 698 : for (auto & material : _kokkos_materials[Moose::FACE_MATERIAL_DATA].getActiveObjects())
245 388 : if (!material->hasRestoredProperties())
246 354 : material->initStatefulProperties(0);
247 :
248 698 : for (auto & material : _kokkos_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveObjects())
249 388 : if (!material->hasRestoredProperties())
250 356 : material->initStatefulProperties(0);
251 :
252 720 : for (auto & material : _kokkos_materials.getActiveObjects())
253 410 : if (!material->hasRestoredProperties() && material->boundaryRestricted())
254 20 : material->initStatefulProperties(0);
255 :
256 : // Copy to old and older states
257 :
258 310 : _kokkos_material_props.copy();
259 310 : _kokkos_bnd_material_props.copy();
260 310 : _kokkos_neighbor_material_props.copy();
261 :
262 : // Clear
263 :
264 930 : for (auto & system : _kokkos_systems)
265 : {
266 620 : system.sync(Moose::Kokkos::MemcpyKind::DEVICE_TO_HOST);
267 :
268 620 : system.clearActiveVariables();
269 620 : system.clearActiveSolutionTags();
270 : }
271 310 : }
272 :
273 : void
274 38387 : FEProblemBase::prepareKokkosMaterials(
275 : const std::unordered_set<unsigned int> & consumer_needed_mat_props)
276 : {
277 38387 : std::unordered_set<unsigned int> needed_mat_props;
278 :
279 78664 : for (auto block : mesh().meshSubdomains())
280 40277 : _kokkos_materials.updateBlockMatPropDependency(block, needed_mat_props);
281 :
282 177560 : for (auto boundary : mesh().meshBoundaryIds())
283 139173 : _kokkos_materials.updateBoundaryMatPropDependency(boundary, needed_mat_props);
284 :
285 38387 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
286 :
287 38387 : setActiveMaterialProperties(needed_mat_props, 0);
288 38387 : }
289 :
290 : void
291 38387 : FEProblemBase::reinitKokkosMaterials()
292 : {
293 38387 : if (hasActiveMaterialProperties(0))
294 : {
295 29815 : for (auto & material : _kokkos_materials.getActiveObjects())
296 16579 : if (!material->boundaryRestricted())
297 15708 : material->computeProperties();
298 :
299 28944 : for (auto & material : _kokkos_materials[Moose::FACE_MATERIAL_DATA].getActiveObjects())
300 15708 : material->computeProperties();
301 :
302 28944 : for (auto & material : _kokkos_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveObjects())
303 15708 : material->computeProperties();
304 :
305 29815 : for (auto & material : _kokkos_materials.getActiveObjects())
306 16579 : if (material->boundaryRestricted())
307 871 : material->computeProperties();
308 : }
309 :
310 38387 : Kokkos::fence();
311 38387 : }
|
