Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://www.mooseframework.org
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "KokkosTypes.h"
11 :
12 : #include "FEProblemBase.h"
13 : #include "NonlinearSystemBase.h"
14 : #include "AuxiliarySystem.h"
15 : #include "MaterialBase.h"
16 :
17 : Moose::Kokkos::System &
18 0 : FEProblemBase::getKokkosSystem(const unsigned int sys_num)
19 : {
20 0 : return _kokkos_systems[sys_num];
21 : }
22 :
23 : const Moose::Kokkos::System &
24 0 : FEProblemBase::getKokkosSystem(const unsigned int sys_num) const
25 : {
26 0 : return _kokkos_systems[sys_num];
27 : }
28 :
29 : void
30 1299 : FEProblemBase::addKokkosKernel(const std::string & kernel_name,
31 : const std::string & name,
32 : InputParameters & parameters)
33 : {
34 : parallel_object_only();
35 :
36 2598 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
37 1299 : if (!isSolverSystemNonlinear(nl_sys_num))
38 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
39 : "supported at the moment!");
40 :
41 1604 : setResidualObjectParamsAndLog(
42 994 : kernel_name, name, parameters, nl_sys_num, "KokkosKernel", _reinit_displaced_elem);
43 :
44 1299 : _nl[nl_sys_num]->addKokkosKernel(kernel_name, name, parameters);
45 :
46 1299 : _has_kokkos_objects = true;
47 1299 : }
48 :
49 : void
50 272 : FEProblemBase::addKokkosNodalKernel(const std::string & kernel_name,
51 : const std::string & name,
52 : InputParameters & parameters)
53 : {
54 : parallel_object_only();
55 :
56 544 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
57 :
58 329 : setResidualObjectParamsAndLog(
59 215 : kernel_name, name, parameters, nl_sys_num, "KokkosNodalKernel", _reinit_displaced_elem);
60 :
61 272 : _nl[nl_sys_num]->addKokkosNodalKernel(kernel_name, name, parameters);
62 :
63 272 : _has_kokkos_objects = true;
64 272 : }
65 :
66 : void
67 1402 : FEProblemBase::addKokkosBoundaryCondition(const std::string & bc_name,
68 : const std::string & name,
69 : InputParameters & parameters)
70 : {
71 : parallel_object_only();
72 :
73 2804 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
74 1402 : if (!isSolverSystemNonlinear(nl_sys_num))
75 0 : mooseError(
76 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
77 : "supported at the moment!");
78 :
79 1735 : setResidualObjectParamsAndLog(
80 1069 : bc_name, name, parameters, nl_sys_num, "KokkosBoundaryCondition", _reinit_displaced_face);
81 :
82 1402 : _nl[nl_sys_num]->addKokkosBoundaryCondition(bc_name, name, parameters);
83 :
84 1402 : _has_kokkos_objects = true;
85 1402 : }
86 :
87 : void
88 294 : FEProblemBase::addKokkosMaterial(const std::string & mat_name,
89 : const std::string & name,
90 : InputParameters & parameters)
91 : {
92 588 : addMaterialHelper({&_kokkos_materials}, mat_name, name, parameters);
93 288 : }
94 :
95 : MaterialData &
96 7601 : FEProblemBase::getKokkosMaterialData(Moose::MaterialDataType type, const MooseObject * object) const
97 : {
98 7601 : switch (type)
99 : {
100 4139 : case Moose::BLOCK_MATERIAL_DATA:
101 4139 : if (object)
102 1616 : _kokkos_material_props.addConsumer(type, object);
103 4139 : return _kokkos_material_props.getMaterialData(0);
104 311 : case Moose::NEIGHBOR_MATERIAL_DATA:
105 311 : if (object)
106 311 : _kokkos_neighbor_material_props.addConsumer(type, object);
107 311 : return _kokkos_neighbor_material_props.getMaterialData(0);
108 3151 : case Moose::BOUNDARY_MATERIAL_DATA:
109 : case Moose::FACE_MATERIAL_DATA:
110 : case Moose::INTERFACE_MATERIAL_DATA:
111 3151 : if (object)
112 525 : _kokkos_bnd_material_props.addConsumer(type, object);
113 3151 : return _kokkos_bnd_material_props.getMaterialData(0);
114 : }
115 :
116 0 : mooseError("FEProblemBase::getKokkosMaterialData(): Invalid MaterialDataType ", type);
117 : }
118 :
119 : const std::set<const MooseObject *> &
120 693 : FEProblemBase::getKokkosMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
121 : {
122 693 : switch (type)
123 : {
124 0 : case Moose::BLOCK_MATERIAL_DATA:
125 0 : return _kokkos_material_props.getConsumers(type);
126 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
127 0 : return _kokkos_neighbor_material_props.getConsumers(type);
128 693 : case Moose::BOUNDARY_MATERIAL_DATA:
129 : case Moose::FACE_MATERIAL_DATA:
130 : case Moose::INTERFACE_MATERIAL_DATA:
131 693 : return _kokkos_bnd_material_props.getConsumers(type);
132 : }
133 :
134 0 : mooseError(
135 : "FEProblemBase::getKokkosMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
136 : type);
137 : }
138 :
139 : void
140 693 : FEProblemBase::initKokkos()
141 : {
142 : // Error on unsupported options
143 :
144 693 : if (haveDisplaced())
145 0 : mooseError("Kokkos does not support displaced mesh yet.");
146 :
147 693 : if (adaptivity().isOn())
148 0 : mooseError("Kokkos does not support adaptivity yet.");
149 :
150 : // Initialize Kokkos assembly
151 :
152 693 : _kokkos_assembly.init();
153 :
154 : // Initialize Kokkos systems
155 :
156 693 : unsigned int max_system_number = 0;
157 :
158 1386 : for (unsigned int s = 0; s < numNonlinearSystems(); ++s)
159 693 : max_system_number = std::max(max_system_number, getNonlinearSystemBase(s).number());
160 :
161 693 : max_system_number = std::max(max_system_number, getAuxiliarySystem().number());
162 :
163 : // Kokkos system does not have a default constructor, so this simply allocates buffer to which we
164 : // should do placement new later
165 693 : _kokkos_systems.create(max_system_number + 1);
166 :
167 1386 : for (unsigned int s = 0; s < numNonlinearSystems(); ++s)
168 : {
169 693 : auto & nl = getNonlinearSystemBase(s);
170 693 : new (&_kokkos_systems[nl.number()]) Moose::Kokkos::System(nl);
171 : }
172 :
173 693 : auto & aux = getAuxiliarySystem();
174 693 : new (&_kokkos_systems[aux.number()]) Moose::Kokkos::System(aux);
175 :
176 : // Initialize Kokkos material properties
177 :
178 693 : _kokkos_material_props.allocateKokkosProperties();
179 693 : _kokkos_bnd_material_props.allocateKokkosProperties();
180 693 : _kokkos_neighbor_material_props.allocateKokkosProperties();
181 693 : }
182 :
183 : void
184 200 : FEProblemBase::initKokkosStatefulProps()
185 : {
186 : // Resolve dependencies
187 :
188 200 : std::set<MooseVariableFieldBase *> needed_moose_vars;
189 200 : std::set<TagID> needed_fe_var_vector_tags;
190 :
191 412 : for (auto block : _mesh.meshSubdomains())
192 : {
193 212 : _kokkos_materials.updateBlockVariableDependency(block, needed_moose_vars);
194 212 : _kokkos_materials.updateBlockFEVariableCoupledVectorTagDependency(block,
195 : needed_fe_var_vector_tags);
196 : }
197 :
198 : // Copy data and preallocate quadature point solution vectors
199 :
200 600 : for (auto & system : _kokkos_systems)
201 : {
202 400 : system.setActiveVariables(needed_moose_vars);
203 400 : system.setActiveVariableTags(needed_fe_var_vector_tags);
204 :
205 400 : system.sync(Moose::Kokkos::MemcpyKind::HOST_TO_DEVICE);
206 400 : system.reinit();
207 : }
208 :
209 200 : _kokkos_systems.copyToDevice();
210 :
211 : // Initialize stateful properties
212 :
213 460 : for (auto & material : _kokkos_materials.getActiveObjects())
214 260 : if (!material->hasRestoredProperties() && !material->boundaryRestricted())
215 216 : material->initStatefulProperties(0);
216 :
217 448 : for (auto & material : _kokkos_materials[Moose::FACE_MATERIAL_DATA].getActiveObjects())
218 248 : if (!material->hasRestoredProperties())
219 214 : material->initStatefulProperties(0);
220 :
221 448 : for (auto & material : _kokkos_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveObjects())
222 248 : if (!material->hasRestoredProperties())
223 216 : material->initStatefulProperties(0);
224 :
225 460 : for (auto & material : _kokkos_materials.getActiveObjects())
226 260 : if (!material->hasRestoredProperties() && material->boundaryRestricted())
227 10 : material->initStatefulProperties(0);
228 :
229 : // Copy to old and older states
230 :
231 200 : _kokkos_material_props.copy();
232 200 : _kokkos_bnd_material_props.copy();
233 200 : _kokkos_neighbor_material_props.copy();
234 :
235 : // Clear
236 :
237 600 : for (auto & system : _kokkos_systems)
238 : {
239 400 : system.sync(Moose::Kokkos::MemcpyKind::DEVICE_TO_HOST);
240 :
241 400 : system.clearActiveVariables();
242 400 : system.clearActiveVariableTags();
243 : }
244 200 : }
245 :
246 : void
247 31587 : FEProblemBase::prepareKokkosMaterials(
248 : const std::unordered_set<unsigned int> & consumer_needed_mat_props)
249 : {
250 31587 : std::unordered_set<unsigned int> needed_mat_props;
251 :
252 65068 : for (auto block : mesh().meshSubdomains())
253 33481 : _kokkos_materials.updateBlockMatPropDependency(block, needed_mat_props);
254 :
255 145150 : for (auto boundary : mesh().meshBoundaryIds())
256 113563 : _kokkos_materials.updateBoundaryMatPropDependency(boundary, needed_mat_props);
257 :
258 31587 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
259 :
260 31587 : setActiveMaterialProperties(needed_mat_props, 0);
261 31587 : }
262 :
263 : void
264 31587 : FEProblemBase::reinitKokkosMaterials()
265 : {
266 31587 : if (hasActiveMaterialProperties(0))
267 : {
268 22011 : for (auto & material : _kokkos_materials.getActiveObjects())
269 12677 : if (!material->boundaryRestricted())
270 11806 : material->computeProperties();
271 :
272 21140 : for (auto & material : _kokkos_materials[Moose::FACE_MATERIAL_DATA].getActiveObjects())
273 11806 : material->computeProperties();
274 :
275 21140 : for (auto & material : _kokkos_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveObjects())
276 11806 : material->computeProperties();
277 :
278 22011 : for (auto & material : _kokkos_materials.getActiveObjects())
279 12677 : if (material->boundaryRestricted())
280 871 : material->computeProperties();
281 : }
282 :
283 31587 : Kokkos::fence();
284 31587 : }
|
