Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://www.mooseframework.org
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "KokkosTypes.h"
11 :
12 : #include "FEProblemBase.h"
13 : #include "NonlinearSystemBase.h"
14 : #include "AuxiliarySystem.h"
15 : #include "MaterialBase.h"
16 :
17 : void
18 1299 : FEProblemBase::addKokkosKernel(const std::string & kernel_name,
19 : const std::string & name,
20 : InputParameters & parameters)
21 : {
22 : parallel_object_only();
23 :
24 2598 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
25 1299 : if (!isSolverSystemNonlinear(nl_sys_num))
26 0 : mooseError("You are trying to add a Kernel to a linear variable/system, which is not "
27 : "supported at the moment!");
28 :
29 1604 : setResidualObjectParamsAndLog(
30 994 : kernel_name, name, parameters, nl_sys_num, "KokkosKernel", _reinit_displaced_elem);
31 :
32 1299 : _nl[nl_sys_num]->addKokkosKernel(kernel_name, name, parameters);
33 :
34 1299 : _has_kokkos_objects = true;
35 1299 : }
36 :
37 : void
38 272 : FEProblemBase::addKokkosNodalKernel(const std::string & kernel_name,
39 : const std::string & name,
40 : InputParameters & parameters)
41 : {
42 : parallel_object_only();
43 :
44 544 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
45 :
46 329 : setResidualObjectParamsAndLog(
47 215 : kernel_name, name, parameters, nl_sys_num, "KokkosNodalKernel", _reinit_displaced_elem);
48 :
49 272 : _nl[nl_sys_num]->addKokkosNodalKernel(kernel_name, name, parameters);
50 :
51 272 : _has_kokkos_objects = true;
52 272 : }
53 :
54 : void
55 1402 : FEProblemBase::addKokkosBoundaryCondition(const std::string & bc_name,
56 : const std::string & name,
57 : InputParameters & parameters)
58 : {
59 : parallel_object_only();
60 :
61 2804 : const auto nl_sys_num = determineSolverSystem(parameters.varName("variable", name), true).second;
62 1402 : if (!isSolverSystemNonlinear(nl_sys_num))
63 0 : mooseError(
64 : "You are trying to add a BoundaryCondition to a linear variable/system, which is not "
65 : "supported at the moment!");
66 :
67 1735 : setResidualObjectParamsAndLog(
68 1069 : bc_name, name, parameters, nl_sys_num, "KokkosBoundaryCondition", _reinit_displaced_face);
69 :
70 1402 : _nl[nl_sys_num]->addKokkosBoundaryCondition(bc_name, name, parameters);
71 :
72 1402 : _has_kokkos_objects = true;
73 1402 : }
74 :
75 : void
76 294 : FEProblemBase::addKokkosMaterial(const std::string & mat_name,
77 : const std::string & name,
78 : InputParameters & parameters)
79 : {
80 588 : addMaterialHelper({&_kokkos_materials}, mat_name, name, parameters);
81 288 : }
82 :
83 : MaterialData &
84 7601 : FEProblemBase::getKokkosMaterialData(Moose::MaterialDataType type, const MooseObject * object) const
85 : {
86 7601 : switch (type)
87 : {
88 4139 : case Moose::BLOCK_MATERIAL_DATA:
89 4139 : if (object)
90 1616 : _kokkos_material_props.addConsumer(type, object);
91 4139 : return _kokkos_material_props.getMaterialData(0);
92 311 : case Moose::NEIGHBOR_MATERIAL_DATA:
93 311 : if (object)
94 311 : _kokkos_neighbor_material_props.addConsumer(type, object);
95 311 : return _kokkos_neighbor_material_props.getMaterialData(0);
96 3151 : case Moose::BOUNDARY_MATERIAL_DATA:
97 : case Moose::FACE_MATERIAL_DATA:
98 : case Moose::INTERFACE_MATERIAL_DATA:
99 3151 : if (object)
100 525 : _kokkos_bnd_material_props.addConsumer(type, object);
101 3151 : return _kokkos_bnd_material_props.getMaterialData(0);
102 : }
103 :
104 0 : mooseError("FEProblemBase::getKokkosMaterialData(): Invalid MaterialDataType ", type);
105 : }
106 :
107 : const std::set<const MooseObject *> &
108 693 : FEProblemBase::getKokkosMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const
109 : {
110 693 : switch (type)
111 : {
112 0 : case Moose::BLOCK_MATERIAL_DATA:
113 0 : return _kokkos_material_props.getConsumers(type);
114 0 : case Moose::NEIGHBOR_MATERIAL_DATA:
115 0 : return _kokkos_neighbor_material_props.getConsumers(type);
116 693 : case Moose::BOUNDARY_MATERIAL_DATA:
117 : case Moose::FACE_MATERIAL_DATA:
118 : case Moose::INTERFACE_MATERIAL_DATA:
119 693 : return _kokkos_bnd_material_props.getConsumers(type);
120 : }
121 :
122 0 : mooseError(
123 : "FEProblemBase::getKokkosMaterialPropertyStorageConsumers(): Invalid MaterialDataType ",
124 : type);
125 : }
126 :
127 : void
128 693 : FEProblemBase::initKokkos()
129 : {
130 : // Error on unsupported options
131 :
132 693 : if (haveDisplaced())
133 0 : mooseError("Kokkos does not support displaced mesh yet.");
134 :
135 693 : if (adaptivity().isOn())
136 0 : mooseError("Kokkos does not support adaptivity yet.");
137 :
138 : // Initialize Kokkos assembly
139 :
140 693 : _kokkos_assembly.init();
141 :
142 : // Initialize Kokkos systems
143 :
144 693 : unsigned int max_system_number = 0;
145 :
146 1386 : for (unsigned int s = 0; s < numNonlinearSystems(); ++s)
147 693 : max_system_number = std::max(max_system_number, getNonlinearSystemBase(s).number());
148 :
149 693 : max_system_number = std::max(max_system_number, getAuxiliarySystem().number());
150 :
151 : // Kokkos system does not have a default constructor, so this simply allocates buffer to which we
152 : // should do placement new later
153 693 : _kokkos_systems.create(max_system_number + 1);
154 :
155 1386 : for (unsigned int s = 0; s < numNonlinearSystems(); ++s)
156 : {
157 693 : auto & nl = getNonlinearSystemBase(s);
158 693 : new (&_kokkos_systems[nl.number()]) Moose::Kokkos::System(nl);
159 : }
160 :
161 693 : auto & aux = getAuxiliarySystem();
162 693 : new (&_kokkos_systems[aux.number()]) Moose::Kokkos::System(aux);
163 :
164 : // Initialize Kokkos material properties
165 :
166 693 : _kokkos_material_props.allocateKokkosProperties();
167 693 : _kokkos_bnd_material_props.allocateKokkosProperties();
168 693 : _kokkos_neighbor_material_props.allocateKokkosProperties();
169 693 : }
170 :
171 : void
172 200 : FEProblemBase::initKokkosStatefulProps()
173 : {
174 : // Resolve dependencies
175 :
176 200 : std::set<MooseVariableFieldBase *> needed_moose_vars;
177 200 : std::set<TagID> needed_fe_var_vector_tags;
178 :
179 412 : for (auto block : _mesh.meshSubdomains())
180 : {
181 212 : _kokkos_materials.updateBlockVariableDependency(block, needed_moose_vars);
182 212 : _kokkos_materials.updateBlockFEVariableCoupledVectorTagDependency(block,
183 : needed_fe_var_vector_tags);
184 : }
185 :
186 : // Copy data and preallocate quadature point solution vectors
187 :
188 600 : for (auto & system : _kokkos_systems)
189 : {
190 400 : system.setActiveVariables(needed_moose_vars);
191 400 : system.setActiveVariableTags(needed_fe_var_vector_tags);
192 :
193 400 : system.sync(Moose::Kokkos::MemcpyKind::HOST_TO_DEVICE);
194 400 : system.reinit();
195 : }
196 :
197 200 : _kokkos_systems.copyToDevice();
198 :
199 : // Initialize stateful properties
200 :
201 460 : for (auto & material : _kokkos_materials.getActiveObjects())
202 260 : if (!material->hasRestoredProperties() && !material->boundaryRestricted())
203 216 : material->initStatefulProperties(0);
204 :
205 448 : for (auto & material : _kokkos_materials[Moose::FACE_MATERIAL_DATA].getActiveObjects())
206 248 : if (!material->hasRestoredProperties())
207 214 : material->initStatefulProperties(0);
208 :
209 448 : for (auto & material : _kokkos_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveObjects())
210 248 : if (!material->hasRestoredProperties())
211 216 : material->initStatefulProperties(0);
212 :
213 460 : for (auto & material : _kokkos_materials.getActiveObjects())
214 260 : if (!material->hasRestoredProperties() && material->boundaryRestricted())
215 10 : material->initStatefulProperties(0);
216 :
217 : // Copy to old and older states
218 :
219 200 : _kokkos_material_props.copy();
220 200 : _kokkos_bnd_material_props.copy();
221 200 : _kokkos_neighbor_material_props.copy();
222 :
223 : // Clear
224 :
225 600 : for (auto & system : _kokkos_systems)
226 : {
227 400 : system.sync(Moose::Kokkos::MemcpyKind::DEVICE_TO_HOST);
228 :
229 400 : system.clearActiveVariables();
230 400 : system.clearActiveVariableTags();
231 : }
232 200 : }
233 :
234 : void
235 31587 : FEProblemBase::prepareKokkosMaterials(
236 : const std::unordered_set<unsigned int> & consumer_needed_mat_props)
237 : {
238 31587 : std::unordered_set<unsigned int> needed_mat_props;
239 :
240 65068 : for (auto block : mesh().meshSubdomains())
241 33481 : _kokkos_materials.updateBlockMatPropDependency(block, needed_mat_props);
242 :
243 145150 : for (auto boundary : mesh().meshBoundaryIds())
244 113563 : _kokkos_materials.updateBoundaryMatPropDependency(boundary, needed_mat_props);
245 :
246 31587 : needed_mat_props.insert(consumer_needed_mat_props.begin(), consumer_needed_mat_props.end());
247 :
248 31587 : setActiveMaterialProperties(needed_mat_props, 0);
249 31587 : }
250 :
251 : void
252 31587 : FEProblemBase::reinitKokkosMaterials()
253 : {
254 31587 : if (hasActiveMaterialProperties(0))
255 : {
256 22011 : for (auto & material : _kokkos_materials.getActiveObjects())
257 12677 : if (!material->boundaryRestricted())
258 11806 : material->computeProperties();
259 :
260 21140 : for (auto & material : _kokkos_materials[Moose::FACE_MATERIAL_DATA].getActiveObjects())
261 11806 : material->computeProperties();
262 :
263 21140 : for (auto & material : _kokkos_materials[Moose::NEIGHBOR_MATERIAL_DATA].getActiveObjects())
264 11806 : material->computeProperties();
265 :
266 22011 : for (auto & material : _kokkos_materials.getActiveObjects())
267 12677 : if (material->boundaryRestricted())
268 871 : material->computeProperties();
269 : }
270 :
271 31587 : Kokkos::fence();
272 31587 : }
|
