https://mooseframework.inl.gov
KokkosAssembly.h
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://www.mooseframework.org
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
10 #pragma once
11 
12 #include "KokkosTypes.h"
13 
14 #include "MooseMesh.h"
15 
16 #include "libmesh/elem_range.h"
17 #include "libmesh/fe_base.h"
18 #include "libmesh/fe_type.h"
19 
20 class FEProblemBase;
21 
22 namespace Moose
23 {
24 namespace Kokkos
25 {
26 
30 class Assembly : public MeshHolder
31 {
32 public:
37  Assembly(FEProblemBase & problem);
41  void init();
42 
43 #ifdef MOOSE_KOKKOS_SCOPE
44 
49  unsigned int getFETypeID(FEType type) const { return libmesh_map_find(_fe_type_map, type); }
54  KOKKOS_FUNCTION unsigned int getDimension() const { return _dimension; }
59  KOKKOS_FUNCTION unsigned int getMaxQpsPerElem() const { return _max_qps_per_elem; }
65  KOKKOS_FUNCTION dof_id_type getNumQps(ContiguousSubdomainID subdomain) const
66  {
67  return _n_subdomain_qps[subdomain];
68  }
74  KOKKOS_FUNCTION unsigned int getNumQps(ElementInfo info) const { return _n_qps[info.id]; }
82  KOKKOS_FUNCTION dof_id_type getNumFaceQps(ContiguousSubdomainID subdomain) const
83  {
84  return _n_subdomain_qps_face[subdomain];
85  }
92  KOKKOS_FUNCTION unsigned int getNumFaceQps(ElementInfo info, unsigned int side) const
93  {
94  return _n_qps_face(side, info.id);
95  }
102  KOKKOS_FUNCTION dof_id_type getQpOffset(ElementInfo info) const { return _qp_offset[info.id]; }
110  KOKKOS_FUNCTION dof_id_type getQpFaceOffset(ElementInfo info, unsigned int side) const
111  {
112  return _qp_offset_face(side, info.id);
113  }
120  KOKKOS_FUNCTION unsigned int getNumDofs(unsigned int elem_type, unsigned int fe_type) const
121  {
122  return _n_dofs(elem_type, fe_type);
123  }
131  KOKKOS_FUNCTION const auto &
132  getPhi(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
133  {
134  return _phi(subdomain, elem_type, fe_type);
135  }
143  KOKKOS_FUNCTION const auto &
144  getPhiFace(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
145  {
146  return _phi_face(subdomain, elem_type, fe_type);
147  }
155  KOKKOS_FUNCTION const auto &
156  getGradPhi(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
157  {
158  return _grad_phi(subdomain, elem_type, fe_type);
159  }
167  KOKKOS_FUNCTION const auto & getGradPhiFace(ContiguousSubdomainID subdomain,
168  unsigned int elem_type,
169  unsigned int fe_type) const
170  {
171  return _grad_phi_face(subdomain, elem_type, fe_type);
172  }
179  KOKKOS_FUNCTION Real33 getJacobian(ElementInfo info, unsigned int qp) const
180  {
181  return _jacobian[info.subdomain][getQpOffset(info) + qp];
182  }
189  KOKKOS_FUNCTION Real getJxW(ElementInfo info, unsigned int qp) const
190  {
191  return _jxw[info.subdomain][getQpOffset(info) + qp];
192  }
199  KOKKOS_FUNCTION Real3 getQPoint(ElementInfo info, unsigned int qp) const
200  {
201  return _xyz[info.subdomain][getQpOffset(info) + qp];
202  }
203 
210  KOKKOS_FUNCTION Real coordTransformFactor(const ContiguousSubdomainID subdomain,
211  const Real3 point) const;
220  KOKKOS_FUNCTION void computePhysicalMap(const ElementInfo info,
221  const unsigned int qp,
222  Real33 * const jacobian,
223  Real * const JxW,
224  Real3 * const q_points) const;
234  KOKKOS_FUNCTION void computePhysicalMap(const ElementInfo info,
235  const unsigned int side,
236  const unsigned int qp,
237  Real33 * const jacobian,
238  Real * const JxW,
239  Real3 * const q_points) const;
240 
244  KOKKOS_FUNCTION void operator()(const ThreadID tid) const;
245 
250  const auto & getMaterialBoundaries() const { return _material_boundaries; }
251 #endif
252 
253 private:
257  void initQuadrature();
261  void initShape();
265  void cachePhysicalMap();
266 
278  std::map<FEType, unsigned int> _fe_type_map;
279 
283  const unsigned int _dimension;
296 
304 
310 
311  unsigned int _max_qps_per_elem = 0;
312 
316 
325 
335 
346 
354 
358  std::set<BoundaryID> _material_boundaries;
359 };
360 
361 #ifdef MOOSE_KOKKOS_SCOPE
362 KOKKOS_FUNCTION inline Real
364 {
365  switch (_coord_type[subdomain])
366  {
367  case Moose::COORD_XYZ:
368  return 1;
369  case Moose::COORD_RZ:
371  return 2 * M_PI *
372  (point - _rz_axis[subdomain].first).cross_product(_rz_axis[subdomain].second).norm();
373  else
374  return 2 * M_PI * point(_rz_radial_coord);
376  return 4 * M_PI * point(0) * point(0);
377  default:
378  return 0;
379  }
380 }
381 
382 KOKKOS_FUNCTION inline void
384  const unsigned int qp,
385  Real33 * const jacobian,
386  Real * const JxW,
387  Real3 * const q_points) const
388 {
389  auto sid = info.subdomain;
390  auto eid = info.id;
391  auto elem_type = info.type;
392  auto num_nodes = kokkosMesh().getNumNodes(elem_type);
393 
394  auto & phi = _map_phi(sid, elem_type);
395  auto & grad_phi = _map_grad_phi(sid, elem_type);
396 
397  Real33 J;
398  Real3 xyz;
399 
400  for (unsigned int node = 0; node < num_nodes; ++node)
401  {
402  auto points = kokkosMesh().getNodePoint(kokkosMesh().getContiguousNodeID(eid, node));
403 
404  if (jacobian || JxW)
405  J += grad_phi(node, qp).cartesian_product(points);
406 
407  xyz += phi(node, qp) * points;
408  }
409 
410  if (jacobian)
411  *jacobian = J.inverse(_dimension);
412  if (JxW)
413  *JxW =
414  J.determinant(_dimension) * _weights(sid, elem_type)[qp] * coordTransformFactor(sid, xyz);
415  if (q_points)
416  *q_points = xyz;
417 }
418 
419 KOKKOS_FUNCTION inline void
421  const unsigned int side,
422  const unsigned int qp,
423  Real33 * const jacobian,
424  Real * const JxW,
425  Real3 * const q_points) const
426 {
427  auto sid = info.subdomain;
428  auto eid = info.id;
429  auto elem_type = info.type;
430  auto num_nodes = kokkosMesh().getNumNodes(elem_type);
431  auto num_side_nodes = kokkosMesh().getNumNodes(elem_type, side);
432 
433  auto & phi = _map_phi_face(sid, elem_type)(side);
434  auto & grad_phi = _map_grad_phi_face(sid, elem_type)(side);
435 
436  Real33 J;
437  Real3 xyz;
438 
439  for (unsigned int node = 0; node < num_nodes; ++node)
440  {
441  auto points = kokkosMesh().getNodePoint(kokkosMesh().getContiguousNodeID(eid, node));
442 
443  if (jacobian)
444  J += grad_phi(node, qp).cartesian_product(points);
445 
446  if (JxW || q_points)
447  xyz += phi(node, qp) * points;
448  }
449 
450  if (jacobian)
451  *jacobian = J.inverse(_dimension);
452  if (q_points)
453  *q_points = xyz;
454 
455  if (JxW)
456  {
457  Real33 J;
458 
459  auto & grad_psi = _map_grad_psi_face(sid, elem_type)(side);
460 
461  for (unsigned int node = 0; node < num_side_nodes; ++node)
462  {
463  auto points = kokkosMesh().getNodePoint(kokkosMesh().getContiguousNodeID(eid, side, node));
464 
465  J += grad_psi(node, qp).cartesian_product(points);
466  }
467 
468  *JxW = ::Kokkos::sqrt((J * J.transpose()).determinant(_dimension - 1)) *
469  _weights_face(sid, elem_type)[side][qp] * coordTransformFactor(sid, xyz);
470  }
471 }
472 #endif
473 
481 {
482 public:
487  AssemblyHolder(const Assembly & assembly) : _assembly_host(assembly), _assembly_device(assembly)
488  {
489  }
495  {
496  }
497 
498 #ifdef MOOSE_KOKKOS_SCOPE
499 
504  KOKKOS_FUNCTION const Assembly & kokkosAssembly() const
505  {
506  KOKKOS_IF_ON_HOST(return _assembly_host;)
507 
508  return _assembly_device;
509  }
510 #endif
511 
512 private:
521 };
522 
523 } // namespace Kokkos
524 } // namespace Moose
Array2D< Array< Real > > _weights
KOKKOS_FUNCTION const auto & getPhi(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
Get the shape functions of a FE type for an element type and subdomain.
std::map< FEType, unsigned int > _fe_type_map
FE type ID map.
AssemblyHolder(const AssemblyHolder &holder)
Copy constructor.
Array2D< Array< Array2D< Real3 > > > _map_grad_psi_face
unsigned int _rz_radial_coord
Radial coordinate index in cylindrical coordinate system.
Array2D< Array< Array2D< Real > > > _map_phi_face
The Kokkos assembly class.
The Kokkos object that contains the information of an element The IDs used in Kokkos are different fr...
Definition: KokkosMesh.h:35
Assembly(FEProblemBase &problem)
Constructor.
Array2D< unsigned int > _n_qps_face
const unsigned int invalid_uint
KOKKOS_FUNCTION dof_id_type getQpOffset(ElementInfo info) const
Get the starting offset of quadrature points of an element into the global quadrature point index...
KOKKOS_FUNCTION const Assembly & kokkosAssembly() const
Get the const reference of the Kokkos assembly.
FEProblemBase & _problem
Reference of the MOOSE problem.
KOKKOS_FUNCTION unsigned int getNumQps(ElementInfo info) const
Get the number of quadrature points of an element.
MPI_Info info
Array3D< Array< Array2D< Real > > > _phi_face
dof_id_type ThreadID
Definition: KokkosThread.h:18
Array< Array< Real > > _jxw
Array2D< dof_id_type > _qp_offset_face
Array< dof_id_type > _n_subdomain_qps_face
KOKKOS_FUNCTION unsigned int getDimension() const
Get the mesh dimension.
KOKKOS_FUNCTION void operator()(const ThreadID tid) const
Kokkos function for caching physical maps on element quadrature points.
const unsigned int _dimension
Mesh dimension.
KOKKOS_FUNCTION const auto & getGradPhiFace(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
Get the gradient of face shape functions of a FE type for an element type and subdomain.
KOKKOS_FUNCTION const auto & getGradPhi(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
Get the gradient of shape functions of a FE type for an element type and subdomain.
unsigned int getFETypeID(FEType type) const
Get the FE type ID.
KOKKOS_FUNCTION unsigned int getNumDofs(unsigned int elem_type, unsigned int fe_type) const
Get the number of DOFs of a FE type for an element type.
Array2D< Array< Array< Real > > > _weights_face
The Kokkos interface that holds the host reference of the Kokkos mesh and copies it to device during ...
Definition: KokkosMesh.h:338
KOKKOS_FUNCTION Real coordTransformFactor(const ContiguousSubdomainID subdomain, const Real3 point) const
Get the coordinate transform factor for a point in a subdomain.
Array< unsigned int > _n_qps
Number of quadrature points.
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
Array2D< unsigned int > _n_dofs
Array2D< Array2D< Real > > _map_phi
Shape functions for computing reference-to-physical maps.
KOKKOS_FUNCTION const auto & getPhiFace(ContiguousSubdomainID subdomain, unsigned int elem_type, unsigned int fe_type) const
Get the face shape functions of a FE type for an element type and subdomain.
void cachePhysicalMap()
Cache physical maps on element quadrature points.
Array2D< Array< Array2D< Real3 > > > _map_grad_phi_face
Array< Pair< Real3, Real3 > > _rz_axis
General axisymmetric axis of each subdomain in cylindrical coordinate system.
KOKKOS_FUNCTION unsigned int getNumNodes(unsigned int elem_type) const
Get the number of nodes of an element type.
Definition: KokkosMesh.h:172
KOKKOS_FUNCTION unsigned int getMaxQpsPerElem() const
Get the maximum number of quadrature points per element in the current partition. ...
Array< Array< Real33 > > _jacobian
Cached physical maps on element quadrature points.
Array2D< Array< Real3 > > _q_points
Quadrature points and weights for reference elements.
Array3D< Array< Array2D< Real3 > > > _grad_phi_face
MooseMesh wraps a libMesh::Mesh object and enhances its capabilities by caching additional data and s...
Definition: MooseMesh.h:92
The Kokkos interface that holds the host reference of the Kokkos assembly and copies it to device dur...
Array3D< Array2D< Real > > _phi
Shape functions for reference elements.
KOKKOS_FUNCTION dof_id_type getNumFaceQps(ContiguousSubdomainID subdomain) const
Get the total number of facial quadrature points in a subdomain Note: this number does not represent ...
unsigned int _max_qps_per_elem
Array2D< Array< Array< Real3 > > > _q_points_face
void initQuadrature()
Initialize quadrature data.
KOKKOS_INLINE_FUNCTION Real33 transpose()
Definition: KokkosTypes.h:113
KOKKOS_FUNCTION Real3 getNodePoint(ContiguousNodeID node) const
Get the coordinate of a node.
Definition: KokkosMesh.h:215
Array3D< Array2D< Real3 > > _grad_phi
KOKKOS_FUNCTION Real getJxW(ElementInfo info, unsigned int qp) const
Get the transformed Jacobian weight of an element quadrature point.
KOKKOS_FUNCTION const Mesh & kokkosMesh() const
Get the const reference of the Kokkos mesh.
Definition: KokkosMesh.h:360
MooseMesh & _mesh
Reference of the MOOSE mesh.
const auto & getMaterialBoundaries() const
Get the list of boundaries to cache face material properties.
DIE A HORRIBLE DEATH HERE typedef LIBMESH_DEFAULT_SCALAR_TYPE Real
KOKKOS_FUNCTION dof_id_type getQpFaceOffset(ElementInfo info, unsigned int side) const
Get the starting offset of quadrature points of a side of an element into the global quadrature point...
KOKKOS_INLINE_FUNCTION Real determinant(const unsigned int dim=3)
Definition: KokkosTypes.h:65
void initShape()
Initialize shape data.
CTSub CT_OPERATOR_BINARY CTMul CTCompareLess CTCompareGreater CTCompareEqual _arg template * sqrt(_arg)) *_arg.template D< dtag >()) CT_SIMPLE_UNARY_FUNCTION(tanh
Array2D< Array2D< Real3 > > _map_grad_phi
Array< dof_id_type > _qp_offset
Starting offset into the global quadrature point index.
KOKKOS_FUNCTION dof_id_type getNumQps(ContiguousSubdomainID subdomain) const
Get the total number of elemental quadrature points in a subdomain.
std::set< BoundaryID > _material_boundaries
Boundaries to cache face material properties.
Array< Moose::CoordinateSystemType > _coord_type
Coordinate system type of each subdomain.
Array< Array< Real3 > > _xyz
MOOSE now contains C++17 code, so give a reasonable error message stating what the user can do to add...
Array< dof_id_type > _n_subdomain_qps
Array2D< Array< Array2D< Real > > > _map_psi_face
void init()
Initialize assembly.
AssemblyHolder(const Assembly &assembly)
Constructor.
const Assembly _assembly_device
Device copy of the Kokkos assembly.
const Assembly & _assembly_host
Host reference of the Kokkos assembly.
KOKKOS_INLINE_FUNCTION Real33 inverse(const unsigned int dim=3)
Definition: KokkosTypes.h:82
KOKKOS_FUNCTION Real33 getJacobian(ElementInfo info, unsigned int qp) const
Get the inverse of Jacobian matrix of an element quadrature point.
uint8_t dof_id_type
KOKKOS_FUNCTION void computePhysicalMap(const ElementInfo info, const unsigned int qp, Real33 *const jacobian, Real *const JxW, Real3 *const q_points) const
Compute physical transformation data for an element.
KOKKOS_FUNCTION unsigned int getNumFaceQps(ElementInfo info, unsigned int side) const
Get the number of quadrature points of a side of an element.
KOKKOS_FUNCTION Real3 getQPoint(ElementInfo info, unsigned int qp) const
Get the coordinate of an element quadrature point.