Line data    Source code

       1             : /********************************************************************/
       2             : /*                  SOFTWARE COPYRIGHT NOTIFICATION                 */
       3             : /*                             Cardinal                             */
       4             : /*                                                                  */
       5             : /*                  (c) 2021 UChicago Argonne, LLC                  */
       6             : /*                        ALL RIGHTS RESERVED                       */
       7             : /*                                                                  */
       8             : /*                 Prepared by UChicago Argonne, LLC                */
       9             : /*               Under Contract No. DE-AC02-06CH11357               */
      10             : /*                With the U. S. Department of Energy               */
      11             : /*                                                                  */
      12             : /*             Prepared by Battelle Energy Alliance, LLC            */
      13             : /*               Under Contract No. DE-AC07-05ID14517               */
      14             : /*                With the U. S. Department of Energy               */
      15             : /*                                                                  */
      16             : /*                 See LICENSE for full restrictions                */
      17             : /********************************************************************/
      18             : 
      19             : #ifdef ENABLE_NEK_COUPLING
      20             : 
      21             : #include "NekRSProblem.h"
      22             : #include "FieldTransferBase.h"
      23             : #include "UserErrorChecking.h"
      24             : 
      25             : std::map<unsigned int, std::string> FieldTransferBase::_field_usrwrk_map;
      26             : 
      27             : InputParameters
      28        2411 : FieldTransferBase::validParams()
      29             : {
      30        2411 :   auto params = NekTransferBase::validParams();
      31        4822 :   params.addParam<std::vector<unsigned int>>(
      32             :       "usrwrk_slot",
      33             :       "When 'direction = to_nek', the slot(s) in the usrwrk array to write the incoming data; "
      34             :       "provide one entry for each quantity being passed");
      35        2411 :   params.addClassDescription("Base class for transferring field data between NekRS and MOOSE.");
      36        2411 :   params.registerBase("FieldTransfer");
      37        2411 :   params.registerSystemAttributeName("FieldTransfer");
      38        2411 :   return params;
      39           0 : }
      40             : 
      41        1213 : FieldTransferBase::FieldTransferBase(const InputParameters & parameters)
      42        1213 :   : NekTransferBase(parameters), _variable(name())
      43             : {
      44        1213 :   if (_direction == "to_nek")
      45             :   {
      46         832 :     checkRequiredParam(parameters, "usrwrk_slot", "writing data 'to_nek'");
      47        1248 :     _usrwrk_slot = getParam<std::vector<unsigned int>>("usrwrk_slot");
      48             : 
      49             :     // there should be no duplicates within a single transfer
      50         416 :     std::set<unsigned int> slots(_usrwrk_slot.begin(), _usrwrk_slot.end());
      51         416 :     if (slots.size() != _usrwrk_slot.size())
      52             :     {
      53             :       std::string slots;
      54           3 :       for (const auto & u : _usrwrk_slot)
      55           4 :         slots += Moose::stringify(u) + " ";
      56           1 :       paramError(
      57             :           "usrwrk_slot",
      58           1 :           "There are duplicate entries in 'usrwrk_slot': " + slots +
      59             :               "; duplicate entries are not allowed because the field transfer will overwrite "
      60           1 :               "itself. Cardinal has allocated " +
      61           1 :               Moose::stringify(_nek_problem.nUsrWrkSlots()) +
      62             :               " usrwrk slots; if you need more, set 'n_usrwrk_slots' in the [Problem] block.");
      63             :     }
      64             : 
      65             :     // each slot should not be greater than the amount allocated
      66         922 :     for (const auto & u : _usrwrk_slot)
      67         508 :       checkAllocatedUsrwrkSlot(u);
      68             :   }
      69             : 
      70        1211 :   _n_per_surf = _nek_mesh->exactMirror() ? std::pow(_nek_mesh->nekNumQuadraturePoints1D(), 2.0)
      71        1111 :                                          : _nek_mesh->numVerticesPerSurface();
      72        1211 :   _n_per_vol = _nek_mesh->exactMirror() ? std::pow(_nek_mesh->nekNumQuadraturePoints1D(), 3.0)
      73        1111 :                                         : _nek_mesh->numVerticesPerVolume();
      74        1211 :   _v_face = (double *)calloc(_n_per_surf, sizeof(double));
      75        1211 :   _v_elem = (double *)calloc(_n_per_vol, sizeof(double));
      76        1211 :   _external_data = (double *)calloc(_nek_problem.nPoints(), sizeof(double));
      77        1211 : }
      78             : 
      79        1176 : FieldTransferBase::~FieldTransferBase()
      80             : {
      81        1176 :   freePointer(_v_face);
      82        1176 :   freePointer(_v_elem);
      83        1176 :   freePointer(_external_data);
      84        1176 : }
      85             : 
      86             : void
      87        1297 : FieldTransferBase::addExternalVariable(const std::string name)
      88             : {
      89        1297 :   InputParameters var_params = _factory.getValidParams("MooseVariable");
      90        2594 :   var_params.set<MooseEnum>("family") = "LAGRANGE";
      91             : 
      92        1297 :   switch (_nek_mesh->order())
      93             :   {
      94             :     case order::first:
      95        1764 :       var_params.set<MooseEnum>("order") = "FIRST";
      96         882 :       break;
      97             :     case order::second:
      98         830 :       var_params.set<MooseEnum>("order") = "SECOND";
      99         415 :       break;
     100           0 :     default:
     101           0 :       mooseError("Unhandled 'NekOrderEnum' in 'FieldTransferBase'!");
     102             :   }
     103             : 
     104        1297 :   _nek_problem.checkDuplicateVariableName(name);
     105        1294 :   _nek_problem.addAuxVariable("MooseVariable", name, var_params);
     106        1294 :   _variable_number.insert(std::pair<std::string, unsigned int>(
     107        1294 :       name, _nek_problem.getAuxiliarySystem().getFieldVariable<Real>(0, name).number()));
     108        1294 : }
     109             : 
     110             : void
     111         500 : FieldTransferBase::addExternalVariable(const unsigned int slot, const std::string name)
     112             : {
     113         999 :   addExternalVariable(name);
     114             : 
     115             :   // check that no other field transfer is trying to write into the same slot;
     116             :   // we don't need to check the scalar transfers, because they will always execute
     117             :   // after the field transfers
     118             :   bool duplicate_field = _field_usrwrk_map.find(slot) != _field_usrwrk_map.end();
     119         499 :   if (duplicate_field)
     120           1 :     paramError("usrwrk_slot",
     121           1 :                "A duplicate slot, " + Moose::stringify(slot) +
     122             :                    ", is being used by another FieldTransfer. Duplicate slots are not allowed for "
     123             :                    "field transfers because these transfers will overwrite all data in that slot. "
     124             :                    "If you need more slots, increase 'n_usrwrk_slots' in the [Problem] block.");
     125             : 
     126         498 :   _field_usrwrk_map.insert({slot, name});
     127         498 : }
     128             : 
     129             : void
     130       88692 : FieldTransferBase::fillAuxVariable(const unsigned int var_number, const double * value)
     131             : {
     132       88692 :   auto & solution = _nek_problem.getAuxiliarySystem().solution();
     133       88692 :   auto sys_number = _nek_problem.getAuxiliarySystem().number();
     134       88692 :   auto pid = _communicator.rank();
     135             : 
     136    16705512 :   for (unsigned int e = 0; e < _nek_mesh->numElems(); e++)
     137             :   {
     138    40342408 :     for (int build = 0; build < _nek_mesh->nMoosePerNek(); ++build)
     139             :     {
     140    23725588 :       auto elem_ptr = _nek_mesh->queryElemPtr(e * _nek_mesh->nMoosePerNek() + build);
     141             : 
     142             :       // Only work on elements we can find on our local chunk of a
     143             :       // distributed mesh
     144    23725588 :       if (!elem_ptr)
     145             :       {
     146             :         libmesh_assert(!_nek_mesh->getMesh().is_serial());
     147     2955858 :         continue;
     148             :       }
     149             : 
     150   303067318 :       for (unsigned int n = 0; n < _nek_mesh->numVerticesPerElem(); n++)
     151             :       {
     152             :         auto node_ptr = elem_ptr->node_ptr(n);
     153             : 
     154             :         // For each face vertex, we can only write into the MOOSE auxiliary fields if that
     155             :         // vertex is "owned" by the present MOOSE process.
     156   282297588 :         if (node_ptr->processor_id() == pid)
     157             :         {
     158    66631160 :           int node_index = _nek_mesh->nodeIndex(n);
     159             :           auto node_offset =
     160    66631160 :               (e * _nek_mesh->nMoosePerNek() + build) * _nek_mesh->numVerticesPerElem() +
     161    66631160 :               node_index;
     162             : 
     163             :           // get the DOF for the auxiliary variable, then use it to set the value in the auxiliary
     164             :           // system
     165    66631160 :           auto dof_idx = node_ptr->dof_number(sys_number, var_number, 0);
     166    66631160 :           solution.set(dof_idx, value[node_offset]);
     167             :         }
     168             :       }
     169             :     }
     170             :   }
     171             : 
     172       88692 :   solution.close();
     173       88692 : }
     174             : #endif