Line data    Source code

       1             : /********************************************************************/
       2             : /*                  SOFTWARE COPYRIGHT NOTIFICATION                 */
       3             : /*                             Cardinal                             */
       4             : /*                                                                  */
       5             : /*                  (c) 2021 UChicago Argonne, LLC                  */
       6             : /*                        ALL RIGHTS RESERVED                       */
       7             : /*                                                                  */
       8             : /*                 Prepared by UChicago Argonne, LLC                */
       9             : /*               Under Contract No. DE-AC02-06CH11357               */
      10             : /*                With the U. S. Department of Energy               */
      11             : /*                                                                  */
      12             : /*             Prepared by Battelle Energy Alliance, LLC            */
      13             : /*               Under Contract No. DE-AC07-05ID14517               */
      14             : /*                With the U. S. Department of Energy               */
      15             : /*                                                                  */
      16             : /*                 See LICENSE for full restrictions                */
      17             : /********************************************************************/
      18             : 
      19             : #ifdef ENABLE_NEK_COUPLING
      20             : 
      21             : #include "NekRSProblem.h"
      22             : #include "FieldTransferBase.h"
      23             : #include "UserErrorChecking.h"
      24             : 
      25             : std::map<unsigned int, std::string> FieldTransferBase::_field_usrwrk_map;
      26             : 
      27             : InputParameters
      28        2320 : FieldTransferBase::validParams()
      29             : {
      30        2320 :   auto params = NekTransferBase::validParams();
      31        4640 :   params.addParam<std::vector<unsigned int>>(
      32             :       "usrwrk_slot",
      33             :       "When 'direction = to_nek', the slot(s) in the usrwrk array to write the incoming data; "
      34             :       "provide one entry for each quantity being passed");
      35        2320 :   params.addClassDescription("Base class for transferring field data between NekRS and MOOSE.");
      36        2320 :   params.registerBase("FieldTransfer");
      37        2320 :   params.registerSystemAttributeName("FieldTransfer");
      38        2320 :   return params;
      39           0 : }
      40             : 
      41        1167 : FieldTransferBase::FieldTransferBase(const InputParameters & parameters)
      42        1167 :   : NekTransferBase(parameters), _variable(name())
      43             : {
      44        1167 :   if (_direction == "to_nek")
      45             :   {
      46         790 :     checkRequiredParam(parameters, "usrwrk_slot", "writing data 'to_nek'");
      47        1185 :     _usrwrk_slot = getParam<std::vector<unsigned int>>("usrwrk_slot");
      48             : 
      49             :     // there should be no duplicates within a single transfer
      50         395 :     std::set<unsigned int> slots(_usrwrk_slot.begin(), _usrwrk_slot.end());
      51         395 :     if (slots.size() != _usrwrk_slot.size())
      52             :     {
      53             :       std::string slots;
      54           3 :       for (const auto & u : _usrwrk_slot)
      55           4 :         slots += Moose::stringify(u) + " ";
      56           1 :       paramError(
      57             :           "usrwrk_slot",
      58           1 :           "There are duplicate entries in 'usrwrk_slot': " + slots +
      59             :               "; duplicate entries are not allowed because the field transfer will overwrite "
      60           1 :               "itself. Cardinal has allocated " +
      61           1 :               Moose::stringify(_nek_problem.nUsrWrkSlots()) +
      62             :               " usrwrk slots; if you need more, set 'n_usrwrk_slots' in the [Problem] block.");
      63             :     }
      64             : 
      65             :     // each slot should not be greater than the amount allocated
      66         879 :     for (const auto & u : _usrwrk_slot)
      67         486 :       checkAllocatedUsrwrkSlot(u);
      68             :   }
      69             : 
      70        1165 :   _n_per_surf = _nek_mesh->exactMirror() ? std::pow(_nek_mesh->nekNumQuadraturePoints1D(), 2.0)
      71        1065 :                                          : _nek_mesh->numVerticesPerSurface();
      72        1165 :   _n_per_vol = _nek_mesh->exactMirror() ? std::pow(_nek_mesh->nekNumQuadraturePoints1D(), 3.0)
      73        1065 :                                         : _nek_mesh->numVerticesPerVolume();
      74        1165 : }
      75             : 
      76             : void
      77        1254 : FieldTransferBase::addExternalVariable(const std::string name)
      78             : {
      79        1254 :   InputParameters var_params = _factory.getValidParams("MooseVariable");
      80        2508 :   var_params.set<MooseEnum>("family") = "LAGRANGE";
      81             : 
      82        1254 :   switch (_nek_mesh->order())
      83             :   {
      84             :     case order::first:
      85        1678 :       var_params.set<MooseEnum>("order") = "FIRST";
      86         839 :       break;
      87             :     case order::second:
      88         830 :       var_params.set<MooseEnum>("order") = "SECOND";
      89         415 :       break;
      90           0 :     default:
      91           0 :       mooseError("Unhandled 'NekOrderEnum' in 'FieldTransferBase'!");
      92             :   }
      93             : 
      94        1254 :   _nek_problem.checkDuplicateVariableName(name);
      95        1251 :   _nek_problem.addAuxVariable("MooseVariable", name, var_params);
      96        1251 :   _variable_number.insert(std::pair<std::string, unsigned int>(
      97        1251 :       name, _nek_problem.getAuxiliarySystem().getFieldVariable<Real>(0, name).number()));
      98        1251 : }
      99             : 
     100             : void
     101         482 : FieldTransferBase::addExternalVariable(const unsigned int slot, const std::string name)
     102             : {
     103         963 :   addExternalVariable(name);
     104             : 
     105             :   // check that no other field transfer is trying to write into the same slot;
     106             :   // we don't need to check the scalar transfers, because they will always execute
     107             :   // after the field transfers
     108             :   bool duplicate_field = _field_usrwrk_map.find(slot) != _field_usrwrk_map.end();
     109         481 :   if (duplicate_field)
     110           1 :     paramError("usrwrk_slot",
     111           1 :                "A duplicate slot, " + Moose::stringify(slot) +
     112             :                    ", is being used by another FieldTransfer. Duplicate slots are not allowed for "
     113             :                    "field transfers because these transfers will overwrite all data in that slot. "
     114             :                    "If you need more slots, increase 'n_usrwrk_slots' in the [Problem] block.");
     115             : 
     116         480 :   _field_usrwrk_map.insert({slot, name});
     117         480 : }
     118             : 
     119             : void
     120       88432 : FieldTransferBase::fillAuxVariable(const unsigned int var_number, const double * value)
     121             : {
     122       88432 :   auto & solution = _nek_problem.getAuxiliarySystem().solution();
     123       88432 :   auto sys_number = _nek_problem.getAuxiliarySystem().number();
     124       88432 :   auto pid = _communicator.rank();
     125             : 
     126    16695292 :   for (unsigned int e = 0; e < _nek_mesh->numElems(); e++)
     127             :   {
     128    40322488 :     for (int build = 0; build < _nek_mesh->nMoosePerNek(); ++build)
     129             :     {
     130    23715628 :       auto elem_ptr = _nek_mesh->queryElemPtr(e * _nek_mesh->nMoosePerNek() + build);
     131             : 
     132             :       // Only work on elements we can find on our local chunk of a
     133             :       // distributed mesh
     134    23715628 :       if (!elem_ptr)
     135             :       {
     136             :         libmesh_assert(!_nek_mesh->getMesh().is_serial());
     137     2951895 :         continue;
     138             :       }
     139             : 
     140   303029581 :       for (unsigned int n = 0; n < _nek_mesh->numVerticesPerElem(); n++)
     141             :       {
     142             :         auto node_ptr = elem_ptr->node_ptr(n);
     143             : 
     144             :         // For each face vertex, we can only write into the MOOSE auxiliary fields if that
     145             :         // vertex is "owned" by the present MOOSE process.
     146   282265848 :         if (node_ptr->processor_id() == pid)
     147             :         {
     148    66611488 :           int node_index = _nek_mesh->nodeIndex(n);
     149             :           auto node_offset =
     150    66611488 :               (e * _nek_mesh->nMoosePerNek() + build) * _nek_mesh->numVerticesPerElem() +
     151    66611488 :               node_index;
     152             : 
     153             :           // get the DOF for the auxiliary variable, then use it to set the value in the auxiliary
     154             :           // system
     155    66611488 :           auto dof_idx = node_ptr->dof_number(sys_number, var_number, 0);
     156    66611488 :           solution.set(dof_idx, value[node_offset]);
     157             :         }
     158             :       }
     159             :     }
     160             :   }
     161             : 
     162       88432 :   solution.close();
     163       88432 : }
     164             : #endif