Line data    Source code

       1             : /********************************************************************/
       2             : /*                  SOFTWARE COPYRIGHT NOTIFICATION                 */
       3             : /*                             Cardinal                             */
       4             : /*                                                                  */
       5             : /*                  (c) 2021 UChicago Argonne, LLC                  */
       6             : /*                        ALL RIGHTS RESERVED                       */
       7             : /*                                                                  */
       8             : /*                 Prepared by UChicago Argonne, LLC                */
       9             : /*               Under Contract No. DE-AC02-06CH11357               */
      10             : /*                With the U. S. Department of Energy               */
      11             : /*                                                                  */
      12             : /*             Prepared by Battelle Energy Alliance, LLC            */
      13             : /*               Under Contract No. DE-AC07-05ID14517               */
      14             : /*                With the U. S. Department of Energy               */
      15             : /*                                                                  */
      16             : /*                 See LICENSE for full restrictions                */
      17             : /********************************************************************/
      18             : 
      19             : #ifdef ENABLE_NEK_COUPLING
      20             : 
      21             : #include "NekRSProblem.h"
      22             : #include "FieldTransferBase.h"
      23             : #include "UserErrorChecking.h"
      24             : 
      25             : std::map<unsigned int, std::string> FieldTransferBase::_field_usrwrk_map;
      26             : std::map<std::string, std::pair<Real, Real>> FieldTransferBase::_field_usrwrk_scales;
      27             : 
      28             : InputParameters
      29        2163 : FieldTransferBase::validParams()
      30             : {
      31        2163 :   auto params = NekTransferBase::validParams();
      32        4326 :   params.addParam<std::vector<unsigned int>>(
      33             :       "usrwrk_slot",
      34             :       "When 'direction = to_nek', the slot(s) in the usrwrk array to write the incoming data; "
      35             :       "provide one entry for each quantity being passed");
      36        2163 :   params.addClassDescription("Base class for transferring field data between NekRS and MOOSE.");
      37        2163 :   params.registerBase("FieldTransfer");
      38        2163 :   params.registerSystemAttributeName("FieldTransfer");
      39        2163 :   return params;
      40           0 : }
      41             : 
      42        1089 : FieldTransferBase::FieldTransferBase(const InputParameters & parameters)
      43        1089 :   : NekTransferBase(parameters), _variable(name())
      44             : {
      45        1089 :   if (_direction == "to_nek")
      46             :   {
      47         796 :     checkRequiredParam(parameters, "usrwrk_slot", "writing data 'to_nek'");
      48        1194 :     _usrwrk_slot = getParam<std::vector<unsigned int>>("usrwrk_slot");
      49             : 
      50             :     // there should be no duplicates within a single transfer
      51         398 :     std::set<unsigned int> slots(_usrwrk_slot.begin(), _usrwrk_slot.end());
      52         398 :     if (slots.size() != _usrwrk_slot.size())
      53             :     {
      54             :       std::string slots;
      55           3 :       for (const auto & u : _usrwrk_slot)
      56           4 :         slots += Moose::stringify(u) + " ";
      57           1 :       paramError(
      58             :           "usrwrk_slot",
      59           1 :           "There are duplicate entries in 'usrwrk_slot': " + slots +
      60             :               "; duplicate entries are not allowed because the field transfer will overwrite "
      61           1 :               "itself. Cardinal has allocated " +
      62           1 :               Moose::stringify(_nek_problem.nUsrWrkSlots()) +
      63             :               " usrwrk slots; if you need more, set 'n_usrwrk_slots' in the [Problem] block.");
      64             :     }
      65             : 
      66             :     // each slot should not be greater than the amount allocated
      67         886 :     for (const auto & u : _usrwrk_slot)
      68         490 :       checkAllocatedUsrwrkSlot(u);
      69             :   }
      70             : 
      71        1087 :   _n_per_surf = _nek_mesh->exactMirror() ? std::pow(_nek_mesh->nekNumQuadraturePoints1D(), 2.0)
      72         999 :                                          : _nek_mesh->numVerticesPerSurface();
      73        1087 :   _n_per_vol = _nek_mesh->exactMirror() ? std::pow(_nek_mesh->nekNumQuadraturePoints1D(), 3.0)
      74         999 :                                         : _nek_mesh->numVerticesPerVolume();
      75        1087 :   _v_face = (double *)calloc(_n_per_surf, sizeof(double));
      76        1087 :   _v_elem = (double *)calloc(_n_per_vol, sizeof(double));
      77        1087 :   _external_data = (double *)calloc(_nek_problem.nPoints(), sizeof(double));
      78        1087 : }
      79             : 
      80        1052 : FieldTransferBase::~FieldTransferBase()
      81             : {
      82        1052 :   freePointer(_v_face);
      83        1052 :   freePointer(_v_elem);
      84        1052 :   freePointer(_external_data);
      85        1052 : }
      86             : 
      87             : void
      88        1172 : FieldTransferBase::addExternalVariable(const std::string name)
      89             : {
      90        1172 :   InputParameters var_params = _factory.getValidParams("MooseVariable");
      91        2344 :   var_params.set<MooseEnum>("family") = "LAGRANGE";
      92             : 
      93        1172 :   switch (_nek_mesh->order())
      94             :   {
      95             :     case order::first:
      96        1634 :       var_params.set<MooseEnum>("order") = "FIRST";
      97         817 :       break;
      98             :     case order::second:
      99         710 :       var_params.set<MooseEnum>("order") = "SECOND";
     100         355 :       break;
     101           0 :     default:
     102           0 :       mooseError("Unhandled 'NekOrderEnum' in 'FieldTransferBase'!");
     103             :   }
     104             : 
     105        1172 :   _nek_problem.checkDuplicateVariableName(name, "FieldTransfer");
     106        1169 :   _nek_problem.addAuxVariable("MooseVariable", name, var_params);
     107        1169 :   _variable_number.insert(std::pair<std::string, unsigned int>(
     108        1169 :       name, _nek_problem.getAuxiliarySystem().getFieldVariable<Real>(0, name).number()));
     109        1169 : }
     110             : 
     111             : void
     112         482 : FieldTransferBase::addExternalVariable(const unsigned int slot,
     113             :                                        const std::string name,
     114             :                                        const Real shift,
     115             :                                        const Real divisor)
     116             : {
     117         963 :   addExternalVariable(name);
     118             : 
     119             :   // check that no other field transfer is trying to write into the same slot;
     120             :   // we don't need to check the scalar transfers, because they will always execute
     121             :   // after the field transfers
     122             :   bool duplicate_field = _field_usrwrk_map.find(slot) != _field_usrwrk_map.end();
     123         481 :   if (duplicate_field)
     124           1 :     paramError("usrwrk_slot",
     125           1 :                "A duplicate slot, " + Moose::stringify(slot) +
     126             :                    ", is being used by another FieldTransfer. Duplicate slots are not allowed for "
     127             :                    "field transfers because these transfers will overwrite all data in that slot. "
     128             :                    "If you need more slots, increase 'n_usrwrk_slots' in the [Problem] block.");
     129             : 
     130         480 :   _field_usrwrk_map.insert({slot, name});
     131         480 :   _field_usrwrk_scales.insert({name, std::make_pair(shift, divisor)});
     132         480 : }
     133             : 
     134             : void
     135       86750 : FieldTransferBase::fillAuxVariable(const unsigned int var_number, const double * value)
     136             : {
     137       86750 :   auto & solution = _nek_problem.getAuxiliarySystem().solution();
     138       86750 :   auto sys_number = _nek_problem.getAuxiliarySystem().number();
     139       86750 :   auto pid = _communicator.rank();
     140             : 
     141    13686522 :   for (unsigned int e = 0; e < _nek_mesh->numElems(); e++)
     142             :   {
     143    36993448 :     for (int build = 0; build < _nek_mesh->nMoosePerNek(); ++build)
     144             :     {
     145    23393676 :       auto elem_ptr = _nek_mesh->queryElemPtr(e * _nek_mesh->nMoosePerNek() + build);
     146             : 
     147             :       // Only work on elements we can find on our local chunk of a
     148             :       // distributed mesh
     149    23393676 :       if (!elem_ptr)
     150             :       {
     151             :         libmesh_assert(!_nek_mesh->getMesh().is_serial());
     152     1577584 :         continue;
     153             :       }
     154             : 
     155   275609996 :       for (unsigned int n = 0; n < _nek_mesh->numVerticesPerElem(); n++)
     156             :       {
     157             :         auto node_ptr = elem_ptr->node_ptr(n);
     158             : 
     159             :         // For each face vertex, we can only write into the MOOSE auxiliary fields if that
     160             :         // vertex is "owned" by the present MOOSE process.
     161   253793904 :         if (node_ptr->processor_id() == pid)
     162             :         {
     163    51067438 :           int node_index = _nek_mesh->nodeIndex(n);
     164             :           auto node_offset =
     165    51067438 :               (e * _nek_mesh->nMoosePerNek() + build) * _nek_mesh->numVerticesPerElem() +
     166    51067438 :               node_index;
     167             : 
     168             :           // get the DOF for the auxiliary variable, then use it to set the value in the auxiliary
     169             :           // system
     170    51067438 :           auto dof_idx = node_ptr->dof_number(sys_number, var_number, 0);
     171    51067438 :           solution.set(dof_idx, value[node_offset]);
     172             :         }
     173             :       }
     174             :     }
     175             :   }
     176             : 
     177       86750 :   solution.close();
     178       86750 : }
     179             : #endif