Line data    Source code

       1             : /********************************************************************/
       2             : /*                  SOFTWARE COPYRIGHT NOTIFICATION                 */
       3             : /*                             Cardinal                             */
       4             : /*                                                                  */
       5             : /*                  (c) 2021 UChicago Argonne, LLC                  */
       6             : /*                        ALL RIGHTS RESERVED                       */
       7             : /*                                                                  */
       8             : /*                 Prepared by UChicago Argonne, LLC                */
       9             : /*               Under Contract No. DE-AC02-06CH11357               */
      10             : /*                With the U. S. Department of Energy               */
      11             : /*                                                                  */
      12             : /*             Prepared by Battelle Energy Alliance, LLC            */
      13             : /*               Under Contract No. DE-AC07-05ID14517               */
      14             : /*                With the U. S. Department of Energy               */
      15             : /*                                                                  */
      16             : /*                 See LICENSE for full restrictions                */
      17             : /********************************************************************/
      18             : 
      19             : #ifdef ENABLE_NEK_COUPLING
      20             : 
      21             : #include "NekVolumetricSource.h"
      22             : 
      23             : registerMooseObject("CardinalApp", NekVolumetricSource);
      24             : 
      25             : InputParameters
      26         190 : NekVolumetricSource::validParams()
      27             : {
      28         190 :   auto params = ConservativeFieldTransfer::validParams();
      29         380 :   params.addParam<Real>(
      30             :       "initial_source_integral",
      31         380 :       0,
      32             :       "Initial value to use for the 'postprocessor_to_conserve'; this initial value will be "
      33             :       "overridden once the coupled app executes its transfer of the volumetric source term "
      34             :       "integral into the 'postprocessor_to_conserve'. You may want to use this parameter if NekRS "
      35             :       "runs first, or if you are running NekRS in isolation but still want to apply a source term "
      36             :       "via Cardinal. Remember that this parameter is only used to normalize the source term "
      37             :       "'source_variable', so you will need to populate an initial shape.");
      38         190 :   params.addClassDescription("Reads/writes volumetric source data between NekRS and MOOSE.");
      39         190 :   return params;
      40           0 : }
      41             : 
      42          98 : NekVolumetricSource::NekVolumetricSource(const InputParameters & parameters)
      43             :   : ConservativeFieldTransfer(parameters),
      44         196 :     _initial_source_integral(getParam<Real>("initial_source_integral"))
      45             : {
      46          98 :   if (_direction == "to_nek")
      47             :   {
      48             :     // TODO: this will need to be generalized if the same transfer is used for fluxes of varying
      49             :     // interpretation
      50          98 :     auto d = nekrs::nondimensionalDivisor(field::heat_source);
      51          98 :     auto a = nekrs::nondimensionalAdditive(field::heat_source);
      52         195 :     addExternalVariable(_usrwrk_slot[0], _variable, a, d);
      53             : 
      54          97 :     if (_usrwrk_slot.size() > 1)
      55           2 :       paramError("usrwrk_slot",
      56             :                  "'usrwrk_slot' must be of length 1 for volumetric source transfers; you have "
      57           1 :                  "entered a vector of length " +
      58           0 :                      Moose::stringify(_usrwrk_slot.size()));
      59             :   }
      60             : 
      61          96 :   if (!_nek_mesh->volume())
      62           1 :     mooseError("The NekVolumetricSource object can only be used when there is volumetric coupling "
      63             :                "of NekRS with MOOSE, i.e. when 'volume = true' in NekRSMesh.");
      64             : 
      65          95 :   if (_direction == "from_nek")
      66           0 :     paramError("direction",
      67             :                "The NekVolumetricSource currently only supports transfers 'to_nek'; contact the "
      68             :                "Cardinal developer team if you require reading of NekRS volumetric source terms.");
      69             : 
      70             :   // Check that there is a udf function providing the source for the passive scalar
      71             :   // equations. NOTE: This check is imperfect, because even if there is a source kernel,
      72             :   // we cannot tell _which_ passive scalar equation that it is applied to (we have
      73             :   // source kernels for the RANS passive scalar equations, for instance).
      74          95 :   if (nekrs::hasTemperatureSolve() && !nekrs::hasHeatSourceKernel())
      75          17 :     mooseWarning(
      76             :         "In order to use the volumetric heat source sent to NekRS, you must have an OCCA source "
      77             :         "kernel in the passive scalar equations! The heat source will currently be unused.");
      78             : 
      79          94 :   if (!nekrs::hasTemperatureVariable())
      80           1 :     mooseError("In order to send a volumetric heat source to NekRS, your case files must have a "
      81           1 :                "[TEMPERATURE] block. Note that you can set 'solver = none' in '" +
      82           1 :                _nek_problem.casename() + ".par' if you don't want to solve for temperature.");
      83             : 
      84          93 :   if (!nekrs::hasTemperatureSolve())
      85           2 :     mooseWarning("By setting 'solver = none' for temperature in '" + _nek_problem.casename() +
      86             :                  ".par', NekRS will not solve for temperature. The volumetric heat source sent by "
      87             :                  "this object will be unused.");
      88             : 
      89          92 :   addExternalPostprocessor(_postprocessor_name, _initial_source_integral);
      90          92 :   _source_integral = &getPostprocessorValueByName(_postprocessor_name);
      91          92 : }
      92             : 
      93             : bool
      94         982 : NekVolumetricSource::normalizeVolumetricSource(const double moose,
      95             :                                                double nek,
      96             :                                                double & normalized_nek)
      97             : {
      98             :   auto dimension_multiplier =
      99         982 :       nekrs::referenceVolume() * nekrs::nondimensionalDivisor(field::heat_source);
     100             : 
     101             :   // scale the nek source to dimensional form for the sake of normalizing against
     102             :   // a dimensional MOOSE source
     103         982 :   nek *= dimension_multiplier;
     104             : 
     105             :   // avoid divide-by-zero
     106         982 :   if (std::abs(nek) < _abs_tol)
     107             :     return true;
     108             : 
     109         958 :   nekrs::scaleUsrwrk(_usrwrk_slot[0] * nekrs::fieldOffset(), moose / nek);
     110             : 
     111             :   // check that the normalization worked properly
     112         958 :   normalized_nek =
     113         958 :       nekrs::usrwrkVolumeIntegral(_usrwrk_slot[0] * nekrs::fieldOffset(), nek_mesh::all) *
     114             :       dimension_multiplier;
     115         958 :   bool low_rel_err = std::abs(normalized_nek - moose) / moose < _rel_tol;
     116         958 :   bool low_abs_err = std::abs(normalized_nek - moose) < _abs_tol;
     117             : 
     118         958 :   return low_rel_err && low_abs_err;
     119             : }
     120             : 
     121             : void
     122         982 : NekVolumetricSource::sendDataToNek()
     123             : {
     124         982 :   _console << "Sending volumetric source to NekRS..." << std::endl;
     125             : 
     126         982 :   auto d = nekrs::nondimensionalDivisor(field::heat_source);
     127         982 :   auto a = nekrs::nondimensionalAdditive(field::heat_source);
     128     2873790 :   for (unsigned int e = 0; e < _nek_mesh->numVolumeElems(); e++)
     129             :   {
     130             :     // We can only write into the nekRS scratch space if that face is "owned" by the current process
     131     2872808 :     if (nekrs::commRank() != _nek_mesh->volumeCoupling().processor_id(e))
     132     2491846 :       continue;
     133             : 
     134      380962 :     _nek_problem.mapVolumeDataToNekVolume(e, _variable_number[_variable], d, a, &_v_elem);
     135      380962 :     _nek_problem.writeVolumeSolution(_usrwrk_slot[0] * nekrs::fieldOffset(), e, _v_elem);
     136             :   }
     137             : 
     138             :   // Because the NekRSMesh may be quite different from that used in the app solving for
     139             :   // the heat source, we will need to normalize the total source on the nekRS side by the
     140             :   // total source computed by the coupled MOOSE app.
     141         982 :   const Real scale_cubed = _nek_mesh->scaling() * _nek_mesh->scaling() * _nek_mesh->scaling();
     142             :   const double nek_source =
     143         982 :       nekrs::usrwrkVolumeIntegral(_usrwrk_slot[0] * nekrs::fieldOffset(), nek_mesh::all);
     144         982 :   const double moose_source = *_source_integral;
     145             : 
     146             :   // For the sake of printing diagnostics to the screen regarding source normalization,
     147             :   // we first scale the nek source by any unit changes and then by the reference source
     148             :   const double nek_source_print_mult =
     149         982 :       scale_cubed * nekrs::nondimensionalDivisor(field::heat_source);
     150         982 :   double normalized_nek_source = 0.0;
     151             :   bool successful_normalization;
     152             : 
     153         982 :   _console << "[volume]: Normalizing total NekRS heat source of "
     154         982 :            << Moose::stringify(nek_source * nek_source_print_mult)
     155        1964 :            << " to the conserved MOOSE value of " + Moose::stringify(moose_source) << std::endl;
     156             : 
     157             :   // Any unit changes (for DIMENSIONAL nekRS runs) are automatically accounted for
     158             :   // here because moose_source is an integral on the MOOSE mesh, while nek_source is
     159             :   // an integral on the nek mesh
     160             :   successful_normalization =
     161         982 :       normalizeVolumetricSource(moose_source, nek_source, normalized_nek_source);
     162             : 
     163             :   // If before normalization, there is a large difference between the nekRS imposed source
     164             :   // and the MOOSE source, this could mean that there is a poor match between the domains,
     165             :   // even if neither value is zero. For instance, if you forgot that the nekRS mesh is in
     166             :   // units of centimeters, but you're coupling to an app based in meters, the sources will
     167             :   // be very different from one another.
     168         982 :   if (moose_source &&
     169         982 :       (std::abs(nek_source * nek_source_print_mult - moose_source) / moose_source) > 0.25)
     170          40 :     mooseDoOnce(
     171             :         mooseWarning("NekRS source differs from MOOSE source by more than 25\%! This is NOT "
     172             :                      "necessarily a problem - but it could indicate that your geometries don't "
     173             :                      "line up properly or something is amiss with your transfer. We recommend "
     174             :                      "opening the output files to visually inspect the volumetric source in both "
     175             :                      "the main and sub applications to check that the fields look correct."));
     176             : 
     177         982 :   if (!successful_normalization)
     178           0 :     mooseError("Volumetric source normalization process failed! NekRS integrated source: ",
     179             :                normalized_nek_source,
     180             :                ". MOOSE integrated source: ",
     181             :                moose_source,
     182             :                ".\n\n",
     183           0 :                normalizationHint());
     184         982 : }
     185             : 
     186             : #endif