Line data Source code
1 : /********************************************************************/
2 : /* SOFTWARE COPYRIGHT NOTIFICATION */
3 : /* Cardinal */
4 : /* */
5 : /* (c) 2021 UChicago Argonne, LLC */
6 : /* ALL RIGHTS RESERVED */
7 : /* */
8 : /* Prepared by UChicago Argonne, LLC */
9 : /* Under Contract No. DE-AC02-06CH11357 */
10 : /* With the U. S. Department of Energy */
11 : /* */
12 : /* Prepared by Battelle Energy Alliance, LLC */
13 : /* Under Contract No. DE-AC07-05ID14517 */
14 : /* With the U. S. Department of Energy */
15 : /* */
16 : /* See LICENSE for full restrictions */
17 : /********************************************************************/
18 :
19 : #pragma once
20 :
21 : #include "CardinalProblem.h"
22 : #include "NekInterface.h"
23 : #include "NekTimeStepper.h"
24 : #include "NekScalarValue.h"
25 : #include "NekRSMesh.h"
26 : #include "Transient.h"
27 :
28 : class FieldTransferBase;
29 :
30 : /**
31 : * \brief Solve nekRS wrapped as a MOOSE app.
32 : *
33 : * This object controls all of the execution of and data transfers to/from nekRS.
34 : * Any number of data transfers can be added using the [FieldTransfers] block.
35 : */
36 : class NekRSProblem : public CardinalProblem
37 : {
38 : public:
39 : NekRSProblem(const InputParameters & params);
40 :
41 : static InputParameters validParams();
42 :
43 : ~NekRSProblem();
44 :
45 : /**
46 : * The number of points (DOFs) on the mesh mirror for this rank; this is used to define the
47 : * size of various allocated terms
48 : * @return number of points on this rank
49 : */
50 769 : int nPoints() const { return _n_points; }
51 :
52 : /**
53 : * Whether a data transfer to/from Nek is occurring
54 : * @param[in] direction direction of data transfer
55 : * @return whether a data transfer to Nek is about to occur
56 : */
57 : bool isDataTransferHappening(ExternalProblem::Direction direction);
58 :
59 537600 : Transient * transientExecutioner() const { return _transient_executioner; }
60 :
61 : /**
62 : * Whether a given slot space is reserved for coupling
63 : * @param[in] slot slot in usrwrk array
64 : * @return whether a usrwrk slot space is reserved by Cardinal
65 : */
66 : bool isUsrWrkSlotReservedForCoupling(const unsigned int & slot) const;
67 :
68 : /**
69 : * Copy an individual slice in the usrwrk array from host to device
70 : * @param[in] slot slot in usrwrk array
71 : */
72 : void copyIndividualScratchSlot(const unsigned int & slot) const;
73 :
74 : /**
75 : * Write NekRS solution field file
76 : * @param[in] time solution time in NekRS (if NekRS is non-dimensional, this will be
77 : * non-dimensional)
78 : * @param[in] step time step index
79 : */
80 : void writeFieldFile(const Real & time, const int & step) const;
81 :
82 : /**
83 : * \brief Whether nekRS should write an output file for the current time step
84 : *
85 : * A nekRS output file (suffix .f000xx) is written if the time step is an integer
86 : * multiple of the output writing interval or if the time step is the last time step.
87 : * \return whether to write a nekRS output file
88 : **/
89 : virtual bool isOutputStep() const;
90 :
91 : virtual void initialSetup() override;
92 :
93 : virtual void externalSolve() override;
94 :
95 33358 : virtual bool converged(unsigned int) override { return true; }
96 :
97 : virtual void addExternalVariables() override;
98 :
99 : virtual void syncSolutions(ExternalProblem::Direction direction) override;
100 :
101 : /**
102 : * Whether the solve is in nondimensional form
103 : * @return whether solve is in nondimensional form
104 : */
105 252 : virtual bool nondimensional() const { return _nondimensional; }
106 :
107 : /**
108 : * Whether the mesh is moving
109 : * @return whether the mesh is moving
110 : */
111 0 : virtual const bool hasMovingNekMesh() const { return false; }
112 :
113 : /**
114 : * Whether data should be synchronized in to nekRS
115 : * \return whether inward data synchronization should occur
116 : */
117 : virtual bool synchronizeIn();
118 :
119 : /**
120 : * Whether data should be synchronized out of nekRS
121 : * \return whether outward data synchronization should occur
122 : */
123 : virtual bool synchronizeOut();
124 :
125 : /**
126 : * Get the number of usrwrk slots allocated
127 : * @return number of allocated usrwrk slots
128 : */
129 592 : unsigned int nUsrWrkSlots() const { return _n_usrwrk_slots; }
130 :
131 : /**
132 : * Write into the NekRS solution space for coupling volumes; for setting a mesh position in terms
133 : * of a displacement, we need to add the displacement to the initial mesh coordinates. For this,
134 : * the 'add' parameter lets you pass in a vector of values (in NekRS's mesh order, i.e. the re2
135 : * order) to add.
136 : * @param[in] elem_id element ID
137 : * @param[in] field field to write
138 : * @param[in] T solution values to write for the field for the given element
139 : * @param[in] add optional vector of values to add to each value set on the NekRS end
140 : */
141 : void writeVolumeSolution(const int elem_id,
142 : const field::NekWriteEnum & field,
143 : double * T,
144 : const std::vector<double> * add = nullptr);
145 :
146 : /**
147 : * Write into the NekRS solution space for coupling boundaries; for setting a mesh position in
148 : * terms of a displacement, we need to add the displacement to the initial mesh coordinates.
149 : * @param[in] elem_id element ID
150 : * @param[in] field field to write
151 : * @param[in] T solution values to write for the field for the given element
152 : */
153 : void writeBoundarySolution(const int elem_id, const field::NekWriteEnum & field, double * T);
154 :
155 : /**
156 : * The casename (prefix) of the NekRS files
157 : * @return casename
158 : */
159 34 : std::string casename() const { return _casename; }
160 :
161 : /**
162 : * Interpolate the NekRS volume solution onto the volume MOOSE mesh mirror (re2 -> mirror)
163 : * @param[in] f field to interpolate
164 : * @param[out] T interpolated volume value
165 : */
166 : void volumeSolution(const field::NekFieldEnum & f, double * T);
167 :
168 : /**
169 : * Interpolate the NekRS boundary solution onto the boundary MOOSE mesh mirror (re2 -> mirror)
170 : * @param[in] f field to interpolate
171 : * @param[out] T interpolated boundary value
172 : */
173 : void boundarySolution(const field::NekFieldEnum & f, double * T);
174 :
175 : /**
176 : * Map nodal points on a MOOSE face element to the GLL points on a Nek face element.
177 : * @param[in] e MOOSE element ID
178 : * @param[in] var_num variable index to fetch MOOSE data from
179 : * @param[in] divisor number to divide MOOSE data by before sending to Nek (to non-dimensionalize
180 : * it)
181 : * @param[in] additive number to subtract from MOOSE data, before dividing by divisor and sending
182 : * to Nek (to non-dimensionalize)
183 : * @param[out] outgoing_data data represented on Nek's GLL points, ready to be applied in Nek
184 : */
185 : void mapFaceDataToNekFace(const unsigned int & e,
186 : const unsigned int & var_num,
187 : const Real & divisor,
188 : const Real & additive,
189 : double ** outgoing_data);
190 :
191 : /**
192 : * Map nodal points on a MOOSE volume element to the GLL points on a Nek volume element.
193 : * @param[in] e MOOSE element ID
194 : * @param[in] var_num variable index to fetch MOOSE data from
195 : * @param[in] divisor number to divide MOOSE data by before sending to Nek (to non-dimensionalize
196 : * it)
197 : * @param[in] additive number to subtract from MOOSE data, before dividing by divisor and sending
198 : * to Nek (to non-dimensionalize)
199 : * @param[out] outgoing_data data represented on Nek's GLL points, ready to be applied in Nek
200 : */
201 : void mapVolumeDataToNekVolume(const unsigned int & e,
202 : const unsigned int & var_num,
203 : const Real & divisor,
204 : const Real & additive,
205 : double ** outgoing_data);
206 :
207 : /**
208 : * \brief Map nodal points on a MOOSE face element to the GLL points on a Nek volume element.
209 : *
210 : * This function is to be used when MOOSE variables are defined over the entire volume
211 : * (maybe the MOOSE transfer only sent meaningful values to the coupling boundaries), so we
212 : * need to do a volume interpolation of the incoming MOOSE data into nrs->usrwrk, rather
213 : * than a face interpolation. This could be optimized in the future to truly only just write
214 : * the boundary values into the nekRS scratch space rather than the volume values, but it
215 : * looks right now that our biggest expense occurs in the MOOSE transfer system, not these
216 : * transfers internally to nekRS.
217 : *
218 : * @param[in] e MOOSE element ID
219 : * @param[in] var_num variable index to fetch MOOSE data from
220 : * @param[in] divisor number to divide MOOSE data by before sending to Nek (to non-dimensionalize
221 : * it)
222 : * @param[in] additive number to subtract from MOOSE data, before dividing by divisor and sending
223 : * to Nek (to non-dimensionalize)
224 : * @param[out] outgoing_data data represented on Nek's GLL points, ready to be applied in Nek
225 : */
226 : void mapFaceDataToNekVolume(const unsigned int & e,
227 : const unsigned int & var_num,
228 : const Real & divisor,
229 : const Real & additive,
230 : double ** outgoing_data);
231 :
232 : protected:
233 : /// Copy the data sent from MOOSE->Nek from host to device.
234 : void copyScratchToDevice();
235 :
236 : /**
237 : * Interpolate the MOOSE mesh mirror solution onto the NekRS boundary mesh (mirror -> re2)
238 : * @param[in] incoming_moose_value MOOSE face values
239 : * @param[out] outgoing_nek_value interpolated MOOSE face values onto the NekRS boundary mesh
240 : */
241 : void interpolateBoundarySolutionToNek(double * incoming_moose_value, double * outgoing_nek_value);
242 :
243 : /**
244 : * Interpolate the MOOSE mesh mirror solution onto the NekRS volume mesh (mirror -> re2)
245 : * @param[in] elem_id element ID
246 : * @param[in] incoming_moose_value MOOSE face values
247 : * @param[out] outgoing_nek_value interpolated MOOSE face values onto the NekRS volume mesh
248 : */
249 : void interpolateVolumeSolutionToNek(const int elem_id,
250 : double * incoming_moose_value,
251 : double * outgoing_nek_value);
252 :
253 : /// Initialize interpolation matrices for transfers in/out of nekRS
254 : void initializeInterpolationMatrices();
255 :
256 : std::unique_ptr<NumericVector<Number>> _serialized_solution;
257 :
258 : /**
259 : * Get a three-character prefix for use in writing output files for repeated
260 : * Nek sibling apps.
261 : * @param[in] number multi-app number
262 : */
263 : std::string fieldFilePrefix(const int & number) const;
264 :
265 : /// NekRS casename
266 : const std::string & _casename;
267 :
268 : /**
269 : * Whether to disable output file writing by NekRS and replace it by output
270 : * file writing in Cardinal. Suppose the case name is 'channel'. If this parameter
271 : * is false, then NekRS will write output files as usual, with names like
272 : * channel0.f00001 for write step 1, channel0.f00002 for write step 2, and so on.
273 : * If true, then NekRS itself does not output any files like this, and instead
274 : * output files are written with names a01channel0.f00001, a01channel0.f00002 (for
275 : * first Nek app), a02channel0.f00001, a02channel0.f00002 (for second Nek app),
276 : * and so on. This feature should only be used when running repeated Nek sub
277 : * apps so that the output from each app is retained. Otherwise, if running N
278 : * repeated Nek sub apps, only a single output file is obtained because each app
279 : * overwrites the output files of the other apps in the order that the apps
280 : * reach the nekrs::outfld function.
281 : */
282 : const bool & _write_fld_files;
283 :
284 : /// Whether to turn off all field file writing
285 : const bool & _disable_fld_file_output;
286 :
287 : /// Whether a dimensionalization action has been added
288 : bool _nondimensional;
289 :
290 : /**
291 : * Number of slices/slots to allocate in nrs->usrwrk to hold fields
292 : * for coupling (i.e. data going into NekRS, written by Cardinal), or
293 : * used for custom user actions, but not for coupling. By default, we just
294 : * allocate 7 slots (no inherent reason, just a fairly big amount). For
295 : * memory-limited cases, you can reduce this number to just the bare
296 : * minimum necessary for your use case.
297 : */
298 : unsigned int _n_usrwrk_slots;
299 :
300 : /// For constant synchronization intervals, the desired frequency (in units of Nek time steps)
301 : const unsigned int & _constant_interval;
302 :
303 : /// Whether to skip writing a field file on NekRS's last time steo
304 : const bool & _skip_final_field_file;
305 :
306 : /// Number of surface elements in the data transfer mesh, across all processes
307 : int _n_surface_elems;
308 :
309 : /// Number of vertices per surface element of the transfer mesh
310 : int _n_vertices_per_surface;
311 :
312 : /// Number of volume elements in the data transfer mesh, across all processes
313 : int _n_volume_elems;
314 :
315 : /// Number of vertices per volume element of the transfer mesh
316 : int _n_vertices_per_volume;
317 :
318 : /// Start time of the simulation based on NekRS's .par file
319 : double _start_time;
320 :
321 : /// Whether the most recent time step was an output file writing step
322 : bool _is_output_step;
323 :
324 : /**
325 : * Underlying mesh object on which NekRS exchanges fields with MOOSE
326 : * or extracts NekRS's solution for I/O features
327 : */
328 : NekRSMesh * _nek_mesh;
329 :
330 : /// The time stepper used for selection of time step size
331 : NekTimeStepper * _timestepper = nullptr;
332 :
333 : /// Underlying executioner
334 : Transient * _transient_executioner = nullptr;
335 :
336 : /**
337 : * Whether an interpolation needs to be performed on the nekRS temperature solution, or
338 : * if we can just grab the solution at specified points
339 : */
340 : bool _needs_interpolation;
341 :
342 : /// Number of points for interpolated fields on the MOOSE mesh
343 : int _n_points;
344 :
345 : /// Postprocessor containing the signal of when a synchronization has occurred
346 : const PostprocessorValue * _transfer_in = nullptr;
347 :
348 : /// Vandermonde interpolation matrix (for outgoing transfers)
349 : double * _interpolation_outgoing = nullptr;
350 :
351 : /// Vandermonde interpolation matrix (for incoming transfers)
352 : double * _interpolation_incoming = nullptr;
353 :
354 : /// For the MOOSE mesh, the number of quadrature points in each coordinate direction
355 : int _moose_Nq;
356 :
357 : /// Slots in the nrs->o_usrwrk array to write to a field file
358 : const std::vector<unsigned int> * _usrwrk_output = nullptr;
359 :
360 : /// Filename prefix to use for naming the field files containing the nrs->o_usrwrk array slots
361 : const std::vector<std::string> * _usrwrk_output_prefix = nullptr;
362 :
363 : /// Sum of the elapsed time in NekRS solves
364 : double _elapsedStepSum;
365 :
366 : /// Sum of the total elapsed time in NekRS solves
367 : double _elapsedTime;
368 :
369 : /// Minimum step solve time
370 : double _tSolveStepMin;
371 :
372 : /// Maximum step solve time
373 : double _tSolveStepMax;
374 :
375 : /**
376 : * \brief When to synchronize the NekRS solution with the mesh mirror
377 : *
378 : * This parameter determines when to synchronize the NekRS solution with the mesh
379 : * mirror - this entails:
380 : *
381 : * - Mapping from the NekRS spectral element mesh to the finite element mesh mirror,
382 : * to extract information from NekRS and make it available to MOOSE
383 : * - Mapping from the finite element mesh mirror into the NekRS spectral element mesh,
384 : * to send information from MOOSE into NekRS
385 : *
386 : * Several options are available:
387 : * - 'constant' will simply keep the NekRS solution and the mesh mirror entirely
388 : * consistent with one another on a given constant frequency of time steps. By
389 : * default, the 'constant_interval' is 1, so that NekRS and MOOSE communicate
390 : * with each other on every single time step
391 : *
392 : * - 'parent_app' will only send data between NekRS and a parent application
393 : * when (1) the main application has just sent "new" information to NekRS, and
394 : * when (2) the main application is just about to run a new time step (with
395 : * updated BCs/source terms from NekRS).
396 : *
397 : * nekRS is often subcycled relative to the application controlling it -
398 : * that is, nekRS may be run with a time step 10x smaller than a conduction MOOSE app.
399 : * If 'interpolate_transfers = false'
400 : * in the master application, then the data going into nekRS is fixed for each
401 : * of the subcycled time steps it takes, so these extra data transfers are
402 : * completely unnecssary. This flag indicates that the information sent from MOOSE
403 : * to NekRS should only be updated if the data from MOOSE is "new", and likewise
404 : * whether the NekRS solution should only be interpolated to the mesh mirror once
405 : * MOOSE is actually "ready" to solve a time step using it.
406 : *
407 : * NOTE: if 'interpolate_transfers = true' in the master application, then the data
408 : * coming into nekRS is _unique_ on each subcycled time step, so setting this to
409 : * true will in effect override `interpolate_transfers` to be false. For the best
410 : * performance, you should set `interpolate_transfers` to false so that you don't
411 : * even bother computing the interpolated data, since it's not used if this parameter
412 : * is set to true.
413 : */
414 : synchronization::SynchronizationEnum _sync_interval;
415 :
416 : /// flag to indicate whether this is the first pass to serialize the solution
417 : static bool _first;
418 :
419 : /// All of the FieldTransfer objects which pass data in/out of NekRS
420 : std::vector<FieldTransferBase *> _field_transfers;
421 :
422 : /// All of the ScalarTransfer objecst which pass data in/out of NekRS
423 : std::vector<ScalarTransferBase *> _scalar_transfers;
424 :
425 : /// Usrwrk slots managed by Cardinal
426 : std::set<unsigned int> _usrwrk_slots;
427 : };
|
