Line data Source code
1 : /********************************************************************/
2 : /* SOFTWARE COPYRIGHT NOTIFICATION */
3 : /* Cardinal */
4 : /* */
5 : /* (c) 2021 UChicago Argonne, LLC */
6 : /* ALL RIGHTS RESERVED */
7 : /* */
8 : /* Prepared by UChicago Argonne, LLC */
9 : /* Under Contract No. DE-AC02-06CH11357 */
10 : /* With the U. S. Department of Energy */
11 : /* */
12 : /* Prepared by Battelle Energy Alliance, LLC */
13 : /* Under Contract No. DE-AC07-05ID14517 */
14 : /* With the U. S. Department of Energy */
15 : /* */
16 : /* See LICENSE for full restrictions */
17 : /********************************************************************/
18 :
19 : #ifdef ENABLE_NEK_COUPLING
20 :
21 : #include "NekInitAction.h"
22 : #include "NekInterface.h"
23 :
24 : #include "MooseApp.h"
25 : #include <chrono>
26 :
27 : registerMooseAction("CardinalApp", NekInitAction, "nek_init");
28 :
29 : int NekInitAction::_n_cases = 0;
30 :
31 : InputParameters
32 1382 : NekInitAction::validParams()
33 : {
34 1382 : InputParameters params = MooseObjectAction::validParams();
35 2764 : params.addParam<std::string>("type", "Problem type");
36 2764 : params.addParam<std::string>(
37 : "casename",
38 : "Case name for the NekRS input files; "
39 : "this is <case> in <case>.par, <case>.udf, <case>.oudf, and <case>.re2.");
40 :
41 2764 : params.addParam<bool>(
42 : "initialize_usrwrk",
43 2764 : true,
44 : "Whether Cardinal should allocate space for the urswrk array; if false, the user is "
45 : "responsible for allocating this array for their usage in the .udf file");
46 2764 : params.addParam<unsigned int>(
47 : "n_usrwrk_slots",
48 2764 : 0,
49 : "If initialize_usrwrk is true, the number of slots that Cardinal will allocate in "
50 : "platform->app->bc->o_usrwrk. These entries can be used to hold fields related to coupling "
51 : "(which will be "
52 : "populated by Cardinal), or other custom usages, such as a distance-to-wall calculation");
53 :
54 1382 : return params;
55 0 : }
56 :
57 954 : NekInitAction::NekInitAction(const InputParameters & parameters)
58 : : MooseObjectAction(parameters),
59 954 : _initialize_usrwrk(getParam<bool>("initialize_usrwrk")),
60 2862 : _n_usrwrk_slots(getParam<unsigned int>("n_usrwrk_slots"))
61 : {
62 954 : if (!_initialize_usrwrk)
63 0 : checkUnusedParam(parameters, "n_usrwrk_slots", "'initialize_usrwrk' is false");
64 954 : }
65 :
66 : void
67 954 : NekInitAction::act()
68 : {
69 954 : if (_type != "NekRSProblem")
70 187 : return;
71 :
72 767 : const auto timeStart = std::chrono::high_resolution_clock::now();
73 :
74 : // NekRS does some checks in main(); because we don't call that function
75 : // directly, repeat those checks here
76 767 : if (!getenv("NEKRS_HOME"))
77 0 : mooseError("Cannot find environment variable NEKRS_HOME!");
78 :
79 767 : std::string bin(getenv("NEKRS_HOME"));
80 : bin += "/bin/nekrs";
81 767 : const char * ptr = realpath(bin.c_str(), NULL);
82 767 : if (!ptr)
83 0 : mooseError("Cannot find '", bin, "'! Did you set NEKRS_HOME to the correct location?");
84 :
85 1534 : std::string setup_file = getParam<std::string>("casename");
86 :
87 : // If the casename is a directory path (i.e. not just a file name), then standalone
88 : // NekRS will cd into that directory so that the casename really is just the case file name,
89 : // i.e. with the path subtracted out. We will replicate this behavior here.
90 767 : std::size_t last_slash = setup_file.rfind('/') + 1;
91 767 : std::string casepath = setup_file.substr(0, last_slash);
92 767 : std::string casename = setup_file.substr(last_slash, setup_file.length() - last_slash);
93 767 : if (casepath.length() > 0)
94 : {
95 5 : int fail = chdir(casepath.c_str());
96 5 : if (fail)
97 1 : mooseError("Failed to find '", casepath.c_str(), "'! Did you set the 'casename' correctly?");
98 : }
99 :
100 : setup_file = casename;
101 :
102 : const int size_target =
103 1532 : _app.isParamValid("nekrs_build_only") ? _app.getParam<int>("nekrs_build_only") : 0;
104 1540 : const int ci_mode = _app.isParamValid("nekrs_cimode") ? _app.getParam<int>("nekrs_cimode") : 0;
105 : const std::string backend =
106 1532 : _app.isParamValid("nekrs_backend") ? _app.getParam<std::string>("nekrs_backend") : "";
107 : const std::string device_id =
108 1532 : _app.isParamValid("nekrs_device_id") ? _app.getParam<std::string>("nekrs_device_id") : "";
109 :
110 766 : const int build_only = size_target > 0 ? 1 : 0;
111 766 : nekrs::buildOnly(build_only);
112 :
113 766 : MPI_Comm comm = *static_cast<const MPI_Comm *>(&_communicator.get());
114 :
115 : // If this MPI communicator has already created one case, then we cannot also create a
116 : // second NekRS case. For instance, if you have 4 Nek sub-apps, but only 3 processes, then
117 : // NekRS doesn't like trying to set up a case with a communicator, then immediately try
118 : // to set up another case with the same communicator. If you un-comment this error message,
119 : // you'll get an error when reading the mesh file that is basically missing a "/".
120 : // This error is probably due to NekRS relying a lot on static variables.
121 766 : if (_n_cases > 0)
122 : {
123 : int size;
124 2 : MPI_Comm_size(comm, &size);
125 2 : mooseError(
126 : "NekRS does not currently support setting up multiple cases with the same "
127 : "MPI communicator.\nThat is, you need at least one MPI process in a master "
128 : "application per Nek sub-application.\n\n"
129 2 : "The MPI communicator has " +
130 2 : std::to_string(size) +
131 : " ranks and is trying to "
132 2 : "construct " +
133 2 : std::to_string(_n_cases + 1) +
134 : "+ cases.\n\n"
135 : "If you are running a Stochastic Tools simulation in either 'normal' or 'batch-reset'\n"
136 : "mode, you will need at least 'min_procs_per_app' * 'num_rows' MPI ranks. OR, we\n"
137 : "recommend using the 'batch-restore' mode, which does not have any such limitations.");
138 : }
139 :
140 764 : auto par = readPar(setup_file, comm);
141 :
142 1528 : nekrs::setup(
143 : comm /* global communicator, like for Nek-Nek : NOT SUPPORTED, so we use same comm */,
144 : comm /* local communicator */,
145 : build_only,
146 : size_target,
147 : ci_mode,
148 : par,
149 : casename,
150 : backend,
151 : device_id,
152 : 1 /* n sessions */,
153 : 0 /* session ID */,
154 : 0 /* debug mode */);
155 :
156 764 : _n_cases++;
157 :
158 : // copy-pasta from NekRS's main()
159 764 : double elapsedTime = 0;
160 764 : const auto timeStop = std::chrono::high_resolution_clock::now();
161 764 : elapsedTime += std::chrono::duration<double, std::milli>(timeStop - timeStart).count() / 1e3;
162 764 : MPI_Allreduce(MPI_IN_PLACE, &elapsedTime, 1, MPI_DOUBLE, MPI_MAX, comm);
163 764 : nekrs::updateTimer("setup", elapsedTime);
164 764 : nekrs::setNekSetupTime(elapsedTime);
165 :
166 764 : _console << "initialization took " << elapsedTime << " s" << std::endl;
167 :
168 : // Initialize host Nek arrays
169 764 : nekrs::initializeNekHostArrays();
170 :
171 764 : if (_initialize_usrwrk)
172 : {
173 764 : if (!nekrs::scratchAvailable())
174 1 : mooseError("Because 'initialize_usrwrk' is true, the platform->app->bc->o_usrwrk array is "
175 : "allocated by Cardinal, but you have tried allocating it separately inside your "
176 : "case files. Please remove the manual allocation of this array in your case "
177 : "files, and be sure to only write such that the space reserved for Cardinal "
178 : "coupling data is untouched. Cardinal can allocate the size of this array by "
179 : "providing the 'n_usrwrk_slots' parameter for NekRSProblem.\n\nIf you prefer to "
180 : "initialize the usrwrk array manually, set 'initialize_usrwrk' to false.");
181 :
182 : // Initialize scratch space in NekRS to write data incoming data from MOOSE
183 763 : nekrs::initializeScratch(_n_usrwrk_slots);
184 : }
185 : }
186 :
187 : std::map<std::string, std::map<std::string, std::string>>
188 764 : NekInitAction::readPar(const std::string & _setupFile, MPI_Comm comm)
189 : {
190 : int rank;
191 764 : MPI_Comm_rank(comm, &rank);
192 :
193 764 : const auto setupFile = _setupFile + ".par";
194 :
195 764 : if (rank == 0)
196 342 : std::cout << "reading " << setupFile << std::endl;
197 :
198 764 : int err = 0;
199 764 : if (rank == 0)
200 : {
201 684 : if (!std::filesystem::exists(setupFile))
202 : {
203 0 : std::cerr << "Cannot find setup file " << setupFile << std::endl;
204 0 : err++;
205 : }
206 : }
207 764 : MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MAX, comm);
208 764 : if (err)
209 : {
210 0 : MPI_Abort(comm, EXIT_FAILURE);
211 : }
212 :
213 : char * rbuf;
214 : long fsize;
215 :
216 764 : if (rank == 0)
217 : {
218 342 : FILE * f = fopen(setupFile.c_str(), "rb");
219 342 : fseek(f, 0, SEEK_END);
220 342 : fsize = ftell(f);
221 342 : fseek(f, 0, SEEK_SET);
222 342 : rbuf = new char[fsize];
223 342 : auto s = fread(rbuf, 1, fsize, f);
224 342 : fclose(f);
225 : }
226 764 : MPI_Bcast(&fsize, sizeof(fsize), MPI_BYTE, 0, comm);
227 :
228 764 : if (rank != 0)
229 422 : rbuf = new char[fsize];
230 764 : MPI_Bcast(rbuf, fsize, MPI_CHAR, 0, comm);
231 :
232 764 : auto par = new inipp::Ini();
233 :
234 764 : std::stringstream is;
235 764 : is.write(rbuf, fsize);
236 :
237 764 : par->parse(is);
238 764 : par->interpolate();
239 :
240 764 : return par->sections;
241 764 : }
242 :
243 : #endif
|
