Line data Source code
1 : /********************************************************************/
2 : /* SOFTWARE COPYRIGHT NOTIFICATION */
3 : /* Cardinal */
4 : /* */
5 : /* (c) 2021 UChicago Argonne, LLC */
6 : /* ALL RIGHTS RESERVED */
7 : /* */
8 : /* Prepared by UChicago Argonne, LLC */
9 : /* Under Contract No. DE-AC02-06CH11357 */
10 : /* With the U. S. Department of Energy */
11 : /* */
12 : /* Prepared by Battelle Energy Alliance, LLC */
13 : /* Under Contract No. DE-AC07-05ID14517 */
14 : /* With the U. S. Department of Energy */
15 : /* */
16 : /* See LICENSE for full restrictions */
17 : /********************************************************************/
18 :
19 : #ifdef ENABLE_NEK_COUPLING
20 :
21 : #include "NekInitAction.h"
22 : #include "NekInterface.h"
23 :
24 : #include "MooseApp.h"
25 : #include <chrono>
26 :
27 : registerMooseAction("CardinalApp", NekInitAction, "nek_init");
28 :
29 : int NekInitAction::_n_cases = 0;
30 :
31 : InputParameters
32 1632 : NekInitAction::validParams()
33 : {
34 1632 : InputParameters params = MooseObjectAction::validParams();
35 3264 : params.addParam<std::string>("type", "Problem type");
36 3264 : params.addParam<std::string>(
37 : "casename",
38 : "Case name for the NekRS input files; "
39 : "this is <case> in <case>.par, <case>.udf, <case>.oudf, and <case>.re2.");
40 :
41 3264 : params.addParam<unsigned int>(
42 : "n_usrwrk_slots",
43 3264 : 7,
44 : "Number of slots to allocate in nrs->usrwrk to hold fields either related to coupling "
45 : "(which will be populated by Cardinal), or other custom usages, such as a distance-to-wall "
46 : "calculation");
47 :
48 1632 : return params;
49 0 : }
50 :
51 1053 : NekInitAction::NekInitAction(const InputParameters & parameters)
52 : : MooseObjectAction(parameters),
53 1053 : _specified_scratch(parameters.isParamSetByUser("n_usrwrk_slots")),
54 3159 : _n_usrwrk_slots(getParam<unsigned int>("n_usrwrk_slots"))
55 : {
56 1053 : }
57 :
58 : void
59 1053 : NekInitAction::act()
60 : {
61 1053 : if (_type != "NekRSProblem")
62 240 : return;
63 :
64 813 : const auto timeStart = std::chrono::high_resolution_clock::now();
65 :
66 : // NekRS does some checks in main(); because we don't call that function
67 : // directly, repeat those checks here
68 813 : if (!getenv("NEKRS_HOME"))
69 0 : mooseError("Cannot find environment variable NEKRS_HOME!");
70 :
71 813 : std::string bin(getenv("NEKRS_HOME"));
72 : bin += "/bin/nekrs";
73 813 : const char * ptr = realpath(bin.c_str(), NULL);
74 813 : if (!ptr)
75 0 : mooseError("Cannot find '", bin, "'! Did you set NEKRS_HOME to the correct location?");
76 :
77 1626 : std::string setup_file = getParam<std::string>("casename");
78 :
79 : // If the casename is a directory path (i.e. not just a file name), then standalone
80 : // NekRS will cd into that directory so that the casename really is just the case file name,
81 : // i.e. with the path subtracted out. We will replicate this behavior here.
82 813 : std::size_t last_slash = setup_file.rfind('/') + 1;
83 813 : std::string casepath = setup_file.substr(0, last_slash);
84 813 : std::string casename = setup_file.substr(last_slash, setup_file.length() - last_slash);
85 813 : if (casepath.length() > 0)
86 : {
87 5 : int fail = chdir(casepath.c_str());
88 5 : if (fail)
89 1 : mooseError("Failed to find '", casepath.c_str(), "'! Did you set the 'casename' correctly?");
90 : }
91 :
92 : const int size_target =
93 1624 : _app.isParamValid("nekrs_build_only") ? _app.getParam<int>("nekrs_build_only") : 0;
94 1632 : const int ci_mode = _app.isParamValid("nekrs_cimode") ? _app.getParam<int>("nekrs_cimode") : 0;
95 : const std::string backend =
96 1624 : _app.isParamValid("nekrs_backend") ? _app.getParam<std::string>("nekrs_backend") : "";
97 : const std::string device_id =
98 1624 : _app.isParamValid("nekrs_device_id") ? _app.getParam<std::string>("nekrs_device_id") : "";
99 :
100 812 : const int build_only = size_target > 0 ? 1 : 0;
101 812 : nekrs::buildOnly(build_only);
102 :
103 812 : MPI_Comm comm = *static_cast<const MPI_Comm *>(&_communicator.get());
104 :
105 : // If this MPI communicator has already created one case, then we cannot also create a
106 : // second NekRS case. For instance, if you have 4 Nek sub-apps, but only 3 processes, then
107 : // NekRS doesn't like trying to set up a case with a communicator, then immediately try
108 : // to set up another case with the same communicator. If you un-comment this error message,
109 : // you'll get an error when reading the mesh file that is basically missing a "/".
110 : // This error is probably due to NekRS relying a lot on static variables.
111 812 : if (_n_cases > 0)
112 : {
113 : int size;
114 2 : MPI_Comm_size(comm, &size);
115 2 : mooseError(
116 : "NekRS does not currently support setting up multiple cases with the same "
117 : "MPI communicator.\nThat is, you need at least one MPI process in a master "
118 : "application per Nek sub-application.\n\n"
119 2 : "The MPI communicator has " +
120 2 : std::to_string(size) +
121 : " ranks and is trying to "
122 2 : "construct " +
123 2 : std::to_string(_n_cases + 1) +
124 : "+ cases.\n\n"
125 : "If you are running a Stochastic Tools simulation in either 'normal' or 'batch-reset'\n"
126 : "mode, you will need at least 'min_procs_per_app' * 'num_rows' MPI ranks. OR, we\n"
127 : "recommend using the 'batch-restore' mode, which does not have any such limitations.");
128 : }
129 :
130 810 : auto par = readPar(casename, comm);
131 :
132 1620 : nekrs::setup(
133 : comm /* global communicator, like for Nek-Nek : NOT SUPPORTED, so we use same comm */,
134 : comm /* local communicator */,
135 : build_only,
136 : size_target,
137 : ci_mode,
138 : par,
139 : casename,
140 : backend,
141 : device_id,
142 : 1 /* n sessions */,
143 : 0 /* session ID */,
144 : 0 /* debug mode */);
145 :
146 810 : _n_cases++;
147 :
148 : // copy-pasta from NekRS's main()
149 810 : double elapsedTime = 0;
150 810 : const auto timeStop = std::chrono::high_resolution_clock::now();
151 810 : elapsedTime += std::chrono::duration<double, std::milli>(timeStop - timeStart).count() / 1e3;
152 810 : MPI_Allreduce(MPI_IN_PLACE, &elapsedTime, 1, MPI_DOUBLE, MPI_MAX, comm);
153 810 : nekrs::updateTimer("setup", elapsedTime);
154 810 : nekrs::setNekSetupTime(elapsedTime);
155 :
156 810 : _console << "initialization took " << elapsedTime << " s" << std::endl;
157 :
158 810 : if (!nekrs::scratchAvailable())
159 1 : mooseError(
160 : "The nrs_t.usrwrk and nrs_t.o_usrwrk arrays are automatically allocated by Cardinal, "
161 : "but you have tried allocating them separately inside your case files. Please remove the "
162 : "manual allocation of the space in your user files, and be sure to only write such that"
163 : "the space reserved for coupling data is untouched.");
164 :
165 : // Initialize scratch space in NekRS to write data incoming data from MOOSE
166 809 : nekrs::initializeScratch(_n_usrwrk_slots);
167 : }
168 :
169 : inipp::Ini *
170 810 : NekInitAction::readPar(const std::string & _setupFile, MPI_Comm comm)
171 : {
172 810 : auto par = new inipp::Ini();
173 :
174 : int rank;
175 810 : MPI_Comm_rank(comm, &rank);
176 :
177 810 : const auto setupFile = _setupFile + ".par";
178 :
179 810 : int err = 0;
180 810 : if (rank == 0)
181 : {
182 670 : if (!std::filesystem::exists(setupFile))
183 : {
184 0 : std::cerr << "Cannot find setup file " << setupFile << std::endl;
185 0 : err++;
186 : }
187 : }
188 810 : MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MAX, comm);
189 810 : if (err)
190 : {
191 0 : MPI_Abort(comm, EXIT_FAILURE);
192 : }
193 :
194 : char * rbuf;
195 : long fsize;
196 :
197 810 : if (rank == 0)
198 : {
199 335 : FILE * f = fopen(setupFile.c_str(), "rb");
200 335 : fseek(f, 0, SEEK_END);
201 335 : fsize = ftell(f);
202 335 : fseek(f, 0, SEEK_SET);
203 335 : rbuf = new char[fsize];
204 335 : auto s = fread(rbuf, 1, fsize, f);
205 335 : fclose(f);
206 : }
207 810 : MPI_Bcast(&fsize, sizeof(fsize), MPI_BYTE, 0, comm);
208 :
209 810 : if (rank != 0)
210 475 : rbuf = new char[fsize];
211 810 : MPI_Bcast(rbuf, fsize, MPI_CHAR, 0, comm);
212 :
213 810 : std::stringstream is;
214 810 : is.write(rbuf, fsize);
215 :
216 810 : par->parse(is);
217 810 : par->interpolate();
218 :
219 810 : return par;
220 810 : }
221 :
222 : #endif
|
