Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "RestartableDataReader.h"
11 :
12 : #include "StringInputStream.h"
13 : #include "FileInputStream.h"
14 : #include "RestartableDataMap.h"
15 : #include "MooseUtils.h"
16 :
17 : #include <fstream>
18 :
19 : // Type hash codes can't be relied on for older clang...
20 : // not sure why, and also don't care why
21 : #if defined(__clang__) && __clang_major__ < 12
22 : #define RESTARTABLE_SKIP_CHECK_HASH_CODE
23 : #endif
24 :
25 3224 : RestartableDataReader::RestartableDataReader(MooseApp & app,
26 : RestartableDataMap & data,
27 3224 : const bool force /* = false */)
28 : : RestartableDataIO(app, data),
29 3224 : _is_restoring(false),
30 3224 : _error_on_different_number_of_processors(true),
31 3224 : _force(force)
32 : {
33 3224 : }
34 :
35 62755 : RestartableDataReader::RestartableDataReader(MooseApp & app,
36 : std::vector<RestartableDataMap> & data,
37 62755 : const bool force /* = false */)
38 : : RestartableDataIO(app, data),
39 62755 : _is_restoring(false),
40 62755 : _error_on_different_number_of_processors(true),
41 62755 : _force(force)
42 : {
43 62755 : }
44 :
45 : void
46 9906 : RestartableDataReader::setInput(std::unique_ptr<std::stringstream> header_stream,
47 : std::unique_ptr<std::stringstream> data_stream)
48 : {
49 : mooseAssert(!_streams.header, "Header input already set");
50 : mooseAssert(!_streams.data, "Data stream already set");
51 9906 : _streams.header = std::make_unique<StringInputStream>(std::move(header_stream));
52 9906 : _streams.data = std::make_unique<StringInputStream>(std::move(data_stream));
53 9906 : }
54 :
55 : void
56 6539 : RestartableDataReader::setInput(const std::filesystem::path & folder_base)
57 : {
58 : mooseAssert(!_streams.header, "Header input already set");
59 : mooseAssert(!_streams.data, "Data stream already set");
60 6539 : _streams.header = std::make_unique<FileInputStream>(restartableHeaderFile(folder_base));
61 6539 : _streams.data = std::make_unique<FileInputStream>(restartableDataFile(folder_base));
62 6539 : }
63 :
64 : RestartableDataReader::InputStreams
65 13182 : RestartableDataReader::clear()
66 : {
67 13182 : _is_restoring = false;
68 13182 : _header.clear();
69 13182 : InputStreams streams;
70 13182 : std::swap(_streams, streams);
71 13182 : return streams;
72 : }
73 :
74 : std::vector<std::unordered_map<std::string, RestartableDataReader::HeaderEntry>>
75 16445 : RestartableDataReader::readHeader(InputStream & header_input) const
76 : {
77 16445 : std::vector<std::unordered_map<std::string, RestartableDataReader::HeaderEntry>> header;
78 :
79 16445 : auto stream_ptr = header_input.get();
80 16445 : auto & stream = *stream_ptr;
81 :
82 16445 : stream.seekg(0);
83 :
84 28 : const auto error = [&header_input](auto... args)
85 : {
86 14 : std::stringstream err_prefix;
87 14 : err_prefix << "While reading restartable data in ";
88 14 : const auto filename = header_input.getFilename();
89 14 : if (filename)
90 14 : err_prefix << std::filesystem::absolute(filename->parent_path());
91 : else
92 0 : err_prefix << "memory";
93 14 : err_prefix << ":\n\n";
94 :
95 14 : mooseError(err_prefix.str(), args...);
96 16445 : };
97 :
98 : // ID
99 : char this_id[2];
100 16445 : stream.read(this_id, 2);
101 16445 : if (this_id[0] != 'R' || this_id[1] != 'D')
102 0 : error("The data is invalid or corrupted (unexpected header)");
103 :
104 : // File version
105 : std::remove_const<decltype(CURRENT_BACKUP_FILE_VERSION)>::type this_file_version;
106 16445 : dataLoad(stream, this_file_version, nullptr);
107 16445 : if (this_file_version != CURRENT_BACKUP_FILE_VERSION)
108 0 : error("There is a mismatch in the backup version\n\n",
109 : "Current backup version: ",
110 : CURRENT_BACKUP_FILE_VERSION,
111 : "\nLoaded backup version: ",
112 : this_file_version);
113 :
114 : // Type id for a basic type
115 : std::size_t this_compare_hash_code;
116 16445 : dataLoad(stream, this_compare_hash_code, nullptr);
117 : #ifndef RESTARTABLE_SKIP_CHECK_HASH_CODE
118 16445 : if (this_compare_hash_code != typeid(COMPARE_HASH_CODE_TYPE).hash_code() && !_force)
119 0 : error("The backup is not compatible\n\nThe hash code check for a basic type (",
120 0 : MooseUtils::prettyCppType<COMPARE_HASH_CODE_TYPE>(),
121 : ") failed.\nIt is possible that this backup was stored with a different architecture or "
122 : "operating system.\n\nTo forcefully attempt loading the backup, use the command line "
123 : "option --force-restart");
124 : #else
125 : (void)_force;
126 : #endif
127 :
128 : // Number of procs
129 16445 : decltype(n_processors()) this_n_procs = 0;
130 16445 : dataLoad(stream, this_n_procs, nullptr);
131 16445 : if (_error_on_different_number_of_processors && this_n_procs != n_processors())
132 7 : error("The number of MPI ranks is not consistent\n\nCurrent MPI ranks: ",
133 : n_processors(),
134 : "\nLoaded MPI ranks: ",
135 : this_n_procs);
136 :
137 : // Number of data
138 16438 : decltype(dataSize()) this_num_data = 0;
139 16438 : dataLoad(stream, this_num_data, nullptr);
140 16438 : if (this_num_data != dataSize())
141 7 : error("The number of threads is not consistent\n\nCurrent threads: ",
142 : dataSize(),
143 : "\nLoaded threads: ",
144 : this_num_data);
145 :
146 16431 : header.resize(dataSize());
147 :
148 : // Size of data for each thread
149 16431 : std::vector<std::size_t> tid_n_data(dataSize());
150 33750 : for (const auto tid : make_range(dataSize()))
151 17319 : dataLoad(stream, tid_n_data[tid], nullptr);
152 :
153 : // The position of the current data that we're loading
154 16431 : std::size_t current_data_position = 0;
155 :
156 : // Load the data header for each thread
157 33750 : for (const auto tid : make_range(dataSize()))
158 552537 : for (const auto i : make_range(tid_n_data[tid]))
159 : {
160 535218 : std::ignore = i;
161 :
162 535218 : std::string name;
163 535218 : dataLoad(stream, name, nullptr);
164 : mooseAssert(name.size(), "Empty name");
165 :
166 : mooseAssert(!header[tid].count(name), "Data '" + name + "' is already inserted");
167 535218 : auto & entry = header[tid][name];
168 :
169 535218 : dataLoad(stream, entry.size, nullptr);
170 535218 : dataLoad(stream, entry.type_hash_code, nullptr);
171 535218 : dataLoad(stream, entry.type, nullptr);
172 535218 : dataLoad(stream, entry.has_context, nullptr);
173 535218 : entry.position = current_data_position;
174 :
175 535218 : current_data_position += entry.size;
176 535218 : }
177 :
178 16431 : stream.seekg(0);
179 :
180 32862 : return header;
181 16431 : }
182 :
183 : void
184 16446 : RestartableDataReader::restore(const DataNames & filter_names /* = {} */)
185 : {
186 16446 : if (!_streams.header || !_streams.data)
187 1 : mooseError("RestartableDataReader::restore(): Cannot restore because an input was not set");
188 :
189 16445 : _is_restoring = true;
190 :
191 : // Set everything as not loaded
192 33786 : for (const auto tid : make_range(dataSize()))
193 553326 : for (auto & value : currentData(tid))
194 535985 : value.setNotLoaded({});
195 :
196 : // Read the header
197 16445 : _header = readHeader(*_streams.header);
198 :
199 33717 : for (const auto tid : index_range(_header))
200 : {
201 17313 : auto & data = currentData(tid);
202 17313 : const auto & header = _header[tid];
203 :
204 : // TODO: Think about what to do with missing data
205 : // Load the data in the order that it was requested
206 551956 : for (auto & value : data)
207 : {
208 534670 : const auto & name = value.name();
209 :
210 534670 : auto find_header = header.find(name);
211 534670 : if (find_header == header.end())
212 1060 : continue;
213 :
214 533610 : auto & header_entry = find_header->second;
215 :
216 : // Only restore values if we're either recovering or the data isn't filtered out
217 533610 : const auto is_data_in_filter = filter_names.find(name) != filter_names.end();
218 533610 : if (!is_data_in_filter)
219 526639 : deserializeValue(*_streams.data, value, header_entry);
220 : }
221 : }
222 16404 : }
223 :
224 : bool
225 21 : RestartableDataReader::hasData(const std::string & data_name,
226 : const std::type_info & type,
227 : const THREAD_ID tid) const
228 : {
229 21 : if (const auto header = queryHeader(data_name, tid))
230 20 : return isSameType(*header, type);
231 1 : return false;
232 : }
233 :
234 : const RestartableDataReader::HeaderEntry *
235 26 : RestartableDataReader::queryHeader(const std::string & data_name, const THREAD_ID tid) const
236 : {
237 26 : requireRestoring();
238 26 : const auto it = _header[tid].find(data_name);
239 26 : if (it == _header[tid].end())
240 1 : return nullptr;
241 25 : return &it->second;
242 : }
243 :
244 : const RestartableDataReader::HeaderEntry &
245 5 : RestartableDataReader::getHeader(const std::string & data_name, const THREAD_ID tid) const
246 : {
247 5 : const auto header = queryHeader(data_name, tid);
248 5 : if (!header)
249 0 : mooseError(
250 : "RestartableDataReader::getHeader(): Failed to find a header entry for data with name '",
251 : data_name,
252 : "'");
253 5 : return *header;
254 : }
255 :
256 : void
257 526644 : RestartableDataReader::deserializeValue(
258 : InputStream & data_input,
259 : RestartableDataValue & value,
260 : const RestartableDataReader::HeaderEntry & header_entry) const
261 : {
262 : mooseAssert(!value.loaded(), value.name() + " is already loaded");
263 :
264 0 : auto error = [&data_input, &value](auto... args)
265 : {
266 0 : std::stringstream err;
267 0 : err << "While loading restartable data\n\n";
268 0 : err << "From: ";
269 0 : const auto filename = data_input.getFilename();
270 0 : if (filename)
271 0 : err << std::filesystem::absolute(filename->parent_path());
272 : else
273 0 : err << "memory";
274 0 : err << "\nData name: " << value.name();
275 0 : err << "\nData type: " << value.type();
276 0 : err << "\n\n";
277 0 : mooseError(err.str(), args...);
278 526644 : };
279 :
280 526644 : if (!isSameType(header_entry, value.typeId()))
281 0 : error("The stored type of '", header_entry.type, "' does not match");
282 :
283 526644 : auto stream_ptr = data_input.get();
284 526644 : auto & stream = *stream_ptr;
285 :
286 526644 : stream.seekg(header_entry.position);
287 526644 : value.load(stream);
288 :
289 526617 : if (stream.tellg() == -1)
290 0 : error("An error was encountered when reading from the stream");
291 :
292 526617 : const std::size_t loaded_size = stream.tellg() - header_entry.position;
293 526617 : if (loaded_size != header_entry.size)
294 0 : error("The data read does not match the data stored\n\n",
295 : "Stored size: ",
296 0 : header_entry.size,
297 : "\nLoaded size: ",
298 : loaded_size);
299 526617 : }
300 :
301 : bool
302 9631 : RestartableDataReader::isAvailable(const std::filesystem::path & folder_base)
303 : {
304 9631 : const auto header_path = restartableDataFile(folder_base);
305 9631 : const auto data_path = restartableDataFile(folder_base);
306 :
307 19262 : const auto available = [](const auto & filename)
308 19262 : { return MooseUtils::pathExists(filename) && MooseUtils::checkFileReadable(filename); };
309 :
310 9631 : const bool header_available = available(header_path);
311 9631 : const bool data_available = available(data_path);
312 :
313 9631 : if (header_available != data_available)
314 0 : mooseError("The restart ",
315 0 : header_available ? "header" : "data",
316 : " is available but the corresponding ",
317 0 : header_available ? "data" : "header",
318 : " is not available\n\n",
319 : "Header (",
320 0 : header_available ? "available" : "missing",
321 : "): ",
322 0 : std::filesystem::absolute(header_path),
323 : "\nData (",
324 0 : data_available ? "available" : "missing",
325 : "): ",
326 0 : std::filesystem::absolute(header_path));
327 :
328 19262 : return header_available && data_available;
329 9631 : }
330 :
331 : void
332 26 : RestartableDataReader::requireRestoring() const
333 : {
334 26 : if (!_is_restoring)
335 0 : mooseError(
336 : "The RestartableDataReader is not available for querying as it is not currently restoring");
337 : mooseAssert(_streams.header, "Header not available");
338 : mooseAssert(_streams.data, "Data not available");
339 26 : }
340 :
341 : bool
342 526664 : RestartableDataReader::isSameType(const RestartableDataReader::HeaderEntry & header_entry,
343 : const std::type_info & type) const
344 : {
345 : #ifndef RESTARTABLE_SKIP_CHECK_HASH_CODE
346 526664 : if (header_entry.type_hash_code == type.hash_code())
347 526654 : return true;
348 : #endif
349 10 : return header_entry.type == type.name();
350 : }
351 :
352 : RestartableDataValue &
353 10 : RestartableDataReader::restoreData(const std::string & data_name,
354 : std::unique_ptr<RestartableDataValue> value,
355 : const THREAD_ID tid)
356 : {
357 10 : auto & data_map = currentData(tid);
358 10 : if (data_map.hasData(data_name))
359 5 : mooseError("RestartableDataReader::restoreData(): Cannot declare restartable data '",
360 : data_name,
361 : "' because it has already been declared");
362 :
363 5 : const auto & header = getHeader(data_name, tid);
364 5 : auto & added_value = currentData(tid).addData(std::move(value));
365 5 : deserializeValue(*_streams.data, added_value, header);
366 5 : return added_value;
367 : }
|
