Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "RestartableDataReader.h"
11 :
12 : #include "StringInputStream.h"
13 : #include "FileInputStream.h"
14 : #include "RestartableDataMap.h"
15 : #include "MooseUtils.h"
16 :
17 : #include <fstream>
18 :
19 : // Type hash codes can't be relied on for older clang...
20 : // not sure why, and also don't care why
21 : #if defined(__clang__) && __clang_major__ < 12
22 : #define RESTARTABLE_SKIP_CHECK_HASH_CODE
23 : #endif
24 :
25 3584 : RestartableDataReader::RestartableDataReader(MooseApp & app,
26 : RestartableDataMap & data,
27 3584 : const bool force /* = false */)
28 : : RestartableDataIO(app, data),
29 3584 : _is_restoring(false),
30 3584 : _error_on_different_number_of_processors(true),
31 3584 : _force(force)
32 : {
33 3584 : }
34 :
35 67232 : RestartableDataReader::RestartableDataReader(MooseApp & app,
36 : std::vector<RestartableDataMap> & data,
37 67232 : const bool force /* = false */)
38 : : RestartableDataIO(app, data),
39 67232 : _is_restoring(false),
40 67232 : _error_on_different_number_of_processors(true),
41 67232 : _force(force)
42 : {
43 67232 : }
44 :
45 : void
46 9720 : RestartableDataReader::setInput(std::unique_ptr<std::stringstream> header_stream,
47 : std::unique_ptr<std::stringstream> data_stream)
48 : {
49 : mooseAssert(!_streams.header, "Header input already set");
50 : mooseAssert(!_streams.data, "Data stream already set");
51 9720 : _streams.header = std::make_unique<StringInputStream>(std::move(header_stream));
52 9720 : _streams.data = std::make_unique<StringInputStream>(std::move(data_stream));
53 9720 : }
54 :
55 : void
56 7322 : RestartableDataReader::setInput(const std::filesystem::path & folder_base)
57 : {
58 : mooseAssert(!_streams.header, "Header input already set");
59 : mooseAssert(!_streams.data, "Data stream already set");
60 7322 : _streams.header = std::make_unique<FileInputStream>(restartableHeaderFile(folder_base));
61 7322 : _streams.data = std::make_unique<FileInputStream>(restartableDataFile(folder_base));
62 7322 : }
63 :
64 : RestartableDataReader::InputStreams
65 13429 : RestartableDataReader::clear()
66 : {
67 13429 : _is_restoring = false;
68 13429 : _header.clear();
69 13429 : InputStreams streams;
70 13429 : std::swap(_streams, streams);
71 13429 : return streams;
72 : }
73 :
74 : std::vector<std::unordered_map<std::string, RestartableDataReader::HeaderEntry>>
75 17042 : RestartableDataReader::readHeader(InputStream & header_input) const
76 : {
77 17042 : std::vector<std::unordered_map<std::string, RestartableDataReader::HeaderEntry>> header;
78 :
79 17042 : auto stream_ptr = header_input.get();
80 17042 : auto & stream = *stream_ptr;
81 :
82 17042 : stream.seekg(0);
83 :
84 8 : const auto error = [&header_input](auto... args)
85 : {
86 8 : std::stringstream err_prefix;
87 8 : err_prefix << "While reading restartable data in ";
88 8 : const auto filename = header_input.getFilename();
89 8 : if (filename)
90 8 : err_prefix << std::filesystem::absolute(filename->parent_path());
91 : else
92 0 : err_prefix << "memory";
93 8 : err_prefix << ":\n\n";
94 :
95 8 : mooseError(err_prefix.str(), args...);
96 17042 : };
97 :
98 : // ID
99 : char this_id[2];
100 17042 : stream.read(this_id, 2);
101 17042 : if (this_id[0] != 'R' || this_id[1] != 'D')
102 0 : error("The data is invalid or corrupted (unexpected header)");
103 :
104 : // File version
105 : std::remove_const<decltype(CURRENT_BACKUP_FILE_VERSION)>::type this_file_version;
106 17042 : dataLoad(stream, this_file_version, nullptr);
107 17042 : if (this_file_version != CURRENT_BACKUP_FILE_VERSION)
108 0 : error("There is a mismatch in the backup version\n\n",
109 : "Current backup version: ",
110 : CURRENT_BACKUP_FILE_VERSION,
111 : "\nLoaded backup version: ",
112 : this_file_version);
113 :
114 : // Type id for a basic type
115 : std::size_t this_compare_hash_code;
116 17042 : dataLoad(stream, this_compare_hash_code, nullptr);
117 : #ifndef RESTARTABLE_SKIP_CHECK_HASH_CODE
118 17042 : if (this_compare_hash_code != typeid(COMPARE_HASH_CODE_TYPE).hash_code() && !_force)
119 0 : error("The backup is not compatible\n\nThe hash code check for a basic type (",
120 0 : MooseUtils::prettyCppType<COMPARE_HASH_CODE_TYPE>(),
121 : ") failed.\nIt is possible that this backup was stored with a different architecture or "
122 : "operating system.\n\nTo forcefully attempt loading the backup, use the command line "
123 : "option --force-restart");
124 : #else
125 : (void)_force;
126 : #endif
127 :
128 : // Number of procs
129 17042 : decltype(n_processors()) this_n_procs = 0;
130 17042 : dataLoad(stream, this_n_procs, nullptr);
131 17042 : if (_error_on_different_number_of_processors && this_n_procs != n_processors())
132 4 : error("The number of MPI ranks is not consistent\n\nCurrent MPI ranks: ",
133 : n_processors(),
134 : "\nLoaded MPI ranks: ",
135 : this_n_procs);
136 :
137 : // Number of data
138 17038 : decltype(dataSize()) this_num_data = 0;
139 17038 : dataLoad(stream, this_num_data, nullptr);
140 17038 : if (this_num_data != dataSize())
141 4 : error("The number of threads is not consistent\n\nCurrent threads: ",
142 : dataSize(),
143 : "\nLoaded threads: ",
144 : this_num_data);
145 :
146 17034 : header.resize(dataSize());
147 :
148 : // Size of data for each thread
149 17034 : std::vector<std::size_t> tid_n_data(dataSize());
150 34969 : for (const auto tid : make_range(dataSize()))
151 17935 : dataLoad(stream, tid_n_data[tid], nullptr);
152 :
153 : // The position of the current data that we're loading
154 17034 : std::size_t current_data_position = 0;
155 :
156 : // Load the data header for each thread
157 34969 : for (const auto tid : make_range(dataSize()))
158 608033 : for (const auto i : make_range(tid_n_data[tid]))
159 : {
160 590098 : std::ignore = i;
161 :
162 590098 : std::string name;
163 590098 : dataLoad(stream, name, nullptr);
164 : mooseAssert(name.size(), "Empty name");
165 :
166 : mooseAssert(!header[tid].count(name), "Data '" + name + "' is already inserted");
167 590098 : auto & entry = header[tid][name];
168 :
169 590098 : dataLoad(stream, entry.size, nullptr);
170 590098 : dataLoad(stream, entry.type_hash_code, nullptr);
171 590098 : dataLoad(stream, entry.type, nullptr);
172 590098 : dataLoad(stream, entry.has_context, nullptr);
173 590098 : entry.position = current_data_position;
174 :
175 590098 : current_data_position += entry.size;
176 590098 : }
177 :
178 17034 : stream.seekg(0);
179 :
180 34068 : return header;
181 17034 : }
182 :
183 : void
184 17044 : RestartableDataReader::restore(const DataNames & filter_names /* = {} */)
185 : {
186 17044 : if (!_streams.header || !_streams.data)
187 2 : mooseError("RestartableDataReader::restore(): Cannot restore because an input was not set");
188 :
189 17042 : _is_restoring = true;
190 :
191 : // Set everything as not loaded
192 34990 : for (const auto tid : make_range(dataSize()))
193 608971 : for (auto & value : currentData(tid))
194 591023 : value.setNotLoaded({});
195 :
196 : // Read the header
197 17042 : _header = readHeader(*_streams.header);
198 :
199 34935 : for (const auto tid : index_range(_header))
200 : {
201 17926 : auto & data = currentData(tid);
202 17926 : const auto & header = _header[tid];
203 :
204 : // TODO: Think about what to do with missing data
205 : // Load the data in the order that it was requested
206 607782 : for (auto & value : data)
207 : {
208 589881 : const auto & name = value.name();
209 :
210 589881 : auto find_header = header.find(name);
211 589881 : if (find_header == header.end())
212 1485 : continue;
213 :
214 588396 : auto & header_entry = find_header->second;
215 :
216 : // Only restore values if we're either recovering or the data isn't filtered out
217 588396 : const auto is_data_in_filter = filter_names.find(name) != filter_names.end();
218 588396 : if (!is_data_in_filter)
219 579924 : deserializeValue(*_streams.data, value, header_entry);
220 : }
221 : }
222 17009 : }
223 :
224 : bool
225 42 : RestartableDataReader::hasData(const std::string & data_name,
226 : const std::type_info & type,
227 : const THREAD_ID tid) const
228 : {
229 42 : if (const auto header = queryHeader(data_name, tid))
230 40 : return isSameType(*header, type);
231 2 : return false;
232 : }
233 :
234 : const RestartableDataReader::HeaderEntry *
235 131 : RestartableDataReader::queryHeader(const std::string & data_name, const THREAD_ID tid) const
236 : {
237 131 : requireRestoring();
238 131 : const auto it = _header[tid].find(data_name);
239 131 : if (it == _header[tid].end())
240 2 : return nullptr;
241 129 : return &it->second;
242 : }
243 :
244 : const RestartableDataReader::HeaderEntry &
245 10 : RestartableDataReader::getHeader(const std::string & data_name, const THREAD_ID tid) const
246 : {
247 10 : const auto header = queryHeader(data_name, tid);
248 10 : if (!header)
249 0 : mooseError(
250 : "RestartableDataReader::getHeader(): Failed to find a header entry for data with name '",
251 : data_name,
252 : "'");
253 10 : return *header;
254 : }
255 :
256 : void
257 580013 : RestartableDataReader::deserializeValue(
258 : InputStream & data_input,
259 : RestartableDataValue & value,
260 : const RestartableDataReader::HeaderEntry & header_entry) const
261 : {
262 : mooseAssert(!value.loaded(), value.name() + " is already loaded");
263 :
264 0 : auto error = [&data_input, &value](auto... args)
265 : {
266 0 : std::stringstream err;
267 0 : err << "While loading restartable data\n\n";
268 0 : err << "From: ";
269 0 : const auto filename = data_input.getFilename();
270 0 : if (filename)
271 0 : err << std::filesystem::absolute(filename->parent_path());
272 : else
273 0 : err << "memory";
274 0 : err << "\nData name: " << value.name();
275 0 : err << "\nData type: " << value.type();
276 0 : err << "\n\n";
277 0 : mooseError(err.str(), args...);
278 580013 : };
279 :
280 580013 : if (!isSameType(header_entry, value.typeId()))
281 0 : error("The stored type of '", header_entry.type, "' does not match");
282 :
283 580013 : auto stream_ptr = data_input.get();
284 580013 : auto & stream = *stream_ptr;
285 :
286 580013 : stream.seekg(header_entry.position);
287 580013 : value.load(stream);
288 :
289 579988 : if (stream.tellg() == -1)
290 0 : error("An error was encountered when reading from the stream");
291 :
292 579988 : const std::size_t loaded_size = stream.tellg() - header_entry.position;
293 579988 : if (loaded_size != header_entry.size)
294 0 : error("The data read does not match the data stored\n\n",
295 : "Stored size: ",
296 0 : header_entry.size,
297 : "\nLoaded size: ",
298 : loaded_size);
299 579988 : }
300 :
301 : bool
302 11526 : RestartableDataReader::isAvailable(const std::filesystem::path & folder_base)
303 : {
304 11526 : const auto header_path = restartableDataFile(folder_base);
305 11526 : const auto data_path = restartableDataFile(folder_base);
306 :
307 23052 : const auto available = [](const auto & filename)
308 23052 : { return MooseUtils::pathExists(filename) && MooseUtils::checkFileReadable(filename); };
309 :
310 11526 : const bool header_available = available(header_path);
311 11526 : const bool data_available = available(data_path);
312 :
313 11526 : if (header_available != data_available)
314 0 : mooseError("The restart ",
315 0 : header_available ? "header" : "data",
316 : " is available but the corresponding ",
317 0 : header_available ? "data" : "header",
318 : " is not available\n\n",
319 : "Header (",
320 0 : header_available ? "available" : "missing",
321 : "): ",
322 0 : std::filesystem::absolute(header_path),
323 : "\nData (",
324 0 : data_available ? "available" : "missing",
325 : "): ",
326 0 : std::filesystem::absolute(header_path));
327 :
328 23052 : return header_available && data_available;
329 11526 : }
330 :
331 : void
332 131 : RestartableDataReader::requireRestoring() const
333 : {
334 131 : if (!_is_restoring)
335 0 : mooseError(
336 : "The RestartableDataReader is not available for querying as it is not currently restoring");
337 : mooseAssert(_streams.header, "Header not available");
338 : mooseAssert(_streams.data, "Data not available");
339 131 : }
340 :
341 : bool
342 82576 : RestartableDataReader::restoreDataIfAvailable(RestartableDataValue & value,
343 : const THREAD_ID tid,
344 : Moose::PassKey<MooseApp>)
345 : {
346 82576 : if (!isRestoring() || value.loaded())
347 82497 : return false;
348 79 : const auto * const header_entry = queryHeader(value.name(), tid);
349 79 : if (!header_entry)
350 0 : return false;
351 79 : deserializeValue(*_streams.data, value, *header_entry);
352 79 : return true;
353 : }
354 :
355 : bool
356 580053 : RestartableDataReader::isSameType(const RestartableDataReader::HeaderEntry & header_entry,
357 : const std::type_info & type) const
358 : {
359 : #ifndef RESTARTABLE_SKIP_CHECK_HASH_CODE
360 580053 : if (header_entry.type_hash_code == type.hash_code())
361 580033 : return true;
362 : #endif
363 20 : return header_entry.type == type.name();
364 : }
365 :
366 : RestartableDataValue &
367 20 : RestartableDataReader::restoreData(const std::string & data_name,
368 : std::unique_ptr<RestartableDataValue> value,
369 : const THREAD_ID tid)
370 : {
371 20 : auto & data_map = currentData(tid);
372 20 : if (data_map.hasData(data_name))
373 10 : mooseError("RestartableDataReader::restoreData(): Cannot declare restartable data '",
374 : data_name,
375 : "' because it has already been declared");
376 :
377 10 : const auto & header = getHeader(data_name, tid);
378 10 : auto & added_value = currentData(tid).addData(std::move(value));
379 10 : deserializeValue(*_streams.data, added_value, header);
380 10 : return added_value;
381 : }
|
