Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "RestartableEquationSystems.h"
11 :
12 : #include "DataIO.h"
13 :
14 : #include "libmesh/dof_map.h"
15 : #include "libmesh/dof_object.h"
16 : #include "libmesh/elem.h"
17 : #include "libmesh/node.h"
18 :
19 : const std::string RestartableEquationSystems::SystemHeader::system_solution_name =
20 : "SYSTEM_SOLUTION";
21 :
22 62453 : RestartableEquationSystems::RestartableEquationSystems(libMesh::MeshBase & mesh)
23 62453 : : _es(mesh), _load_all_vectors(true)
24 : {
25 62453 : }
26 :
27 : RestartableEquationSystems::EquationSystemsHeader
28 46660 : RestartableEquationSystems::buildHeader(
29 : const std::vector<const libMesh::DofObject *> & ordered_objects) const
30 : {
31 46660 : EquationSystemsHeader es_header;
32 :
33 : // Systems
34 141034 : for (const auto sys_num : make_range(_es.n_systems()))
35 : {
36 94374 : const auto & sys = _es.get_system(sys_num);
37 :
38 94374 : SystemHeader sys_header;
39 94374 : sys_header.name = sys.name();
40 94374 : sys_header.type = sys.system_type();
41 :
42 : // Variables in the system
43 188332 : for (const auto var_num : make_range(sys.n_vars()))
44 : {
45 93958 : const auto & var = sys.variable(var_num);
46 :
47 93958 : VariableHeader var_header;
48 93958 : var_header.name = var.name();
49 93958 : var_header.type = var.type();
50 93958 : var_header.size = 0;
51 93958 : var_header.variable = &var;
52 :
53 : mooseAssert(_es.comm().verify("sys_" + sys.name() + "_var_" + var.name()),
54 : "Out of order in parallel");
55 : mooseAssert(!sys_header.variables.count(var.name()), "Already inserted");
56 :
57 : // Non-SCALAR variable
58 93958 : if (var.type().family != SCALAR)
59 : {
60 19408274 : for (const auto & obj : ordered_objects)
61 19315973 : var_header.size += sizeof(Real) * obj->n_comp(sys.number(), var.number());
62 : }
63 : // SCALAR variable on the last rank
64 1657 : else if (_es.processor_id() == _es.n_processors() - 1)
65 : {
66 1611 : std::vector<dof_id_type> scalar_dofs;
67 1611 : sys.get_dof_map().SCALAR_dof_indices(scalar_dofs, var.number());
68 1611 : var_header.size += sizeof(Real) * scalar_dofs.size();
69 1611 : }
70 :
71 93958 : sys_header.variables.emplace(var.name(), var_header);
72 93958 : }
73 :
74 : // System vector
75 94374 : auto & sys_vec_header = sys_header.vectors[SystemHeader::system_solution_name];
76 94374 : sys_vec_header.name = SystemHeader::system_solution_name;
77 94374 : sys_vec_header.vector = sys.solution.get();
78 94374 : sys_vec_header.type = sys_vec_header.vector->type();
79 409699 : for (const auto vec_num : make_range(sys.n_vectors()))
80 : {
81 : mooseAssert(_es.comm().verify("sys_" + sys.name() + "_vec_" + sys.vector_name(vec_num)),
82 : "Out of order in parallel");
83 315325 : const auto & name = sys.vector_name(vec_num);
84 : mooseAssert(!sys_header.vectors.count(name), "Already inserted");
85 315325 : auto & vec_header = sys_header.vectors[name];
86 315325 : vec_header.name = sys.vector_name(vec_num);
87 315325 : vec_header.projections = sys.vector_preservation(vec_header.name);
88 315325 : vec_header.vector = &sys.get_vector(vec_header.name);
89 315325 : vec_header.type = vec_header.vector->type();
90 : }
91 :
92 : // System in this EquationSystems
93 : mooseAssert(!es_header.systems.count(sys.name()), "Already inserted");
94 94374 : es_header.systems.emplace(sys.name(), sys_header);
95 94374 : }
96 :
97 : // Setup the positions in each vector for easy access later
98 46660 : std::size_t offset = 0;
99 141034 : for (auto & sys_name_header_pair : es_header.systems)
100 : {
101 94374 : auto & sys_header = sys_name_header_pair.second;
102 504073 : for (auto & vec_name_header_pair : sys_header.vectors)
103 : {
104 409699 : auto & vec_header = vec_name_header_pair.second;
105 821143 : for (const auto & var_name_header_pair : sys_header.variables)
106 : {
107 411444 : const auto & var_header = var_name_header_pair.second;
108 : mooseAssert(!vec_header.variable_offset.count(var_header.name), "Already inserted");
109 411444 : vec_header.variable_offset[var_header.name] = offset;
110 411444 : offset += var_header.size;
111 : }
112 : }
113 : }
114 :
115 46660 : es_header.data_size = offset;
116 :
117 46660 : return es_header;
118 0 : }
119 :
120 : std::vector<const libMesh::DofObject *>
121 60150 : RestartableEquationSystems::orderDofObjects() const
122 : {
123 60150 : std::vector<const libMesh::DofObject *> objects;
124 120300 : auto add = [&objects](const auto begin, const auto end)
125 : {
126 120300 : std::set<const libMesh::DofObject *, libMesh::CompareDofObjectsByID> ordered(begin, end);
127 120300 : objects.insert(objects.end(), ordered.begin(), ordered.end());
128 180450 : };
129 :
130 60150 : const auto & mesh = _es.get_mesh();
131 60150 : add(mesh.local_elements_begin(), mesh.local_elements_end());
132 60150 : add(mesh.local_nodes_begin(), mesh.local_nodes_end());
133 :
134 120300 : return objects;
135 0 : }
136 :
137 : void
138 46660 : RestartableEquationSystems::store(std::ostream & stream) const
139 : {
140 : // Order objects (elements and then nodes) by ID for storing
141 46660 : const auto ordered_objects = orderDofObjects();
142 :
143 : // Store the header (systems, variables, vectors)
144 46660 : EquationSystemsHeader es_header = buildHeader(ordered_objects);
145 46660 : dataStore(stream, es_header, nullptr);
146 :
147 : // Store the ordered objects so we can do a sanity check on if we're
148 : // loading the same thing
149 : {
150 46660 : std::vector<dof_id_type> ordered_objects_ids(ordered_objects.size());
151 10158367 : for (const auto i : index_range(ordered_objects))
152 10111707 : ordered_objects_ids[i] = ordered_objects[i]->id();
153 46660 : dataStore(stream, ordered_objects_ids, nullptr);
154 46660 : }
155 :
156 : #ifndef NDEBUG
157 : const std::size_t data_initial_position = static_cast<std::size_t>(stream.tellp());
158 : #endif
159 :
160 : // Store each system
161 141034 : for (const auto & sys_name_header_pair : es_header.systems)
162 : {
163 94374 : const auto & sys_header = sys_name_header_pair.second;
164 94374 : const auto & sys = _es.get_system(sys_header.name);
165 :
166 : // Store each vector and variable
167 504073 : for (const auto & vec_name_header_pair : sys_header.vectors)
168 : {
169 409699 : const auto & vec_header = vec_name_header_pair.second;
170 409699 : const auto & vec = *vec_header.vector;
171 821143 : for (const auto & var_name_header_pair : sys_header.variables)
172 : {
173 411444 : const auto & var_header = var_name_header_pair.second;
174 411444 : const auto & var = *var_header.variable;
175 :
176 : #ifndef NDEBUG
177 : const std::size_t var_initial_position = stream.tellp();
178 : #endif
179 :
180 : // Non-SCALAR variable
181 411444 : if (var.type().family != SCALAR)
182 : {
183 : // Store for each component of each element and node
184 77677682 : for (const auto & obj : ordered_objects)
185 116751013 : for (const auto comp : make_range(obj->n_comp(sys.number(), var.number())))
186 : {
187 39476785 : auto val = vec(obj->dof_number(sys.number(), var.number(), comp));
188 39476785 : dataStore(stream, val, nullptr);
189 : }
190 : }
191 : // SCALAR variable on the last rank
192 7990 : else if (_es.processor_id() == _es.n_processors() - 1)
193 : {
194 7864 : const auto & dof_map = sys.get_dof_map();
195 7864 : std::vector<dof_id_type> scalar_dofs;
196 7864 : dof_map.SCALAR_dof_indices(scalar_dofs, var.number());
197 16965 : for (const auto dof : scalar_dofs)
198 : {
199 9101 : auto val = vec(dof);
200 9101 : dataStore(stream, val, nullptr);
201 : }
202 7864 : }
203 :
204 : #ifndef NDEBUG
205 : const std::size_t data_offset = var_initial_position - data_initial_position;
206 : mooseAssert(vec_header.variable_offset.at(var_header.name) == data_offset,
207 : "Invalid offset");
208 :
209 : const std::size_t current_position = static_cast<std::size_t>(stream.tellp());
210 : const std::size_t var_size = current_position - var_initial_position;
211 : mooseAssert(var_size == sys_header.variables.at(var.name()).size, "Incorrect assumed size");
212 : #endif
213 : }
214 : }
215 : }
216 :
217 : mooseAssert((data_initial_position + es_header.data_size) ==
218 : static_cast<std::size_t>(stream.tellp()),
219 : "Incorrect assumed size");
220 46660 : }
221 :
222 : void
223 13490 : RestartableEquationSystems::load(std::istream & stream)
224 : {
225 : // Load the header (systems, variables, vectors)
226 : // We do this first so that the loader can make informed decisions
227 : // on what to put where based on everything that is available
228 13490 : _loaded_header.systems.clear();
229 13490 : dataLoad(stream, _loaded_header, nullptr);
230 :
231 : // Order objects (elements and then node) by ID for storing
232 13490 : _loaded_ordered_objects = orderDofObjects();
233 :
234 : // Clear previously loaded variables to ensure a clean state
235 13490 : _loaded_variables.clear();
236 :
237 : // Sanity check on if we're loading the same thing
238 : {
239 : const std::string mesh_mismatch_message =
240 : ". This means that equation-system data is being restored onto a different mesh topology. "
241 : "If this occurs while restoring a MultiApp sub-application that uses mesh adaptivity, the "
242 : "backup may not contain mesh checkpoint data for the restored topology, or the "
243 : "sub-application mesh may have changed after the backup was made. Use clone_parent_mesh = "
244 : "true when the sub-application should share the parent application's mesh topology, or "
245 13490 : "avoid changing the sub-application mesh between backup and restore.";
246 :
247 13490 : std::vector<dof_id_type> from_ordered_objects_ids;
248 13490 : dataLoad(stream, from_ordered_objects_ids, nullptr);
249 13490 : if (_loaded_ordered_objects.size() != from_ordered_objects_ids.size())
250 0 : mooseError("RestartableEquationSystems::load(): Number of previously stored elements/nodes (",
251 0 : from_ordered_objects_ids.size(),
252 : ") does not match the current number of elements/nodes (",
253 0 : _loaded_ordered_objects.size(),
254 : mesh_mismatch_message);
255 3578428 : for (const auto i : index_range(_loaded_ordered_objects))
256 3564938 : if (_loaded_ordered_objects[i]->id() != from_ordered_objects_ids[i])
257 0 : mooseError("RestartableEquationSystems::load(): Id of previously stored element/node (",
258 0 : from_ordered_objects_ids[i],
259 : ") does not match the current element/node id (",
260 0 : _loaded_ordered_objects[i]->id(),
261 : mesh_mismatch_message);
262 13490 : }
263 :
264 13490 : _loaded_stream_data_begin = static_cast<std::size_t>(stream.tellg());
265 :
266 : // Load everything that we have available at the moment
267 40609 : for (const auto & sys_name_header_pair : _loaded_header.systems)
268 : {
269 27119 : const auto & sys_header = sys_name_header_pair.second;
270 27119 : if (!_es.has_system(sys_header.name))
271 66 : continue;
272 27053 : auto & sys = _es.get_system(sys_header.name);
273 :
274 27053 : bool modified_sys = false;
275 :
276 138446 : for (const auto & vec_name_header_pair : sys_header.vectors)
277 : {
278 111393 : bool modified_vec = false;
279 :
280 111393 : const auto & vec_header = vec_name_header_pair.second;
281 111393 : const bool is_solution = vec_header.name == SystemHeader::system_solution_name;
282 :
283 111393 : if (!is_solution && !sys.have_vector(vec_header.name))
284 : {
285 174 : if (_load_all_vectors)
286 97 : sys.add_vector(vec_header.name, vec_header.projections, vec_header.type);
287 : else
288 77 : continue;
289 : }
290 :
291 111316 : auto & vec = is_solution ? *sys.solution : sys.get_vector(vec_header.name);
292 :
293 221628 : for (const auto & var_name_header_pair : sys_header.variables)
294 : {
295 110312 : const auto & var_header = var_name_header_pair.second;
296 110312 : if (!sys.has_variable(var_header.name))
297 13 : continue;
298 110299 : const auto & var = sys.variable(sys.variable_number(var_header.name));
299 110299 : if (var.type() != var_header.type)
300 0 : continue;
301 :
302 110299 : restore(sys_header, vec_header, var_header, sys, vec, var, stream);
303 110299 : modified_vec = true;
304 : }
305 :
306 111316 : if (modified_vec)
307 : {
308 88813 : vec.close();
309 88813 : modified_sys = true;
310 : }
311 : }
312 :
313 27053 : if (modified_sys)
314 20475 : sys.update();
315 : }
316 :
317 : // Move the stream to the end of our data so that we make RestartableDataReader happy
318 13490 : stream.seekg(_loaded_stream_data_begin + _loaded_header.data_size);
319 13490 : }
320 :
321 : bool
322 0 : RestartableEquationSystems::isVariableRestored(const std::string & system_name,
323 : const std::string & vector_name,
324 : const std::string & variable_name) const
325 : {
326 0 : std::tuple<std::string, std::string, std::string> key(system_name, vector_name, variable_name);
327 0 : return _loaded_variables.count(key);
328 0 : }
329 :
330 : void
331 110299 : RestartableEquationSystems::restore(const SystemHeader & from_sys_header,
332 : const VectorHeader & from_vec_header,
333 : const VariableHeader & from_var_header,
334 : const libMesh::System & to_sys,
335 : libMesh::NumericVector<libMesh::Number> & to_vec,
336 : const libMesh::Variable & to_var,
337 : std::istream & stream)
338 : {
339 : #ifndef NDEBUG
340 : const auto sys_it = _loaded_header.systems.find(from_sys_header.name);
341 : mooseAssert(sys_it != _loaded_header.systems.end(), "System does not exist");
342 : const auto & sys_header = sys_it->second;
343 : mooseAssert(sys_header == from_sys_header, "Not my system");
344 : const auto vec_it = sys_header.vectors.find(from_vec_header.name);
345 : mooseAssert(vec_it != sys_header.vectors.end(), "Vector does not exist");
346 : const auto & vec_header = vec_it->second;
347 : mooseAssert(vec_header == from_vec_header, "Not my vector");
348 : const auto var_it = sys_header.variables.find(from_var_header.name);
349 : mooseAssert(var_it != sys_header.variables.end(), "Variable does not exist");
350 : const auto & var_header = var_it->second;
351 : mooseAssert(var_header == from_var_header, "Not my variable");
352 : mooseAssert(!isVariableRestored(from_sys_header.name, from_vec_header.name, from_var_header.name),
353 : "Variable already restored");
354 : #endif
355 :
356 : const auto error =
357 0 : [&from_sys_header, &from_vec_header, &from_var_header, &to_sys, &to_var](auto... args)
358 : {
359 0 : mooseError("An error occured while restoring a system:\n",
360 : args...,
361 : "\n\nFrom system: ",
362 0 : from_sys_header.name,
363 : "\nFrom vector: ",
364 0 : from_vec_header.name,
365 : "\nFrom variable: ",
366 0 : from_var_header.name,
367 : "\nTo system: ",
368 0 : to_sys.name(),
369 : "\nTo variable: ",
370 0 : to_var.name());
371 110299 : };
372 :
373 110299 : if (from_var_header.type != to_var.type())
374 0 : error("Cannot restore to a variable of a different type");
375 :
376 110299 : const auto offset = from_vec_header.variable_offset.at(from_var_header.name);
377 110299 : stream.seekg(_loaded_stream_data_begin + offset);
378 :
379 : // Non-SCALAR variable
380 110299 : if (to_var.type().family != SCALAR)
381 : {
382 25914880 : for (const auto & obj : _loaded_ordered_objects)
383 39170823 : for (const auto comp : make_range(obj->n_comp(to_sys.number(), to_var.number())))
384 : {
385 : Real val;
386 13365535 : dataLoad(stream, val, nullptr);
387 13365535 : to_vec.set(obj->dof_number(to_sys.number(), to_var.number(), comp), val);
388 : }
389 : }
390 : // SCALAR variable on the last rank
391 707 : else if (_es.processor_id() == _es.n_processors() - 1)
392 : {
393 704 : const auto & dof_map = to_sys.get_dof_map();
394 704 : std::vector<dof_id_type> scalar_dofs;
395 704 : dof_map.SCALAR_dof_indices(scalar_dofs, to_var.number());
396 1651 : for (const auto dof : scalar_dofs)
397 : {
398 : Real val;
399 947 : dataLoad(stream, val, nullptr);
400 947 : to_vec.set(dof, val);
401 : }
402 704 : }
403 :
404 : // insert into the member variable
405 : std::tuple<std::string, std::string, std::string> _loaded_variable = {
406 110299 : from_sys_header.name, from_vec_header.name, from_var_header.name};
407 :
408 110299 : _loaded_variables.insert(_loaded_variable);
409 :
410 : mooseAssert(isVariableRestored(from_sys_header.name, from_vec_header.name, from_var_header.name),
411 : "Variable not marked as restored");
412 110299 : }
413 :
414 : void
415 46660 : dataStore(std::ostream & stream, RestartableEquationSystems & res, void *)
416 : {
417 46660 : res.store(stream);
418 46660 : }
419 :
420 : void
421 13490 : dataLoad(std::istream & stream, RestartableEquationSystems & res, void *)
422 : {
423 13490 : res.load(stream);
424 13490 : }
425 :
426 : void
427 46660 : dataStore(std::ostream & stream, RestartableEquationSystems::EquationSystemsHeader & header, void *)
428 : {
429 46660 : dataStore(stream, header.systems, nullptr);
430 46660 : dataStore(stream, header.data_size, nullptr);
431 46660 : }
432 :
433 : void
434 13490 : dataLoad(std::istream & stream, RestartableEquationSystems::EquationSystemsHeader & header, void *)
435 : {
436 13490 : dataLoad(stream, header.systems, nullptr);
437 13490 : dataLoad(stream, header.data_size, nullptr);
438 13490 : }
439 :
440 : void
441 94374 : dataStore(std::ostream & stream, RestartableEquationSystems::SystemHeader & header, void *)
442 : {
443 94374 : dataStore(stream, header.name, nullptr);
444 94374 : dataStore(stream, header.type, nullptr);
445 94374 : dataStore(stream, header.variables, nullptr);
446 94374 : dataStore(stream, header.vectors, nullptr);
447 94374 : }
448 :
449 : void
450 27119 : dataLoad(std::istream & stream, RestartableEquationSystems::SystemHeader & header, void *)
451 : {
452 27119 : dataLoad(stream, header.name, nullptr);
453 27119 : dataLoad(stream, header.type, nullptr);
454 27119 : dataLoad(stream, header.variables, nullptr);
455 27119 : dataLoad(stream, header.vectors, nullptr);
456 27119 : }
457 :
458 : void
459 93958 : dataStore(std::ostream & stream, RestartableEquationSystems::VariableHeader & header, void *)
460 : {
461 93958 : dataStore(stream, header.name, nullptr);
462 93958 : dataStore(stream, header.type, nullptr);
463 93958 : }
464 : void
465 27638 : dataLoad(std::istream & stream, RestartableEquationSystems::VariableHeader & header, void *)
466 : {
467 27638 : dataLoad(stream, header.name, nullptr);
468 27638 : dataLoad(stream, header.type, nullptr);
469 27638 : }
470 :
471 : void
472 409699 : dataStore(std::ostream & stream, RestartableEquationSystems::VectorHeader & header, void *)
473 : {
474 409699 : dataStore(stream, header.name, nullptr);
475 409699 : dataStore(stream, header.projections, nullptr);
476 409699 : dataStore(stream, header.type, nullptr);
477 409699 : dataStore(stream, header.variable_offset, nullptr);
478 409699 : }
479 : void
480 111521 : dataLoad(std::istream & stream, RestartableEquationSystems::VectorHeader & header, void *)
481 : {
482 111521 : dataLoad(stream, header.name, nullptr);
483 111521 : dataLoad(stream, header.projections, nullptr);
484 111521 : dataLoad(stream, header.type, nullptr);
485 111521 : dataLoad(stream, header.variable_offset, nullptr);
486 111521 : }
487 :
488 : void
489 22 : to_json(nlohmann::json & json, const RestartableEquationSystems & res)
490 : {
491 :
492 22 : nlohmann::json loaded_vars = nlohmann::json::array();
493 :
494 220 : for (const auto & [system, vector, variable] : res.getLoadedVariables())
495 : {
496 1980 : loaded_vars.push_back({{"system", system}, {"vector", vector}, {"variable", variable}});
497 : }
498 :
499 22 : json["loaded_variables"] = loaded_vars;
500 1606 : }
|
