https://mooseframework.inl.gov
RestartableEquationSystems.C
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://mooseframework.inl.gov
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
11 
12 #include "DataIO.h"
13 
14 #include "libmesh/dof_map.h"
15 #include "libmesh/dof_object.h"
16 #include "libmesh/elem.h"
17 #include "libmesh/node.h"
18 
20  "SYSTEM_SOLUTION";
21 
23  : _es(mesh), _load_all_vectors(true)
24 {
25 }
26 
29  const std::vector<const libMesh::DofObject *> & ordered_objects) const
30 {
31  EquationSystemsHeader es_header;
32 
33  // Systems
34  for (const auto sys_num : make_range(_es.n_systems()))
35  {
36  const auto & sys = _es.get_system(sys_num);
37 
38  SystemHeader sys_header;
39  sys_header.name = sys.name();
40  sys_header.type = sys.system_type();
41 
42  // Variables in the system
43  for (const auto var_num : make_range(sys.n_vars()))
44  {
45  const auto & var = sys.variable(var_num);
46 
47  VariableHeader var_header;
48  var_header.name = var.name();
49  var_header.type = var.type();
50  var_header.size = 0;
51  var_header.variable = &var;
52 
53  mooseAssert(_es.comm().verify("sys_" + sys.name() + "_var_" + var.name()),
54  "Out of order in parallel");
55  mooseAssert(!sys_header.variables.count(var.name()), "Already inserted");
56 
57  // Non-SCALAR variable
58  if (var.type().family != SCALAR)
59  {
60  for (const auto & obj : ordered_objects)
61  var_header.size += sizeof(Real) * obj->n_comp(sys.number(), var.number());
62  }
63  // SCALAR variable on the last rank
64  else if (_es.processor_id() == _es.n_processors() - 1)
65  {
66  std::vector<dof_id_type> scalar_dofs;
67  sys.get_dof_map().SCALAR_dof_indices(scalar_dofs, var.number());
68  var_header.size += sizeof(Real) * scalar_dofs.size();
69  }
70 
71  sys_header.variables.emplace(var.name(), var_header);
72  }
73 
74  // System vector
75  auto & sys_vec_header = sys_header.vectors[SystemHeader::system_solution_name];
76  sys_vec_header.name = SystemHeader::system_solution_name;
77  sys_vec_header.vector = sys.solution.get();
78  sys_vec_header.type = sys_vec_header.vector->type();
79  for (const auto vec_num : make_range(sys.n_vectors()))
80  {
81  mooseAssert(_es.comm().verify("sys_" + sys.name() + "_vec_" + sys.vector_name(vec_num)),
82  "Out of order in parallel");
83  const auto & name = sys.vector_name(vec_num);
84  mooseAssert(!sys_header.vectors.count(name), "Already inserted");
85  auto & vec_header = sys_header.vectors[name];
86  vec_header.name = sys.vector_name(vec_num);
87  vec_header.projections = sys.vector_preservation(vec_header.name);
88  vec_header.vector = &sys.get_vector(vec_header.name);
89  vec_header.type = vec_header.vector->type();
90  }
91 
92  // System in this EquationSystems
93  mooseAssert(!es_header.systems.count(sys.name()), "Already inserted");
94  es_header.systems.emplace(sys.name(), sys_header);
95  }
96 
97  // Setup the positions in each vector for easy access later
98  std::size_t offset = 0;
99  for (auto & sys_name_header_pair : es_header.systems)
100  {
101  auto & sys_header = sys_name_header_pair.second;
102  for (auto & vec_name_header_pair : sys_header.vectors)
103  {
104  auto & vec_header = vec_name_header_pair.second;
105  for (const auto & var_name_header_pair : sys_header.variables)
106  {
107  const auto & var_header = var_name_header_pair.second;
108  mooseAssert(!vec_header.variable_offset.count(var_header.name), "Already inserted");
109  vec_header.variable_offset[var_header.name] = offset;
110  offset += var_header.size;
111  }
112  }
113  }
114 
115  es_header.data_size = offset;
116 
117  return es_header;
118 }
119 
120 std::vector<const libMesh::DofObject *>
122 {
123  std::vector<const libMesh::DofObject *> objects;
124  auto add = [&objects](const auto begin, const auto end)
125  {
126  std::set<const libMesh::DofObject *, libMesh::CompareDofObjectsByID> ordered(begin, end);
127  objects.insert(objects.end(), ordered.begin(), ordered.end());
128  };
129 
130  const auto & mesh = _es.get_mesh();
131  add(mesh.local_elements_begin(), mesh.local_elements_end());
132  add(mesh.local_nodes_begin(), mesh.local_nodes_end());
133 
134  return objects;
135 }
136 
137 void
138 RestartableEquationSystems::store(std::ostream & stream) const
139 {
140  // Order objects (elements and then nodes) by ID for storing
141  const auto ordered_objects = orderDofObjects();
142 
143  // Store the header (systems, variables, vectors)
144  EquationSystemsHeader es_header = buildHeader(ordered_objects);
145  dataStore(stream, es_header, nullptr);
146 
147  // Store the ordered objects so we can do a sanity check on if we're
148  // loading the same thing
149  {
150  std::vector<dof_id_type> ordered_objects_ids(ordered_objects.size());
151  for (const auto i : index_range(ordered_objects))
152  ordered_objects_ids[i] = ordered_objects[i]->id();
153  dataStore(stream, ordered_objects_ids, nullptr);
154  }
155 
156 #ifndef NDEBUG
157  const std::size_t data_initial_position = static_cast<std::size_t>(stream.tellp());
158 #endif
159 
160  // Store each system
161  for (const auto & sys_name_header_pair : es_header.systems)
162  {
163  const auto & sys_header = sys_name_header_pair.second;
164  const auto & sys = _es.get_system(sys_header.name);
165 
166  // Store each vector and variable
167  for (const auto & vec_name_header_pair : sys_header.vectors)
168  {
169  const auto & vec_header = vec_name_header_pair.second;
170  const auto & vec = *vec_header.vector;
171  for (const auto & var_name_header_pair : sys_header.variables)
172  {
173  const auto & var_header = var_name_header_pair.second;
174  const auto & var = *var_header.variable;
175 
176 #ifndef NDEBUG
177  const std::size_t var_initial_position = stream.tellp();
178 #endif
179 
180  // Non-SCALAR variable
181  if (var.type().family != SCALAR)
182  {
183  // Store for each component of each element and node
184  for (const auto & obj : ordered_objects)
185  for (const auto comp : make_range(obj->n_comp(sys.number(), var.number())))
186  {
187  auto val = vec(obj->dof_number(sys.number(), var.number(), comp));
188  dataStore(stream, val, nullptr);
189  }
190  }
191  // SCALAR variable on the last rank
192  else if (_es.processor_id() == _es.n_processors() - 1)
193  {
194  const auto & dof_map = sys.get_dof_map();
195  std::vector<dof_id_type> scalar_dofs;
196  dof_map.SCALAR_dof_indices(scalar_dofs, var.number());
197  for (const auto dof : scalar_dofs)
198  {
199  auto val = vec(dof);
200  dataStore(stream, val, nullptr);
201  }
202  }
203 
204 #ifndef NDEBUG
205  const std::size_t data_offset = var_initial_position - data_initial_position;
206  mooseAssert(vec_header.variable_offset.at(var_header.name) == data_offset,
207  "Invalid offset");
208 
209  const std::size_t current_position = static_cast<std::size_t>(stream.tellp());
210  const std::size_t var_size = current_position - var_initial_position;
211  mooseAssert(var_size == sys_header.variables.at(var.name()).size, "Incorrect assumed size");
212 #endif
213  }
214  }
215  }
216 
217  mooseAssert((data_initial_position + es_header.data_size) ==
218  static_cast<std::size_t>(stream.tellp()),
219  "Incorrect assumed size");
220 }
221 
222 void
223 RestartableEquationSystems::load(std::istream & stream)
224 {
225  // Load the header (systems, variables, vectors)
226  // We do this first so that the loader can make informed decisions
227  // on what to put where based on everything that is available
228  _loaded_header.systems.clear();
229  dataLoad(stream, _loaded_header, nullptr);
230 
231  // Order objects (elements and then node) by ID for storing
233 
234  // Sanity check on if we're loading the same thing
235  {
236  std::vector<dof_id_type> from_ordered_objects_ids;
237  dataLoad(stream, from_ordered_objects_ids, nullptr);
238  if (_loaded_ordered_objects.size() != from_ordered_objects_ids.size())
239  mooseError("RestartableEquationSystems::load(): Number of previously stored elements/nodes (",
241  ") does not "
242  "match the current number of elements/nodes (",
243  from_ordered_objects_ids.size(),
244  ")");
245  for (const auto i : index_range(_loaded_ordered_objects))
246  if (_loaded_ordered_objects[i]->id() != from_ordered_objects_ids[i])
247  mooseError("RestartableEquationSystems::load(): Id of previously stored element/node (",
248  _loaded_ordered_objects[i]->id(),
249  ") does not "
250  "match the current element/node id (",
251  from_ordered_objects_ids[i],
252  ")");
253  }
254 
255  _loaded_stream_data_begin = static_cast<std::size_t>(stream.tellg());
256 
257  // Load everything that we have available at the moment
258  for (const auto & sys_name_header_pair : _loaded_header.systems)
259  {
260  const auto & sys_header = sys_name_header_pair.second;
261  if (!_es.has_system(sys_header.name))
262  continue;
263  auto & sys = _es.get_system(sys_header.name);
264 
265  bool modified_sys = false;
266 
267  for (const auto & vec_name_header_pair : sys_header.vectors)
268  {
269  bool modified_vec = false;
270 
271  const auto & vec_header = vec_name_header_pair.second;
272  const bool is_solution = vec_header.name == SystemHeader::system_solution_name;
273 
274  if (!is_solution && !sys.have_vector(vec_header.name))
275  {
276  if (_load_all_vectors)
277  sys.add_vector(vec_header.name, vec_header.projections, vec_header.type);
278  else
279  continue;
280  }
281 
282  auto & vec = is_solution ? *sys.solution : sys.get_vector(vec_header.name);
283 
284  for (const auto & var_name_header_pair : sys_header.variables)
285  {
286  const auto & var_header = var_name_header_pair.second;
287  if (!sys.has_variable(var_header.name))
288  continue;
289  const auto & var = sys.variable(sys.variable_number(var_header.name));
290  if (var.type() != var_header.type)
291  continue;
292 
293  restore(sys_header, vec_header, var_header, sys, vec, var, stream);
294  modified_vec = true;
295  }
296 
297  if (modified_vec)
298  {
299  vec.close();
300  modified_sys = true;
301  }
302  }
303 
304  if (modified_sys)
305  sys.update();
306  }
307 
308  // Move the stream to the end of our data so that we make RestartableDataReader happy
310 }
311 
312 void
314  const VectorHeader & from_vec_header,
315  const VariableHeader & from_var_header,
316  const libMesh::System & to_sys,
318  const libMesh::Variable & to_var,
319  std::istream & stream)
320 {
321 #ifndef NDEBUG
322  const auto sys_it = _loaded_header.systems.find(from_sys_header.name);
323  mooseAssert(sys_it != _loaded_header.systems.end(), "System does not exist");
324  const auto & sys_header = sys_it->second;
325  mooseAssert(sys_header == from_sys_header, "Not my system");
326  const auto vec_it = sys_header.vectors.find(from_vec_header.name);
327  mooseAssert(vec_it != sys_header.vectors.end(), "Vector does not exist");
328  const auto & vec_header = vec_it->second;
329  mooseAssert(vec_header == from_vec_header, "Not my vector");
330  const auto var_it = sys_header.variables.find(from_var_header.name);
331  mooseAssert(var_it != sys_header.variables.end(), "Variable does not exist");
332  const auto & var_header = var_it->second;
333  mooseAssert(var_header == from_var_header, "Not my variable");
334 #endif
335 
336  const auto error =
337  [&from_sys_header, &from_vec_header, &from_var_header, &to_sys, &to_var](auto... args)
338  {
339  mooseError("An error occured while restoring a system:\n",
340  args...,
341  "\n\nFrom system: ",
342  from_sys_header.name,
343  "\nFrom vector: ",
344  from_vec_header.name,
345  "\nFrom variable: ",
346  from_var_header.name,
347  "\nTo system: ",
348  to_sys.name(),
349  "\nTo variable: ",
350  to_var.name());
351  };
352 
353  if (from_var_header.type != to_var.type())
354  error("Cannot restore to a variable of a different type");
355 
356  const auto offset = from_vec_header.variable_offset.at(from_var_header.name);
357  stream.seekg(_loaded_stream_data_begin + offset);
358 
359  // Non-SCALAR variable
360  if (to_var.type().family != SCALAR)
361  {
362  for (const auto & obj : _loaded_ordered_objects)
363  for (const auto comp : make_range(obj->n_comp(to_sys.number(), to_var.number())))
364  {
365  Real val;
366  dataLoad(stream, val, nullptr);
367  to_vec.set(obj->dof_number(to_sys.number(), to_var.number(), comp), val);
368  }
369  }
370  // SCALAR variable on the last rank
371  else if (_es.processor_id() == _es.n_processors() - 1)
372  {
373  const auto & dof_map = to_sys.get_dof_map();
374  std::vector<dof_id_type> scalar_dofs;
375  dof_map.SCALAR_dof_indices(scalar_dofs, to_var.number());
376  for (const auto dof : scalar_dofs)
377  {
378  Real val;
379  dataLoad(stream, val, nullptr);
380  to_vec.set(dof, val);
381  }
382  }
383 }
384 
385 void
386 dataStore(std::ostream & stream, RestartableEquationSystems & res, void *)
387 {
388  res.store(stream);
389 }
390 
391 void
392 dataLoad(std::istream & stream, RestartableEquationSystems & res, void *)
393 {
394  res.load(stream);
395 }
396 
397 void
398 dataStore(std::ostream & stream, RestartableEquationSystems::EquationSystemsHeader & header, void *)
399 {
400  dataStore(stream, header.systems, nullptr);
401  dataStore(stream, header.data_size, nullptr);
402 }
403 
404 void
405 dataLoad(std::istream & stream, RestartableEquationSystems::EquationSystemsHeader & header, void *)
406 {
407  dataLoad(stream, header.systems, nullptr);
408  dataLoad(stream, header.data_size, nullptr);
409 }
410 
411 void
412 dataStore(std::ostream & stream, RestartableEquationSystems::SystemHeader & header, void *)
413 {
414  dataStore(stream, header.name, nullptr);
415  dataStore(stream, header.type, nullptr);
416  dataStore(stream, header.variables, nullptr);
417  dataStore(stream, header.vectors, nullptr);
418 }
419 
420 void
421 dataLoad(std::istream & stream, RestartableEquationSystems::SystemHeader & header, void *)
422 {
423  dataLoad(stream, header.name, nullptr);
424  dataLoad(stream, header.type, nullptr);
425  dataLoad(stream, header.variables, nullptr);
426  dataLoad(stream, header.vectors, nullptr);
427 }
428 
429 void
430 dataStore(std::ostream & stream, RestartableEquationSystems::VariableHeader & header, void *)
431 {
432  dataStore(stream, header.name, nullptr);
433  dataStore(stream, header.type, nullptr);
434 }
435 void
436 dataLoad(std::istream & stream, RestartableEquationSystems::VariableHeader & header, void *)
437 {
438  dataLoad(stream, header.name, nullptr);
439  dataLoad(stream, header.type, nullptr);
440 }
441 
442 void
443 dataStore(std::ostream & stream, RestartableEquationSystems::VectorHeader & header, void *)
444 {
445  dataStore(stream, header.name, nullptr);
446  dataStore(stream, header.projections, nullptr);
447  dataStore(stream, header.type, nullptr);
448  dataStore(stream, header.variable_offset, nullptr);
449 }
450 void
451 dataLoad(std::istream & stream, RestartableEquationSystems::VectorHeader & header, void *)
452 {
453  dataLoad(stream, header.name, nullptr);
454  dataLoad(stream, header.projections, nullptr);
455  dataLoad(stream, header.type, nullptr);
456  dataLoad(stream, header.variable_offset, nullptr);
457 }
std::string name(const ElemQuality q)
libMesh::FEType type
The type of the stored variable.
void store(std::ostream &stream) const
Stores the EquationSystems to the given stream.
void restore(const SystemHeader &from_sys_header, const VectorHeader &from_vec_header, const VariableHeader &from_var_header, const libMesh::System &to_sys, libMesh::NumericVector< libMesh::Number > &to_vec, const libMesh::Variable &to_var, std::istream &stream)
unsigned int n_systems() const
std::size_t data_size
The total size of data for this EquationSystems.
SCALAR
std::size_t _loaded_stream_data_begin
The starting position for the vector data in the input stream.
bool has_system(std::string_view name) const
void mooseError(Args &&... args)
Emit an error message with the given stringified, concatenated args and terminate the application...
Definition: MooseError.h:302
std::size_t size
The size of this variable&#39;s data.
MeshBase & mesh
const Parallel::Communicator & comm() const
const T_sys & get_system(std::string_view name) const
bool projections
The projection flag (whether or not it should be projected or zeroed)
std::string name
The name of the stored vector.
libMesh::EquationSystems _es
The underlying EquationSystems.
std::string name
The name of the stored variable.
processor_id_type n_processors() const
std::map< std::string, RestartableEquationSystems::SystemHeader > systems
The stored systems in the equation systems.
RestartableEquationSystems(libMesh::MeshBase &mesh)
std::string name
The name of the stored system.
std::map< std::string, RestartableEquationSystems::VectorHeader > vectors
The stored vectors in the system.
static const std::string system_solution_name
Special name for a vector that is the system solution vector.
std::map< std::string, RestartableEquationSystems::VariableHeader > variables
The stored variables in the system.
void load(std::istream &stream)
Loads the EquationSystems from the given stream.
Represents a stored system in restart.
std::string type
The type of the stored system.
EquationSystemsHeader _loaded_header
The loaded header.
libMesh::ParallelType type
The type of the stored vector.
DIE A HORRIBLE DEATH HERE typedef LIBMESH_DEFAULT_SCALAR_TYPE Real
bool _load_all_vectors
Whether or not to load all of the vectors, including ones that haven&#39;t been added yet...
timpi_pure bool verify(const T &r) const
void dataLoad(std::istream &stream, RestartableEquationSystems &res, void *)
const libMesh::Variable * variable
The underlying variable (only valid during store, not used in load)
Wrapper class that owns a libMesh EquationSystem and adds advanced restart capability to it...
std::vector< const libMesh::DofObject * > _loaded_ordered_objects
The object ordering for this data.
const MeshBase & get_mesh() const
IntRange< T > make_range(T beg, T end)
void dataStore(std::ostream &stream, RestartableEquationSystems &res, void *)
std::vector< const libMesh::DofObject * > orderDofObjects() const
Internal method for ordering the DofObjects by ID (elems and the nodes)
virtual void set(const numeric_index_type i, const T value)=0
const std::string & name() const
std::map< std::string, std::size_t > variable_offset
The position of each variable for this vector (relative to the start of the data) ...
unsigned int number() const
Represents a stored EquationSystems in restart.
Represents a stored variable in restart.
processor_id_type processor_id() const
EquationSystemsHeader buildHeader(const std::vector< const libMesh::DofObject *> &ordered_objects) const
Internal method for building the header struct.
auto index_range(const T &sizable)
const FEType & type() const
Represents a stored variable in restart.