Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "RedistributeProperties.h"
11 :
12 : #include "MaterialProperty.h"
13 : #include "MaterialPropertyStorage.h"
14 :
15 : #include <timpi/parallel_sync.h>
16 :
17 : registerMooseObject("MooseApp", RedistributeProperties);
18 :
19 : InputParameters
20 14397 : RedistributeProperties::validParams()
21 : {
22 14397 : InputParameters params = RelationshipManager::validParams();
23 14397 : return params;
24 : }
25 :
26 66 : RedistributeProperties::RedistributeProperties(const InputParameters & parameters)
27 66 : : RelationshipManager(parameters)
28 : {
29 66 : }
30 :
31 : void
32 72 : RedistributeProperties::operator()(const MeshBase::const_element_iterator &,
33 : const MeshBase::const_element_iterator &,
34 : processor_id_type,
35 : map_type &)
36 : {
37 72 : }
38 :
39 : std::unique_ptr<libMesh::GhostingFunctor>
40 0 : RedistributeProperties::clone() const
41 : {
42 0 : return _app.getFactory().copyConstruct(*this);
43 : }
44 :
45 : std::string
46 0 : RedistributeProperties::getInfo() const
47 : {
48 0 : return "RedistributeProperties";
49 : }
50 :
51 : // the LHS ("this" object) in MooseApp::addRelationshipManager is the existing RelationshipManager
52 : // object to which we are comparing the rhs to determine whether it should get added
53 : bool
54 0 : RedistributeProperties::operator>=(const RelationshipManager & rhs) const
55 : {
56 0 : const auto * rm = dynamic_cast<const RedistributeProperties *>(&rhs);
57 :
58 : // All RedistributeProperties objects are effectively equivalent
59 0 : return rm;
60 : }
61 :
62 : void
63 198 : RedistributeProperties::addMaterialPropertyStorage(MaterialPropertyStorage & mat_props)
64 : {
65 198 : _materials.emplace_back(&mat_props);
66 198 : }
67 :
68 : void
69 88 : RedistributeProperties::redistribute()
70 : {
71 88 : const MeshBase & mesh = _moose_mesh->getMesh();
72 88 : const processor_id_type pid = mesh.processor_id();
73 :
74 352 : for (auto & mat_prop_store : _materials)
75 : {
76 : // Once we've redistributed data elsewhere we'll delete it here.
77 : //
78 : // We don't need stored properties on non-local elements, and so
79 : // we don't compute them and we don't *bother deleting them when
80 : // non-local elements are coarsened away*, so this may be our last
81 : // chance to avoid dangling pointers in future redistribute()
82 : // calls.
83 264 : std::set<const Elem *> elems_being_migrated;
84 :
85 264 : if (mat_prop_store->hasStatefulProperties())
86 : {
87 : mooseAssert(
88 : !Threads::in_threads,
89 : "This routine has not been implemented for threads. Please query this routine before "
90 : "a threaded region or contact a MOOSE developer to discuss.");
91 :
92 900 : for (const auto state : mat_prop_store->stateIndexRange())
93 : {
94 636 : MaterialPropertyStorage::PropsType & props_map = mat_prop_store->setProps(state);
95 : typedef std::unordered_map<unsigned int, std::string> stored_props_type;
96 : typedef std::tuple<stored_props_type, dof_id_type, int> stored_elem_type;
97 636 : std::map<processor_id_type, std::vector<stored_elem_type>> props_to_push;
98 :
99 : // Take non-const references here for compatibility with dataStore(T&)
100 31141 : for (auto & [elem, elem_props_map] : props_map)
101 : {
102 : // There better be an element here
103 : mooseAssert(elem, "Null element found in material property map?");
104 :
105 : // It had better be a *real* element here, not a dangling
106 : // pointer
107 : mooseAssert(elem->id() < mesh.max_elem_id(),
108 : "Invalid (dangling? corrupted?) element in material property map?");
109 : mooseAssert(elem->processor_id() < mesh.n_processors(),
110 : "Invalid (corrupted?) element in material property map?");
111 :
112 : // If it's not in the particular mesh we're responsible for,
113 : // we can skip it. MOOSE mixes non-displaced with displaced
114 : // mesh properties in their storages.
115 30505 : if (elem != mesh.query_elem_ptr(elem->id()))
116 0 : continue;
117 :
118 30505 : std::set<processor_id_type> target_pids;
119 :
120 30505 : if (elem->active())
121 21465 : target_pids.insert(elem->processor_id());
122 9040 : else if (elem->subactive())
123 : {
124 : mooseAssert(elem->parent(),
125 : "Invalid (corrupted?) subactive element in material property map");
126 : mooseAssert(elem->parent()->refinement_flag() == Elem::JUST_COARSENED,
127 : "Invalid (subactive child of not-just-coarsened) element in material "
128 : "property map");
129 6376 : target_pids.insert(elem->parent()->processor_id());
130 : }
131 : else
132 : {
133 : mooseAssert(elem->ancestor(),
134 : "Unexpected relationship for element in material property map?");
135 : mooseAssert(elem->has_children(),
136 : "Ancestor element with no children in material property map?");
137 17928 : for (const Elem & child : elem->child_ref_range())
138 15264 : target_pids.insert(child.processor_id());
139 : }
140 :
141 : // If we're being migrated elsewhere and we're not needed
142 : // for AMR/C locally then we should erase the local entry
143 30505 : bool remote_pid = false, local_pid = false;
144 61246 : for (processor_id_type target_pid : target_pids)
145 30741 : if (target_pid == pid)
146 24222 : local_pid = true;
147 : else
148 : {
149 6519 : remote_pid = true;
150 :
151 6519 : stored_props_type stored_props;
152 6519 : unsigned int n_q_points = 0;
153 13090 : for (auto & [prop_id, prop_vals] : elem_props_map)
154 : {
155 : mooseAssert(!prop_vals.empty(),
156 : "Empty MaterialProperties in stateful properties map?");
157 :
158 13142 : for (const auto & prop : prop_vals)
159 6571 : n_q_points = std::max(n_q_points, prop.size());
160 :
161 6571 : std::ostringstream oss;
162 6571 : dataStore(oss, prop_vals, nullptr);
163 6571 : stored_props[prop_id] = oss.str();
164 6571 : }
165 :
166 : // Get the stored data ready to push to the element's new
167 : // processor
168 13038 : props_to_push[target_pid].emplace_back(
169 6519 : std::move(stored_props), elem->id(), n_q_points);
170 6519 : }
171 :
172 30505 : if (remote_pid && !local_pid)
173 6283 : elems_being_migrated.insert(elem);
174 30505 : }
175 :
176 773 : auto recv_functor = [&, mat_prop_store_ptr = mat_prop_store](
177 6571 : processor_id_type, const std::vector<stored_elem_type> & data)
178 : {
179 6656 : for (const auto & [elem_hash, elem_id, n_q_points] : data)
180 : {
181 6519 : const Elem * elem = mesh.elem_ptr(elem_id);
182 6519 : auto & elem_props = props_map[elem];
183 :
184 13090 : for (const auto & [prop_id, prop_str] : elem_hash)
185 : {
186 : // This should be called "initPropsIfNecessary"... which
187 : // is confusing but convenient.
188 : //
189 : // We need to *only* initProps for the map we're working
190 : // on, otherwise we might see an
191 : // initialized-but-not-filled entry in the next map and
192 : // foolishly try to send it places.
193 6571 : mat_prop_store_ptr->initProps(0, state, elem, prop_id, n_q_points);
194 :
195 : mooseAssert(elem_props.contains(prop_id),
196 : "Trying to load into a nonexistant property id?");
197 6571 : MaterialProperties & mat_props = elem_props[prop_id];
198 6571 : std::istringstream iss(prop_str);
199 6571 : dataLoad(iss, mat_props, nullptr);
200 6571 : }
201 : }
202 137 : };
203 :
204 636 : Parallel::push_parallel_vector_data(mesh.comm(), props_to_push, recv_functor);
205 636 : }
206 : }
207 :
208 2786 : for (const Elem * elem : elems_being_migrated)
209 2522 : mat_prop_store->eraseProperty(elem);
210 264 : }
211 88 : }
|
