Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "RepeatableRayStudyBase.h"
11 :
12 : // libMesh includes
13 : #include "libmesh/parallel_sync.h"
14 :
15 : #include "DataIO.h"
16 :
17 : InputParameters
18 3668 : RepeatableRayStudyBase::validParams()
19 : {
20 3668 : auto params = RayTracingStudy::validParams();
21 :
22 : // Whether or not the _rays filled by defineRays() are replicated
23 : // (the same across all processors)
24 3668 : params.addPrivateParam<bool>("_define_rays_replicated", true);
25 : // Whether or not Rays need to be claimed after calling defineRays()
26 3668 : params.addPrivateParam<bool>("_claim_after_define_rays", true);
27 :
28 3668 : return params;
29 0 : }
30 :
31 1856 : RepeatableRayStudyBase::RepeatableRayStudyBase(const InputParameters & parameters)
32 : : RayTracingStudy(parameters),
33 1856 : _rays(declareRestartableDataWithContext<std::vector<std::shared_ptr<Ray>>>("rays", this)),
34 1856 : _define_rays_replicated(getParam<bool>("_claim_after_define_rays")
35 3014 : ? getParam<bool>("_define_rays_replicated")
36 : : false),
37 3712 : _claim_after_define_rays(getParam<bool>("_claim_after_define_rays")),
38 3712 : _should_define_rays(declareRestartableData<bool>("should_define_rays", true)),
39 1856 : _local_rays(
40 1856 : declareRestartableDataWithContext<std::vector<std::shared_ptr<Ray>>>("local_rays", this)),
41 1856 : _claim_rays(*this,
42 1856 : _rays,
43 : _local_rays,
44 1856 : /* do_exchange = */ !_define_rays_replicated),
45 1856 : _should_claim_rays(
46 3712 : declareRestartableData<bool>("claim_after_define_rays", _claim_after_define_rays))
47 : {
48 4410 : if (!_claim_after_define_rays && getParam<bool>("_define_rays_replicated"))
49 2 : mooseWarning("The combination of private parameters:",
50 : "\n '_define_rays_replicated' == true",
51 : "\n '_claim_after_define_rays' == false",
52 : "\nis not a valid combination.",
53 : "\n\n_define_rays_replicated is being set to false.");
54 1854 : }
55 :
56 : void
57 5048 : RepeatableRayStudyBase::generateRays()
58 : {
59 : // Initially, the user is to define the Rays that they want to trace by overriding
60 : // defineRays() and filling into _rays within this method. These Rays are not
61 : // the Rays that will actually be traced, they just serve as a template for
62 : // Rays that will be put into the tracer to be traced.
63 : //
64 : // If the private parameter '_claim_after_define_rays' == true, it is assumed
65 : // that the Rays that were filled into _rays from the overridden defineRays()
66 : // do not have their starting element and incoming sides set. The Rays in _rays
67 : // will be "claimed" later and communicated to the processors that will start them
68 : // with their starting element set and incoming side set (if any). An example of this
69 : // is in RepeatableRayStudy.
70 : //
71 : // If the private parameter '_claim_after_define_rays' == false, it is assumed
72 : // that the Rays that were filled into _rays from the overridden defineRays():
73 : // - Have their starting element and incoming side (if applicable) set and it is
74 : // correct
75 : // - Are on the processor that will start them (the processor that contains
76 : // the starting element)
77 : // At this point, the Rays in _rays will be also inserted into _local_rays
78 : // because they are on the right processor with starting information set.
79 : // An example of this is in LotsOfRaysRayStudy.
80 5048 : if (_should_define_rays)
81 : {
82 1565 : _should_define_rays = false;
83 :
84 1565 : defineRaysInternal();
85 : }
86 :
87 5040 : if (_should_claim_rays)
88 : {
89 562 : _should_claim_rays = false;
90 :
91 562 : claimRaysInternal();
92 : }
93 :
94 : // Reserve ahead of time how many Rays we are adding to the buffer
95 5040 : reserveRayBuffer(_local_rays.size());
96 :
97 : // To make this study "repeatable", we will not trace the Rays that
98 : // are ready to go in _local_rays. We will instead create new Rays
99 : // that are duplicates of the ones in _local_rays, and trace those.
100 : // This ensures that on multiple executions of this study, we always
101 : // have the information to create the same Rays.
102 2703459 : for (const auto & ray : _local_rays)
103 : {
104 : // This acquires a new ray that is copied from a Ray that has already
105 : // been claimed to begin on this processor with the user-defined trajectory
106 2698419 : std::shared_ptr<Ray> copied_ray = acquireCopiedRay(*ray);
107 :
108 : // This calls std::move() on the ray, which means that copied_ray in this context
109 : // is no longer valid. We use the move method because copied_ray is a shared_ptr
110 : // and otherwise we would increase the count as we add it to the buffer and also
111 : // decrease the count once this goes out of scope.
112 2698419 : moveRayToBuffer(copied_ray);
113 : }
114 5040 : }
115 :
116 : void
117 128 : RepeatableRayStudyBase::meshChanged()
118 : {
119 128 : RayTracingStudy::meshChanged();
120 :
121 : // Need to reclaim after this
122 128 : _should_claim_rays = true;
123 :
124 : // Invalidate all of the old starting info because we can't be sure those elements still exist
125 31232 : for (const auto & ray : _rays)
126 : {
127 31104 : ray->invalidateStartingElem();
128 31104 : ray->invalidateStartingIncomingSide();
129 : }
130 31232 : for (const auto & ray : _local_rays)
131 : {
132 31104 : ray->invalidateStartingElem();
133 31104 : ray->invalidateStartingIncomingSide();
134 : }
135 128 : }
136 :
137 : void
138 562 : RepeatableRayStudyBase::claimRaysInternal()
139 : {
140 1124 : TIME_SECTION("claimRaysInternal", 2, "Claiming Rays");
141 :
142 562 : _claim_rays.claim();
143 562 : }
144 :
145 : void
146 1565 : RepeatableRayStudyBase::defineRaysInternal()
147 : {
148 : {
149 3130 : TIME_SECTION("defineRaysInternal", 2, "Defining Rays");
150 :
151 1565 : _rays.clear();
152 1565 : _local_rays.clear();
153 :
154 1565 : defineRays();
155 1565 : }
156 :
157 : // Do we actually have Rays
158 1565 : auto num_rays = _rays.size();
159 1565 : _communicator.sum(num_rays);
160 1565 : if (!num_rays)
161 2 : mooseError("No Rays were moved to _rays in defineRays()");
162 2656835 : for (const auto & ray : _rays)
163 2655274 : if (!ray)
164 2 : mooseError("A nullptr Ray was found in _rays after defineRays().");
165 :
166 : // The Rays in _rays are ready to go as is: they have their starting element
167 : // set, their incoming set (if any), and are on the processor that owns said
168 : // starting element. Therefore, we move them right into _local_rays and
169 : // set that we don't need to claim.
170 1561 : if (!_claim_after_define_rays)
171 : {
172 1123 : _local_rays.reserve(_rays.size());
173 2654195 : for (const std::shared_ptr<Ray> & ray : _rays)
174 2653072 : _local_rays.emplace_back(ray);
175 :
176 1123 : _should_claim_rays = false;
177 : }
178 : // Claiming is required after defining. The Rays in _rays should not
179 : // have their starting elems or incoming sides set - verify that.
180 : else
181 : {
182 2636 : for (const std::shared_ptr<Ray> & ray : _rays)
183 2200 : if (ray->currentElem() || !ray->invalidCurrentIncomingSide())
184 2 : mooseError(
185 : "A Ray was found in _rays after defineRays() that has a starting element or "
186 : "incoming side set.\n\n",
187 : "With the mode in which the private param '_claim_after_define_rays' == true,",
188 : "\nthe defined Rays at this point should not have their starting elem/side set.\n",
189 : "\nTheir starting information will be set internally using a claiming process.\n\n",
190 2 : ray->getInfo());
191 : }
192 :
193 : // Sanity checks on if the Rays are actually replicated
194 1559 : if (_define_rays_replicated && verifyRays())
195 436 : verifyReplicatedRays();
196 1557 : }
197 :
198 : void
199 436 : RepeatableRayStudyBase::verifyReplicatedRays()
200 : {
201 872 : TIME_SECTION("verifyReplicatedRays", 2, "Verifying Replicated Rays");
202 :
203 : const std::string error_suffix =
204 : "\n\nThe Rays added in defineRays() must be replicated across all processors\nwith the "
205 436 : "private param '_define_rays_replicated' == true.";
206 :
207 : // First, verify that our _rays have unique IDs beacuse we will do mapping based on Ray ID
208 436 : verifyUniqueRayIDs(_rays.begin(),
209 436 : _rays.end(),
210 : /* global = */ false,
211 870 : "in _rays after calling defineRays()." + error_suffix);
212 :
213 : // Tag for sending rays from rank 0 -> all other ranks
214 434 : const auto tag = comm().get_unique_tag();
215 :
216 : // Send a copy of the rays on rank 0 to all other processors for verification
217 434 : if (_pid == 0)
218 : {
219 313 : std::vector<Parallel::Request> requests(n_processors() - 1);
220 : auto request_it = requests.begin();
221 :
222 747 : for (processor_id_type pid = 0; pid < n_processors(); ++pid)
223 434 : if (pid != 0)
224 121 : comm().send_packed_range(
225 121 : pid, parallelStudy(), _rays.begin(), _rays.end(), *request_it++, tag);
226 :
227 313 : Parallel::wait(requests);
228 313 : }
229 : // All other processors will receive and verify that their rays match the rays on rank 0
230 : else
231 : {
232 : // Map of RayID -> Ray for comparison from the Rays on rank 0 to the local rays
233 : std::unordered_map<RayID, const Ray *> ray_map;
234 121 : ray_map.reserve(_rays.size());
235 803 : for (const auto & ray : _rays)
236 682 : ray_map.emplace(ray->id(), ray.get());
237 :
238 : // Receive the duplicated rays from rank 0
239 : std::vector<std::shared_ptr<Ray>> rank_0_rays;
240 121 : rank_0_rays.reserve(_rays.size());
241 121 : comm().receive_packed_range(
242 : 0, parallelStudy(), std::back_inserter(rank_0_rays), (std::shared_ptr<Ray> *)nullptr, tag);
243 :
244 : // The sizes better match
245 121 : if (rank_0_rays.size() != _rays.size())
246 0 : mooseError("The size of _rays on rank ",
247 0 : _pid,
248 : " does not match the size of rays on rank 0.",
249 : error_suffix);
250 :
251 : // Make sure we have a matching local ray for each ray from rank 0
252 803 : for (const auto & ray : rank_0_rays)
253 : {
254 682 : const auto find = ray_map.find(ray->id());
255 682 : if (find == ray_map.end())
256 0 : mooseError("A Ray was found on rank ",
257 0 : _pid,
258 : " with an ID that does not exist on rank 0.",
259 : error_suffix,
260 : "\n\n",
261 0 : ray->getInfo());
262 :
263 682 : const Ray * root_ray = find->second;
264 682 : if (*root_ray != *ray)
265 : {
266 0 : mooseError("A Ray was found on rank ",
267 0 : _pid,
268 : " that does not exist on rank 0.",
269 : error_suffix,
270 : "\n\nLocal ray:\n\n",
271 0 : ray->getInfo(),
272 : "\n\nRank 0 ray:\n\n",
273 0 : root_ray->getInfo());
274 : }
275 : }
276 121 : }
277 868 : }
|
