Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : #include "RepeatableRayStudyBase.h"
11 :
12 : // libMesh includes
13 : #include "libmesh/parallel_sync.h"
14 :
15 : #include "DataIO.h"
16 :
17 : InputParameters
18 7156 : RepeatableRayStudyBase::validParams()
19 : {
20 7156 : auto params = RayTracingStudy::validParams();
21 :
22 : // Whether or not the _rays filled by defineRays() are replicated
23 : // (the same across all processors)
24 7156 : params.addPrivateParam<bool>("_define_rays_replicated", true);
25 : // Whether or not Rays need to be claimed after calling defineRays()
26 7156 : params.addPrivateParam<bool>("_claim_after_define_rays", true);
27 :
28 7156 : return params;
29 0 : }
30 :
31 3600 : RepeatableRayStudyBase::RepeatableRayStudyBase(const InputParameters & parameters)
32 : : RayTracingStudy(parameters),
33 3600 : _rays(declareRestartableDataWithContext<std::vector<std::shared_ptr<Ray>>>("rays", this)),
34 3600 : _define_rays_replicated(getParam<bool>("_claim_after_define_rays")
35 5526 : ? getParam<bool>("_define_rays_replicated")
36 : : false),
37 7200 : _claim_after_define_rays(getParam<bool>("_claim_after_define_rays")),
38 7200 : _should_define_rays(declareRestartableData<bool>("should_define_rays", true)),
39 3600 : _local_rays(
40 3600 : declareRestartableDataWithContext<std::vector<std::shared_ptr<Ray>>>("local_rays", this)),
41 3600 : _claim_rays(*this,
42 3600 : _rays,
43 : _local_rays,
44 3600 : /* do_exchange = */ !_define_rays_replicated),
45 3600 : _should_claim_rays(
46 7200 : declareRestartableData<bool>("claim_after_define_rays", _claim_after_define_rays))
47 : {
48 8874 : if (!_claim_after_define_rays && getParam<bool>("_define_rays_replicated"))
49 2 : mooseWarning("The combination of private parameters:",
50 : "\n '_define_rays_replicated' == true",
51 : "\n '_claim_after_define_rays' == false",
52 : "\nis not a valid combination.",
53 : "\n\n_define_rays_replicated is being set to false.");
54 3598 : }
55 :
56 : void
57 7569 : RepeatableRayStudyBase::generateRays()
58 : {
59 : // Initially, the user is to define the Rays that they want to trace by overriding
60 : // defineRays() and filling into _rays within this method. These Rays are not
61 : // the Rays that will actually be traced, they just serve as a template for
62 : // Rays that will be put into the tracer to be traced.
63 : //
64 : // If the private parameter '_claim_after_define_rays' == true, it is assumed
65 : // that the Rays that were filled into _rays from the overridden defineRays()
66 : // do not have their starting element and incoming sides set. The Rays in _rays
67 : // will be "claimed" later and communicated to the processors that will start them
68 : // with their starting element set and incoming side set (if any). An example of this
69 : // is in RepeatableRayStudy.
70 : //
71 : // If the private parameter '_claim_after_define_rays' == false, it is assumed
72 : // that the Rays that were filled into _rays from the overridden defineRays():
73 : // - Have their starting element and incoming side (if applicable) set and it is
74 : // correct
75 : // - Are on the processor that will start them (the processor that contains
76 : // the starting element)
77 : // At this point, the Rays in _rays will be also inserted into _local_rays
78 : // because they are on the right processor with starting information set.
79 : // An example of this is in LotsOfRaysRayStudy.
80 7569 : if (_should_define_rays)
81 : {
82 2614 : _should_define_rays = false;
83 :
84 2614 : defineRaysInternal();
85 : }
86 :
87 7561 : if (_should_claim_rays)
88 : {
89 893 : _should_claim_rays = false;
90 :
91 893 : claimRaysInternal();
92 : }
93 :
94 : // Reserve ahead of time how many Rays we are adding to the buffer
95 7561 : reserveRayBuffer(_local_rays.size());
96 :
97 : // To make this study "repeatable", we will not trace the Rays that
98 : // are ready to go in _local_rays. We will instead create new Rays
99 : // that are duplicates of the ones in _local_rays, and trace those.
100 : // This ensures that on multiple executions of this study, we always
101 : // have the information to create the same Rays.
102 4043184 : for (const auto & ray : _local_rays)
103 : {
104 : // This acquires a new ray that is copied from a Ray that has already
105 : // been claimed to begin on this processor with the user-defined trajectory
106 4035623 : std::shared_ptr<Ray> copied_ray = acquireCopiedRay(*ray);
107 :
108 : // This calls std::move() on the ray, which means that copied_ray in this context
109 : // is no longer valid. We use the move method because copied_ray is a shared_ptr
110 : // and otherwise we would increase the count as we add it to the buffer and also
111 : // decrease the count once this goes out of scope.
112 4035623 : moveRayToBuffer(copied_ray);
113 : }
114 7561 : }
115 :
116 : void
117 224 : RepeatableRayStudyBase::meshChanged()
118 : {
119 224 : RayTracingStudy::meshChanged();
120 :
121 : // Need to reclaim after this
122 224 : _should_claim_rays = true;
123 :
124 : // Invalidate all of the old starting info because we can't be sure those elements still exist
125 46880 : for (const auto & ray : _rays)
126 : {
127 46656 : ray->invalidateStartingElem();
128 46656 : ray->invalidateStartingIncomingSide();
129 : }
130 46880 : for (const auto & ray : _local_rays)
131 : {
132 46656 : ray->invalidateStartingElem();
133 46656 : ray->invalidateStartingIncomingSide();
134 : }
135 224 : }
136 :
137 : void
138 893 : RepeatableRayStudyBase::claimRaysInternal()
139 : {
140 1786 : TIME_SECTION("claimRaysInternal", 2, "Claiming Rays");
141 :
142 893 : _claim_rays.claim();
143 893 : }
144 :
145 : void
146 2614 : RepeatableRayStudyBase::defineRaysInternal()
147 : {
148 : {
149 5228 : TIME_SECTION("defineRaysInternal", 2, "Defining Rays");
150 :
151 2614 : _rays.clear();
152 2614 : _local_rays.clear();
153 :
154 2614 : defineRays();
155 : }
156 :
157 : // Do we actually have Rays
158 2614 : auto num_rays = _rays.size();
159 2614 : _communicator.sum(num_rays);
160 2614 : if (!num_rays)
161 2 : mooseError("No Rays were moved to _rays in defineRays()");
162 3975713 : for (const auto & ray : _rays)
163 3973103 : if (!ray)
164 2 : mooseError("A nullptr Ray was found in _rays after defineRays().");
165 :
166 : // The Rays in _rays are ready to go as is: they have their starting element
167 : // set, their incoming set (if any), and are on the processor that owns said
168 : // starting element. Therefore, we move them right into _local_rays and
169 : // set that we don't need to claim.
170 2610 : if (!_claim_after_define_rays)
171 : {
172 1937 : _local_rays.reserve(_rays.size());
173 3971471 : for (const std::shared_ptr<Ray> & ray : _rays)
174 3969534 : _local_rays.emplace_back(ray);
175 :
176 1937 : _should_claim_rays = false;
177 : }
178 : // Claiming is required after defining. The Rays in _rays should not
179 : // have their starting elems or incoming sides set - verify that.
180 : else
181 : {
182 4238 : for (const std::shared_ptr<Ray> & ray : _rays)
183 3567 : if (ray->currentElem() || !ray->invalidCurrentIncomingSide())
184 2 : mooseError(
185 : "A Ray was found in _rays after defineRays() that has a starting element or "
186 : "incoming side set.\n\n",
187 : "With the mode in which the private param '_claim_after_define_rays' == true,",
188 : "\nthe defined Rays at this point should not have their starting elem/side set.\n",
189 : "\nTheir starting information will be set internally using a claiming process.\n\n",
190 2 : ray->getInfo());
191 : }
192 :
193 : // Sanity checks on if the Rays are actually replicated
194 2608 : if (_define_rays_replicated && verifyRays())
195 671 : verifyReplicatedRays();
196 2606 : }
197 :
198 : void
199 671 : RepeatableRayStudyBase::verifyReplicatedRays()
200 : {
201 1342 : TIME_SECTION("verifyReplicatedRays", 2, "Verifying Replicated Rays");
202 :
203 : const std::string error_suffix =
204 : "\n\nThe Rays added in defineRays() must be replicated across all processors\nwith the "
205 671 : "private param '_define_rays_replicated' == true.";
206 :
207 : // First, verify that our _rays have unique IDs beacuse we will do mapping based on Ray ID
208 671 : verifyUniqueRayIDs(_rays.begin(),
209 671 : _rays.end(),
210 : /* global = */ false,
211 1340 : "in _rays after calling defineRays()." + error_suffix);
212 :
213 : // Tag for sending rays from rank 0 -> all other ranks
214 669 : const auto tag = comm().get_unique_tag();
215 :
216 : // Send a copy of the rays on rank 0 to all other processors for verification
217 669 : if (_pid == 0)
218 : {
219 427 : std::vector<Parallel::Request> requests(n_processors() - 1);
220 : auto request_it = requests.begin();
221 :
222 1096 : for (processor_id_type pid = 0; pid < n_processors(); ++pid)
223 669 : if (pid != 0)
224 242 : comm().send_packed_range(
225 242 : pid, parallelStudy(), _rays.begin(), _rays.end(), *request_it++, tag);
226 :
227 427 : Parallel::wait(requests);
228 427 : }
229 : // All other processors will receive and verify that their rays match the rays on rank 0
230 : else
231 : {
232 : // Map of RayID -> Ray for comparison from the Rays on rank 0 to the local rays
233 : std::unordered_map<RayID, const Ray *> ray_map;
234 242 : ray_map.reserve(_rays.size());
235 1622 : for (const auto & ray : _rays)
236 1380 : ray_map.emplace(ray->id(), ray.get());
237 :
238 : // Receive the duplicated rays from rank 0
239 : std::vector<std::shared_ptr<Ray>> rank_0_rays;
240 242 : rank_0_rays.reserve(_rays.size());
241 242 : comm().receive_packed_range(
242 : 0, parallelStudy(), std::back_inserter(rank_0_rays), (std::shared_ptr<Ray> *)nullptr, tag);
243 :
244 : // The sizes better match
245 242 : if (rank_0_rays.size() != _rays.size())
246 0 : mooseError("The size of _rays on rank ",
247 0 : _pid,
248 : " does not match the size of rays on rank 0.",
249 : error_suffix);
250 :
251 : // Make sure we have a matching local ray for each ray from rank 0
252 1622 : for (const auto & ray : rank_0_rays)
253 : {
254 1380 : const auto find = ray_map.find(ray->id());
255 1380 : if (find == ray_map.end())
256 0 : mooseError("A Ray was found on rank ",
257 0 : _pid,
258 : " with an ID that does not exist on rank 0.",
259 : error_suffix,
260 : "\n\n",
261 0 : ray->getInfo());
262 :
263 1380 : const Ray * root_ray = find->second;
264 1380 : if (*root_ray != *ray)
265 : {
266 0 : mooseError("A Ray was found on rank ",
267 0 : _pid,
268 : " that does not exist on rank 0.",
269 : error_suffix,
270 : "\n\nLocal ray:\n\n",
271 0 : ray->getInfo(),
272 : "\n\nRank 0 ray:\n\n",
273 0 : root_ray->getInfo());
274 : }
275 : }
276 242 : }
277 1338 : }
|
