Line data Source code
1 : // The libMesh Finite Element Library.
2 : // Copyright (C) 2002-2026 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 :
4 : // This library is free software; you can redistribute it and/or
5 : // modify it under the terms of the GNU Lesser General Public
6 : // License as published by the Free Software Foundation; either
7 : // version 2.1 of the License, or (at your option) any later version.
8 :
9 : // This library is distributed in the hope that it will be useful,
10 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : // Lesser General Public License for more details.
13 :
14 : // You should have received a copy of the GNU Lesser General Public
15 : // License along with this library; if not, write to the Free Software
16 : // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 :
18 : #ifndef LIBMESH_THREADS_PTHREAD_H
19 : #define LIBMESH_THREADS_PTHREAD_H
20 :
21 : // Do not try to #include this header directly, it is designed to be
22 : // #included directly by threads.h
23 : #ifndef LIBMESH_SQUASH_HEADER_WARNING
24 : # warning "This file is designed to be included through libmesh/threads.h"
25 : #else
26 :
27 : #ifdef LIBMESH_HAVE_PTHREAD
28 :
29 : // C++ includes
30 : #ifdef LIBMESH_HAVE_CXX11_THREAD
31 : # include <thread>
32 : #endif
33 :
34 : #include <pthread.h>
35 : #include <algorithm>
36 : #include <vector>
37 : #include <memory> // std::unique_ptr, std::make_unique
38 :
39 : #ifdef __APPLE__
40 : # ifdef __MAC_10_12
41 : # include <os/lock.h>
42 : #else
43 : # include <libkern/OSAtomic.h>
44 : # endif
45 : #endif
46 :
47 : // Thread-Local-Storage macros
48 : #ifdef LIBMESH_HAVE_CXX11_THREAD
49 : # define LIBMESH_TLS_TYPE(type) thread_local type
50 : # define LIBMESH_TLS_REF(value) (value)
51 : #else // Maybe support gcc __thread eventually?
52 : # define LIBMESH_TLS_TYPE(type) type
53 : # define LIBMESH_TLS_REF(value) (value)
54 : #endif
55 :
56 : namespace libMesh
57 : {
58 :
59 : namespace Threads
60 : {
61 :
62 :
63 : #ifdef LIBMESH_HAVE_CXX11_THREAD
64 : /**
65 : * Use std::thread when available.
66 : */
67 : typedef std::thread Thread;
68 :
69 : #else
70 :
71 : /**
72 : * Use the non-concurrent placeholder.
73 : */
74 : typedef NonConcurrentThread Thread;
75 :
76 : #endif // LIBMESH_HAVE_CXX11_THREAD
77 :
78 :
79 : /**
80 : * Spin mutex. Implements mutual exclusion by busy-waiting in user
81 : * space for the lock to be acquired.
82 : */
83 : #ifdef __APPLE__
84 : #ifdef __MAC_10_12
85 : class spin_mutex
86 : {
87 : public:
88 : spin_mutex() { ulock = OS_UNFAIR_LOCK_INIT; }
89 : ~spin_mutex() = default;
90 :
91 : void lock () { os_unfair_lock_lock(&ulock); }
92 : void unlock () { os_unfair_lock_unlock(&ulock); }
93 :
94 : class scoped_lock
95 : {
96 : public:
97 : scoped_lock () : smutex(nullptr) {}
98 : explicit scoped_lock ( spin_mutex & in_smutex ) : smutex(&in_smutex) { smutex->lock(); }
99 :
100 : ~scoped_lock () { release(); }
101 :
102 : void acquire ( spin_mutex & in_smutex ) { smutex = &in_smutex; smutex->lock(); }
103 : void release () { if (smutex) smutex->unlock(); smutex = nullptr; }
104 :
105 : private:
106 : spin_mutex * smutex;
107 : };
108 :
109 : private:
110 : os_unfair_lock ulock;
111 : };
112 : #else
113 : class spin_mutex
114 : {
115 : public:
116 : spin_mutex() : slock(0) {} // The convention is that the lock being zero is _unlocked_
117 : ~spin_mutex() = default;
118 :
119 : void lock () { OSSpinLockLock(&slock); }
120 : void unlock () { OSSpinLockUnlock(&slock); }
121 :
122 : class scoped_lock
123 : {
124 : public:
125 : scoped_lock () : smutex(nullptr) {}
126 : explicit scoped_lock ( spin_mutex & in_smutex ) : smutex(&in_smutex) { smutex->lock(); }
127 :
128 : ~scoped_lock () { release(); }
129 :
130 : void acquire ( spin_mutex & in_smutex ) { smutex = &in_smutex; smutex->lock(); }
131 : void release () { if (smutex) smutex->unlock(); smutex = nullptr; }
132 :
133 : private:
134 : spin_mutex * smutex;
135 : };
136 :
137 : private:
138 : OSSpinLock slock;
139 : };
140 : #endif
141 : #else
142 : class spin_mutex
143 : {
144 : public:
145 : // Might want to use PTHREAD_MUTEX_ADAPTIVE_NP on Linux, but it's not available on OSX.
146 50562 : spin_mutex() { pthread_spin_init(&slock, PTHREAD_PROCESS_PRIVATE); }
147 157722 : ~spin_mutex() { pthread_spin_destroy(&slock); }
148 :
149 5259285700 : void lock () { pthread_spin_lock(&slock); }
150 4544098770 : void unlock () { pthread_spin_unlock(&slock); }
151 :
152 : class scoped_lock
153 : {
154 : public:
155 : scoped_lock () : smutex(nullptr) {}
156 287796 : explicit scoped_lock ( spin_mutex & in_smutex ) : smutex(&in_smutex) { smutex->lock(); }
157 :
158 287796 : ~scoped_lock () { release(); }
159 :
160 8 : void acquire ( spin_mutex & in_smutex ) { smutex = &in_smutex; smutex->lock(); }
161 766043 : void release () { if (smutex) smutex->unlock(); smutex = nullptr; }
162 :
163 : private:
164 : spin_mutex * smutex;
165 : };
166 :
167 : private:
168 : pthread_spinlock_t slock;
169 : };
170 : #endif // __APPLE__
171 :
172 :
173 :
174 : /**
175 : * Recursive mutex. Implements mutual exclusion by busy-waiting in user
176 : * space for the lock to be acquired.
177 : */
178 : class recursive_mutex
179 : {
180 : public:
181 : // Might want to use PTHREAD_MUTEX_ADAPTIVE_NP on Linux, but it's not available on OSX.
182 16600 : recursive_mutex()
183 16600 : {
184 16600 : pthread_mutexattr_init(&attr);
185 16600 : pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
186 16600 : pthread_mutex_init(&mutex, &attr);
187 16600 : }
188 16600 : ~recursive_mutex() { pthread_mutex_destroy(&mutex); }
189 :
190 : void lock () { pthread_mutex_lock(&mutex); }
191 : void unlock () { pthread_mutex_unlock(&mutex); }
192 :
193 : class scoped_lock
194 : {
195 : public:
196 : scoped_lock () : rmutex(nullptr) {}
197 : explicit scoped_lock ( recursive_mutex & in_rmutex ) : rmutex(&in_rmutex) { rmutex->lock(); }
198 :
199 : ~scoped_lock () { release(); }
200 :
201 : void acquire ( recursive_mutex & in_rmutex ) { rmutex = &in_rmutex; rmutex->lock(); }
202 : void release () { if (rmutex) rmutex->unlock(); rmutex = nullptr; }
203 :
204 : private:
205 : recursive_mutex * rmutex;
206 : };
207 :
208 : private:
209 : pthread_mutex_t mutex;
210 : pthread_mutexattr_t attr;
211 : };
212 :
213 : template <typename Range>
214 6880949 : unsigned int num_pthreads(const Range & range,
215 : unsigned int requested = libMesh::n_threads())
216 : {
217 7043175 : std::size_t mn = std::min((std::size_t)requested, range.size()/range.grainsize());
218 7041381 : return mn > 0 ? cast_int<unsigned int>(mn) : 1;
219 : }
220 :
221 : template <typename Range, typename Body>
222 : class RangeBody
223 : {
224 : public:
225 : Range * range;
226 : Body * body;
227 : };
228 :
229 : template <typename Range, typename Body>
230 6808 : void * run_body(void * args)
231 : {
232 6148 : RangeBody<Range, Body> * range_body = (RangeBody<Range, Body> *)args;
233 :
234 6552 : Body & body = *range_body->body;
235 6808 : Range & range = *range_body->range;
236 :
237 6148 : body(range);
238 :
239 6808 : return nullptr;
240 : }
241 :
242 : /**
243 : * Scheduler to manage threads.
244 : */
245 : class task_scheduler_init
246 : {
247 : public:
248 : static const int automatic = -1;
249 468 : explicit task_scheduler_init (int = automatic) {}
250 : void initialize (int = automatic) {}
251 : void terminate () {}
252 : };
253 :
254 : //-------------------------------------------------------------------
255 : /**
256 : * Dummy "splitting object" used to distinguish splitting constructors
257 : * from copy constructors.
258 : */
259 : class split {};
260 :
261 :
262 :
263 :
264 : //-------------------------------------------------------------------
265 : /**
266 : * Execute the provided function object in parallel on the specified
267 : * range.
268 : */
269 : template <typename Range, typename Body>
270 : inline
271 2686538 : void parallel_for (const Range & range, const Body & body,
272 : unsigned int n_threads = libMesh::n_threads())
273 : {
274 2686538 : libmesh_error_msg_if(n_threads > libMesh::n_threads(),
275 : "Requested n_threads (" << n_threads << ") exceeds the "
276 : "global thread count (" << libMesh::n_threads() << ").");
277 74912 : Threads::BoolAcquire set_in_threads(Threads::in_threads);
278 :
279 2611650 : unsigned int actual_threads = num_pthreads(range, n_threads);
280 :
281 : // If we're running in serial - just run!
282 2614704 : if (actual_threads == 1)
283 : {
284 2646517 : body(range);
285 72908 : return;
286 : }
287 :
288 7985 : Threads::RAIIAcquire<int> set_active_threads(Threads::active_threads, actual_threads);
289 :
290 8008 : DisablePerfLogInScope disable_perf;
291 :
292 9985 : std::vector<std::unique_ptr<Range>> ranges(actual_threads);
293 7985 : std::vector<RangeBody<const Range, const Body>> range_bodies(actual_threads);
294 5981 : std::vector<pthread_t> threads(actual_threads);
295 :
296 : // Create the ranges for each thread
297 5981 : std::size_t range_size = range.size() / actual_threads;
298 :
299 5981 : typename Range::const_iterator current_beginning = range.begin();
300 :
301 17943 : for (unsigned int i=0; i<actual_threads; i++)
302 : {
303 4008 : std::size_t this_range_size = range_size;
304 :
305 11962 : if (i+1 == actual_threads)
306 5981 : this_range_size += range.size() % actual_threads; // Give the last one the remaining work to do
307 :
308 15962 : ranges[i] = std::make_unique<Range>(range, current_beginning, current_beginning + this_range_size);
309 :
310 15962 : current_beginning = current_beginning + this_range_size;
311 : }
312 :
313 : // Create the RangeBody arguments
314 17943 : for (unsigned int i=0; i<actual_threads; i++)
315 : {
316 15962 : range_bodies[i].range = ranges[i].get();
317 11962 : range_bodies[i].body = &body;
318 : }
319 :
320 : // Create the threads. It may seem redundant to wrap a pragma in
321 : // #ifdefs... but GCC warns about an "unknown pragma" if it
322 : // encounters this line of code when -fopenmp is not passed to the
323 : // compiler.
324 : #ifdef LIBMESH_HAVE_OPENMP
325 : // Cap the OpenMP team to the actual number of threads this dispatch uses. Without this clause the
326 : // team defaults to the process-wide libMesh::n_threads(), so omp_get_thread_num()
327 : // could exceed a reduced per-application thread count and index past
328 : // per-thread storage sized to that count.
329 5981 : #pragma omp parallel for schedule (static) num_threads(actual_threads)
330 : #endif
331 : for (int i=0; i<static_cast<int>(actual_threads); i++)
332 : {
333 : #if !LIBMESH_HAVE_OPENMP
334 : pthread_create(&threads[i], nullptr, &run_body<Range, Body>, (void *)&range_bodies[i]);
335 : #else
336 : run_body<Range, Body>((void *)&range_bodies[i]);
337 : #endif
338 : }
339 :
340 : #if !LIBMESH_HAVE_OPENMP
341 : // Wait for them to finish
342 :
343 : // The use of 'int' instead of unsigned for the iteration variable
344 : // is deliberate here. This is an OpenMP loop, and some older
345 : // compilers warn when you don't use int for the loop index. The
346 : // reason has to do with signed vs. unsigned integer overflow
347 : // behavior and optimization.
348 : // http://blog.llvm.org/2011/05/what-every-c-programmer-should-know.html
349 : for (int i=0; i<static_cast<int>(actual_threads); i++)
350 : pthread_join(threads[i], nullptr);
351 : #endif
352 1977 : }
353 :
354 : /**
355 : * Execute the provided function object in parallel on the specified
356 : * range with the specified partitioner.
357 : */
358 : template <typename Range, typename Body, typename Partitioner>
359 : inline
360 : void parallel_for (const Range & range, const Body & body, const Partitioner &,
361 : unsigned int n_threads = libMesh::n_threads())
362 : {
363 : parallel_for (range, body, n_threads);
364 : }
365 :
366 : /**
367 : * Execute the provided reduction operation in parallel on the specified
368 : * range.
369 : */
370 : template <typename Range, typename Body>
371 : inline
372 4356637 : void parallel_reduce (const Range & range, Body & body,
373 : unsigned int n_threads = libMesh::n_threads())
374 : {
375 4356637 : libmesh_error_msg_if(n_threads > libMesh::n_threads(),
376 : "Requested n_threads (" << n_threads << ") exceeds the "
377 : "global thread count (" << libMesh::n_threads() << ").");
378 85550 : Threads::BoolAcquire set_in_threads(Threads::in_threads);
379 :
380 4269299 : unsigned int actual_threads = num_pthreads(range, n_threads);
381 :
382 : // If we're running in serial - just run!
383 4272555 : if (actual_threads == 1)
384 : {
385 4353441 : body(range);
386 84480 : return;
387 : }
388 :
389 4266 : Threads::RAIIAcquire<int> set_active_threads(Threads::active_threads, actual_threads);
390 :
391 4280 : DisablePerfLogInScope disable_perf;
392 :
393 5336 : std::vector<std::unique_ptr<Range>> ranges(actual_threads);
394 5336 : std::vector<std::unique_ptr<Body>> managed_bodies(actual_threads); // bodies we are responsible for
395 4266 : std::vector<Body *> bodies(actual_threads); // dumb pointers to managed_bodies
396 4266 : std::vector<RangeBody<Range, Body>> range_bodies(actual_threads);
397 :
398 : // Create actual_threads-1 copies of "body". We manage the lifetime of
399 : // these copies with std::unique_ptrs.
400 6392 : for (unsigned int i=1; i<actual_threads; i++)
401 3918 : managed_bodies[i] = std::make_unique<Body>(body, Threads::split());
402 :
403 : // Set up the "bodies" vector. Use the passed in body for the first
404 : // one, point to managed_bodies entries for the others.
405 3196 : bodies[0] = &body;
406 6392 : for (unsigned int i=1; i<actual_threads; i++)
407 4266 : bodies[i] = managed_bodies[i].get();
408 :
409 : // Create the ranges for each thread
410 3196 : std::size_t range_size = range.size() / actual_threads;
411 :
412 3196 : typename Range::const_iterator current_beginning = range.begin();
413 :
414 9588 : for (unsigned int i=0; i<actual_threads; i++)
415 : {
416 2140 : std::size_t this_range_size = range_size;
417 :
418 6392 : if (i+1 == actual_threads)
419 3196 : this_range_size += range.size() % actual_threads; // Give the last one the remaining work to do
420 :
421 8532 : ranges[i] = std::make_unique<Range>(range, current_beginning, current_beginning + this_range_size);
422 :
423 8532 : current_beginning = current_beginning + this_range_size;
424 : }
425 :
426 : // Create the RangeBody arguments
427 9588 : for (unsigned int i=0; i<actual_threads; i++)
428 : {
429 8532 : range_bodies[i].range = ranges[i].get();
430 8532 : range_bodies[i].body = bodies[i];
431 : }
432 :
433 : // Create the threads
434 4266 : std::vector<pthread_t> threads(actual_threads);
435 :
436 : // It may seem redundant to wrap a pragma in #ifdefs... but GCC
437 : // warns about an "unknown pragma" if it encounters this line of
438 : // code when -fopenmp is not passed to the compiler.
439 : #ifdef LIBMESH_HAVE_OPENMP
440 : // Cap the OpenMP team to the actual number of threads this dispatch uses. Without this clause the
441 : // team defaults to the process-wide libMesh::n_threads(), so omp_get_thread_num() (consumed by
442 : // MOOSE's ParallelUniqueId) could exceed a reduced per-application thread count and index past
443 : // per-thread storage sized to that count.
444 3196 : #pragma omp parallel for schedule (static) num_threads(actual_threads)
445 : #endif
446 : // The use of 'int' instead of unsigned for the iteration variable
447 : // is deliberate here. This is an OpenMP loop, and some older
448 : // compilers warn when you don't use int for the loop index. The
449 : // reason has to do with signed vs. unsigned integer overflow
450 : // behavior and optimization.
451 : // http://blog.llvm.org/2011/05/what-every-c-programmer-should-know.html
452 : for (int i=0; i<static_cast<int>(actual_threads); i++)
453 : {
454 : #if !LIBMESH_HAVE_OPENMP
455 : pthread_create(&threads[i], nullptr, &run_body<Range, Body>, (void *)&range_bodies[i]);
456 : #else
457 : run_body<Range, Body>((void *)&range_bodies[i]);
458 : #endif
459 : }
460 :
461 : #if !LIBMESH_HAVE_OPENMP
462 : // Wait for them to finish
463 : for (unsigned int i=0; i<actual_threads; i++)
464 : pthread_join(threads[i], nullptr);
465 : #endif
466 :
467 : // Join them all down to the original Body
468 6392 : for (unsigned int i=actual_threads-1; i != 0; i--)
469 5336 : bodies[i-1]->join(*bodies[i]);
470 1056 : }
471 :
472 : /**
473 : * Execute the provided reduction operation in parallel on the specified
474 : * range with the specified partitioner.
475 : */
476 : template <typename Range, typename Body, typename Partitioner>
477 : inline
478 : void parallel_reduce (const Range & range, Body & body, const Partitioner &,
479 : unsigned int n_threads = libMesh::n_threads())
480 : {
481 : parallel_reduce(range, body, n_threads);
482 : }
483 :
484 :
485 : /**
486 : * Defines atomic operations which can only be executed on a
487 : * single thread at a time.
488 : */
489 : template <typename T>
490 : class atomic
491 : {
492 : public:
493 468 : atomic () : val(0) {}
494 33668 : operator T () { return val; }
495 :
496 : T operator=( T value )
497 : {
498 : spin_mutex::scoped_lock lock(smutex);
499 : val = value;
500 : return val;
501 : }
502 :
503 : atomic<T> & operator=( const atomic<T> & value )
504 : {
505 : spin_mutex::scoped_lock lock(smutex);
506 : val = value;
507 : return *this;
508 : }
509 :
510 :
511 : T operator+=(T value)
512 : {
513 : spin_mutex::scoped_lock lock(smutex);
514 : val += value;
515 : return val;
516 : }
517 :
518 : T operator-=(T value)
519 : {
520 : spin_mutex::scoped_lock lock(smutex);
521 : val -= value;
522 : return val;
523 : }
524 :
525 515994617 : T operator++()
526 : {
527 31111246 : spin_mutex::scoped_lock lock(smutex);
528 1203317608 : val++;
529 515994617 : return val;
530 : }
531 :
532 : T operator++(int)
533 : {
534 : spin_mutex::scoped_lock lock(smutex);
535 : val++;
536 : return val;
537 : }
538 :
539 3731552270 : T operator--()
540 : {
541 11200004 : spin_mutex::scoped_lock lock(smutex);
542 3759416209 : val--;
543 3731552270 : return val;
544 : }
545 :
546 : T operator--(int)
547 : {
548 : spin_mutex::scoped_lock lock(smutex);
549 : val--;
550 : return val;
551 : }
552 :
553 : private:
554 : T val;
555 : spin_mutex smutex;
556 : };
557 :
558 : } // namespace Threads
559 :
560 : } // namespace libMesh
561 :
562 : #endif // #ifdef LIBMESH_HAVE_PTHREAD
563 :
564 : #endif // LIBMESH_SQUASH_HEADER_WARNING
565 :
566 : #endif // LIBMESH_THREADS_PTHREAD_H
|
