StarPU Handbook - StarPU Basics
starpu_util.h
Go to the documentation of this file.
1/* StarPU --- Runtime system for heterogeneous multicore architectures.
2 *
3 * Copyright (C) 2008-2025 University of Bordeaux, CNRS (LaBRI UMR 5800), Inria
4 *
5 * StarPU is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU Lesser General Public License as published by
7 * the Free Software Foundation; either version 2.1 of the License, or (at
8 * your option) any later version.
9 *
10 * StarPU is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 *
14 * See the GNU Lesser General Public License in COPYING.LGPL for more details.
15 */
16
17#ifndef __STARPU_UTIL_H__
18#define __STARPU_UTIL_H__
19
20#include <stdio.h>
21#include <stdlib.h>
22#include <stdint.h>
23#include <string.h>
24#include <assert.h>
25
26#include <starpu_config.h>
27
28#ifdef __GLIBC__
29#include <execinfo.h>
30#endif
31
32#ifdef STARPU_SIMGRID_MC
33#include <simgrid/modelchecker.h>
34#endif
35
36#ifdef __cplusplus
37extern "C" {
38#endif
39
51#if defined __GNUC__ && defined __GNUC_MINOR__
52#define STARPU_GNUC_PREREQ(maj, min) \
53 ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
54#else
55#define STARPU_GNUC_PREREQ(maj, min) 0
56#endif
57
62#ifdef __GNUC__
63#define STARPU_UNLIKELY(expr) (__builtin_expect(!!(expr), 0))
64#else
65#define STARPU_UNLIKELY(expr) (expr)
66#endif
67
72#ifdef __GNUC__
73#define STARPU_LIKELY(expr) (__builtin_expect(!!(expr), 1))
74#else
75#define STARPU_LIKELY(expr) (expr)
76#endif
77
81#ifdef __GNUC__
82#define STARPU_ATTRIBUTE_UNUSED __attribute__((unused))
83#else
84#define STARPU_ATTRIBUTE_UNUSED
85#endif
86
90#ifdef __GNUC__
91#define STARPU_ATTRIBUTE_NORETURN __attribute__((noreturn))
92#else
93#define STARPU_ATTRIBUTE_NORETURN
94#endif
95
99#ifdef __GNUC__
100#define STARPU_ATTRIBUTE_VISIBILITY_DEFAULT __attribute__((visibility("default")))
101#else
102#define STARPU_ATTRIBUTE_VISIBILITY_DEFAULT
103#endif
104
108#ifdef __GNUC__
109#define STARPU_VISIBILITY_PUSH_HIDDEN #pragma GCC visibility push(hidden)
110#else
111#define STARPU_VISIBILITY_PUSH_HIDDEN
112#endif
113
117#ifdef __GNUC__
118#define STARPU_VISIBILITY_POP #pragma GCC visibility pop
119#else
120#define STARPU_VISIBILITY_POP
121#endif
122
126#ifdef __GNUC__
127#define STARPU_ATTRIBUTE_MALLOC __attribute__((malloc))
128#else
129#define STARPU_ATTRIBUTE_MALLOC
130#endif
131
135#ifdef __GNUC__
136#define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
137#else
138#define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT
139#endif
140
144#ifdef __GNUC__
145#define STARPU_ATTRIBUTE_PURE __attribute__((pure))
146#else
147#define STARPU_ATTRIBUTE_PURE
148#endif
149
153#ifdef __GNUC__
154#define STARPU_ATTRIBUTE_ALIGNED(size) __attribute__((aligned(size)))
155#else
156#define STARPU_ATTRIBUTE_ALIGNED(size)
157#endif
158
159#ifdef __GNUC__
160#define STARPU_ATTRIBUTE_FORMAT(type, string, first) __attribute__((format(type, string, first)))
161#else
162#define STARPU_ATTRIBUTE_FORMAT(type, string, first)
163#endif
164
165/* Note that if we're compiling C++, then just use the "inline"
166 keyword, since it's part of C++ */
167#if defined(c_plusplus) || defined(__cplusplus)
168#define STARPU_INLINE inline
169#elif defined(_MSC_VER) || defined(__HP_cc)
170#define STARPU_INLINE __inline
171#else
172#define STARPU_INLINE __inline__
173#endif
174
175#if STARPU_GNUC_PREREQ(4, 3)
176#define STARPU_ATTRIBUTE_CALLOC_SIZE(num, size) __attribute__((alloc_size(num, size)))
177#define STARPU_ATTRIBUTE_ALLOC_SIZE(size) __attribute__((alloc_size(size)))
178#else
179#define STARPU_ATTRIBUTE_CALLOC_SIZE(num, size)
180#define STARPU_ATTRIBUTE_ALLOC_SIZE(size)
181#endif
182
183#if STARPU_GNUC_PREREQ(3, 1) && !defined(BUILDING_STARPU) && !defined(STARPU_USE_DEPRECATED_API) && !defined(STARPU_USE_DEPRECATED_ONE_ZERO_API)
184#define STARPU_DEPRECATED __attribute__((__deprecated__))
185#else
186#define STARPU_DEPRECATED
187#endif /* __GNUC__ */
188
189#if STARPU_GNUC_PREREQ(3, 3)
190#define STARPU_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))
191#else
192#define STARPU_WARN_UNUSED_RESULT
193#endif /* __GNUC__ */
194
195#define STARPU_BACKTRACE_LENGTH 32
196#ifdef __GLIBC__
197#define STARPU_DUMP_BACKTRACE() \
198 do { \
199 void *__ptrs[STARPU_BACKTRACE_LENGTH]; \
200 int __n = backtrace(__ptrs, STARPU_BACKTRACE_LENGTH); \
201 backtrace_symbols_fd(__ptrs, __n, 2); \
202 } \
203 while (0)
204#else
205#define STARPU_DUMP_BACKTRACE() \
206 do { \
207 } \
208 while (0)
209#endif
210
211#ifdef STARPU_SIMGRID_MC
212#define STARPU_SIMGRID_ASSERT(x) MC_assert(!!(x))
213#else
214#define STARPU_SIMGRID_ASSERT(x)
215#endif
216
221#ifdef STARPU_NO_ASSERT
222#define STARPU_ASSERT(x) \
223 do { \
224 if (0) { (void)(x); } \
225 } \
226 while (0)
227#else
228#if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS)
229#define STARPU_ASSERT(x) \
230 do { \
231 if (STARPU_UNLIKELY(!(x))) \
232 { \
233 STARPU_DUMP_BACKTRACE(); \
234 STARPU_SIMGRID_ASSERT(0 && #x); \
235 *(int *)NULL = 0; \
236 } \
237 } \
238 while (0)
239#else
240#define STARPU_ASSERT(x) \
241 do { \
242 if (STARPU_UNLIKELY(!(x))) \
243 { \
244 STARPU_DUMP_BACKTRACE(); \
245 STARPU_SIMGRID_ASSERT(0 && #x); \
246 assert(0 && #x); \
247 } \
248 } \
249 while (0)
250#endif
251#endif
252
258#ifdef STARPU_NO_ASSERT
259#define STARPU_ASSERT_ACCESSIBLE(x) \
260 do { \
261 if (0) { (void)(x); } \
262 } \
263 while (0)
264#else
265#define STARPU_ASSERT_ACCESSIBLE(ptr) \
266 do { \
267 volatile char __c STARPU_ATTRIBUTE_UNUSED = *(char *)(ptr); \
268 } \
269 while (0)
270#endif
271
275#if STARPU_GNUC_PREREQ(4, 6) && !defined __cplusplus && !defined(__STRICT_ANSI__)
276#define STARPU_STATIC_ASSERT(x) _Static_assert(x, #x)
277#else
278#define STARPU_STATIC_ASSERT(x) STARPU_ASSERT(x)
279#endif
280
285#if defined(__INTEL_COMPILER)
286#pragma warning disable 279 // otherwise icc triggers "warning #279: controlling expression is constant" (probably because of assert(0 && #x))
287#endif
288#if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS)
289#define STARPU_ASSERT_MSG_ALWAYS(x, msg, ...) \
290 do { \
291 if (STARPU_UNLIKELY(!(x))) \
292 { \
293 STARPU_DUMP_BACKTRACE(); \
294 fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ##__VA_ARGS__); \
295 STARPU_SIMGRID_ASSERT(0 && #x); \
296 *(int *)NULL = 0; \
297 } \
298 } \
299 while (0)
300#else
301#define STARPU_ASSERT_MSG_ALWAYS(x, msg, ...) \
302 do { \
303 if (STARPU_UNLIKELY(!(x))) \
304 { \
305 STARPU_DUMP_BACKTRACE(); \
306 fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ##__VA_ARGS__); \
307 STARPU_SIMGRID_ASSERT(0 && #x); \
308 assert(0 && #x); \
309 abort(); \
310 *(int *)NULL = 0; \
311 } \
312 } \
313 while (0)
314#endif
315
321#ifdef STARPU_NO_ASSERT
322#define STARPU_ASSERT_MSG(x, msg, ...) \
323 do { \
324 if (0) \
325 { \
326 (void)(x); \
327 (void)msg; \
328 } \
329 } \
330 while (0)
331#else
332#define STARPU_ASSERT_MSG(x, msg, ...) \
333 STARPU_ASSERT_MSG_ALWAYS(x, msg, ##__VA_ARGS__)
334#endif
335
336#ifdef __APPLE_CC__
337#ifdef __clang_analyzer__
338#define _starpu_abort() exit(42)
339#else
340#define _starpu_abort() *(volatile int *)NULL = 0
341#endif
342#else
343#define _starpu_abort() abort()
344#endif
345
349#define STARPU_ABORT() \
350 do { \
351 STARPU_DUMP_BACKTRACE(); \
352 fprintf(stderr, "[starpu][abort][%s()@%s:%d]\n", __starpu_func__, __FILE__, __LINE__); \
353 _starpu_abort(); \
354 } \
355 while (0)
356
362#define STARPU_ABORT_MSG(msg, ...) \
363 do { \
364 STARPU_DUMP_BACKTRACE(); \
365 fprintf(stderr, "[starpu][abort][%s()@%s:%d] " msg "\n", __starpu_func__, __FILE__, __LINE__, ##__VA_ARGS__); \
366 _starpu_abort(); \
367 } \
368 while (0)
369
370#if defined(_MSC_VER)
371#undef STARPU_HAVE_STRERROR_R
372#endif
373
374#if defined(STARPU_HAVE_STRERROR_R)
375#if (!defined(__GLIBC__) || !__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && (!defined(_GNU_SOURCE)))
376/* XSI-compliant version of strerror_r returns an int */
377#define starpu_strerror_r(errnum, buf, buflen) \
378 do \
379 { \
380 int _ret = strerror_r((errnum), (buf), (buflen)); \
381 STARPU_ASSERT(_ret == 0); \
382 } \
383 while (0)
384#else
385/* GNU-specific version of strerror_r returns a char * */
386#define starpu_strerror_r(errnum, buf, buflen) \
387 do \
388 { \
389 char *const _user_buf = (buf); \
390 const size_t _user_buflen = (buflen); \
391 /* the GNU-specific behaviour when 'buf' == NULL cannot be emulated with the XSI-compliant version */ \
392 STARPU_ASSERT((buf) != NULL); \
393 char *_tmp_buf = strerror_r((errnum), _user_buf, _user_buflen); \
394 if (_tmp_buf != _user_buf) \
395 { \
396 if (_user_buflen > 0) \
397 { \
398 strncpy(_user_buf, _tmp_buf, _user_buflen - 1); \
399 _user_buf[_user_buflen - 1] = '\0'; \
400 } \
401 } \
402 } \
403 while (0)
404#endif /* strerror_r ABI version */
405#endif /* STARPU_HAVE_STRERROR_R */
406
411#if defined(STARPU_HAVE_STRERROR_R)
412#define STARPU_CHECK_RETURN_VALUE(err, message, ...) \
413 { \
414 if (STARPU_UNLIKELY(err != 0)) \
415 { \
416 char xmessage[256]; \
417 starpu_strerror_r(-err, xmessage, 256); \
418 fprintf(stderr, "[starpu] Unexpected value: <%d:%s> returned for " message "\n", err, xmessage, ##__VA_ARGS__); \
419 STARPU_ABORT(); \
420 } \
421 }
422#else
423#define STARPU_CHECK_RETURN_VALUE(err, message, ...) \
424 { \
425 if (STARPU_UNLIKELY(err != 0)) \
426 { \
427 fprintf(stderr, "[starpu] Unexpected value: <%d> returned for " message "\n", err, ##__VA_ARGS__); \
428 STARPU_ABORT(); \
429 } \
430 }
431#endif
432
437#if defined(STARPU_HAVE_STRERROR_R)
438#define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) \
439 { \
440 if (STARPU_UNLIKELY(err != value)) \
441 { \
442 char xmessage[256]; \
443 starpu_strerror_r(-err, xmessage, 256); \
444 fprintf(stderr, "[starpu] Unexpected value: <%d!=%d:%s> returned for " message "\n", err, value, xmessage, ##__VA_ARGS__); \
445 STARPU_ABORT(); \
446 } \
447 }
448#else
449#define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) \
450 { \
451 if (STARPU_UNLIKELY(err != value)) \
452 { \
453 fprintf(stderr, "[starpu] Unexpected value: <%d != %d> returned for " message "\n", err, value, ##__VA_ARGS__); \
454 STARPU_ABORT(); \
455 } \
456 }
457#endif
458
459/* Note: do not use _starpu_cmpxchg / _starpu_xchg / _starpu_cmpxchgl /
460 * _starpu_xchgl / _starpu_cmpxchg64 / _starpu_xchg64, which only
461 * assembly-hand-written fallbacks used when building with an old gcc.
462 * Rather use STARPU_VAL_COMPARE_AND_SWAP and STARPU_VAL_EXCHANGE available on
463 * all platforms with a recent-enough gcc */
464
465#if defined(__i386__) || defined(__x86_64__)
466static __starpu_inline unsigned _starpu_cmpxchg(unsigned *ptr, unsigned old, unsigned next)
467{
468 __asm__ __volatile__("lock cmpxchgl %2,%1"
469 : "+a"(old), "+m"(*ptr)
470 : "q"(next)
471 : "memory");
472 return old;
473}
474#define STARPU_HAVE_CMPXCHG
475static __starpu_inline unsigned _starpu_xchg(unsigned *ptr, unsigned next)
476{
477 /* Note: xchg is always locked already */
478 __asm__ __volatile__("xchgl %1,%0"
479 : "+m"(*ptr), "+q"(next)
480 :
481 : "memory");
482 return next;
483}
484#define STARPU_HAVE_XCHG
485
486static __starpu_inline uint32_t _starpu_cmpxchg32(uint32_t *ptr, uint32_t old, uint32_t next)
487{
488 __asm__ __volatile__("lock cmpxchgl %2,%1"
489 : "+a"(old), "+m"(*ptr)
490 : "q"(next)
491 : "memory");
492 return old;
493}
494#define STARPU_HAVE_CMPXCHG32
495static __starpu_inline uint32_t _starpu_xchg32(uint32_t *ptr, uint32_t next)
496{
497 /* Note: xchg is always locked already */
498 __asm__ __volatile__("xchgl %1,%0"
499 : "+m"(*ptr), "+q"(next)
500 :
501 : "memory");
502 return next;
503}
504#define STARPU_HAVE_XCHG32
505
506#if defined(__i386__)
507static __starpu_inline unsigned long _starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
508{
509 __asm__ __volatile__("lock cmpxchgl %2,%1"
510 : "+a"(old), "+m"(*ptr)
511 : "q"(next)
512 : "memory");
513 return old;
514}
515#define STARPU_HAVE_CMPXCHGL
516static __starpu_inline unsigned long _starpu_xchgl(unsigned long *ptr, unsigned long next)
517{
518 /* Note: xchg is always locked already */
519 __asm__ __volatile__("xchgl %1,%0"
520 : "+m"(*ptr), "+q"(next)
521 :
522 : "memory");
523 return next;
524}
525#define STARPU_HAVE_XCHGL
526#endif
527
528#if defined(__x86_64__)
529static __starpu_inline unsigned long _starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
530{
531 __asm__ __volatile__("lock cmpxchgq %2,%1"
532 : "+a"(old), "+m"(*ptr)
533 : "q"(next)
534 : "memory");
535 return old;
536}
537#define STARPU_HAVE_CMPXCHGL
538static __starpu_inline unsigned long _starpu_xchgl(unsigned long *ptr, unsigned long next)
539{
540 /* Note: xchg is always locked already */
541 __asm__ __volatile__("xchgq %1,%0"
542 : "+m"(*ptr), "+q"(next)
543 :
544 : "memory");
545 return next;
546}
547#define STARPU_HAVE_XCHGL
548#endif
549
550#if defined(__i386__)
551static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
552{
553 uint32_t next_hi = next >> 32;
554 uint32_t next_lo = next & 0xfffffffful;
555 __asm__ __volatile__("lock cmpxchg8b %1"
556 : "+A"(old), "+m"(*ptr)
557 : "c"(next_hi), "b"(next_lo)
558 : "memory");
559 return old;
560}
561#define STARPU_HAVE_CMPXCHG64
562#endif
563
564#if defined(__x86_64__)
565static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
566{
567 __asm__ __volatile__("lock cmpxchgq %2,%1"
568 : "+a"(old), "+m"(*ptr)
569 : "q"(next)
570 : "memory");
571 return old;
572}
573#define STARPU_HAVE_CMPXCHG64
574static __starpu_inline uint64_t _starpu_xchg64(uint64_t *ptr, uint64_t next)
575{
576 /* Note: xchg is always locked already */
577 __asm__ __volatile__("xchgq %1,%0"
578 : "+m"(*ptr), "+q"(next)
579 :
580 : "memory");
581 return next;
582}
583#define STARPU_HAVE_XCHG64
584#endif
585
586#endif
587
588#define STARPU_ATOMIC_SOMETHING(name, expr) \
589 static __starpu_inline unsigned starpu_atomic_##name(unsigned *ptr, unsigned value) \
590 { \
591 unsigned old, next; \
592 while (1) \
593 { \
594 old = *ptr; \
595 next = expr; \
596 if (_starpu_cmpxchg(ptr, old, next) == old) \
597 break; \
598 }; \
599 return expr; \
600 }
601#define STARPU_ATOMIC_SOMETHINGL(name, expr) \
602 static __starpu_inline unsigned long starpu_atomic_##name##l(unsigned long *ptr, unsigned long value) \
603 { \
604 unsigned long old, next; \
605 while (1) \
606 { \
607 old = *ptr; \
608 next = expr; \
609 if (_starpu_cmpxchgl(ptr, old, next) == old) \
610 break; \
611 }; \
612 return expr; \
613 }
614#define STARPU_ATOMIC_SOMETHING64(name, expr) \
615 static __starpu_inline uint64_t starpu_atomic_##name##64(uint64_t * ptr, uint64_t value) \
616 { \
617 uint64_t old, next; \
618 while (1) \
619 { \
620 old = *ptr; \
621 next = expr; \
622 if (_starpu_cmpxchg64(ptr, old, next) == old) \
623 break; \
624 }; \
625 return expr; \
626 }
627
628/* Atomic addition, returns the new value */
629#if defined(STARPU_HAVE_SYNC_FETCH_AND_ADD)
630#define STARPU_ATOMIC_ADD(ptr, value) (__sync_fetch_and_add((ptr), (value)) + (value))
631#define STARPU_ATOMIC_ADDL(ptr, value) (__sync_fetch_and_add((ptr), (value)) + (value))
632#elif defined(STARPU_HAVE_ATOMIC_FETCH_ADD)
633#define STARPU_ATOMIC_ADD(ptr, value) (__atomic_fetch_add((ptr), (value), __ATOMIC_SEQ_CST) + (value))
634#define STARPU_ATOMIC_ADDL(ptr, value) (__atomic_fetch_add((ptr), (value), __ATOMIC_SEQ_CST) + (value))
635#else
636#if defined(STARPU_HAVE_CMPXCHG)
637STARPU_ATOMIC_SOMETHING(add, old + value)
638#define STARPU_ATOMIC_ADD(ptr, value) starpu_atomic_add(ptr, value)
639#endif
640#if defined(STARPU_HAVE_CMPXCHGL)
641STARPU_ATOMIC_SOMETHINGL(add, old + value)
642#define STARPU_ATOMIC_ADDL(ptr, value) starpu_atomic_addl(ptr, value)
643#endif
644#endif
645
646#if defined(STARPU_HAVE_SYNC_FETCH_AND_ADD_8)
647#define STARPU_ATOMIC_ADD64(ptr, value) (__sync_fetch_and_add((ptr), (value)) + (value))
648#elif defined(STARPU_HAVE_ATOMIC_FETCH_ADD_8)
649#define STARPU_ATOMIC_ADD64(ptr, value) (__atomic_fetch_add((ptr), (value), __ATOMIC_SEQ_CST) + (value))
650#else
651#if defined(STARPU_HAVE_CMPXCHG64)
652STARPU_ATOMIC_SOMETHING64(add, old + value)
653#define STARPU_ATOMIC_ADD64(ptr, value) starpu_atomic_add64(ptr, value)
654#endif
655#endif
656
657/* Atomic OR, returns the *old* value */
658#if defined(STARPU_HAVE_SYNC_FETCH_AND_OR)
659#define STARPU_ATOMIC_OR(ptr, value) (__sync_fetch_and_or((ptr), (value)))
660#define STARPU_ATOMIC_ORL(ptr, value) (__sync_fetch_and_or((ptr), (value)))
661#elif defined(STARPU_HAVE_ATOMIC_FETCH_OR)
662#define STARPU_ATOMIC_OR(ptr, value) (__atomic_fetch_or((ptr), (value), __ATOMIC_SEQ_CST))
663#define STARPU_ATOMIC_ORL(ptr, value) (__atomic_fetch_or((ptr), (value), __ATOMIC_SEQ_CST))
664#else
665#if defined(STARPU_HAVE_CMPXCHG)
666STARPU_ATOMIC_SOMETHING(or, old | value)
667#define STARPU_ATOMIC_OR(ptr, value) starpu_atomic_or(ptr, value)
668#endif
669#if defined(STARPU_HAVE_CMPXCHGL)
670STARPU_ATOMIC_SOMETHINGL(or, old | value)
671#define STARPU_ATOMIC_ORL(ptr, value) starpu_atomic_orl(ptr, value)
672#endif
673#endif
674
675#if defined(STARPU_HAVE_SYNC_FETCH_AND_OR_8)
676#define STARPU_ATOMIC_OR64(ptr, value) (__sync_fetch_and_or((ptr), (value)))
677#elif defined(STARPU_HAVE_ATOMIC_FETCH_OR_8)
678#define STARPU_ATOMIC_OR64(ptr, value) (__atomic_fetch_or((ptr), (value), __ATOMIC_SEQ_CST))
679#else
680#if defined(STARPU_HAVE_CMPXCHG64)
681STARPU_ATOMIC_SOMETHING64(or, old | value)
682#define STARPU_ATOMIC_OR64(ptr, value) starpu_atomic_or64(ptr, value)
683#endif
684#endif
685
686/* Try to replace `old' with `value' at `ptr'. Returns true iff the swap was successful. */
687#ifdef STARPU_HAVE_SYNC_BOOL_COMPARE_AND_SWAP
688#define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (__sync_bool_compare_and_swap((ptr), (old), (value)))
689#else
690#ifdef STARPU_HAVE_CMPXCHG
691#define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)) == (old))
692#endif
693#endif
694
695#ifdef STARPU_HAVE_SYNC_BOOL_COMPARE_AND_SWAP
696#define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (__sync_bool_compare_and_swap((ptr), (old), (value)))
697#else
698#ifdef STARPU_HAVE_CMPXCHG32
699#define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)) == (old))
700#endif
701#endif
702
703#if defined(STARPU_HAVE_SYNC_BOOL_COMPARE_AND_SWAP_8)
704#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (__sync_bool_compare_and_swap((ptr), (old), (value)))
705#elif defined(STARPU_HAVE_ATOMIC_EXCHANGE_N_8) && defined(__GNUC__)
706static __starpu_inline int starpu_bool_compare_and_swap64(uint64_t *ptr, uint64_t old, uint64_t value)
707{
708 uint64_t expected = old;
709 return __atomic_compare_exchange_n(ptr, &expected, value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
710}
711#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) starpu_bool_compare_and_swap64((ptr), (old), (value))
712#else
713#ifdef STARPU_HAVE_CMPXCHG64
714#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)) == (old))
715#endif
716#endif
717
718#if UINTPTR_MAX == UINT64_MAX
719#define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP64((uint64_t*) (ptr), (uint64_t) (old), (uint64_t) (value))
720#else
721#define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value)
722#endif
723
724/* Try to replace `old' with `value' at `ptr'. Returns the value actually seen at `ptr'. */
725#ifdef STARPU_HAVE_SYNC_VAL_COMPARE_AND_SWAP
726#define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (__sync_val_compare_and_swap((ptr), (old), (value)))
727#else
728#ifdef STARPU_HAVE_CMPXCHG
729#define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)))
730#endif
731#endif
732
733#ifdef STARPU_HAVE_SYNC_VAL_COMPARE_AND_SWAP
734#define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (__sync_val_compare_and_swap((ptr), (old), (value)))
735#else
736#ifdef STARPU_HAVE_CMPXCHG32
737#define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)))
738#endif
739#endif
740
741#if defined(STARPU_HAVE_SYNC_VAL_COMPARE_AND_SWAP_8)
742#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (__sync_val_compare_and_swap((ptr), (old), (value)))
743#elif defined(STARPU_HAVE_ATOMIC_EXCHANGE_N_8) && defined(__GNUC__)
744static __starpu_inline uint64_t starpu_val_compare_and_swap64(uint64_t *ptr, uint64_t old, uint64_t value)
745{
746 uint64_t expected = old;
747 if (__atomic_compare_exchange_n(ptr, &expected, value, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
748 return old;
749 else
750 return expected;
751}
752#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) starpu_val_compare_and_swap64((ptr), (old), (value))
753#else
754#ifdef STARPU_HAVE_CMPXCHG64
755#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)))
756#endif
757#endif
758
759#if UINTPTR_MAX == UINT64_MAX
760#define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) ((void*)STARPU_VAL_COMPARE_AND_SWAP64((uint64_t*) (ptr), (uint64_t) (old), (uint64_t) (value)))
761#else
762#define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value)
763#endif
764
765#ifdef STARPU_HAVE_ATOMIC_EXCHANGE_N_8
766#define STARPU_VAL_EXCHANGE64(ptr, value) STARPU_VAL_EXCHANGE((ptr)(value))
767#else
768#ifdef STARPU_HAVE_XCHG64
769#define STARPU_VAL_EXCHANGE64(ptr, value) (_starpu_xchg64((ptr), (value)))
770#endif
771#endif
772
773#ifdef STARPU_HAVE_ATOMIC_EXCHANGE_N
774#define STARPU_VAL_EXCHANGE(ptr, value) (__atomic_exchange_n((ptr), (value), __ATOMIC_SEQ_CST))
775#define STARPU_VAL_EXCHANGEL(ptr, value) STARPU_VAL_EXCHANGE((ptr)(value))
776#define STARPU_VAL_EXCHANGE32(ptr, value) STARPU_VAL_EXCHANGE((ptr)(value))
777#else
778#ifdef STARPU_HAVE_XCHG
779#define STARPU_VAL_EXCHANGE(ptr, value) (_starpu_xchg((ptr), (value)))
780#endif
781#ifdef STARPU_HAVE_XCHGL
782#define STARPU_VAL_EXCHANGEL(ptr, value) (_starpu_xchgl((ptr), (value)))
783#endif
784#ifdef STARPU_HAVE_XCHG32
785#define STARPU_VAL_EXCHANGE32(ptr, value) (_starpu_xchg32((ptr), (value)))
786#endif
787#endif
788
789/* Returns the previous value */
790#ifdef STARPU_HAVE_SYNC_LOCK_TEST_AND_SET
791#define STARPU_TEST_AND_SET(ptr, value) (__sync_lock_test_and_set((ptr), (value)))
792#define STARPU_RELEASE(ptr) (__sync_lock_release((ptr)))
793#elif defined(STARPU_HAVE_XCHG)
794#define STARPU_TEST_AND_SET(ptr, value) (_starpu_xchg((ptr), (value)))
795#define STARPU_RELEASE(ptr) (_starpu_xchg((ptr), 0))
796#endif
797
798#ifdef STARPU_HAVE_SYNC_SYNCHRONIZE
799#define STARPU_SYNCHRONIZE() __sync_synchronize()
800#elif defined(__i386__)
801#define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%esp)" :: \
802 : "memory")
803#elif defined(__KNC__) || defined(__KNF__)
804#define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%rsp)" :: \
805 : "memory")
806#elif defined(__x86_64__)
807#define STARPU_SYNCHRONIZE() __asm__ __volatile__("mfence" :: \
808 : "memory")
809#elif defined(__ppc__) || defined(__ppc64__)
810#define STARPU_SYNCHRONIZE() __asm__ __volatile__("sync" :: \
811 : "memory")
812#endif
813
817#if defined(__x86_64__)
818#define STARPU_RMB() __asm__ __volatile__("lfence" :: \
819 : "memory")
820#elif defined(__aarch64__)
821#define STARPU_RMB() __asm__ __volatile__("dsb ld" :: \
822 : "memory")
823#else
824#define STARPU_RMB() STARPU_SYNCHRONIZE()
825#endif
826
830#if defined(__x86_64__)
831#define STARPU_WMB() __asm__ __volatile__("sfence" :: \
832 : "memory")
833#elif defined(__aarch64__)
834#define STARPU_WMB() __asm__ __volatile__("dsb st" :: \
835 : "memory")
836#else
837#define STARPU_WMB() STARPU_SYNCHRONIZE()
838#endif
839
840#if defined(__i386__) || defined(__x86_64__)
841#define STARPU_CACHELINE_SIZE 64
842#elif defined(__ppc__) || defined(__ppc64__) || defined(__ia64__)
843#define STARPU_CACHELINE_SIZE 128
844#elif defined(__s390__) || defined(__s390x__)
845#define STARPU_CACHELINE_SIZE 256
846#else
847/* Conservative default */
848#define STARPU_CACHELINE_SIZE 1024
849#endif
850
851#ifdef _WIN32
852/* Try to fetch the system definition of timespec */
853#include <sys/types.h>
854#include <sys/stat.h>
855#ifdef HAVE_UNISTD_H
856#include <unistd.h>
857#endif
858#include <time.h>
859#if !defined(_MSC_VER) || defined(BUILDING_STARPU)
860#include <pthread.h>
861#endif
862#if !defined(STARPU_HAVE_STRUCT_TIMESPEC) || (defined(_MSC_VER) && _MSC_VER < 1900)
863/* If it didn't get defined in the standard places, then define it ourself */
864#ifndef STARPU_TIMESPEC_DEFINED
865#define STARPU_TIMESPEC_DEFINED 1
866struct timespec
867{
868 time_t tv_sec; /* Seconds */
869 long tv_nsec; /* Nanoseconds */
870};
871#endif /* STARPU_TIMESPEC_DEFINED */
872#endif /* STARPU_HAVE_STRUCT_TIMESPEC */
873/* Fetch gettimeofday on mingw/cygwin */
874#if defined(__MINGW32__) || defined(__CYGWIN__)
875#include <sys/time.h>
876#endif
877#else
878#include <sys/time.h>
879#endif /* _WIN32 */
880
883#ifdef __cplusplus
884}
885#endif
886
887#endif /* __STARPU_UTIL_H__ */