pyatomic.h 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250
  1. #ifndef Py_ATOMIC_H
  2. #define Py_ATOMIC_H
  3. #ifdef Py_BUILD_CORE
  4. #include "dynamic_annotations.h"
  5. #include "pyconfig.h"
  6. #if defined(HAVE_STD_ATOMIC)
  7. #include <stdatomic.h>
  8. #endif
  9. /* This is modeled after the atomics interface from C1x, according to
  10. * the draft at
  11. * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf.
  12. * Operations and types are named the same except with a _Py_ prefix
  13. * and have the same semantics.
  14. *
  15. * Beware, the implementations here are deep magic.
  16. */
  17. #if defined(HAVE_STD_ATOMIC)
  18. typedef enum _Py_memory_order {
  19. _Py_memory_order_relaxed = memory_order_relaxed,
  20. _Py_memory_order_acquire = memory_order_acquire,
  21. _Py_memory_order_release = memory_order_release,
  22. _Py_memory_order_acq_rel = memory_order_acq_rel,
  23. _Py_memory_order_seq_cst = memory_order_seq_cst
  24. } _Py_memory_order;
  25. typedef struct _Py_atomic_address {
  26. atomic_uintptr_t _value;
  27. } _Py_atomic_address;
  28. typedef struct _Py_atomic_int {
  29. atomic_int _value;
  30. } _Py_atomic_int;
  31. #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
  32. atomic_signal_fence(ORDER)
  33. #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
  34. atomic_thread_fence(ORDER)
  35. #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
  36. atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
  37. #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
  38. atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
  39. /* Use builtin atomic operations in GCC >= 4.7 */
  40. #elif defined(HAVE_BUILTIN_ATOMIC)
  41. typedef enum _Py_memory_order {
  42. _Py_memory_order_relaxed = __ATOMIC_RELAXED,
  43. _Py_memory_order_acquire = __ATOMIC_ACQUIRE,
  44. _Py_memory_order_release = __ATOMIC_RELEASE,
  45. _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL,
  46. _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST
  47. } _Py_memory_order;
  48. typedef struct _Py_atomic_address {
  49. Py_uintptr_t _value;
  50. } _Py_atomic_address;
  51. typedef struct _Py_atomic_int {
  52. int _value;
  53. } _Py_atomic_int;
  54. #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
  55. __atomic_signal_fence(ORDER)
  56. #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
  57. __atomic_thread_fence(ORDER)
  58. #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
  59. (assert((ORDER) == __ATOMIC_RELAXED \
  60. || (ORDER) == __ATOMIC_SEQ_CST \
  61. || (ORDER) == __ATOMIC_RELEASE), \
  62. __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER))
  63. #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
  64. (assert((ORDER) == __ATOMIC_RELAXED \
  65. || (ORDER) == __ATOMIC_SEQ_CST \
  66. || (ORDER) == __ATOMIC_ACQUIRE \
  67. || (ORDER) == __ATOMIC_CONSUME), \
  68. __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER))
  69. #else
  70. typedef enum _Py_memory_order {
  71. _Py_memory_order_relaxed,
  72. _Py_memory_order_acquire,
  73. _Py_memory_order_release,
  74. _Py_memory_order_acq_rel,
  75. _Py_memory_order_seq_cst
  76. } _Py_memory_order;
  77. typedef struct _Py_atomic_address {
  78. Py_uintptr_t _value;
  79. } _Py_atomic_address;
  80. typedef struct _Py_atomic_int {
  81. int _value;
  82. } _Py_atomic_int;
  83. /* Only support GCC (for expression statements) and x86 (for simple
  84. * atomic semantics) for now */
  85. #if defined(__GNUC__) && (defined(__i386__) || defined(__amd64))
  86. static __inline__ void
  87. _Py_atomic_signal_fence(_Py_memory_order order)
  88. {
  89. if (order != _Py_memory_order_relaxed)
  90. __asm__ volatile("":::"memory");
  91. }
  92. static __inline__ void
  93. _Py_atomic_thread_fence(_Py_memory_order order)
  94. {
  95. if (order != _Py_memory_order_relaxed)
  96. __asm__ volatile("mfence":::"memory");
  97. }
  98. /* Tell the race checker about this operation's effects. */
  99. static __inline__ void
  100. _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
  101. {
  102. (void)address; /* shut up -Wunused-parameter */
  103. switch(order) {
  104. case _Py_memory_order_release:
  105. case _Py_memory_order_acq_rel:
  106. case _Py_memory_order_seq_cst:
  107. _Py_ANNOTATE_HAPPENS_BEFORE(address);
  108. break;
  109. case _Py_memory_order_relaxed:
  110. case _Py_memory_order_acquire:
  111. break;
  112. }
  113. switch(order) {
  114. case _Py_memory_order_acquire:
  115. case _Py_memory_order_acq_rel:
  116. case _Py_memory_order_seq_cst:
  117. _Py_ANNOTATE_HAPPENS_AFTER(address);
  118. break;
  119. case _Py_memory_order_relaxed:
  120. case _Py_memory_order_release:
  121. break;
  122. }
  123. }
  124. #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
  125. __extension__ ({ \
  126. __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
  127. __typeof__(atomic_val->_value) new_val = NEW_VAL;\
  128. volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \
  129. _Py_memory_order order = ORDER; \
  130. _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
  131. \
  132. /* Perform the operation. */ \
  133. _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \
  134. switch(order) { \
  135. case _Py_memory_order_release: \
  136. _Py_atomic_signal_fence(_Py_memory_order_release); \
  137. /* fallthrough */ \
  138. case _Py_memory_order_relaxed: \
  139. *volatile_data = new_val; \
  140. break; \
  141. \
  142. case _Py_memory_order_acquire: \
  143. case _Py_memory_order_acq_rel: \
  144. case _Py_memory_order_seq_cst: \
  145. __asm__ volatile("xchg %0, %1" \
  146. : "+r"(new_val) \
  147. : "m"(atomic_val->_value) \
  148. : "memory"); \
  149. break; \
  150. } \
  151. _Py_ANNOTATE_IGNORE_WRITES_END(); \
  152. })
  153. #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
  154. __extension__ ({ \
  155. __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
  156. __typeof__(atomic_val->_value) result; \
  157. volatile __typeof__(result) *volatile_data = &atomic_val->_value; \
  158. _Py_memory_order order = ORDER; \
  159. _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
  160. \
  161. /* Perform the operation. */ \
  162. _Py_ANNOTATE_IGNORE_READS_BEGIN(); \
  163. switch(order) { \
  164. case _Py_memory_order_release: \
  165. case _Py_memory_order_acq_rel: \
  166. case _Py_memory_order_seq_cst: \
  167. /* Loads on x86 are not releases by default, so need a */ \
  168. /* thread fence. */ \
  169. _Py_atomic_thread_fence(_Py_memory_order_release); \
  170. break; \
  171. default: \
  172. /* No fence */ \
  173. break; \
  174. } \
  175. result = *volatile_data; \
  176. switch(order) { \
  177. case _Py_memory_order_acquire: \
  178. case _Py_memory_order_acq_rel: \
  179. case _Py_memory_order_seq_cst: \
  180. /* Loads on x86 are automatically acquire operations so */ \
  181. /* can get by with just a compiler fence. */ \
  182. _Py_atomic_signal_fence(_Py_memory_order_acquire); \
  183. break; \
  184. default: \
  185. /* No fence */ \
  186. break; \
  187. } \
  188. _Py_ANNOTATE_IGNORE_READS_END(); \
  189. result; \
  190. })
  191. #else /* !gcc x86 */
  192. /* Fall back to other compilers and processors by assuming that simple
  193. volatile accesses are atomic. This is false, so people should port
  194. this. */
  195. #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0)
  196. #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0)
  197. #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
  198. ((ATOMIC_VAL)->_value = NEW_VAL)
  199. #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
  200. ((ATOMIC_VAL)->_value)
  201. #endif /* !gcc x86 */
  202. #endif
  203. /* Standardized shortcuts. */
  204. #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \
  205. _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst)
  206. #define _Py_atomic_load(ATOMIC_VAL) \
  207. _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst)
  208. /* Python-local extensions */
  209. #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \
  210. _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed)
  211. #define _Py_atomic_load_relaxed(ATOMIC_VAL) \
  212. _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed)
  213. #endif /* Py_BUILD_CORE */
  214. #endif /* Py_ATOMIC_H */