Skip to content

Commit b6ddfb0

Browse files
kddnewtonkddnewton
committed
gh-155628: Add relaxed _Py_atomic_add_*
Add _Py_atomic_add_*_relaxed() for all arithmetic types supported by _Py_atomic_add_*(). The existing add operations are sequentially consistent, which is stronger (and on ARM, measurably more expensive) than necessary for uses like statistics counters and unique ID allocation, where the add must be atomic but does not need to order surrounding memory accesses. The GCC/Clang backend uses __atomic_fetch_add() with __ATOMIC_RELAXED, and the standard C11/C++11 backend uses atomic_fetch_add_explicit() with memory_order_relaxed. The MSVC backend uses the _InterlockedExchangeAdd*_nf ("no fence") intrinsics on ARM64; on x86 and x64 those intrinsics do not exist, so it falls back to the plain interlocked intrinsics, whose stronger ordering is a conforming implementation of relaxed (x86 has no cheaper atomic read-modify-write). As with the sequentially consistent version, 64-bit adds on 32-bit x86 fall back to a compare-exchange loop. The _testcapi smoke tests for atomic adds now exercise the relaxed variants as well.
1 parent 3444ef9 commit b6ddfb0

5 files changed

Lines changed: 351 additions & 5 deletions

File tree

‎Include/cpython/pyatomic.h‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
// Operations are sequentially consistent unless they have a suffix indicating
55
// otherwise. If in doubt, prefer the sequentially consistent operations.
66
//
7-
// The "_relaxed" suffix for load and store operations indicates the "relaxed"
8-
// memory order. They don't provide synchronization, but (roughly speaking)
9-
// guarantee somewhat sane behavior for races instead of undefined behavior.
10-
// In practice, they correspond to "normal" hardware load and store
7+
// The "_relaxed" suffix indicates the "relaxed" memory order. Relaxed
8+
// operations don't provide synchronization, but (roughly speaking) guarantee
9+
// somewhat sane behavior for races instead of undefined behavior. In practice,
10+
// relaxed loads and stores correspond to "normal" hardware load and store
1111
// instructions, so they are almost as inexpensive as plain loads and stores
12-
// in C.
12+
// in C. Relaxed read-modify-write operations, such as
13+
// _Py_atomic_add_*_relaxed, are still atomic, but do not order surrounding
14+
// memory accesses.
1315
//
1416
// Note that atomic read-modify-write operations like _Py_atomic_add_* return
1517
// the previous value of the atomic variable, not the new value.
@@ -55,6 +57,12 @@
5557
// obj += value
5658
// return old_obj
5759
//
60+
// def _Py_atomic_add_relaxed(obj, value):
61+
// # relaxed consistency
62+
// old_obj = obj
63+
// obj += value
64+
// return old_obj
65+
//
5866
// def _Py_atomic_and(obj, value):
5967
// # sequential consistency
6068
// old_obj = obj
@@ -130,6 +138,50 @@ static inline Py_ssize_t
130138
_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);
131139

132140

141+
// --- _Py_atomic_add_relaxed ------------------------------------------------
142+
// Atomically adds `value` to `obj` and returns the previous value
143+
// (relaxed consistency, i.e., no ordering)
144+
145+
static inline int
146+
_Py_atomic_add_int_relaxed(int *obj, int value);
147+
148+
static inline int8_t
149+
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value);
150+
151+
static inline int16_t
152+
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value);
153+
154+
static inline int32_t
155+
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value);
156+
157+
static inline int64_t
158+
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value);
159+
160+
static inline intptr_t
161+
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value);
162+
163+
static inline unsigned int
164+
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value);
165+
166+
static inline uint8_t
167+
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value);
168+
169+
static inline uint16_t
170+
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value);
171+
172+
static inline uint32_t
173+
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value);
174+
175+
static inline uint64_t
176+
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value);
177+
178+
static inline uintptr_t
179+
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value);
180+
181+
static inline Py_ssize_t
182+
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);
183+
184+
133185
// --- _Py_atomic_compare_exchange -------------------------------------------
134186
// Performs an atomic compare-and-exchange.
135187
//

‎Include/cpython/pyatomic_gcc.h‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,61 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
6363
{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
6464

6565

66+
// --- _Py_atomic_add_relaxed ------------------------------------------------
67+
68+
static inline int
69+
_Py_atomic_add_int_relaxed(int *obj, int value)
70+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
71+
72+
static inline int8_t
73+
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value)
74+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
75+
76+
static inline int16_t
77+
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value)
78+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
79+
80+
static inline int32_t
81+
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value)
82+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
83+
84+
static inline int64_t
85+
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value)
86+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
87+
88+
static inline intptr_t
89+
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value)
90+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
91+
92+
static inline unsigned int
93+
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value)
94+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
95+
96+
static inline uint8_t
97+
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value)
98+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
99+
100+
static inline uint16_t
101+
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value)
102+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
103+
104+
static inline uint32_t
105+
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value)
106+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
107+
108+
static inline uint64_t
109+
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value)
110+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
111+
112+
static inline uintptr_t
113+
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
114+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
115+
116+
static inline Py_ssize_t
117+
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
118+
{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); }
119+
120+
66121
// --- _Py_atomic_compare_exchange -------------------------------------------
67122

68123
static inline int

‎Include/cpython/pyatomic_msc.h‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,132 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
125125
}
126126

127127

128+
// --- _Py_atomic_add_relaxed ------------------------------------------------
129+
130+
// The "_nf" (no fence) intrinsic variants provide relaxed memory order on
131+
// ARM64. On x86 and x86-64 they do not exist; the plain interlocked
132+
// intrinsics are used instead, which have stronger (sequentially consistent)
133+
// ordering. That is a conforming implementation of relaxed memory order;
134+
// x86 simply has no cheaper atomic read-modify-write.
135+
136+
static inline int8_t
137+
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value)
138+
{
139+
_Py_atomic_ASSERT_ARG_TYPE(char);
140+
#if defined(_M_ARM64)
141+
return (int8_t)_InterlockedExchangeAdd8_nf((volatile char *)obj, (char)value);
142+
#else
143+
return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);
144+
#endif
145+
}
146+
147+
static inline int16_t
148+
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value)
149+
{
150+
_Py_atomic_ASSERT_ARG_TYPE(short);
151+
#if defined(_M_ARM64)
152+
return (int16_t)_InterlockedExchangeAdd16_nf((volatile short *)obj, (short)value);
153+
#else
154+
return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);
155+
#endif
156+
}
157+
158+
static inline int32_t
159+
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value)
160+
{
161+
_Py_atomic_ASSERT_ARG_TYPE(long);
162+
#if defined(_M_ARM64)
163+
return (int32_t)_InterlockedExchangeAdd_nf((volatile long *)obj, (long)value);
164+
#else
165+
return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);
166+
#endif
167+
}
168+
169+
static inline int64_t
170+
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value)
171+
{
172+
#if defined(_M_ARM64)
173+
_Py_atomic_ASSERT_ARG_TYPE(__int64);
174+
return (int64_t)_InterlockedExchangeAdd64_nf((volatile __int64 *)obj, (__int64)value);
175+
#elif defined(_M_X64)
176+
_Py_atomic_ASSERT_ARG_TYPE(__int64);
177+
return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);
178+
#else
179+
int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
180+
for (;;) {
181+
int64_t new_value = old_value + value;
182+
if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {
183+
return old_value;
184+
}
185+
}
186+
#endif
187+
}
188+
189+
static inline uint8_t
190+
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value)
191+
{
192+
return (uint8_t)_Py_atomic_add_int8_relaxed((int8_t *)obj, (int8_t)value);
193+
}
194+
195+
static inline uint16_t
196+
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value)
197+
{
198+
return (uint16_t)_Py_atomic_add_int16_relaxed((int16_t *)obj, (int16_t)value);
199+
}
200+
201+
static inline uint32_t
202+
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value)
203+
{
204+
return (uint32_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
205+
}
206+
207+
static inline int
208+
_Py_atomic_add_int_relaxed(int *obj, int value)
209+
{
210+
_Py_atomic_ASSERT_ARG_TYPE(int32_t);
211+
return (int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
212+
}
213+
214+
static inline unsigned int
215+
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value)
216+
{
217+
_Py_atomic_ASSERT_ARG_TYPE(int32_t);
218+
return (unsigned int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
219+
}
220+
221+
static inline uint64_t
222+
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value)
223+
{
224+
return (uint64_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value);
225+
}
226+
227+
static inline intptr_t
228+
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value)
229+
{
230+
#if SIZEOF_VOID_P == 8
231+
_Py_atomic_ASSERT_ARG_TYPE(int64_t);
232+
return (intptr_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value);
233+
#else
234+
_Py_atomic_ASSERT_ARG_TYPE(int32_t);
235+
return (intptr_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value);
236+
#endif
237+
}
238+
239+
static inline uintptr_t
240+
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
241+
{
242+
_Py_atomic_ASSERT_ARG_TYPE(intptr_t);
243+
return (uintptr_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value);
244+
}
245+
246+
static inline Py_ssize_t
247+
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
248+
{
249+
_Py_atomic_ASSERT_ARG_TYPE(intptr_t);
250+
return (Py_ssize_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value);
251+
}
252+
253+
128254
// --- _Py_atomic_compare_exchange -------------------------------------------
129255

130256
static inline int

‎Include/cpython/pyatomic_std.h‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,113 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
112112
}
113113

114114

115+
// --- _Py_atomic_add_relaxed ------------------------------------------------
116+
117+
static inline int
118+
_Py_atomic_add_int_relaxed(int *obj, int value)
119+
{
120+
_Py_USING_STD;
121+
return atomic_fetch_add_explicit((_Atomic(int)*)obj, value,
122+
memory_order_relaxed);
123+
}
124+
125+
static inline int8_t
126+
_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value)
127+
{
128+
_Py_USING_STD;
129+
return atomic_fetch_add_explicit((_Atomic(int8_t)*)obj, value,
130+
memory_order_relaxed);
131+
}
132+
133+
static inline int16_t
134+
_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value)
135+
{
136+
_Py_USING_STD;
137+
return atomic_fetch_add_explicit((_Atomic(int16_t)*)obj, value,
138+
memory_order_relaxed);
139+
}
140+
141+
static inline int32_t
142+
_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value)
143+
{
144+
_Py_USING_STD;
145+
return atomic_fetch_add_explicit((_Atomic(int32_t)*)obj, value,
146+
memory_order_relaxed);
147+
}
148+
149+
static inline int64_t
150+
_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value)
151+
{
152+
_Py_USING_STD;
153+
return atomic_fetch_add_explicit((_Atomic(int64_t)*)obj, value,
154+
memory_order_relaxed);
155+
}
156+
157+
static inline intptr_t
158+
_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value)
159+
{
160+
_Py_USING_STD;
161+
return atomic_fetch_add_explicit((_Atomic(intptr_t)*)obj, value,
162+
memory_order_relaxed);
163+
}
164+
165+
static inline unsigned int
166+
_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value)
167+
{
168+
_Py_USING_STD;
169+
return atomic_fetch_add_explicit((_Atomic(unsigned int)*)obj, value,
170+
memory_order_relaxed);
171+
}
172+
173+
static inline uint8_t
174+
_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value)
175+
{
176+
_Py_USING_STD;
177+
return atomic_fetch_add_explicit((_Atomic(uint8_t)*)obj, value,
178+
memory_order_relaxed);
179+
}
180+
181+
static inline uint16_t
182+
_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value)
183+
{
184+
_Py_USING_STD;
185+
return atomic_fetch_add_explicit((_Atomic(uint16_t)*)obj, value,
186+
memory_order_relaxed);
187+
}
188+
189+
static inline uint32_t
190+
_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value)
191+
{
192+
_Py_USING_STD;
193+
return atomic_fetch_add_explicit((_Atomic(uint32_t)*)obj, value,
194+
memory_order_relaxed);
195+
}
196+
197+
static inline uint64_t
198+
_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value)
199+
{
200+
_Py_USING_STD;
201+
return atomic_fetch_add_explicit((_Atomic(uint64_t)*)obj, value,
202+
memory_order_relaxed);
203+
}
204+
205+
static inline uintptr_t
206+
_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
207+
{
208+
_Py_USING_STD;
209+
return atomic_fetch_add_explicit((_Atomic(uintptr_t)*)obj, value,
210+
memory_order_relaxed);
211+
}
212+
213+
static inline Py_ssize_t
214+
_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
215+
{
216+
_Py_USING_STD;
217+
return atomic_fetch_add_explicit((_Atomic(Py_ssize_t)*)obj, value,
218+
memory_order_relaxed);
219+
}
220+
221+
115222
// --- _Py_atomic_compare_exchange -------------------------------------------
116223

117224
static inline int

0 commit comments

Comments
 (0)