Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * atomics.h
4 : * Atomic operations.
5 : *
6 : * Hardware and compiler dependent functions for manipulating memory
7 : * atomically and dealing with cache coherency. Used to implement locking
8 : * facilities and lockless algorithms/data structures.
9 : *
10 : * To bring up postgres on a platform/compiler at the very least
11 : * implementations for the following operations should be provided:
12 : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
13 : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
14 : * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag()
15 : * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate.
16 : *
17 : * There exist generic, hardware independent, implementations for several
18 : * compilers which might be sufficient, although possibly not optimal, for a
19 : * new platform. If no such generic implementation is available spinlocks (or
20 : * even OS provided semaphores) will be used to implement the API.
21 : *
22 : * Implement _u64 atomics if and only if your platform can use them
23 : * efficiently (and obviously correctly).
24 : *
25 : * Use higher level functionality (lwlocks, spinlocks, heavyweight locks)
26 : * whenever possible. Writing correct code using these facilities is hard.
27 : *
28 : * For an introduction to using memory barriers within the PostgreSQL backend,
29 : * see src/backend/storage/lmgr/README.barrier
30 : *
31 : * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
32 : * Portions Copyright (c) 1994, Regents of the University of California
33 : *
34 : * src/include/port/atomics.h
35 : *
36 : *-------------------------------------------------------------------------
37 : */
38 : #ifndef ATOMICS_H
39 : #define ATOMICS_H
40 :
41 : #ifdef FRONTEND
42 : #error "atomics.h may not be included from frontend code"
43 : #endif
44 :
45 : #define INSIDE_ATOMICS_H
46 :
47 : #include <limits.h>
48 :
49 : /*
50 : * First a set of architecture specific files is included.
51 : *
52 : * These files can provide the full set of atomics or can do pretty much
53 : * nothing if all the compilers commonly used on these platforms provide
54 : * usable generics.
55 : *
56 : * Don't add an inline assembly of the actual atomic operations if all the
57 : * common implementations of your platform provide intrinsics. Intrinsics are
58 : * much easier to understand and potentially support more architectures.
59 : *
60 : * It will often make sense to define memory barrier semantics here, since
61 : * e.g. generic compiler intrinsics for x86 memory barriers can't know that
62 : * postgres doesn't need x86 read/write barriers do anything more than a
63 : * compiler barrier.
64 : *
65 : */
66 : #if defined(__arm__) || defined(__arm) || defined(__aarch64__)
67 : #include "port/atomics/arch-arm.h"
68 : #elif defined(__i386__) || defined(__i386) || defined(__x86_64__)
69 : #include "port/atomics/arch-x86.h"
70 : #elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__)
71 : #include "port/atomics/arch-ppc.h"
72 : #elif defined(__hppa) || defined(__hppa__)
73 : #include "port/atomics/arch-hppa.h"
74 : #endif
75 :
76 : /*
77 : * Compiler specific, but architecture independent implementations.
78 : *
79 : * Provide architecture independent implementations of the atomic
80 : * facilities. At the very least compiler barriers should be provided, but a
81 : * full implementation of
82 : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
83 : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
84 : * using compiler intrinsics are a good idea.
85 : */
86 : /*
87 : * gcc or compatible, including clang and icc. Exclude xlc. The ppc64le "IBM
88 : * XL C/C++ for Linux, V13.1.2" emulates gcc, but __sync_lock_test_and_set()
89 : * of one-byte types elicits SIGSEGV. That bug was gone by V13.1.5 (2016-12).
90 : */
91 : #if (defined(__GNUC__) || defined(__INTEL_COMPILER)) && !(defined(__IBMC__) || defined(__IBMCPP__))
92 : #include "port/atomics/generic-gcc.h"
93 : #elif defined(_MSC_VER)
94 : #include "port/atomics/generic-msvc.h"
95 : #elif defined(__SUNPRO_C) && !defined(__GNUC__)
96 : #include "port/atomics/generic-sunpro.h"
97 : #else
98 : /*
99 : * Unsupported compiler, we'll likely use slower fallbacks... At least
100 : * compiler barriers should really be provided.
101 : */
102 : #endif
103 :
104 : /*
105 : * Provide a full fallback of the pg_*_barrier(), pg_atomic**_flag and
106 : * pg_atomic_* APIs for platforms without sufficient spinlock and/or atomics
107 : * support. In the case of spinlock backed atomics the emulation is expected
108 : * to be efficient, although less so than native atomics support.
109 : */
110 : #include "port/atomics/fallback.h"
111 :
112 : /*
113 : * Provide additional operations using supported infrastructure. These are
114 : * expected to be efficient if the underlying atomic operations are efficient.
115 : */
116 : #include "port/atomics/generic.h"
117 :
118 :
119 : /*
120 : * pg_compiler_barrier - prevent the compiler from moving code across
121 : *
122 : * A compiler barrier need not (and preferably should not) emit any actual
123 : * machine code, but must act as an optimization fence: the compiler must not
124 : * reorder loads or stores to main memory around the barrier. However, the
125 : * CPU may still reorder loads or stores at runtime, if the architecture's
126 : * memory model permits this.
127 : */
128 : #define pg_compiler_barrier() pg_compiler_barrier_impl()
129 :
130 : /*
131 : * pg_memory_barrier - prevent the CPU from reordering memory access
132 : *
133 : * A memory barrier must act as a compiler barrier, and in addition must
134 : * guarantee that all loads and stores issued prior to the barrier are
135 : * completed before any loads or stores issued after the barrier. Unless
136 : * loads and stores are totally ordered (which is not the case on most
137 : * architectures) this requires issuing some sort of memory fencing
138 : * instruction.
139 : */
140 : #define pg_memory_barrier() pg_memory_barrier_impl()
141 :
142 : /*
143 : * pg_(read|write)_barrier - prevent the CPU from reordering memory access
144 : *
145 : * A read barrier must act as a compiler barrier, and in addition must
146 : * guarantee that any loads issued prior to the barrier are completed before
147 : * any loads issued after the barrier. Similarly, a write barrier acts
148 : * as a compiler barrier, and also orders stores. Read and write barriers
149 : * are thus weaker than a full memory barrier, but stronger than a compiler
150 : * barrier. In practice, on machines with strong memory ordering, read and
151 : * write barriers may require nothing more than a compiler barrier.
152 : */
153 : #define pg_read_barrier() pg_read_barrier_impl()
154 : #define pg_write_barrier() pg_write_barrier_impl()
155 :
156 : /*
157 : * Spinloop delay - Allow CPU to relax in busy loops
158 : */
159 : #define pg_spin_delay() pg_spin_delay_impl()
160 :
161 : /*
3118 andres 162 ECB : * pg_atomic_init_flag - initialize atomic flag.
163 : *
164 : * No barrier semantics.
165 : */
166 : static inline void
3118 andres 167 GIC 5481 : pg_atomic_init_flag(volatile pg_atomic_flag *ptr)
168 : {
169 5481 : pg_atomic_init_flag_impl(ptr);
170 5481 : }
171 :
172 : /*
173 : * pg_atomic_test_set_flag - TAS()
174 : *
3118 andres 175 ECB : * Returns true if the flag has successfully been set, false otherwise.
176 : *
177 : * Acquire (including read barrier) semantics.
178 : */
179 : static inline bool
3118 andres 180 GIC 325 : pg_atomic_test_set_flag(volatile pg_atomic_flag *ptr)
181 : {
182 325 : return pg_atomic_test_set_flag_impl(ptr);
183 : }
184 :
185 : /*
186 : * pg_atomic_unlocked_test_flag - Check if the lock is free
187 : *
3118 andres 188 ECB : * Returns true if the flag currently is not set, false otherwise.
189 : *
190 : * No barrier semantics.
191 : */
192 : static inline bool
3118 andres 193 GIC 742 : pg_atomic_unlocked_test_flag(volatile pg_atomic_flag *ptr)
194 : {
195 742 : return pg_atomic_unlocked_test_flag_impl(ptr);
196 : }
197 :
198 : /*
3118 andres 199 ECB : * pg_atomic_clear_flag - release lock set by TAS()
200 : *
201 : * Release (including write barrier) semantics.
202 : */
203 : static inline void
3118 andres 204 GIC 22 : pg_atomic_clear_flag(volatile pg_atomic_flag *ptr)
205 : {
206 22 : pg_atomic_clear_flag_impl(ptr);
207 22 : }
208 :
209 :
210 : /*
211 : * pg_atomic_init_u32 - initialize atomic variable
212 : *
3118 andres 213 ECB : * Has to be done before any concurrent usage..
214 : *
215 : * No barrier semantics.
216 : */
2804 217 : static inline void
3118 andres 218 CBC 44166133 : pg_atomic_init_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
219 : {
3118 andres 220 GIC 44166133 : AssertPointerAlignment(ptr, 4);
221 :
222 44166133 : pg_atomic_init_u32_impl(ptr, val);
223 44166133 : }
224 :
225 : /*
226 : * pg_atomic_read_u32 - unlocked read from atomic variable.
227 : *
228 : * The read is guaranteed to return a value as it has been written by this or
229 : * another process at some point in the past. There's however no cache
230 : * coherency interaction guaranteeing the value hasn't since been written to
2804 andres 231 ECB : * again.
232 : *
3118 233 : * No barrier semantics.
234 : */
235 : static inline uint32
3118 andres 236 GIC 512772941 : pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr)
237 : {
238 512772941 : AssertPointerAlignment(ptr, 4);
239 512772941 : return pg_atomic_read_u32_impl(ptr);
240 : }
241 :
242 : /*
243 : * pg_atomic_write_u32 - write to atomic variable.
244 : *
245 : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
246 : * observe a partial write for any reader. Note that this correctly interacts
247 : * with pg_atomic_compare_exchange_u32, in contrast to
2375 andres 248 ECB : * pg_atomic_unlocked_write_u32().
249 : *
3118 250 : * No barrier semantics.
251 : */
2804 252 : static inline void
3118 andres 253 CBC 51791743 : pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
254 : {
3118 andres 255 GIC 51791743 : AssertPointerAlignment(ptr, 4);
256 :
257 51791743 : pg_atomic_write_u32_impl(ptr, val);
258 51791743 : }
259 :
260 : /*
261 : * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable.
262 : *
263 : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
264 : * observe a partial write for any reader. But note that writing this way is
265 : * not guaranteed to correctly interact with read-modify-write operations like
266 : * pg_atomic_compare_exchange_u32. This should only be used in cases where
2375 andres 267 ECB : * minor performance regressions due to atomics emulation are unacceptable.
268 : *
269 : * No barrier semantics.
270 : */
271 : static inline void
2375 andres 272 CBC 1917968 : pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
273 : {
2375 andres 274 GIC 1917968 : AssertPointerAlignment(ptr, 4);
275 :
276 1917968 : pg_atomic_unlocked_write_u32_impl(ptr, val);
277 1917968 : }
278 :
279 : /*
280 : * pg_atomic_exchange_u32 - exchange newval with current value
281 : *
3118 andres 282 ECB : * Returns the old value of 'ptr' before the swap.
283 : *
284 : * Full barrier semantics.
285 : */
2804 286 : static inline uint32
3118 andres 287 GIC 10910 : pg_atomic_exchange_u32(volatile pg_atomic_uint32 *ptr, uint32 newval)
288 : {
289 10910 : AssertPointerAlignment(ptr, 4);
290 :
291 10910 : return pg_atomic_exchange_u32_impl(ptr, newval);
292 : }
293 :
294 : /*
295 : * pg_atomic_compare_exchange_u32 - CAS operation
296 : *
297 : * Atomically compare the current value of ptr with *expected and store newval
298 : * iff ptr and *expected have the same value. The current value of *ptr will
299 : * always be stored in *expected.
300 : *
3118 andres 301 ECB : * Return true if values have been exchanged, false otherwise.
302 : *
303 : * Full barrier semantics.
304 : */
2804 305 : static inline bool
3118 andres 306 GIC 457749013 : pg_atomic_compare_exchange_u32(volatile pg_atomic_uint32 *ptr,
3118 andres 307 ECB : uint32 *expected, uint32 newval)
308 : {
3118 andres 309 GIC 457749013 : AssertPointerAlignment(ptr, 4);
310 457749013 : AssertPointerAlignment(expected, 4);
311 :
312 457749013 : return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval);
313 : }
314 :
315 : /*
316 : * pg_atomic_fetch_add_u32 - atomically add to variable
317 : *
3117 andres 318 ECB : * Returns the value of ptr before the arithmetic operation.
319 : *
3118 320 : * Full barrier semantics.
321 : */
322 : static inline uint32
3118 andres 323 GIC 5315983 : pg_atomic_fetch_add_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
324 : {
325 5315983 : AssertPointerAlignment(ptr, 4);
326 5315983 : return pg_atomic_fetch_add_u32_impl(ptr, add_);
327 : }
328 :
329 : /*
330 : * pg_atomic_fetch_sub_u32 - atomically subtract from variable
331 : *
332 : * Returns the value of ptr before the arithmetic operation. Note that sub_
3117 andres 333 ECB : * may not be INT_MIN due to platform limitations.
334 : *
3118 335 : * Full barrier semantics.
336 : */
2804 337 : static inline uint32
3118 andres 338 GIC 585947 : pg_atomic_fetch_sub_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
339 : {
340 585947 : AssertPointerAlignment(ptr, 4);
341 585947 : Assert(sub_ != INT_MIN);
342 585947 : return pg_atomic_fetch_sub_u32_impl(ptr, sub_);
343 : }
344 :
345 : /*
346 : * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable
347 : *
3117 andres 348 ECB : * Returns the value of ptr before the arithmetic operation.
349 : *
3118 350 : * Full barrier semantics.
351 : */
352 : static inline uint32
3118 andres 353 GIC 23209419 : pg_atomic_fetch_and_u32(volatile pg_atomic_uint32 *ptr, uint32 and_)
354 : {
355 23209419 : AssertPointerAlignment(ptr, 4);
356 23209419 : return pg_atomic_fetch_and_u32_impl(ptr, and_);
357 : }
358 :
359 : /*
360 : * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable
361 : *
3117 andres 362 ECB : * Returns the value of ptr before the arithmetic operation.
363 : *
3118 364 : * Full barrier semantics.
365 : */
366 : static inline uint32
3118 andres 367 GIC 75071978 : pg_atomic_fetch_or_u32(volatile pg_atomic_uint32 *ptr, uint32 or_)
368 : {
369 75071978 : AssertPointerAlignment(ptr, 4);
370 75071978 : return pg_atomic_fetch_or_u32_impl(ptr, or_);
371 : }
372 :
373 : /*
374 : * pg_atomic_add_fetch_u32 - atomically add to variable
375 : *
3117 andres 376 ECB : * Returns the value of ptr after the arithmetic operation.
377 : *
3118 378 : * Full barrier semantics.
379 : */
380 : static inline uint32
3118 andres 381 GIC 438 : pg_atomic_add_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
382 : {
383 438 : AssertPointerAlignment(ptr, 4);
384 438 : return pg_atomic_add_fetch_u32_impl(ptr, add_);
385 : }
386 :
387 : /*
388 : * pg_atomic_sub_fetch_u32 - atomically subtract from variable
389 : *
390 : * Returns the value of ptr after the arithmetic operation. Note that sub_ may
3117 andres 391 ECB : * not be INT_MIN due to platform limitations.
392 : *
3118 393 : * Full barrier semantics.
394 : */
2804 395 : static inline uint32
3118 andres 396 GIC 295493445 : pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
397 : {
398 295493445 : AssertPointerAlignment(ptr, 4);
399 295493445 : Assert(sub_ != INT_MIN);
400 295493445 : return pg_atomic_sub_fetch_u32_impl(ptr, sub_);
401 : }
402 :
403 : /* ----
404 : * The 64 bit operations have the same semantics as their 32bit counterparts
3118 andres 405 ECB : * if they are available. Check the corresponding 32bit function for
406 : * documentation.
407 : * ----
408 : */
409 : static inline void
3118 andres 410 GIC 2561651 : pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
411 : {
412 : /*
2193 andres 413 ECB : * Can't necessarily enforce alignment - and don't need it - when using
414 : * the spinlock based fallback implementation. Therefore only assert when
415 : * not using it.
416 : */
417 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 418 GIC 2561651 : AssertPointerAlignment(ptr, 8);
2193 andres 419 ECB : #endif
3118 andres 420 GIC 2561651 : pg_atomic_init_u64_impl(ptr, val);
421 2561651 : }
3118 andres 422 ECB :
423 : static inline uint64
3118 andres 424 CBC 14079661 : pg_atomic_read_u64(volatile pg_atomic_uint64 *ptr)
425 : {
426 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 427 GIC 14079661 : AssertPointerAlignment(ptr, 8);
2193 andres 428 ECB : #endif
3118 andres 429 GIC 14079661 : return pg_atomic_read_u64_impl(ptr);
430 : }
3118 andres 431 ECB :
432 : static inline void
3118 andres 433 CBC 3135764 : pg_atomic_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
3118 andres 434 ECB : {
435 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 436 GIC 3135764 : AssertPointerAlignment(ptr, 8);
2193 andres 437 ECB : #endif
3118 andres 438 GIC 3135764 : pg_atomic_write_u64_impl(ptr, val);
439 3135764 : }
3118 andres 440 ECB :
441 : static inline uint64
3118 andres 442 CBC 6 : pg_atomic_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 newval)
443 : {
444 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 445 GIC 6 : AssertPointerAlignment(ptr, 8);
2193 andres 446 ECB : #endif
3118 andres 447 GIC 6 : return pg_atomic_exchange_u64_impl(ptr, newval);
448 : }
449 :
2804 andres 450 ECB : static inline bool
3118 andres 451 CBC 1481951 : pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr,
452 : uint64 *expected, uint64 newval)
3118 andres 453 ECB : {
454 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 455 GIC 1481951 : AssertPointerAlignment(ptr, 8);
456 1481951 : AssertPointerAlignment(expected, 8);
2193 andres 457 ECB : #endif
3118 andres 458 GIC 1481951 : return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval);
459 : }
3118 andres 460 ECB :
461 : static inline uint64
3118 andres 462 CBC 93056 : pg_atomic_fetch_add_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
463 : {
464 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 465 GIC 93056 : AssertPointerAlignment(ptr, 8);
2193 andres 466 ECB : #endif
3118 andres 467 GIC 93056 : return pg_atomic_fetch_add_u64_impl(ptr, add_);
468 : }
3118 andres 469 ECB :
470 : static inline uint64
3118 andres 471 CBC 660 : pg_atomic_fetch_sub_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
3118 andres 472 ECB : {
473 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 474 GIC 660 : AssertPointerAlignment(ptr, 8);
475 : #endif
2929 andres 476 CBC 660 : Assert(sub_ != PG_INT64_MIN);
3118 andres 477 GIC 660 : return pg_atomic_fetch_sub_u64_impl(ptr, sub_);
478 : }
3118 andres 479 ECB :
480 : static inline uint64
3118 andres 481 CBC 9 : pg_atomic_fetch_and_u64(volatile pg_atomic_uint64 *ptr, uint64 and_)
482 : {
483 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 484 GIC 9 : AssertPointerAlignment(ptr, 8);
2193 andres 485 ECB : #endif
3118 andres 486 GIC 9 : return pg_atomic_fetch_and_u64_impl(ptr, and_);
487 : }
3118 andres 488 ECB :
489 : static inline uint64
3118 andres 490 CBC 6 : pg_atomic_fetch_or_u64(volatile pg_atomic_uint64 *ptr, uint64 or_)
491 : {
492 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 493 GIC 6 : AssertPointerAlignment(ptr, 8);
2193 andres 494 ECB : #endif
3118 andres 495 GIC 6 : return pg_atomic_fetch_or_u64_impl(ptr, or_);
496 : }
3118 andres 497 ECB :
498 : static inline uint64
3118 andres 499 CBC 59 : pg_atomic_add_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
500 : {
501 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 502 GIC 59 : AssertPointerAlignment(ptr, 8);
2193 andres 503 ECB : #endif
3118 andres 504 GIC 59 : return pg_atomic_add_fetch_u64_impl(ptr, add_);
505 : }
3118 andres 506 ECB :
507 : static inline uint64
3118 andres 508 CBC 3 : pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
3118 andres 509 ECB : {
510 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
3118 andres 511 GIC 3 : AssertPointerAlignment(ptr, 8);
512 : #endif
2929 513 3 : Assert(sub_ != PG_INT64_MIN);
3118 514 3 : return pg_atomic_sub_fetch_u64_impl(ptr, sub_);
515 : }
516 :
517 : #undef INSIDE_ATOMICS_H
518 :
519 : #endif /* ATOMICS_H */
|
