From 376fa7c58c0292c785b3bfc5679063185e796a34 Mon Sep 17 00:00:00 2001 From: Kamil Rytarowski Date: Thu, 7 May 2020 04:46:17 +0200 Subject: [PATCH] Add libatomic Based on: https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/atomic.c --- distrib/sets/lists/base/shl.mi | 3 + distrib/sets/lists/comp/mi | 2 + distrib/sets/lists/comp/shl.mi | 1 + distrib/sets/lists/debug/mi | 1 + distrib/sets/lists/debug/shl.mi | 1 + lib/Makefile | 2 +- lib/libatomic/Makefile | 7 + lib/libatomic/assembly.h | 202 +++++++++++++++ lib/libatomic/atomic.c | 426 ++++++++++++++++++++++++++++++++ lib/libatomic/shlib_version | 5 + 10 files changed, 649 insertions(+), 1 deletion(-) create mode 100644 lib/libatomic/Makefile create mode 100644 lib/libatomic/assembly.h create mode 100644 lib/libatomic/atomic.c create mode 100644 lib/libatomic/shlib_version diff --git a/distrib/sets/lists/base/shl.mi b/distrib/sets/lists/base/shl.mi index 3205ee031f42..958bd604a158 100644 --- a/distrib/sets/lists/base/shl.mi +++ b/distrib/sets/lists/base/shl.mi @@ -219,6 +219,9 @@ ./usr/lib/libatf-c.so base-atf-shlib compatfile,atf ./usr/lib/libatf-c.so.0 base-atf-shlib compatfile,atf ./usr/lib/libatf-c.so.0.0 base-atf-shlib compatfile,atf +./usr/lib/libatomic.so base-sys-shlib compatfile +./usr/lib/libatomic.so.0 base-sys-shlib compatfile +./usr/lib/libatomic.so.0.0 base-sys-shlib compatfile ./usr/lib/libavl.so base-zfs-shlib compatfile,zfs ./usr/lib/libavl.so.0 base-zfs-shlib compatfile,zfs ./usr/lib/libavl.so.0.0 base-zfs-shlib compatfile,zfs diff --git a/distrib/sets/lists/comp/mi b/distrib/sets/lists/comp/mi index 452da7fe4870..8f79da498025 100644 --- a/distrib/sets/lists/comp/mi +++ b/distrib/sets/lists/comp/mi @@ -3424,6 +3424,8 @@ ./usr/lib/libatf-c_p.a comp-atf-lib compatfile,atf,profile ./usr/lib/libatf.a comp-obsolete obsolete ./usr/lib/libatf_p.a comp-obsolete obsolete +./usr/lib/libatomic.a comp-c-lib compatfile +./usr/lib/libatomic_p.a comp-c-proflib compatfile,profile ./usr/lib/libavl.a comp-zfs-lib compatfile,zfs ./usr/lib/libavl_p.a comp-zfs-proflib compatfile,profile,zfs ./usr/lib/libbfd.a comp-obsolete obsolete diff --git a/distrib/sets/lists/comp/shl.mi b/distrib/sets/lists/comp/shl.mi index 0b8a973e5215..b9951f0e3def 100644 --- a/distrib/sets/lists/comp/shl.mi +++ b/distrib/sets/lists/comp/shl.mi @@ -12,6 +12,7 @@ ./usr/lib/libatf-c++_pic.a comp-atf-piclib compatfile,picinstall,atf ./usr/lib/libatf-c_pic.a comp-atf-piclib compatfile,picinstall,atf ./usr/lib/libatf_pic.a comp-obsolete obsolete +./usr/lib/libatomic_pic.a comp-c-lib compatfile,picinstall ./usr/lib/libavl_pic.a comp-zfs-piclib compatfile,picinstall,zfs ./usr/lib/libbind9_pic.a comp-bind-piclib compatfile,picinstall ./usr/lib/libblacklist_pic.a comp-c-piclib compatfile,picinstall diff --git a/distrib/sets/lists/debug/mi b/distrib/sets/lists/debug/mi index b1bf4785e7b9..d13ba69869c1 100644 --- a/distrib/sets/lists/debug/mi +++ b/distrib/sets/lists/debug/mi @@ -30,6 +30,7 @@ ./usr/lib/libasn1_g.a comp-c-debuglib debuglib,compatfile,kerberos ./usr/lib/libatf-c++_g.a comp-c-debuglib debuglib,compatfile,atf ./usr/lib/libatf-c_g.a comp-c-debuglib debuglib,compatfile,atf +./usr/lib/libatomic_g.a comp-c-debuglib debuglib,compatfile ./usr/lib/libavl_g.a comp-c-debuglib debuglib,compatfile,zfs ./usr/lib/libbind9_g.a comp-c-debuglib debuglib,compatfile ./usr/lib/libblacklist_g.a comp-c-debuglib debuglib,compatfile diff --git a/distrib/sets/lists/debug/shl.mi b/distrib/sets/lists/debug/shl.mi index c8bc3e6ecb90..34bd7987179e 100644 --- a/distrib/sets/lists/debug/shl.mi +++ b/distrib/sets/lists/debug/shl.mi @@ -72,6 +72,7 @@ ./usr/libdata/debug/usr/lib/libasn1.so.10.0.debug comp-krb5-debug debug,compatfile,kerberos ./usr/libdata/debug/usr/lib/libatf-c++.so.2.0.debug comp-atf-debug debug,compatfile,atf ./usr/libdata/debug/usr/lib/libatf-c.so.0.0.debug comp-atf-debug debug,compatfile,atf +./usr/libdata/debug/usr/lib/libatomic.so.0.0.debug comp-sys-debug debug,compatfile ./usr/libdata/debug/usr/lib/libavl.so.0.0.debug comp-zfs-debug debug,compatfile,zfs ./usr/libdata/debug/usr/lib/libbfd.so.16.0.debug comp-sys-debug debug,compatfile,binutils=231 ./usr/libdata/debug/usr/lib/libbfd.so.17.0.debug comp-sys-debug debug,compatfile,binutils=234 diff --git a/lib/Makefile b/lib/Makefile index e231b1fa605c..c6ee13077153 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -20,7 +20,7 @@ SUBDIR+= .WAIT SUBDIR+= i18n_module -SUBDIR+= libarch \ +SUBDIR+= libarch libatomic \ libbluetooth libbsdmalloc libbz2 \ libcompat libcrypt \ libintl libipsec libkvm libm \ diff --git a/lib/libatomic/Makefile b/lib/libatomic/Makefile new file mode 100644 index 000000000000..3cacd7543de6 --- /dev/null +++ b/lib/libatomic/Makefile @@ -0,0 +1,7 @@ +# $NetBSD$ + +LIB= atomic + +SRCS= atomic.c + +.include diff --git a/lib/libatomic/assembly.h b/lib/libatomic/assembly.h new file mode 100644 index 000000000000..4a35946a3dcc --- /dev/null +++ b/lib/libatomic/assembly.h @@ -0,0 +1,202 @@ +/* $NetBSD$ */ + +//===-- assembly.h - compiler-rt assembler support macros -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines macros for use in compiler-rt assembler source. +// This file is not part of the interface of this library. +// +//===----------------------------------------------------------------------===// + +#ifndef COMPILERRT_ASSEMBLY_H +#define COMPILERRT_ASSEMBLY_H + +#if defined(__POWERPC__) || defined(__powerpc__) || defined(__ppc__) +#define SEPARATOR @ +#else +#define SEPARATOR ; +#endif + +#if defined(__APPLE__) +#define HIDDEN(name) .private_extern name +#define LOCAL_LABEL(name) L_##name +// tell linker it can break up file at label boundaries +#define FILE_LEVEL_DIRECTIVE .subsections_via_symbols +#define SYMBOL_IS_FUNC(name) +#define CONST_SECTION .const + +#define NO_EXEC_STACK_DIRECTIVE + +#elif defined(__ELF__) + +#define HIDDEN(name) .hidden name +#define LOCAL_LABEL(name) .L_##name +#define FILE_LEVEL_DIRECTIVE +#if defined(__arm__) +#define SYMBOL_IS_FUNC(name) .type name,%function +#else +#define SYMBOL_IS_FUNC(name) .type name,@function +#endif +#define CONST_SECTION .section .rodata + +#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ + defined(__linux__) +#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits +#else +#define NO_EXEC_STACK_DIRECTIVE +#endif + +#else // !__APPLE__ && !__ELF__ + +#define HIDDEN(name) +#define LOCAL_LABEL(name) .L ## name +#define FILE_LEVEL_DIRECTIVE +#define SYMBOL_IS_FUNC(name) \ + .def name SEPARATOR \ + .scl 2 SEPARATOR \ + .type 32 SEPARATOR \ + .endef +#define CONST_SECTION .section .rdata,"rd" + +#define NO_EXEC_STACK_DIRECTIVE + +#endif + +#if defined(__arm__) + +// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros: +// - for '-mthumb -march=armv6' compiler defines '__thumb__' +// - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__' +#if defined(__thumb2__) || defined(__thumb__) +#define DEFINE_CODE_STATE .thumb SEPARATOR +#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR +#if defined(__thumb2__) +#define USE_THUMB_2 +#define IT(cond) it cond +#define ITT(cond) itt cond +#define ITE(cond) ite cond +#else +#define USE_THUMB_1 +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif // defined(__thumb__2) +#else // !defined(__thumb2__) && !defined(__thumb__) +#define DEFINE_CODE_STATE .arm SEPARATOR +#define DECLARE_FUNC_ENCODING +#define IT(cond) +#define ITT(cond) +#define ITE(cond) +#endif + +#if defined(USE_THUMB_1) && defined(USE_THUMB_2) +#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together." +#endif + +#if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5 +#define ARM_HAS_BX +#endif +#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \ + (__ARM_ARCH >= 6 || (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__))) +#define __ARM_FEATURE_CLZ +#endif + +#ifdef ARM_HAS_BX +#define JMP(r) bx r +#define JMPc(r, c) bx##c r +#else +#define JMP(r) mov pc, r +#define JMPc(r, c) mov##c pc, r +#endif + +// pop {pc} can't switch Thumb mode on ARMv4T +#if __ARM_ARCH >= 5 +#define POP_PC() pop {pc} +#else +#define POP_PC() \ + pop {ip}; \ + JMP(ip) +#endif + +#if defined(USE_THUMB_2) +#define WIDE(op) op.w +#else +#define WIDE(op) op +#endif +#else // !defined(__arm) +#define DECLARE_FUNC_ENCODING +#define DEFINE_CODE_STATE +#endif + +#define GLUE2(a, b) a##b +#define GLUE(a, b) GLUE2(a, b) +#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name) + +#ifdef VISIBILITY_HIDDEN +#define DECLARE_SYMBOL_VISIBILITY(name) \ + HIDDEN(SYMBOL_NAME(name)) SEPARATOR +#else +#define DECLARE_SYMBOL_VISIBILITY(name) +#endif + +#define DEFINE_COMPILERRT_FUNCTION(name) \ + DEFINE_CODE_STATE \ + FILE_LEVEL_DIRECTIVE SEPARATOR \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(name) \ + DECLARE_FUNC_ENCODING \ + SYMBOL_NAME(name): + +#define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \ + DEFINE_CODE_STATE \ + FILE_LEVEL_DIRECTIVE SEPARATOR \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \ + .thumb_func SEPARATOR \ + SYMBOL_NAME(name): + +#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \ + DEFINE_CODE_STATE \ + FILE_LEVEL_DIRECTIVE SEPARATOR \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + HIDDEN(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_FUNC_ENCODING \ + SYMBOL_NAME(name): + +#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \ + DEFINE_CODE_STATE \ + .globl name SEPARATOR \ + SYMBOL_IS_FUNC(name) SEPARATOR \ + HIDDEN(name) SEPARATOR \ + DECLARE_FUNC_ENCODING \ + name: + +#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \ + .globl SYMBOL_NAME(name) SEPARATOR \ + SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(SYMBOL_NAME(name)) SEPARATOR \ + .set SYMBOL_NAME(name), SYMBOL_NAME(target) SEPARATOR + +#if defined(__ARM_EABI__) +#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) \ + DEFINE_COMPILERRT_FUNCTION_ALIAS(aeabi_name, name) +#else +#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) +#endif + +#ifdef __ELF__ +#define END_COMPILERRT_FUNCTION(name) \ + .size SYMBOL_NAME(name), . - SYMBOL_NAME(name) +#else +#define END_COMPILERRT_FUNCTION(name) +#endif + +#endif // COMPILERRT_ASSEMBLY_H diff --git a/lib/libatomic/atomic.c b/lib/libatomic/atomic.c new file mode 100644 index 000000000000..0edc4c12cb17 --- /dev/null +++ b/lib/libatomic/atomic.c @@ -0,0 +1,426 @@ +/* $NetBSD$ */ + +// Local changes by Kamil Rytarowski: +// - ported to NetBSD +// - switched to C11 stdatomic.h and fixed compatibility with GCC +// - added __atomic_fetch_nand_##n functions +// - fixed __atomic_compare_exchange_##n + +//===-- atomic.c - Implement support functions for atomic operations.------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// atomic.c defines a set of functions for performing atomic accesses on +// arbitrary-sized memory locations. This design uses locks that should +// be fast in the uncontended case, for two reasons: +// +// 1) This code must work with C programs that do not link to anything +// (including pthreads) and so it should not depend on any pthread +// functions. +// 2) Atomic operations, rather than explicit mutexes, are most commonly used +// on code where contended operations are rate. +// +// To avoid needing a per-object lock, this code allocates an array of +// locks and hashes the object pointers to find the one that it should use. +// For operations that must be atomic on two locations, the lower lock is +// always acquired first, to avoid deadlock. +// +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include + +#include "assembly.h" + +// Clang objects if you redefine a builtin. This little hack allows us to +// define a function with the same name as an intrinsic. +#pragma redefine_extname __atomic_load_c SYMBOL_NAME(__atomic_load) +#pragma redefine_extname __atomic_store_c SYMBOL_NAME(__atomic_store) +#pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange) +#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME( \ + __atomic_compare_exchange) + +/// Number of locks. This allocates one page on 32-bit platforms, two on +/// 64-bit. This can be specified externally if a different trade between +/// memory usage and contention probability is required for a given platform. +#ifndef SPINLOCK_COUNT +#define SPINLOCK_COUNT (1 << 10) +#endif +static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; + +//////////////////////////////////////////////////////////////////////////////// +// Platform-specific lock implementation. Falls back to spinlocks if none is +// defined. Each platform should define the Lock type, and corresponding +// lock() and unlock() functions. +//////////////////////////////////////////////////////////////////////////////// +#ifdef __FreeBSD__ +#include +// clang-format off +#include +#include +#include +// clang-format on +typedef struct _usem Lock; +__inline static void unlock(Lock *l) { + __c11_atomic_store((_Atomic(uint32_t) *)&l->_count, 1, __ATOMIC_RELEASE); + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); + if (l->_has_waiters) + _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); +} +__inline static void lock(Lock *l) { + uint32_t old = 1; + while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t) *)&l->_count, + &old, 0, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) { + _umtx_op(l, UMTX_OP_SEM_WAIT, 0, 0, 0); + old = 1; + } +} +/// locks for atomic operations +static Lock locks[SPINLOCK_COUNT] = {[0 ... SPINLOCK_COUNT - 1] = {0, 1, 0}}; + +#elif defined(__NetBSD__) +// clang-format off +#include +#include +#include +#include +#include +// clang-format on + +__inline static int +__futex(volatile uint32_t *uaddr, int op, int val, const struct timespec *timeout, + volatile uint32_t *uaddr2, int val2, int val3) +{ + return syscall(SYS___futex, uaddr, op, val, timeout, uaddr2, + val2, val3); +} + +typedef volatile uint32_t Lock; + +__inline static void unlock(Lock *l) { + if (atomic_dec_32_nv(l) != 0) { + *l = 0; + __futex(l, FUTEX_WAKE, 1, NULL, NULL, 0, 0); + } +} + +__inline static void lock(Lock *l) { + if (atomic_cas_32(l, 0, 1) != 0) { + while (atomic_swap_32(l, 2) != 1) + __futex(l, FUTEX_WAIT, 2, NULL, NULL, 0, 0); + } +} +/// locks for atomic operations +static Lock locks[SPINLOCK_COUNT]; + +#elif defined(__APPLE__) +#include +typedef OSSpinLock Lock; +__inline static void unlock(Lock *l) { OSSpinLockUnlock(l); } +/// Locks a lock. In the current implementation, this is potentially +/// unbounded in the contended case. +__inline static void lock(Lock *l) { OSSpinLockLock(l); } +static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 + +#else +typedef _Atomic(uintptr_t) Lock; +/// Unlock a lock. This is a release operation. +__inline static void unlock(Lock *l) { + atomic_store_explicit(l, 0, memory_order_release); +} +/// Locks a lock. In the current implementation, this is potentially +/// unbounded in the contended case. +__inline static void lock(Lock *l) { + uintptr_t old = 0; + while (!atomic_compare_exchange_weak_explicit(l, &old, 1, memory_order_acquire, + memory_order_relaxed)) + old = 0; +} +/// locks for atomic operations +static Lock locks[SPINLOCK_COUNT]; +#endif + +/// Returns a lock to use for a given pointer. +static __inline Lock *lock_for_pointer(void *ptr) { + intptr_t hash = (intptr_t)ptr; + // Disregard the lowest 4 bits. We want all values that may be part of the + // same memory operation to hash to the same value and therefore use the same + // lock. + hash >>= 4; + // Use the next bits as the basis for the hash + intptr_t low = hash & SPINLOCK_MASK; + // Now use the high(er) set of bits to perturb the hash, so that we don't + // get collisions from atomic fields in a single object + hash >>= 16; + hash ^= low; + // Return a pointer to the word to use + return locks + (hash & SPINLOCK_MASK); +} + +/// Macros for determining whether a size is lock free. Clang can not yet +/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are +/// not lock free. +#define IS_LOCK_FREE_1 atomic_is_lock_free((uint8_t *)NULL) +#define IS_LOCK_FREE_2 atomic_is_lock_free((uint16_t *)NULL) +#define IS_LOCK_FREE_4 atomic_is_lock_free((uint32_t *)NULL) +#define IS_LOCK_FREE_8 atomic_is_lock_free((uint64_t *)NULL) +#define IS_LOCK_FREE_16 0 + +/// Macro that calls the compiler-generated lock-free versions of functions +/// when they exist. +#define LOCK_FREE_CASES() \ + do { \ + switch (size) { \ + case 1: \ + if (IS_LOCK_FREE_1) { \ + LOCK_FREE_ACTION(uint8_t); \ + } \ + break; \ + case 2: \ + if (IS_LOCK_FREE_2) { \ + LOCK_FREE_ACTION(uint16_t); \ + } \ + break; \ + case 4: \ + if (IS_LOCK_FREE_4) { \ + LOCK_FREE_ACTION(uint32_t); \ + } \ + break; \ + case 8: \ + if (IS_LOCK_FREE_8) { \ + LOCK_FREE_ACTION(uint64_t); \ + } \ + break; \ + case 16: \ + if (IS_LOCK_FREE_16) { \ + /* FIXME: __uint128_t isn't available on 32 bit platforms. \ + LOCK_FREE_ACTION(__uint128_t);*/ \ + } \ + break; \ + } \ + } while (0) + +/// An atomic load operation. This is atomic with respect to the source +/// pointer only. +void __atomic_load_c(int size, void *src, void *dest, memory_order model); +void __atomic_load_c(int size, void *src, void *dest, memory_order model) { +#define LOCK_FREE_ACTION(type) \ + *((type *)dest) = atomic_load_explicit((_Atomic(type) *)src, model); \ + return; + LOCK_FREE_CASES(); +#undef LOCK_FREE_ACTION + Lock *l = lock_for_pointer(src); + lock(l); + memcpy(dest, src, size); + unlock(l); +} + +/// An atomic store operation. This is atomic with respect to the destination +/// pointer only. +void __atomic_store_c(int size, void *dest, void *src, memory_order model); +void __atomic_store_c(int size, void *dest, void *src, memory_order model) { +#define LOCK_FREE_ACTION(type) \ + atomic_store_explicit((_Atomic(type) *)dest, *(type *)src, model); \ + return; + LOCK_FREE_CASES(); +#undef LOCK_FREE_ACTION + Lock *l = lock_for_pointer(dest); + lock(l); + memcpy(dest, src, size); + unlock(l); +} + +/// Atomic compare and exchange operation. If the value at *ptr is identical +/// to the value at *expected, then this copies value at *desired to *ptr. If +/// they are not, then this stores the current value from *ptr in *expected. +/// +/// This function returns 1 if the exchange takes place or 0 if it fails. +int __atomic_compare_exchange_c(int size, void *ptr, void *expected, + void *desired, int success, int failure); +int __atomic_compare_exchange_c(int size, void *ptr, void *expected, + void *desired, int success, int failure) { +#define LOCK_FREE_ACTION(type) \ + return atomic_compare_exchange_strong_explicit( \ + (_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \ + failure) + LOCK_FREE_CASES(); +#undef LOCK_FREE_ACTION + Lock *l = lock_for_pointer(ptr); + lock(l); + if (memcmp(ptr, expected, size) == 0) { + memcpy(ptr, desired, size); + unlock(l); + return 1; + } + memcpy(expected, ptr, size); + unlock(l); + return 0; +} + +/// Performs an atomic exchange operation between two pointers. This is atomic +/// with respect to the target address. +void __atomic_exchange_c(int size, void *ptr, void *val, void *old, memory_order model); +void __atomic_exchange_c(int size, void *ptr, void *val, void *old, memory_order model) { +#define LOCK_FREE_ACTION(type) \ + *(type *)old = \ + atomic_exchange_explicit((_Atomic(type) *)ptr, *(type *)val, model); \ + return; + LOCK_FREE_CASES(); +#undef LOCK_FREE_ACTION + Lock *l = lock_for_pointer(ptr); + lock(l); + memcpy(old, ptr, size); + memcpy(ptr, val, size); + unlock(l); +} + +//////////////////////////////////////////////////////////////////////////////// +// Where the size is known at compile time, the compiler may emit calls to +// specialised versions of the above functions. +//////////////////////////////////////////////////////////////////////////////// +#ifdef __SIZEOF_INT128__ +#define OPTIMISED_CASES \ + OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ + OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ + OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ + OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) \ + OPTIMISED_CASE(16, IS_LOCK_FREE_16, __uint128_t) +#else +#define OPTIMISED_CASES \ + OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ + OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ + OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ + OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) +#endif + +#define OPTIMISED_CASE(n, lockfree, type) \ + type __atomic_load_##n(type *src, memory_order model); \ + type __atomic_load_##n(type *src, memory_order model) { \ + if (lockfree) \ + return atomic_load_explicit((_Atomic(type) *)src, model); \ + Lock *l = lock_for_pointer(src); \ + lock(l); \ + type val = *src; \ + unlock(l); \ + return val; \ + } +OPTIMISED_CASES +#undef OPTIMISED_CASE + +#define OPTIMISED_CASE(n, lockfree, type) \ + void __atomic_store_##n(type *dest, type val, memory_order model); \ + void __atomic_store_##n(type *dest, type val, memory_order model) { \ + if (lockfree) { \ + atomic_store_explicit((_Atomic(type) *)dest, val, model); \ + return; \ + } \ + Lock *l = lock_for_pointer(dest); \ + lock(l); \ + *dest = val; \ + unlock(l); \ + return; \ + } +OPTIMISED_CASES +#undef OPTIMISED_CASE + +#define OPTIMISED_CASE(n, lockfree, type) \ + type __atomic_exchange_##n(type *dest, type val, memory_order model); \ + type __atomic_exchange_##n(type *dest, type val, memory_order model) { \ + if (lockfree) \ + return atomic_exchange_explicit((_Atomic(type) *)dest, val, model); \ + Lock *l = lock_for_pointer(dest); \ + lock(l); \ + type tmp = *dest; \ + *dest = val; \ + unlock(l); \ + return tmp; \ + } +OPTIMISED_CASES +#undef OPTIMISED_CASE + +#define OPTIMISED_CASE(n, lockfree, type) \ + bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ + bool weak, memory_order success, \ + memory_order failure); \ + bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ + bool weak, memory_order success, \ + memory_order failure) { \ + if (lockfree) { \ + if (weak) \ + return atomic_compare_exchange_weak_explicit( \ + (_Atomic(type) *)ptr, expected, desired, success, failure); \ + else \ + return atomic_compare_exchange_strong_explicit( \ + (_Atomic(type) *)ptr, expected, desired, success, failure); \ + } \ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + if (*ptr == *expected) { \ + *ptr = desired; \ + unlock(l); \ + return 1; \ + } \ + *expected = *ptr; \ + unlock(l); \ + return 0; \ + } +OPTIMISED_CASES +#undef OPTIMISED_CASE + +//////////////////////////////////////////////////////////////////////////////// +// Atomic read-modify-write operations for integers of various sizes. +//////////////////////////////////////////////////////////////////////////////// +#define ATOMIC_RMW(n, lockfree, type, opname, op) \ + type __atomic_fetch_##opname##_##n(_Atomic(type) *ptr, type val, \ + memory_order model); \ + type __atomic_fetch_##opname##_##n(_Atomic(type) *ptr, type val, \ + memory_order model) { \ + if (lockfree) \ + return atomic_fetch_##opname##_explicit((_Atomic(type) *)ptr, val, model);\ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + type tmp = *ptr; \ + *ptr = tmp op val; \ + unlock(l); \ + return tmp; \ + } + +// No lockfree atomic_fetch_*_explicit variation. +#define ATOMIC_RMW_NEG(n, lockfree, type, opname, op) \ + type __atomic_fetch_##opname##_##n(_Atomic(type) *ptr, type val, \ + memory_order model); \ + type __atomic_fetch_##opname##_##n(_Atomic(type) *ptr, type val, \ + memory_order model) { \ + Lock *l = lock_for_pointer(ptr); \ + lock(l); \ + type tmp = *ptr; \ + *ptr = ~ (tmp op val); \ + unlock(l); \ + return tmp; \ + } + +#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +) +OPTIMISED_CASES +#undef OPTIMISED_CASE +#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, sub, -) +OPTIMISED_CASES +#undef OPTIMISED_CASE +#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, and, &) +OPTIMISED_CASES +#undef OPTIMISED_CASE +#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, or, |) +OPTIMISED_CASES +#undef OPTIMISED_CASE +#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, xor, ^) +OPTIMISED_CASES +#undef OPTIMISED_CASE +#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW_NEG(n, lockfree, type, nand, &) +OPTIMISED_CASES +#undef OPTIMISED_CASE diff --git a/lib/libatomic/shlib_version b/lib/libatomic/shlib_version new file mode 100644 index 000000000000..41273f317c1c --- /dev/null +++ b/lib/libatomic/shlib_version @@ -0,0 +1,5 @@ +# $NetBSD$ +# Remember to update distrib/sets/lists/base/shl.* when changing +# +major=0 +minor=0 -- 2.25.0