From d310c0d833b098732dc50aed035fc4672fe5ab57 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Tue, 19 Jul 2022 22:07:02 +0000
Subject: [PATCH 1/2] sys/atomic.h: Fix atomic_store_* on sparcv7, sparcv8.

These did not cooperate with the hash-locked scheme of the other
atomic operations, with the effect that, for instance, a typical
naive spin lock based on atomic_*,

volatile unsigned locked = 0;
lock()
{
	while (atomic_swap_uint(&locked, 1))
		continue;
	membar_acquire();
}
unlock()
{
	membar_release();
	atomic_store_relaxed(&locked, 0);
}

would fail to achieve mutual exclusion.

For this case, we need to use atomic_swap_* (or, for 8- or 16-bit
objects, atomic_cas_32 loops, since there is no atomic_swap_8 or
atomic_swap_16).

The new machine/types.h macro __HAVE_HASHLOCKED_ATOMICS says whether
these contortions are necessary.

Note that this _requires_ the use of atomic_store_*(p, v), not
regular stores *p = v, to work with the r/m/w atomic operations.
---
 sys/arch/sparc/include/types.h |  4 +++
 sys/kern/subr_csan.c           |  4 +++
 sys/sys/atomic.h               | 50 ++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+)

diff --git a/sys/arch/sparc/include/types.h b/sys/arch/sparc/include/types.h
index 87852daa2b9c..c2a6294b8709 100644
--- a/sys/arch/sparc/include/types.h
+++ b/sys/arch/sparc/include/types.h
@@ -48,6 +48,7 @@
 #endif
 
 #if defined(_KERNEL_OPT)
+#include "opt_multiprocessor.h"
 #include "opt_sparc_arch.h"
 #endif
 
@@ -135,6 +136,9 @@ typedef unsigned long int	__register_t;
 #define	__HAVE_FAST_SOFTINTS
 #else
 #define	__HAVE_MM_MD_READWRITE
+#ifdef MULTIPROCESSOR
+#define	__HAVE_HASHLOCKED_ATOMICS
+#endif
 #endif
 
 #define	__HAVE_CPU_LWP_SETPRIVATE
diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c
index 28b71fce1038..b834bb3b5ea8 100644
--- a/sys/kern/subr_csan.c
+++ b/sys/kern/subr_csan.c
@@ -615,12 +615,16 @@ void
 kcsan_atomic_store(volatile void *p, const void *v, int size)
 {
 	kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR);
+#ifdef __HAVE_HASHLOCKED_ATOMICS
+	__do_atomic_store(p, v, size);
+#else
 	switch (size) {
 	case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break;
 	case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break;
 	case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break;
 	case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break;
 	}
+#endif
 }
 
 /* -------------------------------------------------------------------------- */
diff --git a/sys/sys/atomic.h b/sys/sys/atomic.h
index c1f50286aa85..e22ba46aead4 100644
--- a/sys/sys/atomic.h
+++ b/sys/sys/atomic.h
@@ -433,9 +433,14 @@ void kcsan_atomic_store(volatile void *, const void *, int);
 	__typeof__(*(p)) v = *(p)
 #define __END_ATOMIC_LOAD(v) \
 	v
+#ifdef __HAVE_HASHLOCKED_ATOMICS
+#define __DO_ATOMIC_STORE(p, v)						      \
+	__do_atomic_store(p, __UNVOLATILE(&v), sizeof(v))
+#else  /* !__HAVE_HASHLOCKED_ATOMICS */
 #define __DO_ATOMIC_STORE(p, v) \
 	*p = v
 #endif
+#endif
 
 #define	atomic_load_relaxed(p)						      \
 ({									      \
@@ -480,6 +485,51 @@ void kcsan_atomic_store(volatile void *, const void *, int);
 	__DO_ATOMIC_STORE(__as_ptr, __as_val);				      \
 })
 
+#ifdef __HAVE_HASHLOCKED_ATOMICS
+static void __inline __always_inline
+__do_atomic_store(volatile void *p, const void *q, size_t size)
+{
+	switch (size) {
+	case 1: {
+		uint8_t v;
+		unsigned s = 8 * ((uintptr_t)p & 3);
+		uint32_t o, n, m = ~(0xffU << s);
+		memcpy(&v, q, 1);
+		do {
+			o = atomic_load_relaxed((const volatile uint32_t *)p);
+			n = (o & m) | ((uint32_t)v << s);
+		} while (atomic_cas_32((volatile uint32_t *)p, o, n) != o);
+		break;
+	}
+	case 2: {
+		uint16_t v;
+		unsigned s = 8 * (((uintptr_t)p & 2) >> 1);
+		uint32_t o, n, m = ~(0xffffU << s);
+		memcpy(&v, q, 2);
+		do {
+			o = atomic_load_relaxed((const volatile uint32_t *)p);
+			n = (o & m) | ((uint32_t)v << s);
+		} while (atomic_cas_32((volatile uint32_t *)p, o, n) != o);
+		break;
+	}
+	case 4: {
+		uint32_t v;
+		memcpy(&v, q, 4);
+		(void)atomic_swap_32(p, v);
+		break;
+	}
+#ifdef __HAVE_ATOMIC64_LOADSTORE
+	case 8: {
+		uint64_t v;
+		memcpy(&v, q, 8);
+		(void)atomic_swap_64(p, v);
+		break;
+	}
+#endif
+	}
+}
+#endif	/* __HAVE_HASHLOCKED_ATOMICS */
+
 #else  /* __STDC_VERSION__ >= 201112L */
 
 /* C11 definitions, not yet available */

From 5f4187a510acd9c54731ab35936d51e548b9147a Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Tue, 19 Jul 2022 22:07:03 +0000
Subject: [PATCH 2/2] membar(3): Fix t_spinlock for machines with hash-locked
 atomics.

Regular stores don't participate in the hash-locking scheme, so use
atomic_swap instead of a regular store here.
---
 tests/lib/libc/membar/t_spinlock.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/lib/libc/membar/t_spinlock.c b/tests/lib/libc/membar/t_spinlock.c
index 7a2f910267f0..32d0f517e1ce 100644
--- a/tests/lib/libc/membar/t_spinlock.c
+++ b/tests/lib/libc/membar/t_spinlock.c
@@ -76,7 +76,11 @@ unlock(void)
 {
 
 	membar_release();
+#ifdef __HAVE_HASHLOCKED_ATOMICS
+	(void)atomic_cas_uint(&lockbit, 1, 0);
+#else
 	lockbit = 0;
+#endif
 }
 
 static void *