From 54c1082f5f50904ccefd0639215a2880abeca923 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Fri, 11 Feb 2022 21:13:51 +0000 Subject: [PATCH 01/49] sparc: Fix membar_sync with SWAP on sparcv8, NOP on sparcv7. membar_sync is required to be a full sequential consistency barrier, i.e., load/store-before-load/store. On sparcv9, we could just do `membar #LoadLoad|LoadStore|StoreLoad|StoreStore'. But we don't have this on sparcv<9. For sparcv8, we can use a SWAP instruction. (Neither TSO nor STBAR is enough -- they don't order store-before-load.) For sparcv7, there's no multiprocessor so this can be a NOP. But we need the same code to work on sparcv7 and sparcv8. So: - For userland, libc's _membar_sync issues NOP, and libsparcv8.so overrides it by a function that issues SWAP. Note that this is not optional for better performance, but mandatory for correctnesss. - For kernel, we start with SWAP and patch the kernel text overwriting it with NOP on sparcv7. --- .../lib/libc/arch/sparc/atomic/membar_ops.S | 54 +++++++++++++++---- lib/libarch/sparc/v8/Makefile | 1 + lib/libarch/sparc/v8/membar_sync.S | 41 ++++++++++++++ sys/arch/sparc/sparc/locore.s | 1 + 4 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 lib/libarch/sparc/v8/membar_sync.S diff --git a/common/lib/libc/arch/sparc/atomic/membar_ops.S b/common/lib/libc/arch/sparc/atomic/membar_ops.S index f3317c83a600..2454a822719e 100644 --- a/common/lib/libc/arch/sparc/atomic/membar_ops.S +++ b/common/lib/libc/arch/sparc/atomic/membar_ops.S @@ -15,7 +15,7 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -29,27 +29,61 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include #include "atomic_op_asm.h" +#ifdef _KERNEL +#include "opt_multiprocessor.h" +#endif + .text -/* These assume Total Store Order (TSO) */ +/* + * These assume Total Store Order (TSO), which may reorder + * store-before-load but nothing else. Hence, only membar_sync must + * issue anything -- specifically, a SWAP, which (along with LDSTUB) + * is the only instruction that implies a sequential consistency + * barrier. + * + * If we ran with Partial Store Order (PSO), we would also need to + * issue STBAR for membar_exit (load/store-before-store) and + * membar_producer (store-before-store). + */ -ENTRY(_membar_producer) +ENTRY(_membar_consumer) retl nop +END(_membar_consumer) -ENTRY(_membar_consumer) - add %sp, -112, %sp - st %g0, [%sp+100] +/* + * This code must run on sparcv7, which has no SWAP -- but also no + * multiprocessing, so NOP is fine. + * + * For userland, the sparc8 version is actually in libsparcv8.so at + * lib/libarch/sparc/v8/membar_sync.S. + * + * For kernel, we will patch the code on boot in locore.s. + */ +ENTRY(_membar_sync) retl - sub %sp, -112, %sp +#ifndef _KERNEL + nop +#else + .globl NOP_ON_4M_MEMBAR_SYNC +NOP_ON_4M_MEMBAR_SYNC: +# ifdef MULTIPROCESSOR + swap [%sp - 4], %i0 +# else + nop +# endif +#endif +END(_membar_sync) -ATOMIC_OP_ALIAS(membar_producer,_membar_producer) +ATOMIC_OP_ALIAS(membar_producer,_membar_consumer) +STRONG_ALIAS(_membar_producer,_membar_consumer) ATOMIC_OP_ALIAS(membar_consumer,_membar_consumer) ATOMIC_OP_ALIAS(membar_enter,_membar_consumer) STRONG_ALIAS(_membar_enter,_membar_consumer) ATOMIC_OP_ALIAS(membar_exit,_membar_consumer) STRONG_ALIAS(_membar_exit,_membar_consumer) -ATOMIC_OP_ALIAS(membar_sync,_membar_consumer) -STRONG_ALIAS(_membar_sync,_membar_consumer) +ATOMIC_OP_ALIAS(membar_sync,_membar_sync) diff --git a/lib/libarch/sparc/v8/Makefile b/lib/libarch/sparc/v8/Makefile index 199d745cf839..9f1d47309d2d 100644 --- a/lib/libarch/sparc/v8/Makefile +++ b/lib/libarch/sparc/v8/Makefile @@ -13,5 +13,6 @@ LDLIBC= -nodefaultlibs AFLAGS+= -Wa,-Av8 SRCS= sparc_v8.S +SRCS+= membar_sync.S .include diff --git a/lib/libarch/sparc/v8/membar_sync.S b/lib/libarch/sparc/v8/membar_sync.S new file mode 100644 index 000000000000..64ce58b599b4 --- /dev/null +++ b/lib/libarch/sparc/v8/membar_sync.S @@ -0,0 +1,41 @@ +/* $NetBSD$ */ + +/*- + * Copyright (c) 2022 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .text + +/* + * membar_sync for sparcv8 userland -- overrides the sparcv7-only + * membar_sync in common/lib/libc/arch/sparc/atomic/membar_ops.S. + */ + +ENTRY(_membar_sync) + retl + swap [%sp - 4], %i0 +END(_membar_sync) diff --git a/sys/arch/sparc/sparc/locore.s b/sys/arch/sparc/sparc/locore.s index 31f5af82c64e..e5b752a225e5 100644 --- a/sys/arch/sparc/sparc/locore.s +++ b/sys/arch/sparc/sparc/locore.s @@ -4411,6 +4411,7 @@ Lgandul: nop MUNGE(NOP_ON_4M_10) MUNGE(NOP_ON_4M_11) MUNGE(NOP_ON_4M_12) + MUNGE(NOP_ON_4M_MEMBAR_SYNC) b,a 2f 1: -- 2.33.0