diff --git a/common/lib/libc/arch/mips/atomic/atomic_add.S b/common/lib/libc/arch/mips/atomic/atomic_add.S index 886b6d54cf1b..1eac9a3c9c1e 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_add.S +++ b/common/lib/libc/arch/mips/atomic/atomic_add.S @@ -49,6 +49,7 @@ LEAF(_atomic_add_32) #if defined(_MIPS_ARCH_OCTEONP) || defined(_MIPS_ARCH_OCTEON2) saa a1, (a0) #else + LLSCSYNC 1: INT_LL t0, 0(a0) nop INT_ADDU t0, a1 @@ -62,6 +63,7 @@ END(_atomic_add_32) ATOMIC_OP_ALIAS(atomic_add_32, _atomic_add_32) LEAF(_atomic_add_32_nv) + LLSCSYNC 1: INT_LL v0, 0(a0) nop INT_ADDU v0, a1 @@ -79,6 +81,7 @@ LEAF(_atomic_add_64) #if defined(_MIPS_ARCH_OCTEONP) || defined(_MIPS_ARCH_OCTEON2) saad a1, (a0) #else + LLSCSYNC 1: REG_LL t0, 0(a0) nop REG_ADDU t0, a1 @@ -92,6 +95,7 @@ END(_atomic_add_64) ATOMIC_OP_ALIAS(atomic_add_64, _atomic_add_64) LEAF(_atomic_add_64_nv) + LLSCSYNC 1: REG_LL v0, 0(a0) nop REG_ADDU v0, a1 diff --git a/common/lib/libc/arch/mips/atomic/atomic_and.S b/common/lib/libc/arch/mips/atomic/atomic_and.S index cb1eb4848b23..e707f5f37cca 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_and.S +++ b/common/lib/libc/arch/mips/atomic/atomic_and.S @@ -46,6 +46,7 @@ RCSID("$NetBSD: atomic_and.S,v 1.4 2012/03/14 16:50:34 christos Exp $") LEAF(_atomic_and_32) + LLSCSYNC 1: INT_LL t0, 0(a0) nop and t0, a1 @@ -58,6 +59,7 @@ END(_atomic_and_32) ATOMIC_OP_ALIAS(atomic_and_32, _atomic_and_32) LEAF(_atomic_and_32_nv) + LLSCSYNC 1: INT_LL v0, 0(a0) nop and v0, a1 @@ -72,6 +74,7 @@ ATOMIC_OP_ALIAS(atomic_and_32_nv, _atomic_and_32_nv) #if !defined(__mips_o32) LEAF(_atomic_and_64) + LLSCSYNC 1: REG_LL t0, 0(a0) nop and t0, a1 @@ -84,6 +87,7 @@ END(_atomic_and_64) ATOMIC_OP_ALIAS(atomic_and_64, _atomic_and_64) LEAF(_atomic_and_64_nv) + LLSCSYNC 1: REG_LL v0, 0(a0) nop and v0, a1 diff --git a/common/lib/libc/arch/mips/atomic/atomic_cas.S b/common/lib/libc/arch/mips/atomic/atomic_cas.S index 5afcae13e48c..416d00e72907 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_cas.S +++ b/common/lib/libc/arch/mips/atomic/atomic_cas.S @@ -37,6 +37,7 @@ RCSID("$NetBSD: atomic_cas.S,v 1.6 2019/02/20 05:25:12 rin Exp $") .set nomacro LEAF(_atomic_cas_32) + LLSCSYNC 1: INT_LL v0, 0(a0) nop bne v0, a1, 2f @@ -58,6 +59,7 @@ ATOMIC_OP_ALIAS(atomic_cas_32_ni, _atomic_cas_32) #if !defined(__mips_o32) LEAF(_atomic_cas_64) + LLSCSYNC 1: REG_LL v0, 0(a0) nop bne v0, a1, 2f diff --git a/common/lib/libc/arch/mips/atomic/atomic_dec.S b/common/lib/libc/arch/mips/atomic/atomic_dec.S index 641ad9b3c73d..0dadebfc6d37 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_dec.S +++ b/common/lib/libc/arch/mips/atomic/atomic_dec.S @@ -49,6 +49,7 @@ LEAF(_atomic_dec_32) li t0, -1 saa t0, (a0) #else + LLSCSYNC 1: INT_LL t0, 0(a0) nop INT_ADDU t0, -1 @@ -62,6 +63,7 @@ END(_atomic_dec_32) ATOMIC_OP_ALIAS(atomic_dec_32, _atomic_dec_32) LEAF(_atomic_dec_32_nv) + LLSCSYNC 1: INT_LL v0, 0(a0) nop INT_ADDU v0, -1 @@ -80,6 +82,7 @@ LEAF(_atomic_dec_64) li t0, -1 saad t0, (a0) #else + LLSCSYNC 1: REG_LL t0, 0(a0) nop REG_ADDU t0, -1 @@ -93,6 +96,7 @@ END(_atomic_dec_64) ATOMIC_OP_ALIAS(atomic_dec_64, _atomic_dec_64) LEAF(_atomic_dec_64_nv) + LLSCSYNC 1: REG_LL v0, 0(a0) nop REG_ADDU v0, -1 diff --git a/common/lib/libc/arch/mips/atomic/atomic_inc.S b/common/lib/libc/arch/mips/atomic/atomic_inc.S index 40fadb8c45d1..6ef59d8c37bd 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_inc.S +++ b/common/lib/libc/arch/mips/atomic/atomic_inc.S @@ -50,6 +50,7 @@ LEAF(_atomic_inc_32) li t0, 1 saa t0, (a0) #else + LLSCSYNC 1: INT_LL t0, 0(a0) nop INT_ADDU t0, 1 @@ -63,6 +64,7 @@ END(_atomic_inc_32) ATOMIC_OP_ALIAS(atomic_inc_32, _atomic_inc_32) LEAF(_atomic_inc_32_nv) + LLSCSYNC 1: INT_LL v0, 0(a0) nop INT_ADDU v0, 1 @@ -81,6 +83,7 @@ LEAF(_atomic_inc_64) li t0, 1 saad t0, (a0) #else + LLSCSYNC 1: REG_LL t0, 0(a0) nop REG_ADDU t0, 1 @@ -94,6 +97,7 @@ END(_atomic_inc_64) ATOMIC_OP_ALIAS(atomic_inc_64, _atomic_inc_64) LEAF(_atomic_inc_64_nv) + LLSCSYNC 1: REG_LL v0, 0(a0) nop REG_ADDU v0, 1 diff --git a/common/lib/libc/arch/mips/atomic/atomic_or.S b/common/lib/libc/arch/mips/atomic/atomic_or.S index 595bc4ec7ce9..f891c71f3931 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_or.S +++ b/common/lib/libc/arch/mips/atomic/atomic_or.S @@ -43,6 +43,7 @@ #endif /* _KERNEL_OPT */ LEAF(_atomic_or_32) + LLSCSYNC 1: INT_LL t0, 0(a0) nop or t0, a1 @@ -55,6 +56,7 @@ END(_atomic_or_32) ATOMIC_OP_ALIAS(atomic_or_32, _atomic_or_32) LEAF(_atomic_or_32_nv) + LLSCSYNC 1: INT_LL v0, 0(a0) nop or v0, a1 @@ -69,6 +71,7 @@ ATOMIC_OP_ALIAS(atomic_or_32_nv, _atomic_or_32_nv) #if !defined(__mips_o32) LEAF(_atomic_or_64) + LLSCSYNC 1: REG_LL t0, 0(a0) nop or t0, a1 @@ -81,6 +84,7 @@ END(_atomic_or_64) ATOMIC_OP_ALIAS(atomic_or_64, _atomic_or_64) LEAF(_atomic_or_64_nv) + LLSCSYNC 1: REG_LL v0, 0(a0) nop or v0, a1 diff --git a/common/lib/libc/arch/mips/atomic/atomic_swap.S b/common/lib/libc/arch/mips/atomic/atomic_swap.S index 591665ff67ad..45a2782e85f4 100644 --- a/common/lib/libc/arch/mips/atomic/atomic_swap.S +++ b/common/lib/libc/arch/mips/atomic/atomic_swap.S @@ -44,8 +44,8 @@ RCSID("$NetBSD: atomic_swap.S,v 1.5 2015/06/01 23:16:54 matt Exp $") .set nomacro #endif /* _KERNEL_OPT */ - LEAF(_atomic_swap_32) + LLSCSYNC 1: INT_LL v0, 0(a0) nop move t0, a1 @@ -60,6 +60,7 @@ ATOMIC_OP_ALIAS(atomic_swap_32, _atomic_swap_32) #if !defined(__mips_o32) LEAF(_atomic_swap_64) + LLSCSYNC 1: REG_LL v0, 0(a0) nop move t0, a1 diff --git a/sys/arch/evbmips/conf/OCTEON.MP b/sys/arch/evbmips/conf/OCTEON.MP index e5c24923ac56..33da849771ae 100644 --- a/sys/arch/evbmips/conf/OCTEON.MP +++ b/sys/arch/evbmips/conf/OCTEON.MP @@ -2,8 +2,8 @@ include "arch/evbmips/conf/OCTEON" -options LOCKDEBUG +#options LOCKDEBUG options DEBUG # extra kernel debugging support - +options UVMHIST options MULTIPROCESSOR diff --git a/sys/arch/mips/include/asm.h b/sys/arch/mips/include/asm.h index c0bdfc5a8b50..53fe45639486 100644 --- a/sys/arch/mips/include/asm.h +++ b/sys/arch/mips/include/asm.h @@ -54,6 +54,10 @@ #ifndef _MIPS_ASM_H #define _MIPS_ASM_H +#include "opt_cputype.h" +#include "opt_lockdebug.h" +#include "opt_multiprocessor.h" + #include /* for API selection */ #include @@ -513,6 +517,23 @@ _C_LABEL(x): #define NOP_L /* nothing */ #endif +#if defined(MULTIPROCESSOR) +#if defined(MIPS64_OCTEON) +#define LLSCSYNC sync 4; sync 4 +#define SYNC sync 4 /* aka syncw */ +#define BDSYNC sync 4 /* aka syncw */ +#else +#define LLSCSYNC /* nothing (something?) */ +#define SYNC sync +#define BDSYNC sync +#endif +#else +#define LLSCSYNC /* nothing */ +#define SYNC /* nothing */ +#define BDSYNC nop +#endif /* defined(MULTIPROCESSOR) */ + + /* CPU dependent hook for cp0 load delays */ #if defined(MIPS1) || defined(MIPS2) || defined(MIPS3) #define MFC0_HAZARD sll $0,$0,1 /* super scalar nop */ diff --git a/sys/arch/mips/mips/db_disasm.c b/sys/arch/mips/mips/db_disasm.c index cfee99054377..2ca7910d9a4b 100644 --- a/sys/arch/mips/mips/db_disasm.c +++ b/sys/arch/mips/mips/db_disasm.c @@ -59,7 +59,7 @@ static const char * const op_name[64] = { /*24 */ "daddi","daddiu","ldl", "ldr", "op34", "op35", "op36", "op37", /*32 */ "lb", "lh", "lwl", "lw", "lbu", "lhu", "lwr", "lwu", /*40 */ "sb", "sh", "swl", "sw", "sdl", "sdr", "swr", "cache", -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON /*48 */ "ll", "lwc1", "bbit0", "lwc3", "lld", "ldc1", "bbit032", "ld", /*56 */ "sc", "swc1", "bbit1", "swc3", "scd", "sdc1", "bbit132", "sd" #else @@ -83,7 +83,7 @@ static const char * const spec2_name[64] = { /* QED RM4650, R5000, etc. */ [OP_MADD] = "madd", [OP_MADDU] = "maddu", [OP_MUL] = "mul", -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON [OP_CVM_DMUL] = "baddu", #endif [OP_MSUB] = "msub", @@ -92,7 +92,7 @@ static const char * const spec2_name[64] = { /* QED RM4650, R5000, etc. */ [OP_CLO] = "clo", [OP_DCLZ] = "dclz", [OP_DCLO] = "dclo", -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON [OP_CVM_BADDU] = "baddu", [OP_CVM_POP] = "pop", [OP_CVM_DPOP] = "dpop", @@ -377,7 +377,7 @@ db_disasm_insn(int insn, db_addr_t loc, bool altfmt) break; } if (i.RType.func == OP_MUL -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON || i.RType.func == OP_CVM_DMUL || i.RType.func == OP_CVM_SEQ || i.RType.func == OP_CVM_SNE @@ -390,7 +390,7 @@ db_disasm_insn(int insn, db_addr_t loc, bool altfmt) reg_name[i.RType.rt]); break; } -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON if (i.RType.func == OP_CVM_CINS || i.RType.func == OP_CVM_CINS32 || i.RType.func == OP_CVM_EXTS @@ -423,7 +423,7 @@ db_disasm_insn(int insn, db_addr_t loc, bool altfmt) #endif if (i.RType.func == OP_CLO || i.RType.func == OP_DCLO -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON || i.RType.func == OP_CVM_POP || i.RType.func == OP_CVM_DPOP #endif @@ -735,7 +735,7 @@ db_disasm_insn(int insn, db_addr_t loc, bool altfmt) bdslot = true; break; -#ifdef __OCTEON__ +#ifdef MIPS64_OCTEON case OP_CVM_BBIT0: case OP_CVM_BBIT032: case OP_CVM_BBIT1: diff --git a/sys/arch/mips/mips/lock_stubs_llsc.S b/sys/arch/mips/mips/lock_stubs_llsc.S index 12c99b01be0f..6704df7ef65f 100644 --- a/sys/arch/mips/mips/lock_stubs_llsc.S +++ b/sys/arch/mips/mips/lock_stubs_llsc.S @@ -46,14 +46,17 @@ RCSID("$NetBSD: lock_stubs_llsc.S,v 1.9 2019/04/06 03:06:26 thorpej Exp $") #endif #if defined(MULTIPROCESSOR) -#if defined(__OCTEON__) -#define SYNC sync 4 -#define BDSYNC sync 4 +#if defined(MIPS64_OCTEON) +#define LLSCSYNC sync 4; sync 4 +#define SYNC sync 4 /* aka syncw */ +#define BDSYNC sync 4 /* aka syncw */ #else +#define LLSCSYNC /* nothing (something?) */ #define SYNC sync #define BDSYNC sync #endif #else +#define LLSCSYNC /* nothing */ #define SYNC /* nothing */ #define BDSYNC nop #endif /* defined(MULTIPROCESSOR) */ @@ -84,6 +87,7 @@ RCSID("$NetBSD: lock_stubs_llsc.S,v 1.9 2019/04/06 03:06:26 thorpej Exp $") * unsigned long old, unsigned long new); */ STATIC_LEAF(llsc_atomic_cas_ulong) + LLSCSYNC 1: LONG_LL t0, (a0) bne t0, a1, 2f @@ -104,6 +108,7 @@ END(llsc_atomic_cas_ulong) * unsigned int old, unsigned int new); */ STATIC_LEAF(llsc_atomic_cas_uint) + LLSCSYNC 1: INT_LL t0, (a0) bne t0, a1, 2f @@ -133,6 +138,7 @@ STATIC_LEAF(llsc_ucas_32) nop move v0, zero + LLSCSYNC 1: ll t0, 0(a0) bne t0, a1, 2f move t1, a2 @@ -161,6 +167,7 @@ STATIC_LEAF(llsc_ucas_64) nop move v0, zero + LLSCSYNC 1: lld t0, 0(a0) bne t0, a1, 2f move t1, a2 @@ -187,6 +194,7 @@ END(llsc_ucaserr) * void mutex_enter(kmutex_t *mtx); */ STATIC_LEAF(llsc_mutex_enter) + LLSCSYNC PTR_LL t0, MTX_OWNER(a0) 1: bnez t0, 2f @@ -205,6 +213,7 @@ END(llsc_mutex_enter) * void mutex_exit(kmutex_t *mtx); */ STATIC_LEAF(llsc_mutex_exit) + LLSCSYNC PTR_LL t0, MTX_OWNER(a0) SYNC 1: @@ -268,6 +277,7 @@ STATIC_NESTED(llsc_mutex_spin_enter, CALLFRAME_SIZ, ra) nop #endif /* PARANOIA */ #if defined(FULL) + LLSCSYNC INT_LL t3, MTX_LOCK(t0) 3: bnez t3, 4f