From b752fd75083fe3ad3e9d95c4cb543fafd7b3e58c Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Sun, 27 Feb 2022 19:43:11 +0000 Subject: [PATCH 1/3] mips/cavium: Insert appropriate membars around IPIs. --- sys/arch/mips/cavium/octeon_intr.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/arch/mips/cavium/octeon_intr.c b/sys/arch/mips/cavium/octeon_intr.c index a4afc88bef80..68b3b1768a89 100644 --- a/sys/arch/mips/cavium/octeon_intr.c +++ b/sys/arch/mips/cavium/octeon_intr.c @@ -545,6 +545,7 @@ octeon_ipi_intr(void *arg) ipi_mask &= mips3_ld(cpu->cpu_mbox_set); if (ipi_mask == 0) return 0; + membar_enter(); mips3_sd(cpu->cpu_mbox_clr, ipi_mask); @@ -563,8 +564,9 @@ octeon_ipi_intr(void *arg) #endif /* if the request is clear, it was previously processed */ - if ((ci->ci_request_ipis & ipi_mask) == 0) + if ((atomic_load_relaxed(&ci->ci_request_ipis) & ipi_mask) == 0) return 0; + membar_enter(); atomic_or_64(&ci->ci_active_ipis, ipi_mask); atomic_and_64(&ci->ci_request_ipis, ~ipi_mask); @@ -597,8 +599,10 @@ octeon_send_ipi(struct cpu_info *ci, int req) const u_int ipi_shift = ipi_prio[req] == IPL_SCHED ? 16 : 0; const uint32_t ipi_mask = __BIT(req + ipi_shift); + membar_exit(); atomic_or_64(&ci->ci_request_ipis, ipi_mask); + membar_exit(); mips3_sd(cpu->cpu_mbox_set, ipi_mask); return 0; From b3513665c8bfe69c2440810b12457fe94e58834c Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Wed, 2 Mar 2022 18:14:07 +0000 Subject: [PATCH 2/3] mips/cavium: Fix membars around establishing interrupt handlers. --- sys/arch/mips/cavium/octeon_intr.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/sys/arch/mips/cavium/octeon_intr.c b/sys/arch/mips/cavium/octeon_intr.c index 68b3b1768a89..f8c4342c8a28 100644 --- a/sys/arch/mips/cavium/octeon_intr.c +++ b/sys/arch/mips/cavium/octeon_intr.c @@ -54,6 +54,7 @@ __KERNEL_RCSID(0, "$NetBSD: octeon_intr.c,v 1.24 2020/08/18 07:41:41 skrll Exp $ #include #include #include +#include #include @@ -353,6 +354,14 @@ octeon_intr_establish(int irq, int ipl, int (*func)(void *), void *arg) ih->ih_irq = irq; ih->ih_ipl = ipl; + /* + * Make sure the initialization is visible on all CPUs before + * we expose it in octciu_intrs. This way we don't need to + * issue any membar for a load-acquire when handling the + * interrupt. + */ + xc_barrier(0); + mutex_enter(&octeon_intr_lock); /* @@ -361,8 +370,7 @@ octeon_intr_establish(int irq, int ipl, int (*func)(void *), void *arg) KASSERTMSG(octciu_intrs[irq] == NULL, "irq %d in use! (%p)", irq, octciu_intrs[irq]); - octciu_intrs[irq] = ih; - membar_producer(); + atomic_store_relaxed(&octciu_intrs[irq], ih); /* * Now enable it. @@ -462,13 +470,15 @@ octeon_intr_disestablish(void *cookie) break; } - /* - * Now remove it since we shouldn't get interrupts for it. - */ - octciu_intrs[irq] = NULL; + atomic_store_relaxed(&octciu_intrs[irq], NULL); mutex_exit(&octeon_intr_lock); + /* + * Wait until the interrupt handler is no longer running on all + * CPUs before freeing ih and returning. + */ + xc_barrier(0); kmem_free(ih, sizeof(*ih)); } @@ -507,7 +517,8 @@ octeon_iointr(int ipl, vaddr_t pc, uint32_t ipending) const int irq = (bank * 64) + bit; hwpend[bank] &= ~__BIT(bit); - struct octeon_intrhand * const ih = octciu_intrs[irq]; + struct octeon_intrhand * const ih = + atomic_load_relaxed(&octciu_intrs[irq]); cpu->cpu_intr_evs[irq].ev_count++; if (__predict_true(ih != NULL)) { #ifdef MULTIPROCESSOR From 09bdffd156b248451e9c4e733338fd2386d8aadb Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Wed, 2 Mar 2022 18:18:27 +0000 Subject: [PATCH 3/3] mips/rmixl: Insert appropriate membars around IPIs. --- sys/arch/mips/rmi/rmixl_intr.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sys/arch/mips/rmi/rmixl_intr.c b/sys/arch/mips/rmi/rmixl_intr.c index 3e9b49731bdf..c37f13508097 100644 --- a/sys/arch/mips/rmi/rmixl_intr.c +++ b/sys/arch/mips/rmi/rmixl_intr.c @@ -967,6 +967,7 @@ rmixl_send_ipi(struct cpu_info *ci, int tag) | (RMIXL_INTRVEC_IPI + tag); mutex_enter(&rmixl_ipi_lock); + membar_exit(); atomic_or_64(&ci->ci_request_ipis, req); RMIXL_PICREG_WRITE(RMIXL_PIC_IPIBASE, r); mutex_exit(&rmixl_ipi_lock); @@ -984,8 +985,9 @@ rmixl_ipi_intr(void *arg) KASSERT((uintptr_t)arg < NIPIS); /* if the request is clear, it was previously processed */ - if ((ci->ci_request_ipis & ipi_mask) == 0) + if ((atomic_load_relaxed(&ci->ci_request_ipis) & ipi_mask) == 0) return 0; + membar_enter(); atomic_or_64(&ci->ci_active_ipis, ipi_mask); atomic_and_64(&ci->ci_request_ipis, ~ipi_mask);