From b7816d1fc65155390d7609f511583bfa60f9619a Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Sun, 13 Feb 2022 14:05:51 +0000
Subject: [PATCH] eqos(4): Membar audit.

---
 sys/dev/ic/dwc_eqos.c | 50 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/sys/dev/ic/dwc_eqos.c b/sys/dev/ic/dwc_eqos.c
index 264037e55299..725d7f65eb7a 100644
--- a/sys/dev/ic/dwc_eqos.c
+++ b/sys/dev/ic/dwc_eqos.c
@@ -269,7 +269,7 @@ eqos_setup_txbuf(struct eqos_softc *sc, int index, struct mbuf *m)
 {
 	bus_dma_segment_t *segs;
 	int error, nsegs, cur, i;
-	uint32_t flags;
+	uint32_t flags, tdes3;
 	bool nospace;
 
 	/* at least one descriptor free ? */
@@ -322,10 +322,20 @@ eqos_setup_txbuf(struct eqos_softc *sc, int index, struct mbuf *m)
 
 	/*
 	 * Defer setting OWN bit on the first descriptor until all
-	 * descriptors have been updated.
+	 * descriptors have been updated.  The hardware will not try to
+	 * process any descriptors past the first one still owned by
+	 * software (i.e., with the OWN bit clear).
+	 *
+	 * Store-release here ensures all the descriptors are updated
+	 * before we set the OWN bit transferring ownership to
+	 * hardware; this pairs with the logical equivalent of a
+	 * `load-acquire' somewhere in the harware/firmware so it
+	 * doesn't load the content of the descriptors until after
+	 * seeing OWN.
 	 */
-	membar_sync();
-	sc->sc_tx.desc_ring[index].tdes3 |= htole32(EQOS_TDES3_OWN);
+	tdes3 = sc->sc_tx.desc_ring[index].tdes3;
+	tdes3 |= htole32(EQOS_TDES3_OWN);
+	atomic_store_release(&sc->sc_tx.desc_ring[index].tdes3, tdes3);
 
 	return nsegs;
 }
@@ -333,12 +343,19 @@ eqos_setup_txbuf(struct eqos_softc *sc, int index, struct mbuf *m)
 static void
 eqos_setup_rxdesc(struct eqos_softc *sc, int index, bus_addr_t paddr)
 {
+	uint32_t tdes3;
+
 	sc->sc_rx.desc_ring[index].tdes0 = htole32((uint32_t)paddr);
 	sc->sc_rx.desc_ring[index].tdes1 = htole32((uint32_t)(paddr >> 32));
 	sc->sc_rx.desc_ring[index].tdes2 = htole32(0);
-	membar_sync();
-	sc->sc_rx.desc_ring[index].tdes3 =
-	    htole32(EQOS_TDES3_OWN | EQOS_TDES3_IOC | EQOS_TDES3_BUF1V);
+
+	/*
+	 * Store-release ensures descriptor is updated before we set
+	 * the OWN bit transferring ownership to the hardware, like
+	 * eqos_setup_txbuf.
+	 */
+	tdes3 = htole32(EQOS_TDES3_OWN | EQOS_TDES3_IOC | EQOS_TDES3_BUF1V);
+	atomic_store_release(&sc->sc_rx.desc_ring[index].tdes3, tdes3);
 }
 
 static int
@@ -694,7 +711,15 @@ eqos_rxintr(struct eqos_softc *sc, int qid)
 		    index, index + 1, RX_DESC_COUNT,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
-		tdes3 = le32toh(sc->sc_rx.desc_ring[index].tdes3);
+		/*
+		 * Load-acquire here pairs with the logical equivalent
+		 * of a `store-release' in the hardware/firmware to
+		 * clear the OWN bit, and ensures we don't load any
+		 * stale content of the descriptor until the OWN bit
+		 * has been cleared.
+		 */
+		tdes3 = atomic_load_acquire(&sc->sc_rx.desc_ring[index].tdes3);
+		tdes3 = le32toh(tdes3);
 		if ((tdes3 & EQOS_TDES3_OWN) != 0) {
 			break;
 		}
@@ -760,7 +785,14 @@ eqos_txintr(struct eqos_softc *sc, int qid)
 		    i, i + 1, TX_DESC_COUNT,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		desc = &sc->sc_tx.desc_ring[i];
-		tdes3 = le32toh(desc->tdes3);
+		/*
+		 * Load-acquire here pairs with the logical equivalent
+		 * of a `store-release' in the hardware/firmware to
+		 * clear the OWN bit, and ensures we don't load any
+		 * stale content of the descriptor until the OWN bit
+		 * has been cleared.
+		 */
+		tdes3 = le32toh(atomic_load_acquire(&desc->tdes3));
 		if ((tdes3 & EQOS_TDES3_OWN) != 0) {
 			break;
 		}