From 4e9be1d3fa67b27f3ed3cf9a9e3de755927953f9 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Sun, 23 Jan 2022 19:41:12 +0000
Subject: [PATCH 29/36] specfs: Prevent new opens while close is waiting to
 drain.

Otherwise, bdev/cdev_close could have cancelled all _existing_ opens,
and waited for them to complete (and freed resources used by them) --
but a new one could start, and hang (e.g., a tty), at the same time
spec_close tries to drain all pending I/O operations, one of which
(the new open) is now hanging indefinitely.

Preventing the new open from even starting until bdev/cdev_close is
finished and all I/O operations have drained avoids this deadlock.
---
 sys/miscfs/specfs/spec_vnops.c | 28 ++++++++++++++++++++++++++--
 sys/miscfs/specfs/specdev.h    |  1 +
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/sys/miscfs/specfs/spec_vnops.c b/sys/miscfs/specfs/spec_vnops.c
index a52134f7cdde..c741d26ced9e 100644
--- a/sys/miscfs/specfs/spec_vnops.c
+++ b/sys/miscfs/specfs/spec_vnops.c
@@ -380,6 +380,7 @@ spec_node_init(vnode_t *vp, dev_t rdev)
 		sd->sd_bdevvp = NULL;
 		sd->sd_iocnt = 0;
 		sd->sd_opened = false;
+		sd->sd_closing = false;
 		sn->sn_dev = sd;
 		sd = NULL;
 	} else {
@@ -684,8 +685,17 @@ spec_open(void *v)
 	case VCHR:
 		/*
 		 * Character devices can accept opens from multiple
-		 * vnodes.
+		 * vnodes.  But first, wait for any close to finish.
+		 * Wait under the vnode lock so we don't have to worry
+		 * about the vnode being revoked while we wait.
 		 */
+		while (sd->sd_closing) {
+			error = cv_wait_sig(&specfs_iocv, &device_lock);
+			if (error)
+				break;
+		}
+		if (error)
+			break;
 		sd->sd_opencnt++;
 		sn->sn_opencnt++;
 		break;
@@ -1563,8 +1573,10 @@ spec_close(void *v)
 		    count + 1);
 		sd->sd_bdevvp = NULL;
 	}
-	if (count == 0)
+	if (count == 0) {
 		sd->sd_opened = false;
+		sd->sd_closing = true;
+	}
 	mutex_exit(&device_lock);
 
 	if (count != 0)
@@ -1589,6 +1601,18 @@ spec_close(void *v)
 	 */
 	spec_io_drain(sd);
 
+	/*
+	 * Wake any spec_open calls waiting for close to finish -- do
+	 * this before reacquiring the vnode lock, because spec_open
+	 * holds the vnode lock while waiting, so doing this after
+	 * reacquiring the lock would deadlock.
+	 */
+	mutex_enter(&device_lock);
+	KASSERT(sd->sd_closing);
+	sd->sd_closing = false;
+	cv_broadcast(&specfs_iocv);
+	mutex_exit(&device_lock);
+
 	if (!(flags & FNONBLOCK))
 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
diff --git a/sys/miscfs/specfs/specdev.h b/sys/miscfs/specfs/specdev.h
index 91e7af9bf09b..8b44cf4339f4 100644
--- a/sys/miscfs/specfs/specdev.h
+++ b/sys/miscfs/specfs/specdev.h
@@ -80,6 +80,7 @@ typedef struct specdev {
 	dev_t		sd_rdev;
 	volatile u_int	sd_iocnt;	/* # bdev/cdev_* operations active */
 	bool		sd_opened;	/* true if successfully opened */
+	bool		sd_closing;	/* true when bdev/cdev_close ongoing */
 } specdev_t;
 
 /*