From 6477bb712b71efb63abb79c259b9567f91dbc978 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh@NetBSD.org>
Date: Thu, 20 Jan 2022 23:24:42 +0000
Subject: [PATCH 35/37] autoconf(9): New function config_detach_commit.

When a driver's .ca_detach function has committed to detaching -- it
definitely won't back out with EBUSY, for instance -- it can call
this to wake all pending calls to device_lookup_acquire and make them
fail immediately.

This is necessary to break a deadlock if the device_lookup_acquire
calls happen inside I/O operations which the driver's .ca_detach
function waits for the completion of -- without config_detach_commit,
I/O operations would be stuck in device_lookup_acquire waiting for
.ca_detach and .ca_detach would be stuck waiting for I/O operations
to return.

Most drivers won't need to call this: for autoconf drivers used the
traditional way by devsw for userland device nodes, the .ca_detach
routine uses vdevgone, and we will arrange to make vdevgone call
config_detach_commit automagically in such drivers anyway.

XXX kernel ABI change to struct device requires bump -- later change
will make struct device opaque to ABI, but we're not there yet
---
 sys/kern/subr_autoconf.c | 45 ++++++++++++++++++++++++++++++++++------
 sys/sys/device.h         |  2 ++
 2 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/sys/kern/subr_autoconf.c b/sys/kern/subr_autoconf.c
index 74f71dca0bcf..03afc7f54b55 100644
--- a/sys/kern/subr_autoconf.c
+++ b/sys/kern/subr_autoconf.c
@@ -2045,10 +2045,17 @@ config_detach(device_t dev, int flags)
 	 * If it was possible to detach the device, ensure that the
 	 * device is deactivated.
 	 */
-	if (rv == 0)
-		dev->dv_flags &= ~DVF_ACTIVE;
-	else if ((flags & DETACH_FORCE) == 0) {
-		/* Detach failed -- likely EBUSY.  */
+	if (rv == 0) {
+		config_detach_commit(dev);
+		dev->dv_flags &= ~DVF_ACTIVE; /* XXXSMP */
+	} else if ((flags & DETACH_FORCE) == 0) {
+		/*
+		 * Detach failed -- likely EBUSY.  Driver must not have
+		 * called config_detach_commit.
+		 */
+		KASSERTMSG(!dev->dv_detached,
+		    "%s committed to detaching and then backed out",
+		    device_xname(dev));
 		goto out;
 	} else {
 		panic("config_detach: forced detach of %s failed (%d)",
@@ -2065,7 +2072,8 @@ config_detach(device_t dev, int flags)
 	 * responsibility of .ca_detach to ensure anything with open
 	 * references will be interrupted and release them promptly,
 	 * not block indefinitely.  All new attempts to acquire
-	 * references will block until dv_detaching clears.
+	 * references will fail, as config_detach_commit has arranged
+	 * by now.
 	 */
 	mutex_enter(&config_misc_lock);
 	localcount_drain(dev->dv_localcount,
@@ -2139,6 +2147,30 @@ out:
 	return rv;
 }
 
+/*
+ * config_detach_commit(dev)
+ *
+ *	Issued by a driver's .ca_detach routine to notify anyone
+ *	waiting in device_lookup_acquire that the driver is committed
+ *	to detaching the device, which allows device_lookup_acquire to
+ *	wake up and fail immediately.
+ *
+ *	Safe to call multiple times -- idempotent.  Must be called
+ *	during config_detach_enter/exit.  Safe to use with
+ *	device_lookup because the device is not actually removed from
+ *	the table until after config_detach_exit.
+ */
+void
+config_detach_commit(device_t dev)
+{
+
+	mutex_enter(&config_misc_lock);
+	KASSERT(dev->dv_detaching == curlwp);
+	dev->dv_detached = true;
+	cv_broadcast(&config_misc_cv);
+	mutex_exit(&config_misc_lock);
+}
+
 int
 config_detach_children(device_t parent, int flags)
 {
@@ -2646,7 +2678,8 @@ device_lookup_acquire(cfdriver_t cd, int unit)
 	mutex_enter(&alldevs_lock);
 retry:	if (unit < 0 || unit >= cd->cd_ndevs ||
 	    (dv = cd->cd_devs[unit]) == NULL ||
-	    dv->dv_del_gen != 0) {
+	    dv->dv_del_gen != 0 ||
+	    dv->dv_detached) {
 		dv = NULL;
 	} else {
 		/*
diff --git a/sys/sys/device.h b/sys/sys/device.h
index 9f95c98ffa42..cfd324a1ac2d 100644
--- a/sys/sys/device.h
+++ b/sys/sys/device.h
@@ -281,6 +281,7 @@ struct device {
 
 	struct lwp	*dv_attaching;	/* thread not yet finished in attach */
 	struct lwp	*dv_detaching;	/* detach lock (config_misc_lock/cv) */
+	bool		dv_detached;	/* config_misc_lock */
 
 	size_t		dv_activity_count;
 	void		(**dv_activity_handlers)(device_t, devactive_t);
@@ -631,6 +632,7 @@ device_t config_attach_pseudo(cfdata_t);
 
 int	config_detach(device_t, int);
 int	config_detach_children(device_t, int flags);
+void	config_detach_commit(device_t);
 bool	config_detach_all(int);
 int	config_deactivate(device_t);
 void	config_defer(device_t, void (*)(device_t));