# HG changeset patch
# User Taylor R Campbell <riastradh@NetBSD.org>
# Date 1610740581 0
#      Fri Jan 15 19:56:21 2021 +0000
# Branch trunk
# Node ID befe3938450a8e0e03febf16a5a9e29273b975ab
# Parent  6ff549b20c2b99584abcc649b09b0031bf87cfb1
# EXP-Topic riastradh-hardclockentropy
entropy: Sample lock spinning on multiprocessor systems.

This is only in slow paths where we are spinning or about to sleep
anyway -- this aims to mitigate performance overhead, and only take
samples when two CPUs are racing anyway.

diff -r 6ff549b20c2b -r befe3938450a sys/kern/kern_entropy.c
--- a/sys/kern/kern_entropy.c	Sat Jan 16 02:21:26 2021 +0000
+++ b/sys/kern/kern_entropy.c	Fri Jan 15 19:56:21 2021 +0000
@@ -93,6 +93,7 @@
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/kthread.h>
+#include <sys/lock.h>
 #include <sys/module_hook.h>
 #include <sys/mutex.h>
 #include <sys/percpu.h>
@@ -191,6 +192,9 @@ static struct lwp	*entropy_lwp __read_mo
 int rnd_initial_entropy __read_mostly; /* XXX legacy */
 
 static struct krndsource seed_rndsource __read_mostly;
+#ifdef MULTIPROCESSOR
+static struct krndsource lockspin_rndsource __read_mostly;
+#endif
 
 /*
  * Event counters
@@ -380,6 +384,15 @@ entropy_init(void)
 	if (!E->seeded)
 		printf("entropy: no seed from bootloader\n");
 
+	/*
+	 * Attach the lockspin random source, if this is an MP build.
+	 * (If !MULTIPROCESSOR, then we never spin anyway.)
+	 */
+#ifdef MULTIPROCESSOR
+	rnd_attach_source(&lockspin_rndsource, "lockspin", RND_TYPE_SKEW,
+	    RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME);
+#endif
+
 	/* Allocate the per-CPU records for all early entropy sources.  */
 	LIST_FOREACH(rs, &E->sources, list)
 		rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
@@ -1922,6 +1935,33 @@ rnd_add_data_sync(struct krndsource *rs,
 }
 
 /*
+ * leekspin_rndsource
+ *
+ *	Lock spinning rndsource.  We take a sample whenever we've just
+ *	spun for a lock -- so this happens only when there is
+ *	contention anyway, in order to mitigate the overhead of
+ *	entering a sample.  We incorporate the CPU number to get some
+ *	entropy from the observation of who won the race.  We also
+ *	avoid recursion for any internal locks involved in entropy
+ *	gathering.
+ */
+void
+rnd_lockspin_sample(const void *sobj, unsigned count)
+{
+
+#ifdef MULTIPROCESSOR
+	if (__predict_true(count <= SPINLOCK_BACKOFF_MIN))
+		return;
+	if (__predict_false(sobj == &E->lock))
+		return;
+	if (__predict_false(E->stage == ENTROPY_COLD))
+		return;
+	count ^= cpu_number() << 8;
+	rnd_add_uint32(&lockspin_rndsource, count);
+#endif
+}
+
+/*
  * rndsource_entropybits(rs)
  *
  *	Return approximately the number of bits of entropy that have
diff -r 6ff549b20c2b -r befe3938450a sys/kern/kern_mutex.c
--- a/sys/kern/kern_mutex.c	Sat Jan 16 02:21:26 2021 +0000
+++ b/sys/kern/kern_mutex.c	Fri Jan 15 19:56:21 2021 +0000
@@ -56,6 +56,7 @@
 #include <sys/types.h>
 #include <sys/cpu.h>
 #include <sys/pserialize.h>
+#include <sys/rnd.h>
 
 #include <dev/lockstat.h>
 
@@ -498,6 +499,7 @@ mutex_vector_enter(kmutex_t *mtx)
 			    LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
 		}
 		LOCKSTAT_EXIT(lsflag);
+		rnd_lockspin_sample(mtx, count);
 #endif	/* !MULTIPROCESSOR */
 #endif	/* FULL */
 		MUTEX_LOCKED(mtx);
@@ -559,6 +561,7 @@ mutex_vector_enter(kmutex_t *mtx)
 			} while (mutex_oncpu(owner));
 			LOCKSTAT_STOP_TIMER(lsflag, spintime);
 			LOCKSTAT_COUNT(spincnt, 1);
+			rnd_lockspin_sample(mtx, count);
 			if (!MUTEX_OWNED(owner))
 				continue;
 		}
diff -r 6ff549b20c2b -r befe3938450a sys/kern/kern_rwlock.c
--- a/sys/kern/kern_rwlock.c	Sat Jan 16 02:21:26 2021 +0000
+++ b/sys/kern/kern_rwlock.c	Fri Jan 15 19:56:21 2021 +0000
@@ -62,6 +62,7 @@
 #include <sys/atomic.h>
 #include <sys/lock.h>
 #include <sys/pserialize.h>
+#include <sys/rnd.h>
 
 #include <dev/lockstat.h>
 
@@ -376,6 +377,7 @@ rw_vector_enter(krwlock_t *rw, const krw
 			LOCKSTAT_COUNT(spincnt, 1);
 			if ((owner & need_wait) == 0)
 				continue;
+			rnd_lockspin_sample(rw, count);
 		}
 
 		/*
diff -r 6ff549b20c2b -r befe3938450a sys/sys/rnd.h
--- a/sys/sys/rnd.h	Sat Jan 16 02:21:26 2021 +0000
+++ b/sys/sys/rnd.h	Fri Jan 15 19:56:21 2021 +0000
@@ -46,6 +46,7 @@ void		rnd_init(void);
 void		rnd_init_softint(void);
 void		rnd_seed(void *, size_t);
 int		rnd_system_ioctl(struct file *, u_long, void *);
+void		rnd_lockspin_sample(const void *, u_int);
 
 extern int	rnd_initial_entropy;