Add percpu_alloc_movable(size, movefn) and use it in psref(9).

percpu_alloc(size) is now percpu_alloc_movable(size, &memcpy).

This lets you define the function to move data from one buffer to
another buffer when someone else does percpu_alloc(9).

XXX This does not help to add initialization/finalization actions for
per-CPU objects on dynamic CPU attachment and detachment.  We really
need additional initfn/finifn routines for that case.


Index: sys/kern/subr_percpu.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_percpu.c,v
retrieving revision 1.18
diff -p -u -r1.18 subr_percpu.c
--- sys/kern/subr_percpu.c	31 May 2017 23:54:17 -0000	1.18
+++ sys/kern/subr_percpu.c	2 Oct 2017 01:47:04 -0000
@@ -47,19 +47,18 @@ __KERNEL_RCSID(0, "$NetBSD: subr_percpu.
 #define	PERCPU_QCACHE_MAX	0
 #define	PERCPU_IMPORT_SIZE	2048
 
-#if defined(DIAGNOSTIC)
-#define	MAGIC	0x50435055	/* "PCPU" */
-#define	percpu_encrypt(pc)	((pc) ^ MAGIC)
-#define	percpu_decrypt(pc)	((pc) ^ MAGIC)
-#else /* defined(DIAGNOSTIC) */
-#define	percpu_encrypt(pc)	(pc)
-#define	percpu_decrypt(pc)	(pc)
-#endif /* defined(DIAGNOSTIC) */
+struct percpu {
+	unsigned		pc_offset;
+	unsigned		pc_size;
+	void			*(*pc_move)(void *, const void *, size_t);
+	LIST_ENTRY(percpu)	pc_entry;
+};
 
 static krwlock_t	percpu_swap_lock	__cacheline_aligned;
 static kmutex_t		percpu_allocation_lock	__cacheline_aligned;
 static vmem_t *		percpu_offset_arena	__cacheline_aligned;
 static unsigned int	percpu_nextoff		__cacheline_aligned;
+static LIST_HEAD(, percpu)	percpu_list	__cacheline_aligned;
 
 static percpu_cpu_t *
 cpu_percpu(struct cpu_info *ci)
@@ -71,7 +70,7 @@ cpu_percpu(struct cpu_info *ci)
 static unsigned int
 percpu_offset(percpu_t *pc)
 {
-	const unsigned int off = percpu_decrypt((uintptr_t)pc);
+	const unsigned int off = pc->pc_offset;
 
 	KASSERT(off < percpu_nextoff);
 	return off;
@@ -95,6 +94,7 @@ percpu_cpu_swap(void *p1, void *p2)
 	 */
 	rw_enter(&percpu_swap_lock, RW_WRITER);
 	if (newpcc->pcc_size > pcc->pcc_size) {
+		struct percpu *pc;
 		percpu_cpu_t tmp;
 		int s;
 
@@ -110,7 +110,12 @@ percpu_cpu_swap(void *p1, void *p2)
 		 * copy data to new storage.
 		 */
 
-		memcpy(newpcc->pcc_data, pcc->pcc_data, pcc->pcc_size);
+		LIST_FOREACH(pc, &percpu_list, pc_entry) {
+			const void *oldp = (const char *)pcc->pcc_data +
+			    pc->pc_offset;
+			void *newp = (char *)newpcc->pcc_data + pc->pc_offset;
+			(void)(*pc->pc_move)(newp, oldp, pc->pc_size);
+		}
 
 		/*
 		 * this assignment needs to be atomic for percpu_getptr_remote.
@@ -243,7 +248,7 @@ percpu_init_cpu(struct cpu_info *ci)
 }
 
 /*
- * percpu_alloc: allocate percpu storage
+ * percpu_alloc_movable: allocate percpu storage
  *
  * => called in thread context.
  * => considered as an expensive and rare operation.
@@ -251,19 +256,40 @@ percpu_init_cpu(struct cpu_info *ci)
  */
 
 percpu_t *
-percpu_alloc(size_t size)
+percpu_alloc_movable(size_t size, void *(*move)(void *, const void *, size_t))
 {
 	vmem_addr_t offset;
 	percpu_t *pc;
 
 	ASSERT_SLEEPABLE();
+	KASSERT(size <= UINT_MAX);
+
 	(void)vmem_alloc(percpu_offset_arena, size, VM_SLEEP | VM_BESTFIT,
 	    &offset);
-	pc = (percpu_t *)percpu_encrypt((uintptr_t)offset);
+
+	pc = kmem_alloc(sizeof(*pc), KM_SLEEP);
+	KASSERT(offset < percpu_nextoff);
+	KASSERT(offset <= UINT_MAX);
+	pc->pc_offset = offset;
+	pc->pc_size = size;
+	pc->pc_move = move;
+
+	mutex_enter(&percpu_allocation_lock);
+	LIST_INSERT_HEAD(&percpu_list, pc, pc_entry);
+	mutex_exit(&percpu_allocation_lock);
+
 	percpu_zero(pc, size);
+
 	return pc;
 }
 
+percpu_t *
+percpu_alloc(size_t size)
+{
+
+	return percpu_alloc_movable(size, &memcpy);
+}
+
 /*
  * percpu_free: free percpu storage
  *
@@ -276,7 +302,14 @@ percpu_free(percpu_t *pc, size_t size)
 {
 
 	ASSERT_SLEEPABLE();
+
 	vmem_free(percpu_offset_arena, (vmem_addr_t)percpu_offset(pc), size);
+
+	mutex_enter(&percpu_allocation_lock);
+	LIST_REMOVE(pc, pc_entry);
+	mutex_exit(&percpu_allocation_lock);
+
+	kmem_free(pc, sizeof(*pc));
 }
 
 /*
Index: sys/kern/subr_psref.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_psref.c,v
retrieving revision 1.7
diff -p -u -r1.7 subr_psref.c
--- sys/kern/subr_psref.c	1 Jun 2017 02:45:13 -0000	1.7
+++ sys/kern/subr_psref.c	2 Oct 2017 01:47:04 -0000
@@ -107,6 +107,26 @@ struct psref_cpu {
 };
 
 /*
+ * psref_cpu_move(new, old, size)
+ *
+ *	Move psref per-CPU data from old buffer to new buffer during
+ *	percpu_alloc.
+ */
+static void *
+psref_cpu_move(void *vnew, const void *vold, size_t size)
+{
+	struct psref_cpu *new = vnew;
+	const struct psref_cpu *old = vold;
+
+	KASSERT(size == sizeof(*new));
+	KASSERT(size == sizeof(*old));
+
+	LIST_MOVE(&old->pcpu_head, &new->pcpu_head);
+
+	return vnew;
+}
+
+/*
  * psref_class_create(name, ipl)
  *
  *	Create a new passive reference class, with the given wchan name
@@ -120,7 +140,8 @@ psref_class_create(const char *name, int
 	ASSERT_SLEEPABLE();
 
 	class = kmem_alloc(sizeof(*class), KM_SLEEP);
-	class->prc_percpu = percpu_alloc(sizeof(struct psref_cpu));
+	class->prc_percpu = percpu_alloc_movable(sizeof(struct psref_cpu),
+	    &psref_cpu_move);
 	mutex_init(&class->prc_lock, MUTEX_DEFAULT, ipl);
 	cv_init(&class->prc_cv, name);
 	class->prc_iplcookie = makeiplcookie(ipl);
Index: sys/sys/percpu.h
===================================================================
RCS file: /cvsroot/src/sys/sys/percpu.h,v
retrieving revision 1.3
diff -p -u -r1.3 percpu.h
--- sys/sys/percpu.h	9 Apr 2008 05:11:20 -0000	1.3
+++ sys/sys/percpu.h	2 Oct 2017 01:47:04 -0000
@@ -33,6 +33,8 @@
 
 void	percpu_init(void);
 void	percpu_init_cpu(struct cpu_info *);
+percpu_t *
+	percpu_alloc_movable(size_t, void *(*)(void *, const void *, size_t));
 percpu_t *percpu_alloc(size_t);
 void	percpu_free(percpu_t *, size_t);
 void	*percpu_getref(percpu_t *);