From 4dcb7c7efdaa92437647c88e911fa028c475f9f5 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Tue, 4 Feb 2020 16:47:12 +0000 Subject: [PATCH 1/3] Include a smaller header file in to reduce recompilation. --- sys/net/if.h | 3 +-- sys/net/if_stats.c | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/net/if.h b/sys/net/if.h index 46041cbb245c..43087af42f29 100644 --- a/sys/net/if.h +++ b/sys/net/if.h @@ -226,10 +226,9 @@ struct ifqueue { }; #ifdef _KERNEL -#include +#include #include #include - #endif /* _KERNEL */ /* diff --git a/sys/net/if_stats.c b/sys/net/if_stats.c index a1a7dcd574d5..902160dc870f 100644 --- a/sys/net/if_stats.c +++ b/sys/net/if_stats.c @@ -34,6 +34,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_stats.c,v 1.1 2020/01/29 03:16:28 thorpej Exp $") #include #include +#include #include #include From 75da4efcb7fc100d3f0289b18b5f354ff0afc818 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Tue, 4 Feb 2020 16:47:51 +0000 Subject: [PATCH 2/3] Add percpu_foreach_xcall(xcflags, percpu, callback, cookie). Calls callback(per-cpu ptr, cookie, curcpu()) on each CPU with an xcflags cross-call -- e.g., XC_HIGHPRI_IPL(IPL_NET). Documentation to come. --- sys/kern/subr_percpu.c | 37 +++++++++++++++++++++++++++++++++++++ sys/sys/percpu.h | 1 + 2 files changed, 38 insertions(+) diff --git a/sys/kern/subr_percpu.c b/sys/kern/subr_percpu.c index 4cd8a3a1529f..56b4404e8d74 100644 --- a/sys/kern/subr_percpu.c +++ b/sys/kern/subr_percpu.c @@ -430,3 +430,40 @@ percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg) } percpu_traverse_exit(); } + +/* + * percpu_foreach_xcall: call the specified callback function on each cpu. + * + * => calls are serialized with one another. + * => caller should not rely on the cpu iteration order. + * => callback function MUST NOT sleep, not even on adaptive locks. + */ + +struct percpu_foreach_xcall { + percpu_callback_t cb; + void *cookie; +}; + +static void +foreach_xc(void *vpercpu, void *vpfx) +{ + struct percpu *percpu = vpercpu; + struct percpu_foreach_xcall *pfx = vpfx; + void *ptr; + + ptr = percpu_getref(percpu); + (*pfx->cb)(ptr, pfx->cookie, curcpu()); + percpu_putref(percpu); +} + +void +percpu_foreach_xcall(unsigned xcflags, struct percpu *percpu, + percpu_callback_t cb, void *cookie) +{ + struct percpu_foreach_xcall pfx = { .cb = cb, .cookie = cookie }; + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + + for (CPU_INFO_FOREACH(cii, ci)) + xc_wait(xc_unicast(xcflags, &foreach_xc, percpu, &pfx, ci)); +} diff --git a/sys/sys/percpu.h b/sys/sys/percpu.h index d15341414682..d19b760534c2 100644 --- a/sys/sys/percpu.h +++ b/sys/sys/percpu.h @@ -40,6 +40,7 @@ void percpu_putref(percpu_t *); typedef void (*percpu_callback_t)(void *, void *, struct cpu_info *); void percpu_foreach(percpu_t *, percpu_callback_t, void *); +void percpu_foreach_xcall(unsigned, percpu_t *, percpu_callback_t, void *); percpu_t *percpu_create(size_t, percpu_callback_t, percpu_callback_t, void *); From 1c7f344d2f1b7080f177b4841d593ab6ccd8b2d5 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Tue, 4 Feb 2020 16:51:17 +0000 Subject: [PATCH 3/3] Use percpu_foreach_xcall(XC_HIGHPRI_IPL(IPL_NET)). This is called from: assert_sleepable+0xc8 percpu_foreach+0x20 if_stats_to_if_data+0x44 if_export_if_data+0x18 rt_ifmsg+0x74 if_link_state_change_softint+0xb4 if_link_state_change_si+0x68 softint_dispatch+0x300 Can't sleep in softint! Gotta use a high-priority xcall. --- sys/net/if_stats.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sys/net/if_stats.c b/sys/net/if_stats.c index 902160dc870f..be243f91f703 100644 --- a/sys/net/if_stats.c +++ b/sys/net/if_stats.c @@ -36,6 +36,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_stats.c,v 1.1 2020/01/29 03:16:28 thorpej Exp $") #include #include #include +#include #include @@ -85,8 +86,6 @@ if_stats_to_if_data_cb(void *v1, void *v2, struct cpu_info *ci) const uint64_t * const local_counters = v1; struct if_stats_to_if_data_ctx *ctx = v2; - int s = splnet(); - if (ctx->ifi) { ctx->ifi->ifi_ipackets += local_counters[if_ipackets]; ctx->ifi->ifi_ierrors += local_counters[if_ierrors]; @@ -104,8 +103,6 @@ if_stats_to_if_data_cb(void *v1, void *v2, struct cpu_info *ci) if (ctx->zero_stats) { memset(v1, 0, IF_STATS_SIZE); } - - splx(s); } /* @@ -116,7 +113,7 @@ if_stats_to_if_data_cb(void *v1, void *v2, struct cpu_info *ci) */ void if_stats_to_if_data(ifnet_t * const ifp, struct if_data * const ifi, - const bool zero_stats) + const bool zero_stats) { struct if_stats_to_if_data_ctx ctx = { .ifi = ifi, @@ -124,7 +121,8 @@ if_stats_to_if_data(ifnet_t * const ifp, struct if_data * const ifi, }; memset(ifi, 0, sizeof(*ifi)); - percpu_foreach(ifp->if_stats, if_stats_to_if_data_cb, &ctx); + percpu_foreach_xcall(XC_HIGHPRI_IPL(IPL_NET), ifp->if_stats, + if_stats_to_if_data_cb, &ctx); } #else /* ! __IF_STATS_PERCPU */