/* $NetBSD$ */
/*-
* Copyright (c) 2015--2016 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Taylor R. Campbell.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD$");
#include <sys/types.h>
#include <sys/condvar.h>
#include <sys/intr.h>
#include <sys/mutex.h>
#include <sys/percpu.h>
#include <sys/time.h>
#include <sys/xcall.h>
#ifdef __HAVE_CPU_COUNTER
#include <machine/cpu_counter.h>
#endif
#include <lib/libkern/keccak.h>
#define ENTROPY_MAXDEBT 32
#define ENTROPY_MAXEXTRACT 32
CTASSERT(ENTROPY_MAXEXTRACT <= ENTPOOL_OUTRATE_BYTES);
/*
* Per-CPU entropy state.
*/
struct entropy_cpu {
struct entpool ec_pool; /* entropy pool */
unsigned ec_credit; /* pending credit */
bool ec_locked; /* lock against hardintr:
* hardintr drops samples if
* locked, so we need not
* actually block hardintrs */
uint64_t ec_bytesdropped;
uint64_t ec_bytesentered;
};
/*
* Global state.
*/
struct percpu *entropy_percpu __read_mostly; /* struct entropy_cpu */
void *entropy_softint_cookie __read_mostly;
struct {
kmutex_t lock;
struct entpool pool;
unsigned debt;
kcondvar_t cv;
LIST_HEAD(krndsource) sources;
rndsave_t *seed;
bool initialized;
} entropy_global __cacheline_aligned;
/*
* Parameters.
*/
bool entropy_collection = ENTROPY_COLLECTION;
bool entropy_depletion = ENTROPY_DEPLETION;
bool entropy_hwrngtest = ENTROPY_HWRNGTEST;
/*
* XXX entropy_depletion is not supported by any cryptography theory
* and defends mainly against bogeymen.
*
* XXX rngtest is a nice side channel for storing what's going into the
* entropy pool on the kernel stack and in extra buffers, and exposing
* it to variable-time operations.
*/
static const struct sysctlnode *entropy_sysctlnode;
static struct sysctllog *entropy_sysctllog;
/*
* entropy_timer()
*
* Cycle counter, time counter, or anything that changes a wee bit
* unpredictably.
*/
static inline uint32_t
entropy_timer(void)
{
struct bintime bt;
uint32_t v;
#ifdef __HAVE_CPU_COUNTER
if (__predict_true(cpu_hascounter()))
return cpu_counter32();
#endif
if (__predict_false(cold))
return 0;
binuptime(&bt);
v = bt->bt_frac;
v ^= bt->bt_frac >> 32;
v ^= bt->bt_sec;
v ^= bt->bt_sec >> 32;
return v;
}
/*
* entropy_init()
*
* Initialize the entropy subsystem. Returns 0 on success, error
* on failure.
*/
int
entropy_init(void)
{
uint32_t extra[11];
unsigned i = 0;
int error;
/* Grab some cycle counts early at boot. */
extra[i++] = entropy_timer();
/* Allocate the per-CPU state. */
entropy_percpu = percpu_alloc(sizeof(struct entropy_cpu));
if (entropy_percpu == NULL) {
error = ENOMEM;
goto fail0;
}
extra[i++] = entropy_timer();
/* Establish the softint. */
entropy_softintcookie = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE,
&entropy_softintr, NULL);
if (entropy_softintcookie == NULL) {
error = ENOMEM;
goto fail1;
}
extra[i++] = entropy_timer();
/* Create the sysctl directory. */
sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlnode,
CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
SYSCTL_DESCR("Entropy (random number sources) options"),
NULL, 0, NULL, 0,
CTL_KERN, CTL_CREATE, CTL_EOL);
if (entropy_sysctlnode == NULL)
goto fail2;
extra[i++] = entropy_timer();
/* Create the sysctl knobs. */
/* XXX These probably shouldn't be writable at securelevel>0. */
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlnode, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collect",
SYSCTL_DESCR("Automatically collect entropy from hardware"),
NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
extra[i++] = entropy_timer();
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlnode, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "deplete",
SYSCTL_DESCR("Voodoo: `deplete' entropy pool when observed"),
NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
extra[i++] = entropy_timer();
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlnode, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "hwrngtest",
SYSCTL_DESCR("Statistically test output of hardware RNG devices"),
NULL, 0, &entropy_hwrngtest, 0, CTL_CREATE, CTL_EOL);
extra[i++] = entropy_timer();
/* Initialize the global state. */
mutex_init(&entropy_global.lock, MUTEX_DEFAULT, IPL_SOFTCLOCK);
extra[i++] = entropy_timer();
entropy_global.debt = ENTROPY_MAXDEBT;
extra[i++] = entropy_timer();
cv_init(&entropy_global.cv, "entropy");
extra[i++] = entropy_timer();
LIST_INIT(&entropy_global.sources);
extra[i++] = entropy_timer();
/* Enter the cycle counts we grabbed into the global pool. */
KASSERT(i == __arraycount(extra));
entpool_enter(&entropy_global.pool, extra, sizeof extra);
/* If the boot loader already provided a seed, use it. */
if (entropy_global.seed == NULL) {
rnd_printf("no entropy seed\n");
} else {
rndsave_t *const seed = entropy_global.seed;
entpool_enter(&entropy_global.pool, seed->data,
sizeof(seed->data));
entropy_global.debt -= MIN(entropy_global.debt,
seed->entropy/NBBY);
explicit_memset(seed, 0, sizeof(*seed));
}
/* Success! */
entropy_global.initialized = true;
return 0;
fail2: softint_disestablish(entropy_softintcookie);
fail1: percpu_free(entropy_percpu, sizeof(struct entropy_cpu));
fail0: KASSERT(error);
return error;
}
/*
* entropy_seed(seed, len)
*
* Seed the entropy pool with len bytes from seed. To be called
* via the bootloader before entropy_init.
*/
void
entropy_seed(void *seed, size_t len)
{
SHA1_CTX ctx;
uint8_t digest[SHA1_DIGEST_LENGTH];
if (len != sizeof(*seed)) {
rnd_printf("invalid seed length: %zu\n", len);
return;
}
SHA1Init(&ctx);
SHA1Update(&ctx, &seed->entropy, sizeof(seed->entropy));
SHA1Update(&ctx, &seed->data, sizeof(seed->data));
SHA1Final(digest, &ctx);
if (consttime_memcmp(digest, seed->digest, sizeof(digest)) != 0) {
rnd_printf("invalid seed checksum\n");
return;
}
if (entropy_global.initialized) {
mutex_enter(&entropy_global.lock);
if (entropy_global.seed) {
rnd_printf("already seeded\n");
return;
}
entropy_global.seed = seed;
entpool_enter(&entropy_global.pool, seed->data,
sizeof(seed->data));
entropy_global.debt -= MIN(entropy_global.debt,
seed->entropy/NBBY);
if (entropy_global.debt == 0)
cv_broadcast(&entropy_global.cv, &entropy_global.lock);
explicit_memset(seed, 0, sizeof(*seed));
mutex_exit(&entropy_global.lock);
} else {
if (entropy_global.seed) {
rnd_printf("already seeded\n");
return;
}
entropy_global.seed = seed;
}
}
/*
* entropy_bootrequest()
*
* Request entropy from all sources at boot, once config is
* complete and interrupts are running.
*/
void
entropy_bootrequest(void)
{
KASSERT(entropy_global.initialized);
/*
* Simply request enough to satisfy the maximum entropy debt.
* This is harmless overkill if the bootloader provided a seed.
*/
rndsources_request(ENTROPY_MAXDEBT);
}
/*
* entropy_credit(len, entrate)
*
* If entrate is nonzero, it is a lower bound on the estimated
* bits of data per bit of entropy in a len-byte sample recently
* fed into the system entropy pool. In that case, credit
* len/entrate bytes of entropy to the entropy debt. Otherwise,
* if entrate is zero, do nothing.
*/
static inline void
entropy_credit(size_t len, unsigned entrate)
{
/*
* If there's no entropy debt, or the entropy rate of the
* source is too low, do nothing.
*/
if (__predict_true(entropy_global.debt == 0 && !entropy_depletion))
return;
if ((entrate == 0) || (len/entrate == 0))
return;
/*
* Otherwise, credit to the entropy debt and wake waiters if we
* reduced it to zero.
*/
mutex_enter(&entropy_global.lock);
entropy_global.debt -= MIN(entropy_global.debt, len/entrate);
if (entropy_global.debt == 0)
/* XXX How to make sure /dev/urandom gets it ASAP? */
cv_broadcast(&entropy_global.cv);
mutex_exit(&entropy_global.lock);
}
/*
* entropy_enter(buf, len, entrate)
*
* Enter len bytes of data from buf into the system's entropy
* pool, permuting the state as necessary when the internal buffer
* fills up. If entrate is nonzero, it is a lower bound on the
* estimated bits of data per bit of entropy in buf. In that
* case, credit len/entrate bytes of entropy to the entropy debt.
*/
void
entropy_enter(const void *buf, size_t len, unsigned entrate)
{
const unsigned rate = ENTPOOL_INRATE_BYTES;
struct entropy_cpu *ec;
int s;
KASSERT(entropy_global.initialized);
KASSERTMSG(!cpu_intr_p(),
"use entropy_enter_intr from interrupt context");
/* Acquire the per-CPU state, blocking hard and soft interrupts. */
ec = percpu_getref(entropy_percpu);
s = splsoftclock();
KASSERT(!ec->ec_locked);
ec->ec_locked = true;
/* Enter into the per-CPU pool. */
entpool_enter(&ec->ec_pool, buf, len);
/* Release the per-CPU state. */
KASSERT(ec->ec_locked);
ec->ec_locked = false;
splx(s);
percpu_putref(entropy_percpu);
/* Credit any entropy. */
entropy_credit(len, entrate);
}
/*
* entropy_extract(buf, len, flags)
*
* Gather entropy on all CPUs into the global pool with a
* cross-call and extract an output from it. Flags may have:
*
* ENTROPY_WAIT Wait for entropy if not available yet.
* ENTROPY_SIG Allow interruption by a signal during wait.
*
* Return zero on success, or error on failure:
*
* EWOULDBLOCK No entropy and ENTROPY_WAIT not set.
* EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted.
*
* Always fill buf, even on error.
*/
int
entropy_extract(void *buf, size_t len, int flags)
{
uint32_t extra[5];
uint64_t ticket;
bool waited;
uint8_t wait;
int error;
ASSERT_SLEEPABLE();
KASSERT(entropy_global.initialized);
KASSERT(len <= ENTROPY_MAXEXTRACT);
if (!entropy_depletion)
waited = false;
/* Grab CPU number and cycle counter to mix extra into the pool. */
extra[i++] = cpu_number();
extra[i++] = entropy_timer();
/*
* First, wait until there is enough entropy in the system, and
* lock extraction.
*/
mutex_enter(&entropy_global.lock);
error = 0;
while (0 < entropy_global.debt) {
/* Request entropy from sources. */
mutex_exit(&entropy_global.lock);
rndsources_request(len);
mutex_enter(&entropy_global.lock);
/* If we got enough already, we're done. */
if (entropy_global.debt == 0) {
KASSERT(error == 0);
break;
}
/*
* Otherwise, wait for some to come in. If not doing
* entropy depletion, note that we will have already
* waited if we exit the loop.
*/
if (!entropy_depletion)
waited = true;
if (!ISSET(flags, ENTROPY_WAIT)) {
error = EWOULDBLOCK;
break;
}
if (ISSET(flags, ENTROPY_SIG)) {
error = cv_wait_sig(&entropy_global.cv,
&entropy_global.lock);
if (error)
break;
} else {
cv_wait(&entropy_global.cv, &entropy_global.lock);
}
}
mutex_exit(&entropy_global.lock);
/* Gather entropy on all CPUs with a broadcast xcall. */
ticket = xc_broadcast(0, &entropy_extract_xc, NULL, NULL);
extra[i++] = entropy_timer();
xc_wait(ticket);
extra[i++] = entropy_timer();
/*
* Mix the extra into the pool, extract the requested data, and
* unlock extraction.
*/
mutex_enter(&entropy_global.lock);
extra[i++] = entropy_timer();
KASSERT(i == __arraycount(extra));
entpool_enter(&entropy_global.pool, extra, sizeof extra);
entpool_extract(&entropy_global.pool, buf, len);
if (entropy_depletion) {
/*
* If we're depleting entropy, deplete it.
*
* We always deplete it in full rather than partially
* in an attempt to thwart iterative guessing attacks:
* if the actual entropy is less than the estimated
* entropy, it may be feasible for an attacker to guess
* the state by querying the entropy pool and trying
* guesses offline. In that case, if we only accrue a
* little more entropy before giving more out, the
* attacker must guess only that little more entropy in
* order to keep up.
*
* If, instead, we always gather another 256 bits of
* entropy, and somehow cease to be wrong in our
* estimate of how much entropy is coming in (...),
* the attacker will have a hard time keeping up.
*/
entropy_global.debt = ENTROPY_MAXDEBT;
} else {
/*
* If we don't deplete entropy, and we didn't wait for
* boot entropy, get an extra byte to decide whether to
* artificially wait and, if so, for how long.
*/
if (!waited) {
KASSERT(error == 0);
entpool_extract(&entropy_global.pool, &wait,
sizeof wait);
}
}
mutex_exit(&entropy_global.lock);
if (!entropy_depletion) {
/*
* If we're not depleting entropy, then after the boot
* entropy is available (if !waited), deliberately wait
* anyway in half the invocations (if wait & 1). The
* number of ticks to wait is distributed uniformly in
* [0, 128). This keeps both blocking and nonblocking
* code paths exercised.
*/
if (!waited) {
KASSERT(error == 0);
if (wait & 1) {
if (!ISSET(flags, ENTROPY_WAIT))
return EWOULDBLOCK;
error = kpause("entropy",
ISSET(flags, ENTROPY_SIG),
wait >> 1 /* timeout in ticks */,
NULL);
if (error)
return error;
}
}
}
return error;
}
/*
* entropy_extract_xc(arg1, arg2)
*
* Extract output from the local CPU's entropy pool and enter it
* into the global pool.
*/
static void
entropy_extract_xc(void *arg1 __unused, void *arg2 __unused)
{
struct entropy_cpu *ec;
uint8_t buf[ENTPOOL_OUTRATE_BYTES];
uint32_t extra[7];
unsigned i;
int s;
/* Grab CPU number and cycle counter to mix extra into the pool. */
extra[i++] = cpu_number();
extra[i++] = entropy_timer();
/* Acquire the per-CPU state, blocking hard and soft interrupts. */
ec = percpu_getref(entropy_percpu);
s = splsoftclock();
KASSERT(!ec->ec_locked);
ec->ec_locked = true;
extra[i++] = entropy_timer();
/* Extract the data. */
entpool_extract(&ec->ec_pool, buf, sizeof buf);
extra[i++] = entropy_timer();
/* Release the per-CPU state. */
KASSERT(!ec->ec_locked);
ec->ec_locked = false;
splx(s);
percpu_putref(entropy_percpu);
extra[i++] = entropy_timer();
/* Enter the data and extra into the global pool. */
mutex_enter(&entropy_global.lock);
extra[i++] = entropy_timer();
entpool_enter(&entropy_global.pool, buf, sizeof buf);
explicit_memset(buf, 0, sizeof buf);
extra[i++] = entropy_timer();
KASSERT(i == __arraycount(extra));
entpool_enter(&entropy_global.pool, extra, sizeof extra);
explicit_memset(extra, 0, sizeof extra);
mutex_exit(&entropy_global.lock);
}
/*
* entropy_enter_intr(buf, len, entrate)
*
* Enter up to len bytes of data from buf into the system's
* entropy pool. If entrate is nonzero, it is a lower bound on
* the estimated bits of data per bit of entropy in buf. In that
* case, credit n/entrate bytes of entropy to the entropy debt,
* where n is the number of bytes actually used from buf.
* Schedule a softint to stir the entropy pool if we filled the
* input buffer. Return the number of bytes actually used.
*
* Using this in thread context will work, but you might as well
* use entropy_enter in that case to contribute all the entropy.
*/
size_t
entropy_enter_intr(const void *buf, size_t len, unsigned entrate)
{
const unsigned rate = ENTPOOL_INRATE_BYTES;
struct entropy_cpu *ec;
size_t nused = 0;
int s;
KASSERT(entropy_global.initialized);
/*
* Acquire the per-CPU state. If someone is in the middle of
* using it, drop the sample. Otherwise, block higher-priority
* interrupts.
*/
ec = percpu_getref(entropy_percpu);
if (ec->ec_locked)
goto out;
ec->ec_locked = true;
/* Enter as much as we can into the per-CPU pool. */
nused = entpool_enter_nostir(&ec->ec_pool, buf, len);
/* If not all got used, need to stir. */
/* XXX What if all got used, but we filled the buffer? Plugh. */
if (nused < len)
softint_schedule(entropy_softintcookie);
/* If there's entropy debt and we provided any data, credit it. */
if (__predict_false(entropy_depletion || 0 < entropy_global.debt) &&
0 < entrate && 0 < nused/entrate) {
softint_schedule(entropy_softintcookie);
ec->ec_credit += MIN(ENTROPY_MAXDEBT - ec->ec_credit,
nused/entrate);
}
out: ec->ec_bytesdropped += len - nused;
ec->ec_bytesentered += nused;
/* Release the per-CPU state. */
KASSERT(ec->ec_locked);
ec->ec_locked = false;
percpu_putref(entropy_percpu);
return nused;
}
/*
* entropy_softintr(cookie)
*
* Softint handler for processing interrupt entropy. Stir the
* pool and credit the entropy if the interrupt provided any.
*/
static void
entropy_softintr(void *cookie __unused)
{
struct entropy_cpu *ec;
unsigned credit;
/* Grab the per-CPU entropy state and lock out interrupts. */
ec = percpu_getref(entropy_percpu);
KASSERT(!ec->ec_locked);
ec->ec_locked = true;
/* Stir the pool. */
entpool_stir(&ec->ec_pool);
/* Find the amount of entropy to credit. */
credit = ec->ec_credit;
ec->ec_credit = 0;
/* Unlock interrupts and release the per-CPU state. */
KASSERT(ec->ec_locked);
ec->ec_locked = false;
percpu_putref(entropy_percpu);
/* Credit to the entropy debt. */
entropy_credit(credit, 1);
}
/*
* Entropy sources. Usage:
*
* - Allocate and zero a struct krndsource.
* - Optionally, set a callback with rndsource_setcb.
* - Attach it with rnd_attach_source.
* - Use rnd_add_data with it.
* - Detach it with rnd_detach_source.
* - Free the struct krndsource.
*/
/*
* rndsource_setcb(rs, request, requestarg)
*
* Set the request callback for the entropy source rs, if it can
* provide entropy on demand. Must be done before
* rnd_attach_source.
*/
void
rndsource_setcb(struct krndsource *rs, void (*request)(size_t, void *),
void *requestarg)
{
rs->rs_request = request;
rs->rs_requestarg = requestarg;
}
/*
* rnd_attach_source(rs, name, type, flags)
*
* Attach the entropy source rs. Must be done after
* rndsource_setcb, if any, and before any calls to rnd_add_data.
*/
void
rnd_attach_source(struct krndsource *rs, const char *name, int type, int flags)
{
uint32_t extra[8];
unsigned i = 0;
uint8_t *cotest = NULL;
rngtest_t *rngtest = NULL;
extra[i++] = cpu_number();
extra[i++] = entropy_timer();
flags |= RND_FLAG_NO_ESTIMATE;
switch (type) {
case RND_TYPE_NET:
flags |= RND_FLAG_NO_COLLECT;
break;
case RND_TYPE_RNG:
cotest = kmem_alloc(HWRNG_COTEST_BYTES, KM_SLEEP);
rngtest = kmem_alloc(sizeof(*rngtest), KM_SLEEP);
strlcpy(rngtest->rt_name, name, sizeof(rngtest->rt_name));
break;
}
extra[i++] = entropy_timer();
refcount_init(&rs->rs_refcount);
strlcpy(rs->rs_name, name, sizeof(rs->rs_name));
rs->rs_type = type;
rs->rs_flags = flags;
rs->rs_bad = 0;
rs->rs_bits = 0;
rs->rs_rngtest = rngtest;
rs->rs_rngtest_resid = (rngtest == NULL ? 0 : FIPS140_RNG_TEST_BYTES);
rs->rs_cotest = cotest;
rs->rs_cotest_resid = (cotest == NULL ? 0 : HWRNG_COTEST_BYTES);
extra[i++] = entropy_timer();
mutex_enter(&entropy_global.lock);
extra[i++] = entropy_timer();
LIST_INSERT_HEAD(&entropy_global.sources, rs, rs_list);
extra[i++] = entropy_timer();
mutex_exit(&entropy_global.lock);
extra[i++] = entropy_timer();
rndsource_request(rs);
extra[i++] = entropy_timer();
KASSERT(i == __arraycount(extra));
entropy_enter(extra, sizeof extra, 0);
}
/*
* rnd_detach_source(rs)
*
* Detach the entropy source rs. May sleep waiting for users to
* drain. Further use is not allowed.
*/
void
rnd_detach_source(struct krndsource *rs)
{
/* We may wait for other users drain. */
ASSERT_SLEEPABLE();
/* Remove it from the list and wait for users to drain. */
mutex_enter(&entropy_global.lock);
LIST_REMOVE(rs, rs_list);
refcount_dec_drain(&rs->rs_refcount, &entropy_global.lock,
&entropy_global.cv);
mutex_exit(&entropy_global.lock);
/* Free the rngtest if there was one. */
if (rs->rs_rngtest != NULL) {
explicit_memset(rs->rs_rngtest, 0, sizeof(*rs->rs_rngtest));
kmem_free(rs->rs_rngtest, sizeof(*rs->rs_rngtest));
rs->rs_rngtest = NULL; /* paranoia */
}
}
/*
* rndsources_request(n)
*
* Request n bytes of entropy from all sources in the system. OK
* if we overdo it.
*/
static void
rndsources_request(size_t n)
{
struct krndsource *rs, *next;
/* Walk the list of sources under the global entropy state lock. */
mutex_enter(&entropy_global.lock);
LIST_FOREACH_SAFE(rs, &entropy_global.sources, rs_list, next) {
/* Try to acquire a reference. If we can't, skip it. */
if (refcount_inc(&rs->rs_refcount) != 0)
continue;
/* Drop the lock while we call the callback. */
mutex_exit(&entropy_global.lock);
rndsource_request(rs, n);
mutex_enter(&entropy_global.lock);
/*
* Release the reference. If we held the last one,
* notify rnd_detach_source. We use
* refcount_dec_local, not refcount_dec_lock or
* refcount_dec_broadcast, because all access is
* serialized under the lock already.
*/
if (refcount_dec_local(&rs->rs_refcount))
cv_broadcast(&entropy_global.cv);
}
mutex_exit(&entropy_global.lock);
}
/*
* rndsource_request(rs, n)
*
* Request that rs try to contribute n bytes to the entropy pool.
*/
static void
rndsource_request(struct krndsource *rs, size_t n)
{
size_t nreq = n;
KASSERT(refcount_referenced_p(&rs->rs_refcount));
if (rs->rs_type == RND_TYPE_RNG) {
/*
* Hardware entropy sources enter data in units of
* COTEST_BYTES, and must first enter
* FIPS140_RNG_TEST_BYTES for testing.
*
* XXX If entropy_hwrngtest is switched from off to on,
* there is a window in which a request here will fail
* to ask for enough bytes.
*/
if (entropy_hwrngtest && rs->rs_rngtest_resid) {
if (nreq > SIZE_MAX - FIPS140_RNG_TEST_BYTES)
nreq = SIZE_MAX;
else
nreq += FIPS140_RNG_TEST_BYTES;
}
if (entropy_hwrngtest) {
if (nreq > SIZE_MAX - COTEST_BYTES)
nreq = SIZE_MAX;
else
nreq = roundup(nreq, COTEST_BYTES);
}
}
/* Call the callback with the new requested number of bytes. */
(*rs->rs_request)(nreq, rs->rs_requestarg);
}
/*
* rnd_add_uint32(rs, value)
*
* Enter 32 bits of data from an entropy source into the pool.
*
* If rs is NULL, may not be called from interrupt context.
*
* If rs is non-NULL, may be called from any context. May drop
* data if called from interrupt context.
*/
void
rnd_add_uint32(struct krndsource *rs, uint32_t value)
{
rnd_add_data(rs, &value, sizeof value, 0);
}
/*
* rnd_add_data(rs, buf, len, entropybits)
*
* Enter data from an entropy source into the pool.
*
* If rs is NULL, may not be called from interrupt context.
*
* If rs is non-NULL, may be called from any context. May drop
* data if called from interrupt context.
*/
void
rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
uint32_t entropybits)
{
uint32_t extra[3];
unsigned i = 0;
unsigned entrate;
size_t nused;
/* If there's no rndsource, just enter the data and time now. */
if (rs == NULL) {
extra[i++] = cpu_number();
extra[i++] = entropy_timer();
entrate = NBBY*(len/entropybits);
entropy_enter(buf, len, entrate);
extra[i++] = entropy_timer();
KASSERT(i == __arraycount(extra));
entropy_enter(extra, sizeof extra, 0);
return;
}
/*
* Skip if:
* - we're not collecting entropy, or
* - this source has been marked bad, or
* - the operator doesn't want to collect entropy from this, or
* - neither data nor timings are being collected from this.
*/
if (!entropy_collection ||
rs->rs_bad ||
ISSET(rs->rs_flags, RND_FLAG_NO_COLLECT) ||
!ISSET(rs->rs_flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
return;
/* Hardware RNG devices get special treatment. */
if (rs->rs_type == RND_TYPE_RNG && entropy_hwrngtest) {
hwrng_add_data(rs, buf, len, entrate);
return;
}
/* Grab CPU number and cycle counter to mix extra into the pool. */
extra[i++] = cpu_number();
extra[i++] = entropy_timer();
/* If we are collecting data, enter them. */
if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_VALUE)) {
entrate = NBBY*(len/entropybits);
if (cpu_intr_p()) {
nused = entropy_enter_intr(buf, len, entrate);
} else {
entropy_enter(buf, len, entrate);
nused = len;
}
if (0 < entrate)
rs->rs_bits += NBBY*nused/entrate;
}
extra[i++] = entropy_timer();
/* If we are collecting timings, enter them. */
KASSERT(i == __arraycount(extra));
if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_TIME)) {
if (cpu_intr_p())
(void)entropy_enter_intr(extra, sizeof extra, 0);
else
entropy_enter(extra, sizeof extra, 0);
}
}
/*
* hwrng_add_data(rs, buf, len, entropybits)
*
* Subject hardware RNGs to statistical tests before allowing
* anything from them into the entropy pool.
*/
static void
hwrng_add_data(struct krndsource *rs, const void *buf, size_t len,
uint32_t entropybits)
{
const unsigned entrate = NBBY*(len/entropybits);
uint32_t extra = entropy_timer();
const uint8_t *p = buf;
size_t n = len;
size_t s, m;
int r;
KASSERT(rs->rs_cotest != NULL);
KASSERT(rs->rs_rngtest != NULL);
/*
* Continuous output test: gather buffers of HWRNG_COTEST_BYTES
* and make sure the first half doesn't match the second half
* each time we fill the buffer before contributing the buffer
* to the entropy pool.
*/
KASSERT(0 < rs->rs_cotest_resid);
while (rs->rs_cotest_resid <= n) {
/* Fill the cotest buffer and check it. */
s = HWRNG_COTEST_BYTES - rs->rs_cotest_resid;
memcpy(rs->rs_cotest + s, p, rs->rs_cotest_resid);
p += rs->rs_cotest_resid;
n -= rs->rs_cotest_resid;
m = HWRNG_COTEST_BYTES/2;
if (consttime_memcmp(rs->rs_cotest, rs->rs_cotest + m, m)
== 0) {
rnd_printf("%s: continuous output test failed\n",
rs->rs_name);
rs->rs_bad = 1;
return;
}
/*
* We filled a buffer and passed the continuous-output
* test. If we haven't applied a FIPS-140 statistical
* test, contribute it to the rngtest buffer.
*/
if (0 < rs->rs_rngtest_resid) {
s = FIPS140_RNG_TEST_BYTES - rs->rs_rngtest_resid;
m = MIN(rs->rs_rngtest_resid, HWRNG_COTEST_BYTES);
memcpy(rs->rs_rngtest->rt_b + s, rs->rs_cotest, m);
rs->rs_rngtest_resid -= m;
/* If we filled the rngtest buffer, test it. */
if (rs->rs_rngtest_resid == 0) {
if (rngtest(rs->rs_rngtest)) {
rnd_printf("%s"
": failed statistical test",
rs->rs_name);
rs->rs_bad = 1;
return;
}
}
}
/* Enter the data and/or timing as requested. */
if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_VALUE)) {
/*
* Assume the hardware has a constant entropy
* rate and compute it from the current sample.
*/
m = HWRNG_COTEST_BYTES;
if (cpu_intr_p())
(void)entropy_enter_intr(rs->rs_cotest, m,
entrate);
else
entropy_enter(rs->rs_cotest, m, entrate);
}
if (ISSET(rs->rs_flags, RND_FLAG_COLLECT_TIME)) {
if (cpu_intr_p())
(void)entropy_enter_intr(&extra, sizeof extra,
0);
else
entropy_enter(&extra, sizeof extra, 0);
extra = entropy_timer();
}
/* Don't let the inputs to the entropy pool hang around. */
explicit_memset(rs->rs_cotest, 0, HWRNG_COTEST_BYTES);
rs->rs_cotest_resid = HWRNG_COTEST_BYTES;
}
/*
* Can't fill the cotest buffer. Put what we have left in it
* and be done.
*/
s = HWRNG_COTEST_BYTES - rs->rs_cotest_resid;
memcpy(rs->rs_cotest + s, p, n);
rs->rs_cotest_resid -= n;
KASSERT(0 < rs->rs_cotest_resid);
}