Index: sys/uvm/uvm_swap.c =================================================================== RCS file: /cvsroot/src/sys/uvm/uvm_swap.c,v retrieving revision 1.175 diff -p -u -r1.175 uvm_swap.c --- sys/uvm/uvm_swap.c 28 Oct 2017 00:37:13 -0000 1.175 +++ sys/uvm/uvm_swap.c 7 Dec 2017 07:30:42 -0000 @@ -41,6 +41,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v #include #include #include +#include #include #include #include @@ -142,6 +143,11 @@ struct swapdev { int swd_maxactive; /* max active i/o reqs */ struct bufq_state *swd_tab; /* buffer list */ int swd_active; /* number of active buffers */ + + unsigned *swd_encrypted; /* bit map of encrypted pgs */ + unsigned swd_nencrypted; /* # of encrypted pgs */ + uint8_t swd_encryptkey[1024/8]; + /* Threefish1024 key */ }; /* @@ -223,6 +229,7 @@ static struct workqueue *sw_reg_workqueu /* tuneables */ u_int uvm_swapisfull_factor = 99; +bool uvm_swap_encrypt = false; /* * prototypes @@ -244,6 +251,9 @@ static void sw_reg_start(struct swapdev static int uvm_swap_io(struct vm_page **, int, int, int); +static void swencrypt(struct swapdev *, struct buf *); +static void swdecrypt(struct swapdev *, struct buf *); + /* * uvm_swap_init: init the swap system data structures and locks * @@ -1009,6 +1019,12 @@ swap_on(struct lwp *l, struct swapdev *s } } + sdp->swd_encrypted = kmem_alloc(howmany(size, + CHAR_BIT*sizeof(sdp->swd_encrypted[0])), KM_SLEEP); + sdp->swd_nencrypted = 0; + cprng_strong(kern_cprng, sdp->swd_encryptkey, + sizeof sdp->swd_encryptkey, 0); + sdp->swd_drumoffset = (int)result; sdp->swd_drumsize = npages; sdp->swd_npages = size; @@ -1113,6 +1129,10 @@ swap_off(struct lwp *l, struct swapdev * /* * free all resources! */ + explicit_memset(sdp->swd_encryptkey, 0, sizeof sdp->swd_encryptkey); + KASSERT(sdp->swd_nencrypted == 0); + kmem_free(sdp->swd_encrypted, howmany(sdp->swd_npages, + CHAR_BIT*sizeof(sdp->swd_encrypted[0]))); vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); blist_destroy(sdp->swd_blist); bufq_free(sdp->swd_tab); @@ -1203,6 +1223,10 @@ swstrategy(struct buf *bp) return; } + if (uvm_swap_encrypt && (bp->b_flags & (B_READ|B_WRITE)) == B_WRITE) + swencrypt(sdp, bp); + + /* * convert drum page number to block number on this swapdev. */ @@ -1765,6 +1789,30 @@ uvm_swap_free(int startslot, int nslots) blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); sdp->swd_npginuse -= nslots; uvmexp.swpginuse -= nslots; + + /* + * Mark any encrypted slots as no longer encrypted and decrease + * the number of encrypted slots. + */ + if (sdp->swd_nencrypted) { + const unsigned bpw = CHAR_BIT * sizeof(sdp->swd_encrypted[0]); + int slot = startslot; + + for (slot -= sdp->swd_drumoffset; nslots --> 0; slot++) { + const unsigned word = slot / bpw; + const unsigned bit = slot % bpw; + + if (sdp->swd_encrypted[word] & __BIT(bit)) { + KASSERT(sdp->swd_nencrypted); + sdp->swd_nencrypted--; + sdp->swd_encrypted[word] &= ~__BIT(bit); + } + } + + if (sdp->swd_nencrypted == 0) + explicit_memset(sdp->swd_encryptkey, 0, + sizeof sdp->swd_encryptkey); + } mutex_exit(&uvm_swap_data_lock); } @@ -1837,6 +1885,8 @@ uvm_swap_io(struct vm_page **pps, int st write = (flags & B_READ) == 0; async = (flags & B_ASYNC) != 0; + KASSERT(write || !async); + /* * allocate a buf for the i/o. */ @@ -1919,6 +1969,28 @@ uvm_swap_io(struct vm_page **pps, int st error = biowait(bp); + if (error) + goto out; + + /* + * XXX Can't skip this even if uvm_swap_encrypt == false + * because it might have once been true, in which case some + * swap slots may still be encrypted. + */ + if (!write) { + struct swapdev *sdp; + int pageno; + + pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; + mutex_enter(&uvm_swap_data_lock); + sdp = swapdrum_getsdp(pageno); + mutex_exit(&uvm_swap_data_lock); + + KASSERT(sdp != NULL); + swdecrypt(sdp, bp); + } + +out: /* * kill the pager mapping */ @@ -1939,3 +2011,122 @@ uvm_swap_io(struct vm_page **pps, int st return (error); } + +/* + * swencrypt(sdp, bp) + * + * Generate a key for sdp if it currently has no encrypted pages. + * Encrypt bp in place with it, mark its corresponding pages as + * encrypted, and account the number of encrypted pages. + */ +void threefish1024_encrypt(const uint8_t[1024/8], const uint8_t[16], + const uint8_t[1024/8], uint8_t[1024/8]); +static void +swencrypt(struct swapdev *sdp, struct buf *bp) +{ + const unsigned bpw = CHAR_BIT * sizeof(sdp->swd_encrypted[0]); + const uint8_t *const K = sdp->swd_encryptkey; + uint8_t *buf = bp->b_data; + int startslot = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; + int nslots = bp->b_bcount >> PAGE_SHIFT; + int i, j; + + KASSERT((bp->b_bcount % PAGE_SIZE) == 0); + KASSERT((PAGE_SIZE % (1024/8)) == 0); + KASSERT((bp->b_bcount % (1024/8)) == 0); + + startslot -= sdp->swd_drumoffset; + + mutex_enter(&uvm_swap_data_lock); + + /* If there are no encrypted pages now, get a fresh key. */ + if (sdp->swd_nencrypted == 0) + cprng_strong(kern_cprng, sdp->swd_encryptkey, + sizeof sdp->swd_encryptkey, 0); + + /* Mark all of the pages encrypted. */ + KASSERT(nslots < sdp->swd_npginuse); + KASSERT(sdp->swd_nencrypted < sdp->swd_npginuse - nslots); + for (i = 0; i < nslots; i++) { + const int slot = startslot + i; + const unsigned word = slot / bpw; + const unsigned bit = slot % bpw; + + KASSERT((sdp->swd_encrypted[word] & __BIT(bit)) == 0); + sdp->swd_encrypted[word] |= __BIT(bit); + } + + /* Count the encrypted pages. */ + sdp->swd_nencrypted += nslots; + + mutex_exit(&uvm_swap_data_lock); + + /* Encrypt all the pages. */ + for (i = 0; i < nslots; i++) { + const int slot = startslot + i; + + /* Encrypt all the 1024-bit blocks in the page. */ + for (j = 0; j < PAGE_SIZE/(1024/8); j++) { + const uint64_t T[2] = { slot, j }; + const uint8_t *P = &buf[i*PAGE_SIZE + j*1024/8]; + uint8_t *C = &buf[i*PAGE_SIZE + j*1024/8]; + + /* + * Native byte order for tweak since this is + * not a persistent format. + */ + threefish1024_encrypt(K, (const uint8_t *)T, P, C); + } + } +} + +/* + * swdecrypt(sdp, bp) + * + * Decrypt any pages in bp that are encrypted. + */ +void threefish1024_decrypt(const uint8_t[1024/8], const uint8_t[16], + const uint8_t[1024/8], uint8_t[1024/8]); +static void +swdecrypt(struct swapdev *sdp, struct buf *bp) +{ + const unsigned bpw = CHAR_BIT * sizeof(sdp->swd_encrypted[0]); + const uint8_t *const K = sdp->swd_encryptkey; + uint8_t *buf = bp->b_data; + int startslot = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; + int nslots = bp->b_bcount >> PAGE_SHIFT; + int i, j; + + KASSERT((bp->b_bcount % PAGE_SIZE) == 0); + KASSERT((PAGE_SIZE % (1024/8)) == 0); + KASSERT((bp->b_bcount % (1024/8)) == 0); + + startslot -= sdp->swd_drumoffset; + + /* Go through each of the pages. */ + for (i = 0; i < nslots; i++) { + const int slot = startslot + i; + const unsigned word = slot / bpw; + const unsigned bit = slot % bpw; + + /* Check whether it's encrypted. If not, skip it. */ + if ((sdp->swd_encrypted[word] & __BIT(bit)) == 0) + continue; + + /* There had better be a positive count of encrypted pages. */ + KASSERT(sdp->swd_nencrypted > 0); + + /* Decrypt all the 1024-bit blocks in the page. */ + for (j = 0; j < PAGE_SIZE/(1024/8); j++) { + const uint64_t T[2] = { slot, j }; + const uint8_t *C = &buf[i*PAGE_SIZE + j*1024/8]; + uint8_t *P = &buf[i*PAGE_SIZE + j*1024/8]; + + /* + * Native byte order for tweak since this is + * not a persistent format. + */ + threefish1024_decrypt(K, (const uint8_t *)T, C, P); + } + } +}