# HG changeset patch # User Taylor R Campbell # Date 1589931391 0 # Tue May 19 23:36:31 2020 +0000 # Branch trunk # Node ID 98bf588501cb99babdd717ccc0da97c2a71de3a5 # Parent 32735fb57443cc6d26b4f38fc6d969eb695a6f47 # EXP-Topic riastradh-swapencrypt-mpsafe Try to make swap encryption MP-safe. diff -r 32735fb57443 -r 98bf588501cb sys/uvm/uvm_swap.c --- a/sys/uvm/uvm_swap.c Mon May 18 09:52:30 2020 +0000 +++ b/sys/uvm/uvm_swap.c Tue May 19 23:36:31 2020 +0000 @@ -147,7 +147,7 @@ struct swapdev { struct bufq_state *swd_tab; /* buffer list */ int swd_active; /* number of active buffers */ - uint8_t *swd_encmap; /* bitmap of encrypted slots */ + volatile uint32_t *swd_encmap; /* bitmap of encrypted slots */ keyInstance swd_enckey; /* AES key expanded for enc */ keyInstance swd_deckey; /* AES key expanded for dec */ bool swd_encinit; /* true if keys initialized */ @@ -304,6 +304,9 @@ swaplist_insert(struct swapdev *sdp, str struct swappri *spp, *pspp; UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); + KASSERT(rw_write_held(&swap_syscall_lock)); + KASSERT(mutex_owned(&uvm_swap_data_lock)); + /* * find entry at or after which to insert the new device. */ @@ -356,6 +359,10 @@ swaplist_find(struct vnode *vp, bool rem struct swapdev *sdp; struct swappri *spp; + KASSERT(rw_lock_held(&swap_syscall_lock)); + KASSERT(remove ? rw_write_held(&swap_syscall_lock) : 1); + KASSERT(mutex_owned(&uvm_swap_data_lock)); + /* * search the lists for the requested vp */ @@ -386,6 +393,9 @@ swaplist_trim(void) { struct swappri *spp, *nextspp; + KASSERT(rw_write_held(&swap_syscall_lock)); + KASSERT(mutex_owned(&uvm_swap_data_lock)); + LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp) { if (!TAILQ_EMPTY(&spp->spi_swapdev)) continue; @@ -407,6 +417,8 @@ swapdrum_getsdp(int pgno) struct swapdev *sdp; struct swappri *spp; + KASSERT(mutex_owned(&uvm_swap_data_lock)); + LIST_FOREACH(spp, &swap_priority, spi_swappri) { TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { if (sdp->swd_flags & SWF_FAKE) @@ -420,6 +432,23 @@ swapdrum_getsdp(int pgno) return NULL; } +/* + * swapdrum_sdp_is: true iff the swap device for pgno is sdp + * + * => for use in positive assertions only; result is not stable + */ +static bool __debugused +swapdrum_sdp_is(int pgno, struct swapdev *sdp) +{ + bool result; + + mutex_enter(&uvm_swap_data_lock); + result = swapdrum_getsdp(pgno) == sdp; + mutex_exit(&uvm_swap_data_lock); + + return result; +} + void swapsys_lock(krw_t op) { rw_enter(&swap_syscall_lock, op); @@ -904,7 +933,8 @@ swap_on(struct lwp *l, struct swapdev *s * allocate space to for swap encryption state and mark the * keys uninitialized so we generate them lazily */ - sdp->swd_encmap = kmem_zalloc(howmany(npages, NBBY), KM_SLEEP); + size_t bpw = NBBY * sizeof(sdp->swd_encmap[0]); + sdp->swd_encmap = kmem_zalloc(howmany(npages, bpw), KM_SLEEP); sdp->swd_encinit = false; /* @@ -1011,6 +1041,9 @@ swap_off(struct lwp *l, struct swapdev * UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); UVMHIST_LOG(pdhist, " dev=%jx, npages=%jd", sdp->swd_dev,npages, 0, 0); + KASSERT(rw_write_held(&swap_syscall_lock)); + KASSERT(mutex_owned(&uvm_swap_data_lock)); + /* disable the swap area being removed */ sdp->swd_flags &= ~SWF_ENABLE; uvmexp.swpgavail -= npages; @@ -1079,7 +1112,9 @@ swap_off(struct lwp *l, struct swapdev * vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); blist_destroy(sdp->swd_blist); bufq_free(sdp->swd_tab); - kmem_free(sdp->swd_encmap, howmany(sdp->swd_npages, NBBY)); + size_t bpw = NBBY * sizeof(sdp->swd_encmap[0]); + kmem_free(__UNVOLATILE(sdp->swd_encmap), + howmany(sdp->swd_npages, bpw)); explicit_memset(&sdp->swd_enckey, 0, sizeof sdp->swd_enckey); explicit_memset(&sdp->swd_deckey, 0, sizeof sdp->swd_deckey); kmem_free(sdp, sizeof(*sdp)); @@ -1228,7 +1263,6 @@ swstrategy(struct buf *bp) sw_reg_strategy(sdp, bp, bn); return; } - /* NOTREACHED */ } /* @@ -1856,6 +1890,7 @@ uvm_swap_io(struct vm_page **pps, int st * 4. KEY, ENCRYPTION: Encrypt and mark the slots * encrypted. */ + mutex_enter(&uvm_swap_data_lock); sdp = swapdrum_getsdp(startslot); if (!sdp->swd_encinit) { if (!swap_encrypt) @@ -1863,26 +1898,29 @@ uvm_swap_io(struct vm_page **pps, int st uvm_swap_genkey(sdp); } KASSERT(sdp->swd_encinit); + mutex_exit(&uvm_swap_data_lock); if (swap_encrypt) { for (i = 0; i < npages; i++) { int s = startslot + i; - KDASSERT(swapdrum_getsdp(s) == sdp); + KDASSERT(swapdrum_sdp_is(s, sdp)); KASSERT(s >= sdp->swd_drumoffset); s -= sdp->swd_drumoffset; KASSERT(s < sdp->swd_drumsize); uvm_swap_encryptpage(sdp, (void *)(kva + (vsize_t)i*PAGE_SIZE), s); - setbit(sdp->swd_encmap, s); + atomic_or_32(&sdp->swd_encmap[s/32], + (uint32_t)1 << (s%32)); } } else { for (i = 0; i < npages; i++) { int s = startslot + i; - KDASSERT(swapdrum_getsdp(s) == sdp); + KDASSERT(swapdrum_sdp_is(s, sdp)); KASSERT(s >= sdp->swd_drumoffset); s -= sdp->swd_drumoffset; KASSERT(s < sdp->swd_drumsize); - clrbit(sdp->swd_encmap, s); + atomic_and_32(&sdp->swd_encmap[s/32], + ~((uint32_t)1 << (s%8))); } } } while (0); @@ -1949,10 +1987,22 @@ uvm_swap_io(struct vm_page **pps, int st if (!write) do { struct swapdev *sdp; + bool encinit; int i; + /* + * Get the sdp. Everything about it except the encinit + * bit, saying whether the encryption key is + * initialized or not, and the encrypted bit for each + * page, is stable until all swap pages have been + * released and the device is removed. + */ + mutex_enter(&uvm_swap_data_lock); sdp = swapdrum_getsdp(startslot); - if (!sdp->swd_encinit) + encinit = sdp->swd_encinit; + mutex_exit(&uvm_swap_data_lock); + + if (!encinit) /* * If there's no encryption key, there's no way * any of these slots can be encrypted, so @@ -1961,11 +2011,12 @@ uvm_swap_io(struct vm_page **pps, int st break; for (i = 0; i < npages; i++) { int s = startslot + i; - KDASSERT(swapdrum_getsdp(s) == sdp); + KDASSERT(swapdrum_sdp_is(s, sdp)); KASSERT(s >= sdp->swd_drumoffset); s -= sdp->swd_drumoffset; KASSERT(s < sdp->swd_drumsize); - if (isclr(sdp->swd_encmap, s)) + if ((atomic_load_relaxed(&sdp->swd_encmap[s/32]) & + ((uint32_t)1 << (s%32))) == 0) continue; uvm_swap_decryptpage(sdp, (void *)(kva + (vsize_t)i*PAGE_SIZE), s);