commit a37a729c1eed0b2e3240e1b9bf3c5d666864ee32 Author: Kamil Rytarowski Date: Mon Jul 6 00:14:23 2020 +0200 Add UVM KM JIT (W^X) allocator and switch SLJIT to it diff --git a/external/bsd/sljit/lib/Makefile b/external/bsd/sljit/lib/Makefile index f7b0facd6..3484ac267 100644 --- a/external/bsd/sljit/lib/Makefile +++ b/external/bsd/sljit/lib/Makefile @@ -15,5 +15,6 @@ SLJITDIST= ${NETBSDSRCDIR}/sys/external/bsd/sljit/dist .PATH: ${SLJITDIST}/sljit_src SRCS= sljitLir.c +CPPFLAGS+= -DSLJIT_PROT_EXECUTABLE_ALLOCATOR .include diff --git a/lib/librumpuser/rumpuser_mem.c b/lib/librumpuser/rumpuser_mem.c index 880640ee9..a429987dd 100644 --- a/lib/librumpuser/rumpuser_mem.c +++ b/lib/librumpuser/rumpuser_mem.c @@ -102,6 +102,56 @@ rumpuser_anonmmap(void *prefaddr, size_t size, int alignbit, ET(rv); } +int +rumpuser_anonmmapjit(void *prefaddr, size_t size, int alignbit, + void **datap, void **codep) +{ +#if defined(__NetBSD__) + void *data, *code; + int prot, rv; + + prot = PROT_READ | PROT_WRITE | PROT_MPROTECT(PROT_EXEC); + + data = mmap(prefaddr, size, prot, + MAP_PRIVATE | MAP_ANON | MAP_ALIGNED(alignbit), -1, 0); + if (data == MAP_FAILED) { + rv = errno; + ET(rv); + } + + code = mremap(data, size, NULL, size, MAP_REMAPDUP); + if (code == MAP_FAILED) { + rv = errno; + munmap(data, size); + ET(rv); + } + + if (mprotect(code, size, PROT_READ | PROT_EXEC) == -1) { + rv = errno; + munmap(code, size); + munmap(data, size); + ET(rv); + } + + *datap = data; + *codep = code; + + return 0; +#else + /* Fallback to a single RWX mapping. */ + void *mem; + int error, rv; + + error = rumpuser_anonmmap(prefaddr, size, alignbit, 1, &mem); + if (error) + return error; + + *datap = mem; + *codep = mem; + return 0; +#endif +} + void rumpuser_unmap(void *addr, size_t len) { @@ -109,3 +159,15 @@ rumpuser_unmap(void *addr, size_t len) munmap(addr, len); } +void +rumpuser_unmapjit(void *data, void *code, size_t len) +{ + +#if defined(__NetBSD__) + munmap(data, len); + munmap(code, len); +#else + /* Fallback to a single RWX mapping. */ + munmap(data, len); +#endif +} diff --git a/sys/external/bsd/sljit/conf/files.sljit b/sys/external/bsd/sljit/conf/files.sljit index 12ff18be1..e57155d3b 100644 --- a/sys/external/bsd/sljit/conf/files.sljit +++ b/sys/external/bsd/sljit/conf/files.sljit @@ -3,6 +3,7 @@ defflag SLJIT makeoptions sljit CPPFLAGS+="-I$S/external/bsd/sljit/dist/sljit_src" +makeoptions sljit CPPFLAGS+="-DSLJIT_PROT_EXECUTABLE_ALLOCATOR" file external/bsd/sljit/sljit/sljit_mod.c sljit file external/bsd/sljit/dist/sljit_src/sljitLir.c sljit diff --git a/sys/external/bsd/sljit/dist/sljit_src/sljitProtExecAllocator.c b/sys/external/bsd/sljit/dist/sljit_src/sljitProtExecAllocator.c index 8a5b2b3cf..5d3ec0188 100644 --- a/sys/external/bsd/sljit/dist/sljit_src/sljitProtExecAllocator.c +++ b/sys/external/bsd/sljit/dist/sljit_src/sljitProtExecAllocator.c @@ -84,6 +84,70 @@ struct chunk_header { as it only uses local variables */ +#if defined(__NetBSD__) && (defined(_KERNEL) || defined(_RUMPKERNEL)) +#include +#include /* for module_map */ +#include + +static SLJIT_INLINE struct chunk_header* alloc_chunk(sljit_uw size) +{ + struct chunk_header *retval; + vaddr_t data, code; + int rv; + + rv = uvm_km_allocjit(module_map, &data, &code, size, + PAGE_SIZE, UVM_KMF_WIRED | UVM_KMF_ZERO); + if (rv != 0) + return NULL; + + retval = (struct chunk_header *)data; + retval->executable = (void *)code; + + return retval; +} + +static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) +{ + struct chunk_header *header = ((struct chunk_header *)chunk) - 1; + + uvm_km_freejit(module_map, (vaddr_t)header, (vaddr_t)header->executable, + size, UVM_KMF_WIRED); +} +#elif defined(__NetBSD__) +static SLJIT_INLINE struct chunk_header* alloc_chunk(sljit_uw size) +{ + struct chunk_header *retval; + + retval = (struct chunk_header *)mmap(NULL, size, + PROT_READ | PROT_WRITE | PROT_MPROTECT(PROT_EXEC), + MAP_ANON | MAP_SHARED, -1, 0); + + if (retval == MAP_FAILED) + return NULL; + + retval->executable = mremap(retval, size, NULL, size, MAP_REMAPDUP); + + if (retval->executable == MAP_FAILED) { + munmap((void *)retval, size); + return NULL; + } + + if (mprotect(retval->executable, size, PROT_READ | PROT_EXEC) == -1) { + munmap(retval->executable, size); + munmap((void *)retval, size); + return NULL; + } + + return retval; +} + +static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) +{ + struct chunk_header *header = ((struct chunk_header *)chunk) - 1; + munmap(header->executable, size); + munmap((void *)header, size); +} +#else #include #ifndef O_NOATIME @@ -207,6 +271,7 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) munmap(header, size); close(fd); } +#endif /* --------------------------------------------------------------------- */ /* Common functions */ @@ -385,7 +450,9 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) if (total_size - free_block->size > (allocated_size * 3 / 2)) { total_size -= free_block->size; sljit_remove_free_block(free_block); - free_chunk(free_block, free_block->size + sizeof(struct block_header)); + free_chunk(free_block, free_block->size + + sizeof(struct chunk_header) + + sizeof(struct block_header)); } } @@ -406,7 +473,9 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) AS_BLOCK_HEADER(free_block, free_block->size)->size == 1) { total_size -= free_block->size; sljit_remove_free_block(free_block); - free_chunk(free_block, free_block->size + sizeof(struct block_header)); + free_chunk(free_block, free_block->size + + sizeof(struct chunk_header) + + sizeof(struct block_header)); } free_block = next_free_block; } diff --git a/sys/modules/sljit/Makefile b/sys/modules/sljit/Makefile index 2b3edd468..df0f798d9 100644 --- a/sys/modules/sljit/Makefile +++ b/sys/modules/sljit/Makefile @@ -7,5 +7,6 @@ KMOD= sljit SRCS= sljitLir.c sljit_mod.c +CPPFLAGS+= -DSLJIT_PROT_EXECUTABLE_ALLOCATOR .include diff --git a/sys/rump/include/rump/rumpuser.h b/sys/rump/include/rump/rumpuser.h index 8e5fe50a7..2118f3e6f 100644 --- a/sys/rump/include/rump/rumpuser.h +++ b/sys/rump/include/rump/rumpuser.h @@ -74,7 +74,10 @@ int rumpuser_init(int, const struct rumpuser_hyperup *); int rumpuser_malloc(size_t, int, void **); void rumpuser_free(void *, size_t); int rumpuser_anonmmap(void *, size_t, int, int, void **); +int rumpuser_anonmmapjit(void *, size_t, int, void **, void **); void rumpuser_unmap(void *, size_t); +void rumpuser_unmapjit(void *, void *, size_t); + /* * files and I/O diff --git a/sys/rump/kern/lib/libsljit/Makefile b/sys/rump/kern/lib/libsljit/Makefile index f503fd03e..49aad707b 100644 --- a/sys/rump/kern/lib/libsljit/Makefile +++ b/sys/rump/kern/lib/libsljit/Makefile @@ -13,6 +13,8 @@ COMMENT=Stackless JIT compiler SRCS= sljitLir.c sljit_mod.c +CPPFLAGS+= -DSLJIT_PROT_EXECUTABLE_ALLOCATOR + # NOTE This is not the best place for icache sync routine but only # sljit uses it at the moment. # XXX Think about a good hypercall interface (hi, pooka!) and move diff --git a/sys/rump/librump/rumpkern/vm.c b/sys/rump/librump/rumpkern/vm.c index e585bee35..3647d6282 100644 --- a/sys/rump/librump/rumpkern/vm.c +++ b/sys/rump/librump/rumpkern/vm.c @@ -854,6 +854,36 @@ uvm_km_alloc(struct vm_map *map, vsize_t size, vsize_t align, uvm_flag_t flags) return (vaddr_t)rv; } +int +uvm_km_allocjit(struct vm_map *map, vaddr_t *rdata, vaddr_t *rcode, + vsize_t size, vsize_t align, uvm_flag_t flags) +{ + void *data, *code; + int alignbit, error; + + alignbit = 0; + if (align) { + alignbit = ffs(align)-1; + } + error = rumpuser_anonmmapjit(NULL, size, alignbit, + &data, &code); + + if (error) { + if (flags & (UVM_KMF_CANFAIL | UVM_KMF_NOWAIT)) + return 0; + else + panic("uvm_km_allocjit failed"); + } + + *rdata = (vaddr_t)data; + *rcode = (vaddr_t)code; + + if (flags & UVM_KMF_ZERO) + memset(data, 0, size); + + return 0; +} + void uvm_km_free(struct vm_map *map, vaddr_t vaddr, vsize_t size, uvm_flag_t flags) { @@ -864,6 +894,14 @@ uvm_km_free(struct vm_map *map, vaddr_t vaddr, vsize_t size, uvm_flag_t flags) rumpuser_free((void *)vaddr, size); } +void +uvm_km_freejit(struct vm_map *map, vaddr_t data, vaddr_t code, vsize_t size, + uvm_flag_t flags) +{ + + rumpuser_unmapjit((void *)data, (void *)code, size); +} + int uvm_km_protect(struct vm_map *map, vaddr_t vaddr, vsize_t size, vm_prot_t prot) { diff --git a/sys/uvm/uvm_extern.h b/sys/uvm/uvm_extern.h index 80523d4ba..abf11332a 100644 --- a/sys/uvm/uvm_extern.h +++ b/sys/uvm/uvm_extern.h @@ -701,10 +701,14 @@ int uvm_io(struct vm_map *, struct uio *, int); /* uvm_km.c */ vaddr_t uvm_km_alloc(struct vm_map *, vsize_t, vsize_t, uvm_flag_t); +int uvm_km_allocjit(struct vm_map *, vaddr_t *, vaddr_t *, + vsize_t, vsize_t, uvm_flag_t); int uvm_km_protect(struct vm_map *, vaddr_t, vsize_t, vm_prot_t); void uvm_km_free(struct vm_map *, vaddr_t, vsize_t, uvm_flag_t); +void uvm_km_freejit(struct vm_map *, vaddr_t, vaddr_t, + vsize_t, uvm_flag_t); struct vm_map *uvm_km_suballoc(struct vm_map *, vaddr_t *, vaddr_t *, vsize_t, int, bool, diff --git a/sys/uvm/uvm_km.c b/sys/uvm/uvm_km.c index e5a06fec8..94c2a5f1e 100644 --- a/sys/uvm/uvm_km.c +++ b/sys/uvm/uvm_km.c @@ -721,6 +721,63 @@ uvm_km_alloc(struct vm_map *map, vsize_t size, vsize_t align, uvm_flag_t flags) return(kva); } +/* + * uvm_km_allocjit: allocate an area of kernel JIT memory. + * + * => NOTE: we can return 0 even if we can wait if there is not enough + * free VM space in the map... caller should be prepared to handle + * this case. + * => We return writable KVA of memory allocated in data and executable + * in code. On a successful allocation, return 0. + */ + +int +uvm_km_allocjit(struct vm_map *map, vaddr_t *rdata, vaddr_t *rcode, + vsize_t size, vsize_t align, uvm_flag_t flags) +{ + struct pmap *pmap; + vaddr_t data, code; + vaddr_t kva, end; + paddr_t pa; + + KASSERT((flags & UVM_KMF_TYPEMASK) == UVM_KMF_WIRED); + /* KASSERT(map == kernel_map || map == module_map); XXX? */ + KASSERT(rdata != NULL); + KASSERT(rcode != NULL); + + data = uvm_km_alloc(map, size, align, flags); + if (data == 0) + goto fail1; + + code = uvm_km_alloc(map, size, align, UVM_KMF_VAONLY); + if (code == 0) + goto fail2; + + pmap = map->pmap; + for (kva = code, end = code + size; kva < end; kva += PAGE_SIZE) { + if (!pmap_extract(pmap, kva, &pa)) + goto fail3; + pmap_kenter_pa(kva, pa, VM_PROT_READ | VM_PROT_EXECUTE, + PMAP_WIRED); + } + pmap_update(pmap); + + *rdata = data; + *rcode = code; + return 0; + +fail3: + if (kva > code) { + pmap_kremove(code, kva - code); + pmap_update(pmap); + } + uvm_km_free(map, data, size, UVM_KMF_VAONLY); +fail2: + uvm_km_free(map, code, size, UVM_KMF_WIRED); +fail1: + return -1; +} + /* * uvm_km_protect: change the protection of an allocated area */ @@ -766,6 +823,25 @@ uvm_km_free(struct vm_map *map, vaddr_t addr, vsize_t size, uvm_flag_t flags) uvm_unmap1(map, addr, addr + size, UVM_FLAG_VAONLY); } +/* + * uvm_km_freejit: free a JIT area of kernel memory + */ + +void +uvm_km_freejit(struct vm_map *map, vaddr_t data, vaddr_t code, vsize_t size, + uvm_flag_t flags) +{ + + KASSERT((flags & UVM_KMF_TYPEMASK) == UVM_KMF_WIRED); + /* KASSERT(map == kernel_map || map == module_map); XXX? */ + + pmap_kremove(code, size); + pmap_update(map->pmap); + + uvm_km_free(map, data, size, UVM_KMF_VAONLY); + uvm_km_free(map, code, size, UVM_KMF_WIRED); +} + /* Sanity; must specify both or none. */ #if (defined(PMAP_MAP_POOLPAGE) || defined(PMAP_UNMAP_POOLPAGE)) && \ (!defined(PMAP_MAP_POOLPAGE) || !defined(PMAP_UNMAP_POOLPAGE)) diff --git a/tests/lib/libbpfjit/Makefile b/tests/lib/libbpfjit/Makefile index c673cb31a..4b267b60f 100644 --- a/tests/lib/libbpfjit/Makefile +++ b/tests/lib/libbpfjit/Makefile @@ -9,8 +9,6 @@ TESTS_C+= t_bpfjit TESTS_C+= t_extmem TESTS_C+= t_cop -PAXCTL_FLAGS= +m - # XXX this variable doesn't belong to here LIBBPFJITDIR!= cd ${NETBSDSRCDIR}/lib/libbpfjit && ${PRINTOBJDIR} diff --git a/tests/lib/libsljit/Makefile b/tests/lib/libsljit/Makefile index 20bff288c..6b20f8635 100644 --- a/tests/lib/libsljit/Makefile +++ b/tests/lib/libsljit/Makefile @@ -14,7 +14,6 @@ BINDIR= ${TESTSDIR} PROG= h_sljit MAN= # defined SRCS= sljitMain.c sljitTest.c -PAXCTL_FLAGS.h_sljit += +m WARNS= 3 LDADD+= -L${LIBSLJITDIR} -lsljit diff --git a/tests/net/bpfjit/Makefile b/tests/net/bpfjit/Makefile index bb995e702..4df501cf3 100644 --- a/tests/net/bpfjit/Makefile +++ b/tests/net/bpfjit/Makefile @@ -10,8 +10,6 @@ TESTS_C+= t_cop TESTS_C+= t_extmem TESTS_C+= t_mbuf -PAXCTL_FLAGS= +m - LDADD+= -lrumpnet_bpfjit -lrumpkern_sljit LDADD+= -lrumpdev_bpf -lrumpnet_net -lrumpnet LDADD+= ${LIBRUMPBASE}