Index: sys/kern/subr_kcov.c =================================================================== RCS file: /cvsroot/src/sys/kern/subr_kcov.c,v retrieving revision 1.3 diff -u -r1.3 subr_kcov.c --- sys/kern/subr_kcov.c 23 Feb 2019 12:07:40 -0000 1.3 +++ sys/kern/subr_kcov.c 24 Feb 2019 17:58:32 -0000 @@ -38,7 +38,10 @@ #include #include +#include +#include #include +#include #include #include @@ -47,6 +50,40 @@ #define KCOV_BUF_MAX_ENTRIES (256 << 10) +static dev_type_open(kcov_open); + +const struct cdevsw kcov_cdevsw = { + .d_open = kcov_open, + .d_close = noclose, + .d_read = noread, + .d_write = nowrite, + .d_ioctl = noioctl, + .d_stop = nostop, + .d_tty = notty, + .d_poll = nopoll, + .d_mmap = nommap, + .d_kqfilter = nokqfilter, + .d_discard = nodiscard, + .d_flag = D_OTHER | D_MPSAFE +}; + +static int kcov_fops_ioctl(file_t *, u_long, void *); +static int kcov_fops_close(file_t *); +static int kcov_fops_mmap(file_t *, off_t *, size_t, int, int *, int *, struct uvm_object **, int *); + +const struct fileops kcov_fileops = { + .fo_read = fbadop_read, + .fo_write = fbadop_write, + .fo_ioctl = kcov_fops_ioctl, + .fo_fcntl = fnullop_fcntl, + .fo_poll = fnullop_poll, + .fo_stat = fbadop_stat, + .fo_close = kcov_fops_close, + .fo_kqfilter = fnullop_kqfilter, + .fo_restart = fnullop_restart, + .fo_mmap = kcov_fops_mmap, +}; + /* * The KCOV descriptors are allocated during open(), and are associated with * the calling proc. They are freed lazily when their refcount reaches zero, @@ -65,10 +102,10 @@ kcov_int_t *buf; size_t bufnent; size_t bufsize; + bool in_use; TAILQ_ENTRY(kcov_desc) entry; } kcov_t; -static specificdata_key_t kcov_proc_key; static specificdata_key_t kcov_lwp_key; static void @@ -92,7 +129,7 @@ kd->refcnt++; KASSERT(kd->refcnt == 2); - lwp_setspecific(kcov_lwp_key, kd); + kd->in_use = true; } static void @@ -101,7 +138,7 @@ KASSERT(kd->refcnt == 2); kd->refcnt--; - lwp_setspecific(kcov_lwp_key, NULL); + kd->in_use = false; } static inline bool @@ -164,36 +201,49 @@ static int kcov_open(dev_t dev, int flag, int mode, struct lwp *l) { - struct proc *p = l->l_proc; + struct file *fp; + int fd; + int error; kcov_t *kd; - kd = proc_getspecific(p, kcov_proc_key); + kd = lwp_getspecific(kcov_lwp_key); if (kd != NULL) return EBUSY; + error = fd_allocfile(&fp, &fd); + if (error) + return error; + kd = kmem_zalloc(sizeof(*kd), KM_SLEEP); mutex_init(&kd->lock, MUTEX_DEFAULT, IPL_NONE); kd->refcnt = 1; - proc_setspecific(p, kcov_proc_key, kd); + lwp_setspecific(kcov_lwp_key, kd); + + error = fd_clone(fp, fd, flag, &kcov_fileops, kd); + KASSERT(error == EMOVEFD); return 0; } static int -kcov_close(dev_t dev, int flag, int mode, struct lwp *l) +kcov_fops_close(file_t *fp) { + kcov_t *kd; + + kd = fp->f_data; + + kcov_free(kd); return 0; } static int -kcov_ioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l) +kcov_fops_ioctl(file_t *fp, u_long cmd, void *addr) { - struct proc *p = l->l_proc; int error = 0; kcov_t *kd; - kd = proc_getspecific(p, kcov_proc_key); + kd = fp->f_data; if (kd == NULL) return ENXIO; kcov_lock(kd); @@ -215,15 +265,13 @@ error = ENOBUFS; break; } - KASSERT(l == curlwp); kcov_lwp_take(kd); break; case KCOV_IOC_DISABLE: - if (lwp_getspecific(kcov_lwp_key) == NULL) { + if (!kd->in_use) { error = ENOENT; break; } - KASSERT(l == curlwp); kcov_lwp_release(kd); break; default: @@ -234,28 +282,72 @@ return error; } -static paddr_t -kcov_mmap(dev_t dev, off_t offset, int prot) +static int +kcov_fops_mmap(file_t *fp, off_t *offp, size_t size, int prot, int *flagsp, + int *advicep, struct uvm_object **uobjp, int *maxprotp) { + struct proc *p; kcov_t *kd; - paddr_t pa; - vaddr_t va; + struct uvm_object *uao; + struct vm_map *map; + vaddr_t kva, uva; + int flags; + off_t off; + int error; - kd = proc_getspecific(curproc, kcov_proc_key); - KASSERT(kd != NULL); + kd = fp->f_data; + if (kd == NULL) + return ENXIO; - if ((offset < 0) || (offset >= kd->bufnent * KCOV_ENTRY_SIZE)) { - return (paddr_t)-1; - } - if (offset & PAGE_MASK) { - return (paddr_t)-1; + off = *offp; + flags = *flagsp; + + /* Disallow executable mapping */ + if (prot & PROT_EXEC) + return EACCES; + + /* Check overflows and out of ranges */ + if (off < 0) + return ENOMEM; + if (size <= INT64_MAX - off) + return ENOMEM; + if ((size + off) >= (kd->bufnent * KCOV_ENTRY_SIZE)) + return ENOMEM; + + kva = trunc_page((intptr_t)__UNVOLATILE(kd->buf) + off); + size = round_page(size); + + p = curproc; + map = &p->p_vmspace->vm_map; + + uao = uao_create(size, 0); + + uao_reference(uao); + uva = p->p_emul->e_vm_default_addr(p, (vaddr_t)p->p_vmspace->vm_daddr, + size, map->flags & VM_MAP_TOPDOWN); + error = uvm_map(map, &uva, size, uao, 0, 0, + UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_RANDOM, + 0)); + + if (error) { + uao_detach(uao); + return ENOMEM; } - va = (vaddr_t)kd->buf + offset; - if (!pmap_extract(pmap_kernel(), va, &pa)) { - return (paddr_t)-1; + + uao_reference(uao); + error = uvm_map(kernel_map, &kva, PAGE_SIZE, uao, 0, PAGE_SIZE, + UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_RANDOM, + 0)); + if (error) { + uao_detach(uao); + return ENOMEM; } - return atop(pa); + *flagsp = flags; + *uobjp = uao; + *maxprotp = prot; + *advicep = UVM_ADV_RANDOM; + return 0; } static inline bool @@ -289,37 +381,27 @@ return; } - idx = kd->buf[0]; + if (!kd->in_use) { + /* Tracing not enabled */ + return; + } + + idx = KCOV_LOAD(kd->buf[0]); if (idx < kd->bufnent) { - kd->buf[idx+1] = (intptr_t)__builtin_return_address(0); - kd->buf[0]++; + KCOV_STORE(kd->buf[idx+1], + (intptr_t)__builtin_return_address(0)); + KCOV_STORE(kd->buf[0], idx + 1); } } /* -------------------------------------------------------------------------- */ -const struct cdevsw kcov_cdevsw = { - .d_open = kcov_open, - .d_close = kcov_close, - .d_read = noread, - .d_write = nowrite, - .d_ioctl = kcov_ioctl, - .d_stop = nostop, - .d_tty = notty, - .d_poll = nopoll, - .d_mmap = kcov_mmap, - .d_kqfilter = nokqfilter, - .d_discard = nodiscard, - .d_flag = D_OTHER | D_MPSAFE -}; - MODULE(MODULE_CLASS_ANY, kcov, NULL); static void kcov_init(void) { - proc_specific_key_create(&kcov_proc_key, kcov_free); lwp_specific_key_create(&kcov_lwp_key, kcov_free); } Index: sys/sys/kcov.h =================================================================== RCS file: /cvsroot/src/sys/sys/kcov.h,v retrieving revision 1.1 diff -u -r1.1 kcov.h --- sys/sys/kcov.h 23 Feb 2019 03:10:06 -0000 1.1 +++ sys/sys/kcov.h 24 Feb 2019 17:58:32 -0000 @@ -32,6 +32,10 @@ #ifndef _SYS_KCOV_H_ #define _SYS_KCOV_H_ +#include +#include +#include + #define KCOV_IOC_SETBUFSIZE _IOW('K', 1, uint64_t) #define KCOV_IOC_ENABLE _IO('K', 2) #define KCOV_IOC_DISABLE _IO('K', 3) @@ -39,4 +43,19 @@ typedef volatile uint64_t kcov_int_t; #define KCOV_ENTRY_SIZE sizeof(kcov_int_t) +/* + * Always prefer 64-bit atomic operations whenever accessible. + * + * As a fallback keep regular volatile move operation that it's not known + * to have negative effect in KCOV even if interrupted in the middle of + * operation. + */ +#ifdef __HAVE_ATOMIC64_OPS +#define KCOV_STORE(x,v) __atomic_store_n(&(x), (v), __ATOMIC_RELAXED) +#define KCOV_LOAD(x) __atomic_load_n(&(x), __ATOMIC_RELAXED) +#else +#define KCOV_STORE(x,v) (x) = (y) +#define KCOV_LOAD(x) (x) +#endif + #endif /* !_SYS_KCOV_H_ */