diff --git a/sys/arch/aarch64/aarch64/aarch64_tlb.c b/sys/arch/aarch64/aarch64/aarch64_tlb.c new file mode 100644 index 000000000000..c12e1a5fd279 --- /dev/null +++ b/sys/arch/aarch64/aarch64/aarch64_tlb.c @@ -0,0 +1,118 @@ +/*- + * Copyright (c) 2021 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Nick Hudson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "opt_cputypes.h" +#include "opt_multiprocessor.h" + +#include +__KERNEL_RCSID(1, "$NetBSD$"); + +#include +#include + +#include + +#include + +#include + +tlb_asid_t +tlb_get_asid(void) +{ + + return __SHIFTOUT(reg_ttbr0_el1_read(), TTBR_ASID); +} + +void +tlb_set_asid(tlb_asid_t asid, pmap_t pm) +{ + const uint64_t ttbr = + __SHIFTIN(asid, TTBR_ASID) | + __SHIFTIN(pmap_l0pa(pm), TTBR_BADDR); + + cpu_set_ttbr0(ttbr); +} + +void +tlb_invalidate_all(void) +{ + + aarch64_tlbi_all(); +} + +void +tlb_invalidate_globals(void) +{ + tlb_invalidate_all(); +} + +void +tlb_invalidate_asids(tlb_asid_t lo, tlb_asid_t hi) +{ + for (; lo <= hi; lo++) { + aarch64_tlbi_by_asid(lo); + } +} + +void +tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid) +{ + KASSERT((va & PAGE_MASK) == 0); + + aarch64_tlbi_by_asid_va(asid, va); +} + +bool +tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert_p) +{ + KASSERT((va & PAGE_MASK) == 0); + + tlb_invalidate_addr(va, asid); + + return true; +} + +u_int +tlb_record_asids(u_long *mapp, tlb_asid_t asid_max) +{ + KASSERT(asid_max == pmap_md_tlb_asid_max()); + +#if DIAGNOSTIC + memset(mapp, 0xff, (asid_max + 1) / (NBBY * sizeof(u_long))); + mapp[0] ^= __BITS(0, KERNEL_PID); +#endif + return asid_max; +} + +void +tlb_walk(void *ctx, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t)) +{ + + /* no way to view the TLB */ +} diff --git a/sys/arch/aarch64/aarch64/cpu.c b/sys/arch/aarch64/aarch64/cpu.c index 1ae3700026f4..ee2cc95b956d 100644 --- a/sys/arch/aarch64/aarch64/cpu.c +++ b/sys/arch/aarch64/aarch64/cpu.c @@ -153,6 +153,7 @@ cpu_attach(device_t dv, cpuid_t id) #ifdef MULTIPROCESSOR if (unit != 0) { mi_cpu_attach(ci); + pmap_tlb_info_attach(&pmap_tlb0_info, ci); return; } #endif /* MULTIPROCESSOR */ @@ -681,6 +682,7 @@ void cpu_hatch(struct cpu_info *ci) { KASSERT(curcpu() == ci); + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); mutex_enter(&cpu_hatch_lock); diff --git a/sys/arch/aarch64/aarch64/pmap.c b/sys/arch/aarch64/aarch64/pmap.c index 52207d908246..cd606d834870 100644 --- a/sys/arch/aarch64/aarch64/pmap.c +++ b/sys/arch/aarch64/aarch64/pmap.c @@ -41,6 +41,7 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.116 2021/09/30 21:19:16 skrll Exp $"); #include #include +#include #include #include @@ -181,10 +182,10 @@ PMAP_COUNTER(unwire_failure, "pmap_unwire failure"); * change the pte to accessible temporarly before cpu_icache_sync_range(). * this macro modifies PTE (*ptep). need to update PTE after this. */ -#define PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, ll) \ +#define PTE_ICACHE_SYNC_PAGE(pte, ptep, asid, va, ll) \ do { \ atomic_swap_64((ptep), (pte) | LX_BLKPAG_AF); \ - AARCH64_TLBI_BY_ASID_VA((pm)->pm_asid, (va), (ll)); \ + AARCH64_TLBI_BY_ASID_VA((asid), (va), (ll)); \ cpu_icache_sync_range((va), PAGE_SIZE); \ } while (0/*CONSTCOND*/) @@ -336,7 +337,6 @@ pmap_map_chunk(vaddr_t va, paddr_t pa, vsize_t size, attr = _pmap_pte_adjust_prot(0, prot, VM_PROT_ALL, false); attr = _pmap_pte_adjust_cacheflags(attr, flags); pmapboot_enter_range(va, pa, resid, attr, printf); - aarch64_tlbi_all(); return resid; } @@ -472,18 +472,23 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t vend) virtual_end = vend; pmap_maxkvaddr = vstart; - aarch64_tlbi_all(); - l0pa = reg_ttbr1_el1_read(); l0 = (void *)AARCH64_PA_TO_KVA(l0pa); + pmap_tlb_info_init(&pmap_tlb0_info); + memset(&kernel_pmap, 0, sizeof(kernel_pmap)); + kpm = pmap_kernel(); - kpm->pm_asid = 0; + struct pmap_asid_info * const pai = PMAP_PAI(kpm, cpu_tlb_info(ci)); + + pai->pai_asid = KERNEL_PID; kpm->pm_refcnt = 1; kpm->pm_idlepdp = 0; kpm->pm_l0table = l0; kpm->pm_l0table_pa = l0pa; + kpm->pm_onproc = kcpuset_running; + kpm->pm_active = kcpuset_running; kpm->pm_activated = true; LIST_INIT(&kpm->pm_vmlist); LIST_INIT(&kpm->pm_pvlist); /* not used for kernel pmap */ @@ -493,6 +498,12 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t vend) CTASSERT(sizeof(kpm->pm_stats.resident_count) == sizeof(long)); } +void +pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) +{ + /* nothing */ +} + static inline void _pmap_adj_wired_count(struct pmap *pm, int adj) { @@ -555,10 +566,6 @@ pmap_init(void) pool_cache_bootstrap(&_pmap_pv_pool, sizeof(struct pv_entry), 32, 0, PR_LARGECACHE, "pvpl", NULL, IPL_NONE, _pmap_pv_ctor, NULL, NULL); - - int nmaxproc = cpu_maxproc(); - if (maxproc > nmaxproc) - maxproc = nmaxproc; } void @@ -674,18 +681,16 @@ pmap_free_pdp(struct pmap *pm, struct vm_page *pg) } /* free empty page table pages */ -static int +static void _pmap_sweep_pdp(struct pmap *pm) { struct vm_page *pg, *tmp; pd_entry_t *ptep_in_parent, opte __diagused; paddr_t pa, pdppa; - int nsweep; uint16_t wirecount __diagused; KASSERT(mutex_owned(&pm->pm_lock) || pm->pm_refcnt == 0); - nsweep = 0; LIST_FOREACH_SAFE(pg, &pm->pm_vmlist, pageq.list, tmp) { if (pg->wire_count != 1) continue; @@ -698,7 +703,6 @@ _pmap_sweep_pdp(struct pmap *pm) if (ptep_in_parent == NULL) { /* no parent */ pmap_free_pdp(pm, pg); - nsweep++; continue; } @@ -708,7 +712,6 @@ _pmap_sweep_pdp(struct pmap *pm) wirecount = --pg->wire_count; /* 1 -> 0 */ KASSERT(wirecount == 0); pmap_free_pdp(pm, pg); - nsweep++; /* L3->L2->L1. no need for L0 */ pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent)); @@ -722,12 +725,10 @@ _pmap_sweep_pdp(struct pmap *pm) /* decrement wire_count of parent */ wirecount = --pg->wire_count; KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1), - "pm=%p[%d], pg=%p, wire_count=%d", - pm, pm->pm_asid, pg, pg->wire_count); + "pm=%p, pg=%p, wire_count=%d", + pm, pg, pg->wire_count); } pm->pm_idlepdp = 0; - - return nsweep; } static void @@ -977,11 +978,14 @@ pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) * change to accessible temporally * to do cpu_icache_sync_range() */ + struct pmap_asid_info * const pai = PMAP_PAI(pm, + cpu_tlb_info(ci)); + atomic_swap_64(ptep, pte | LX_BLKPAG_AF); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); cpu_icache_sync_range(va, len); atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); } } @@ -1179,14 +1183,15 @@ pmap_db_mdpg_print(struct vm_page *pg, void (*pr)(const char *, ...) __printflik KASSERT(pv == &pp->pp_pv); continue; } - pr(" pv[%d] pv=%p\n", - i, pv); - pr(" pv[%d].pv_pmap = %p (asid=%d)\n", - i, pv->pv_pmap, pv->pv_pmap->pm_asid); - pr(" pv[%d].pv_va = %016lx (color=%d)\n", - i, trunc_page(pv->pv_va), _pmap_color(pv->pv_va)); - pr(" pv[%d].pv_ptep = %p\n", - i, pv->pv_ptep); + struct pmap * const pm = pv->pv_pmap; + struct pmap_asid_info * const pai = PMAP_PAI(pm, + cpu_tlb_info(ci)); + + pr(" pv[%d] pv=%p\n", i, pv); + pr(" pv[%d].pv_pmap = %p (asid=%d)\n", i, pm, pai->pai_asid); + pr(" pv[%d].pv_va = %016lx (color=%d)\n", i, + trunc_page(pv->pv_va), _pmap_color(pv->pv_va)); + pr(" pv[%d].pv_ptep = %p\n", i, pv->pv_ptep); i++; } } @@ -1298,7 +1303,11 @@ _pmap_protect_pv(struct pmap_page *pp, struct pv_entry *pv, vm_prot_t prot) /* new prot = prot & pteprot & mdattr */ pte = _pmap_pte_adjust_prot(pte, prot & pteprot, mdattr, user); atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, trunc_page(pv->pv_va), + + struct pmap * const pm = pv->pv_pmap; + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); + + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, trunc_page(pv->pv_va), true); } @@ -1392,23 +1401,27 @@ pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) executable = l3pte_executable(pte, user); pte = _pmap_pte_adjust_prot(pte, prot, mdattr, user); + struct pmap_asid_info * const pai = PMAP_PAI(pm, + cpu_tlb_info(ci)); if (!executable && (prot & VM_PROT_EXECUTE)) { /* non-exec -> exec */ UVMHIST_LOG(pmaphist, "icache_sync: " "pm=%p, va=%016lx, pte: %016lx -> %016lx", pm, va, opte, pte); + if (!l3pte_readable(pte)) { - PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, true); + PTE_ICACHE_SYNC_PAGE(pte, ptep, pai->pai_asid, + va, true); atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); } else { atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); cpu_icache_sync_range(va, PAGE_SIZE); } } else { atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); } } @@ -1420,11 +1433,11 @@ void pmap_activate(struct lwp *l) { struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap; - uint64_t ttbr0, tcr; + uint64_t tcr; UVMHIST_FUNC(__func__); - UVMHIST_CALLARGS(pmaphist, "lwp=%p asid=%d (pid=%d)", l, pm->pm_asid, - l->l_proc->p_pid, 0); + UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, kernel=%u)", l, + l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0); KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); @@ -1435,15 +1448,13 @@ pmap_activate(struct lwp *l) KASSERT(pm->pm_l0table != NULL); + /* this calls tlb_set_asid which calls cpu_set_ttbr0 */ + pmap_tlb_asid_acquire(pm, l); - /* XXX: allocate asid, and regenerate if needed */ - if (pm->pm_asid == -1) - pm->pm_asid = l->l_proc->p_pid; + struct pmap_asid_info * const pai __debugused = PMAP_PAI(pm, + cpu_tlb_info(ci)); - ttbr0 = - __SHIFTIN(pm->pm_asid, TTBR_ASID) | - __SHIFTIN(pm->pm_l0table_pa, TTBR_BADDR); - cpu_set_ttbr0(ttbr0); + UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, pai->pai_asid, 0, 0); /* Re-enable translation table walks using TTBR0 */ tcr = reg_tcr_el1_read(); @@ -1462,17 +1473,21 @@ pmap_deactivate(struct lwp *l) uint64_t tcr; UVMHIST_FUNC(__func__); - UVMHIST_CALLARGS(pmaphist, "lwp=%p, asid=%d", l, pm->pm_asid, 0, 0); - - if (pm == pmap_kernel()) - return; + UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, (kernel=%u))", l, + l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0); /* Disable translation table walks using TTBR0 */ tcr = reg_tcr_el1_read(); reg_tcr_el1_write(tcr | TCR_EPD0); isb(); - /* XXX */ + struct pmap_asid_info * const pai __debugused = PMAP_PAI(pm, + cpu_tlb_info(ci)); + + UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, pai->pai_asid, 0, 0); + + pmap_tlb_asid_deactivate(pm); + pm->pm_activated = false; PMAP_COUNT(deactivate); @@ -1490,11 +1505,14 @@ pmap_create(void) memset(pm, 0, sizeof(*pm)); pm->pm_refcnt = 1; pm->pm_idlepdp = 0; - pm->pm_asid = -1; LIST_INIT(&pm->pm_vmlist); LIST_INIT(&pm->pm_pvlist); mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_NONE); + kcpuset_create(&pm->pm_active, true); + kcpuset_create(&pm->pm_onproc, true); + pm->pm_remove_all = false; + pm->pm_l0table_pa = pmap_alloc_pdp(pm, NULL, 0, true); KASSERT(pm->pm_l0table_pa != POOL_PADDR_INVALID); pm->pm_l0table = (pd_entry_t *)AARCH64_PA_TO_KVA(pm->pm_l0table_pa); @@ -1514,8 +1532,8 @@ pmap_destroy(struct pmap *pm) UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(pmaphist, - "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx, refcnt=%d", - pm, pm->pm_l0table, pm->pm_l0table_pa, pm->pm_refcnt); + "pm=%p, pm_l0table=%016lx, pm_remove_all=%jd, refcnt=%jd", + pm, pm->pm_l0table, pm->pm_remove_all, pm->pm_refcnt); if (pm == NULL) return; @@ -1523,19 +1541,23 @@ pmap_destroy(struct pmap *pm) if (pm == pmap_kernel()) panic("cannot destroy kernel pmap"); + if (pm->pm_remove_all) { + pmap_tlb_asid_release_all(pm); + pm->pm_remove_all = false; + } + refcnt = atomic_dec_uint_nv(&pm->pm_refcnt); if (refcnt > 0) return; KASSERT(LIST_EMPTY(&pm->pm_pvlist)); - /* - * no need to call aarch64_tlbi_by_asid(pm->pm_asid). - * TLB should already be invalidated in pmap_remove_all() - */ _pmap_free_pdp_all(pm, true); mutex_destroy(&pm->pm_lock); + kcpuset_destroy(pm->pm_active); + kcpuset_destroy(pm->pm_onproc); + pool_cache_put(&_pmap_cache, pm); PMAP_COUNT(destroy); @@ -1653,8 +1675,8 @@ _pmap_pdp_delref(struct pmap *pm, paddr_t pdppa, bool do_free_pdp) /* decrement wire_count of parent */ wirecount = atomic_add_32_nv(&pg->wire_count, -1); KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1), - "pm=%p[%d], pg=%p, wire_count=%d", - pm, pm->pm_asid, pg, pg->wire_count); + "pm=%p, pg=%p, wire_count=%d", + pm, pg, pg->wire_count); } return removed; @@ -1822,10 +1844,8 @@ _pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, } pm_lock(pm); - if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER && - _pmap_sweep_pdp(pm) != 0) { - /* several L1-L3 page table pages have been freed */ - aarch64_tlbi_by_asid(pm->pm_asid); + if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER) { + _pmap_sweep_pdp(pm); } } @@ -1937,7 +1957,10 @@ _pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, bool pdpremoved = _pmap_pdp_delref(pm, AARCH64_KVA_TO_PA(trunc_page( (vaddr_t)ptep)), true); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, + struct pmap_asid_info * const pai = PMAP_PAI(pm, + cpu_tlb_info(ci)); + + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, !pdpremoved); } PMAP_COUNT(pv_entry_cannotalloc); @@ -1963,7 +1986,8 @@ _pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, } #endif - attr = _pmap_pte_adjust_prot(L3_PAGE, prot, mdattr, user); + attr = L3_PAGE | (kenter ? 0 : LX_BLKPAG_NG); + attr = _pmap_pte_adjust_prot(attr, prot, mdattr, user); attr = _pmap_pte_adjust_cacheflags(attr, flags); if (VM_MAXUSER_ADDRESS > va) attr |= LX_BLKPAG_APUSER; @@ -1975,23 +1999,27 @@ _pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, pte = pa | attr; + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); + const tlb_asid_t asid = pai->pai_asid; + if (need_sync_icache) { /* non-exec -> exec */ UVMHIST_LOG(pmaphist, "icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx", pm, va, opte, pte); + if (!l3pte_readable(pte)) { - PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, l3only); + PTE_ICACHE_SYNC_PAGE(pte, ptep, asid, va, l3only); atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va ,true); + AARCH64_TLBI_BY_ASID_VA(asid, va ,true); } else { atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only); + AARCH64_TLBI_BY_ASID_VA(asid, va, l3only); cpu_icache_sync_range(va, PAGE_SIZE); } } else { atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only); + AARCH64_TLBI_BY_ASID_VA(asid, va, l3only); } if (pte & LX_BLKPAG_OS_WIRED) { @@ -2025,6 +2053,42 @@ pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) return _pmap_enter(pm, va, pa, prot, flags, false); } + + +void +pmap_update(pmap_t pm) +{ + + UVMHIST_FUNC(__func__); + UVMHIST_CALLARGS(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, + pm->pm_remove_all, 0, 0); + + kpreempt_disable(); + /* + * If pmap_remove_all was called, we deactivated ourselves and released + * our ASID. Now we have to reactivate ourselves. + */ + if (__predict_false(pm->pm_remove_all)) { + pm->pm_remove_all = false; + + KASSERT(pm != pmap_kernel()); + + /* this calls tlb_set_asid which calls cpu_set_ttbr0 */ + pmap_tlb_asid_acquire(pm, curlwp); + + /* Enable translation table walks using TTBR0 */ + uint64_t tcr = reg_tcr_el1_read(); + reg_tcr_el1_write(tcr & ~TCR_EPD0); + isb(); + + pm->pm_activated = true; + } + + kpreempt_enable(); + + UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); +} + bool pmap_remove_all(struct pmap *pm) { @@ -2036,8 +2100,28 @@ pmap_remove_all(struct pmap *pm) UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(pmaphist, "pm=%p", pm, 0, 0, 0); - if (pm == pmap_kernel()) - return false; + KASSERT(pm != pmap_kernel()); + + struct cpu_info * const ci = curcpu(); + // This should be the last CPU with this pmap onproc + KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); + if (kcpuset_isset(pm->pm_onproc, cpu_index(ci))) { + /* Disable translation table walks using TTBR0 */ + uint64_t tcr = reg_tcr_el1_read(); + reg_tcr_el1_write(tcr | TCR_EPD0); + isb(); + + pmap_tlb_asid_deactivate(pm); + } + + KASSERT(kcpuset_iszero(pm->pm_onproc)); + + pmap_tlb_asid_release_all(pm); + pm->pm_remove_all = true; + + struct pmap_asid_info * const pai __debugused = PMAP_PAI(pm, + cpu_tlb_info(ci)); + UVMHIST_LOG(pmaphist, "pm=%p, asid=%d", pm, pai->pai_asid, 0, 0); pm_lock(pm); @@ -2046,16 +2130,16 @@ pmap_remove_all(struct pmap *pm) pte = *ptep; KASSERTMSG(lxpde_valid(pte), - "pte is not valid: pmap=%p, asid=%d, va=%016lx", - pm, pm->pm_asid, pv->pv_va); + "pte is not valid: pmap=%p, va=%016lx", + pm, pv->pv_va); pa = lxpde_pa(pte); pp = phys_to_pp(pa); KASSERTMSG(pp != NULL, "no pmap_page of physical address:%016lx, " - "pmap=%p, asid=%d, va=%016lx", - pa, pm, pm->pm_asid, pv->pv_va); + "pmap=%p, va=%016lx", + pa, pm, pv->pv_va); pmap_pv_lock(pp); opv = _pmap_remove_pv(pp, pm, trunc_page(pv->pv_va), pte); @@ -2071,7 +2155,6 @@ pmap_remove_all(struct pmap *pm) /* clear L0 page table page */ pmap_zero_page(pm->pm_l0table_pa); - aarch64_tlbi_by_asid(pm->pm_asid); /* free L1-L3 page table pages, but not L0 */ _pmap_free_pdp_all(pm, false); @@ -2135,10 +2218,12 @@ _pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva, bool kremove, pte = atomic_swap_64(ptep, 0); if (!lxpde_valid(pte)) continue; + struct pmap_asid_info * const pai = PMAP_PAI(pm, + cpu_tlb_info(ci)); pdpremoved = _pmap_pdp_delref(pm, AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep)), true); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, !pdpremoved); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, !pdpremoved); if (pdpremoved) { /* @@ -2196,12 +2281,14 @@ pmap_page_remove(struct pmap_page *pp, vm_prot_t prot) continue; } opte = atomic_swap_64(pv->pv_ptep, 0); + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); const vaddr_t va = trunc_page(pv->pv_va); + if (lxpde_valid(opte)) { _pmap_pdp_delref(pm, AARCH64_KVA_TO_PA(trunc_page( (vaddr_t)pv->pv_ptep)), false); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); if ((opte & LX_BLKPAG_OS_WIRED) != 0) { _pmap_adj_wired_count(pm, -1); @@ -2441,7 +2528,8 @@ pmap_fault_fixup(struct pmap *pm, vaddr_t va, vm_prot_t accessprot, bool user) pmap_pv_unlock(pp); atomic_swap_64(ptep, pte); - AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true); + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); fixed = true; @@ -2510,7 +2598,9 @@ pmap_clear_modify(struct vm_page *pg) goto tryagain; } - AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true); + struct pmap * const pm = pv->pv_pmap; + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); UVMHIST_LOG(pmaphist, "va=%016llx, ptep=%p, pa=%016lx, RW -> RO", @@ -2568,7 +2658,9 @@ pmap_clear_reference(struct vm_page *pg) goto tryagain; } - AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true); + struct pmap * const pm = pv->pv_pmap; + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); + AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true); UVMHIST_LOG(pmaphist, "va=%016llx, ptep=%p, pa=%016lx, unse AF", va, ptep, l3pte_pa(pte), 0); @@ -2610,8 +2702,9 @@ void pmap_db_pmap_print(struct pmap *pm, void (*pr)(const char *, ...) __printflike(1, 2)) { + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); - pr(" pm_asid = %d\n", pm->pm_asid); + pr(" pm_asid = %d\n", pai->pai_asid); pr(" pm_l0table = %p\n", pm->pm_l0table); pr(" pm_l0table_pa = %lx\n", pm->pm_l0table_pa); pr(" pm_activated = %d\n\n", pm->pm_activated); diff --git a/sys/arch/aarch64/conf/files.aarch64 b/sys/arch/aarch64/conf/files.aarch64 index 87f9e27dae21..79b852fbf00e 100644 --- a/sys/arch/aarch64/conf/files.aarch64 +++ b/sys/arch/aarch64/conf/files.aarch64 @@ -112,9 +112,11 @@ file arch/aarch64/aarch64/vectors.S file arch/aarch64/aarch64/vm_machdep.c # pmap +file arch/aarch64/aarch64/aarch64_tlb.c file arch/aarch64/aarch64/pmap.c file arch/aarch64/aarch64/pmapboot.c file arch/aarch64/aarch64/pmap_page.S +file uvm/pmap/pmap_tlb.c file uvm/pmap/pmap_pvt.c # EFI runtime (machdep) diff --git a/sys/arch/aarch64/include/cpu.h b/sys/arch/aarch64/include/cpu.h index f166119f2cf3..e51f39a31c17 100644 --- a/sys/arch/aarch64/include/cpu.h +++ b/sys/arch/aarch64/include/cpu.h @@ -113,6 +113,8 @@ struct cpu_info { int ci_kfpu_spl; + tlb_asid_t ci_pmap_asid_cur; + /* event counters */ struct evcnt ci_vfp_use; struct evcnt ci_vfp_reuse; @@ -158,22 +160,6 @@ static __inline struct cpu_info *lwp_getcpu(struct lwp *); #undef curlwp #define curlwp (aarch64_curlwp()) -static inline int -cpu_maxproc(void) -{ - /* - * the pmap uses PID for ASID. - */ - switch (__SHIFTOUT(reg_id_aa64mmfr0_el1_read(), ID_AA64MMFR0_EL1_ASIDBITS)) { - case ID_AA64MMFR0_EL1_ASIDBITS_8BIT: - return (1U << 8) - 1; - case ID_AA64MMFR0_EL1_ASIDBITS_16BIT: - return (1U << 16) - 1; - default: - return 0; - } -} - void cpu_signotify(struct lwp *l); void cpu_need_proftick(struct lwp *l); diff --git a/sys/arch/aarch64/include/pmap.h b/sys/arch/aarch64/include/pmap.h index 5c6b7ece6290..69b4dc2accbf 100644 --- a/sys/arch/aarch64/include/pmap.h +++ b/sys/arch/aarch64/include/pmap.h @@ -42,6 +42,7 @@ #include #include #include + #include #include @@ -54,6 +55,43 @@ #define __HAVE_VM_PAGE_MD #define __HAVE_PMAP_PV_TRACK 1 +#define PMAP_HWPAGEWALKER 1 + +#define PMAP_TLB_MAX 1 +#if PMAP_TLB_MAX > 1 +#define PMAP_TLB_NEED_SHOOTDOWN 1 +#endif + +#define PMAP_TLB_FLUSH_ASID_ON_RESET (true) + +/* Maximum number of ASIDs. Some CPUs have less.*/ +#define PMAP_TLB_NUM_PIDS 65536 +#define PMAP_TLB_BITMAP_LENGTH PMAP_TLB_NUM_PIDS +#define cpu_set_tlb_info(ci, ti) ((void)((ci)->ci_tlb_info = (ti))) +#if PMAP_TLB_MAX > 1 +#define cpu_tlb_info(ci) ((ci)->ci_tlb_info) +#else +#define cpu_tlb_info(ci) (&pmap_tlb0_info) +#endif + +static inline tlb_asid_t +pmap_md_tlb_asid_max(void) +{ + switch (__SHIFTOUT(reg_id_aa64mmfr0_el1_read(), ID_AA64MMFR0_EL1_ASIDBITS)) { + case ID_AA64MMFR0_EL1_ASIDBITS_8BIT: + return (1U << 8) - 1; + case ID_AA64MMFR0_EL1_ASIDBITS_16BIT: + return (1U << 16) - 1; + default: + return 0; + } +} + +#include +#include + +#define KERNEL_PID 0 /* The kernel uses ASID 0 */ + #ifndef KASAN #define PMAP_MAP_POOLPAGE(pa) AARCH64_PA_TO_KVA(pa) #define PMAP_UNMAP_POOLPAGE(va) AARCH64_KVA_TO_PA(va) @@ -81,10 +119,21 @@ struct pmap { struct pmap_statistics pm_stats; unsigned int pm_refcnt; unsigned int pm_idlepdp; - int pm_asid; + + kcpuset_t *pm_onproc; + kcpuset_t *pm_active; + + struct pmap_asid_info pm_pai[PMAP_TLB_MAX]; bool pm_activated; + bool pm_remove_all; }; +static inline paddr_t +pmap_l0pa(struct pmap *pm) +{ + return pm->pm_l0table_pa; +} + /* * should be kept <=32 bytes sized to reduce memory consumption & cache misses, * but it doesn't... @@ -173,6 +222,20 @@ struct vm_page_md { #define l3pte_is_page(pde) (((pde) & LX_TYPE) == L3_TYPE_PAG) /* l3pte contains always page entries */ + +static inline uint64_t +pte_value(pt_entry_t pte) +{ + return pte; +} + +static inline bool +pte_valid_p(pt_entry_t pte) +{ + + return l3pte_valid(pte); +} + void pmap_bootstrap(vaddr_t, vaddr_t); bool pmap_fault_fixup(struct pmap *, vaddr_t, vm_prot_t, bool user); @@ -322,7 +385,6 @@ aarch64_mmap_flags(paddr_t mdpgno) #define pmap_phys_address(pa) aarch64_ptob((pa)) #define pmap_mmap_flags(ppn) aarch64_mmap_flags((ppn)) -#define pmap_update(pmap) ((void)0) #define pmap_copy(dp,sp,d,l,s) ((void)0) #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) diff --git a/sys/arch/aarch64/include/pte.h b/sys/arch/aarch64/include/pte.h index 530ec3283532..cb4d37436970 100644 --- a/sys/arch/aarch64/include/pte.h +++ b/sys/arch/aarch64/include/pte.h @@ -110,7 +110,7 @@ typedef uint64_t pt_entry_t; /* L3(4k) table entry */ #define L1_SIZE (1UL << L1_SHIFT) #define L1_OFFSET (L1_SIZE - 1UL) #define L1_FRAME (~L1_OFFSET) -#define L1_BLOCK (LX_BLKPAG_NG | LX_TYPE_BLK | LX_VALID) +#define L1_BLOCK (LX_TYPE_BLK | LX_VALID) #define L1_TABLE (LX_TYPE_TBL | LX_VALID) /* L2 table, 2MB/entry * 512 */ @@ -119,7 +119,7 @@ typedef uint64_t pt_entry_t; /* L3(4k) table entry */ #define L2_SIZE (1UL << L2_SHIFT) #define L2_OFFSET (L2_SIZE - 1UL) #define L2_FRAME (~L2_OFFSET) -#define L2_BLOCK (LX_BLKPAG_NG | LX_TYPE_BLK | LX_VALID) +#define L2_BLOCK (LX_TYPE_BLK | LX_VALID) #define L2_TABLE (LX_TYPE_TBL | LX_VALID) #define L2_BLOCK_MASK __BITS(47,21) @@ -129,7 +129,7 @@ typedef uint64_t pt_entry_t; /* L3(4k) table entry */ #define L3_SIZE (1UL << L3_SHIFT) #define L3_OFFSET (L3_SIZE - 1UL) #define L3_FRAME (~L3_OFFSET) -#define L3_PAGE (LX_BLKPAG_NG | L3_TYPE_PAG | LX_VALID) +#define L3_PAGE (L3_TYPE_PAG | LX_VALID) #define Ln_ENTRIES_SHIFT 9 #define Ln_ENTRIES (1 << Ln_ENTRIES_SHIFT) diff --git a/sys/arch/aarch64/include/types.h b/sys/arch/aarch64/include/types.h index b9c6df96ead1..47f3fb647c20 100644 --- a/sys/arch/aarch64/include/types.h +++ b/sys/arch/aarch64/include/types.h @@ -97,7 +97,6 @@ typedef __uint64_t __register_t; #define __HAVE_COMMON___TLS_GET_ADDR #define __HAVE_CPU_COUNTER #define __HAVE_CPU_DATA_FIRST -#define __HAVE_CPU_MAXPROC #define __HAVE_FAST_SOFTINTS #define __HAVE_MINIMAL_EMUL #define __HAVE_MM_MD_DIRECT_MAPPED_PHYS