Index: sys/arch/riscv/include/machdep.h =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/include/machdep.h,v retrieving revision 1.3 diff -u -p -r1.3 machdep.h --- sys/arch/riscv/include/machdep.h 16 Oct 2022 06:14:53 -0000 1.3 +++ sys/arch/riscv/include/machdep.h 18 Oct 2022 06:47:36 -0000 @@ -66,4 +66,7 @@ paddr_t init_mmu(paddr_t); void init_riscv(register_t, paddr_t); +void pt_dump(void (*)(const char *, ...)); + + #endif /* _RISCV_MACHDEP_H_ */ Index: sys/arch/riscv/include/pte.h =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/include/pte.h,v retrieving revision 1.10 diff -u -p -r1.10 pte.h --- sys/arch/riscv/include/pte.h 18 Oct 2022 06:44:43 -0000 1.10 +++ sys/arch/riscv/include/pte.h 18 Oct 2022 06:47:36 -0000 @@ -75,6 +75,9 @@ typedef uint32_t pd_entry_t; #define PTE_KERN (PTE_V | PTE_G | PTE_A | PTE_D) #define PTE_RW (PTE_R | PTE_W) #define PTE_RX (PTE_R | PTE_X) +#define PTE_RWX (PTE_R | PTE_W | PTE_X) + +#define PTE_IS_LEAF(pte) (((pte) & PTE_RWX) != 0) #define PA_TO_PTE(pa) (((pa) >> PAGE_SHIFT) << PTE_PPN_SHIFT) #define PTE_TO_PA(pte) (((pte) >> PTE_PPN_SHIFT) << PAGE_SHIFT) @@ -309,4 +312,9 @@ pte_value(pt_entry_t pte) return pte; } +#ifdef __PTE_PRIVATE +extern pt_entry_t l2_pte[PAGE_SIZE / sizeof(pt_entry_t)]; +extern pt_entry_t l1_pte[PAGE_SIZE / sizeof(pt_entry_t)]; +#endif /* __PTE_PRIVATE */ + #endif /* _RISCV_PTE_H_ */ Index: sys/arch/riscv/include/vmparam.h =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/include/vmparam.h,v retrieving revision 1.13 diff -u -p -r1.13 vmparam.h --- sys/arch/riscv/include/vmparam.h 16 Oct 2022 06:14:53 -0000 1.13 +++ sys/arch/riscv/include/vmparam.h 18 Oct 2022 06:47:36 -0000 @@ -150,10 +150,21 @@ #define RISCV_DIRECTMAP_SIZE (-RISCV_DIRECTMAP_MASK - PAGE_SIZE) /* 128GiB */ #define RISCV_DIRECTMAP_START RISCV_DIRECTMAP_MASK #define RISCV_DIRECTMAP_END (RISCV_DIRECTMAP_START + RISCV_DIRECTMAP_SIZE) -#define RISCV_KVA_P(va) (((vaddr_t) (va) & RISCV_DIRECTMAP_MASK) != 0) +#define RISCV_KVA_P(va) (((vaddr_t) (va) & RISCV_DIRECTMAP_MASK) != 0) +#if 1 +#define RISCV_PA_TO_KVA(pa) PMAP_DIRECT_MAP(pa) +#define RISCV_KVA_TO_PA(va) PMAP_DIRECT_UNMAP(va) +#else #define RISCV_PA_TO_KVA(pa) ((vaddr_t) ((pa) | RISCV_DIRECTMAP_START)) #define RISCV_KVA_TO_PA(va) ((paddr_t) ((va) & ~RISCV_DIRECTMAP_MASK)) #endif +#else +/* + * As we don't have the address space only map 2GB of physical memory (RAM) + * at VA 0 in early boot. + */ +#define RISCV_DIRECTMAP_START ((vaddr_t) 0x0L) +#endif /* * The address to which unspecified mapping requests default Index: sys/arch/riscv/riscv/genassym.cf =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/riscv/genassym.cf,v retrieving revision 1.13 diff -u -p -r1.13 genassym.cf --- sys/arch/riscv/riscv/genassym.cf 16 Oct 2022 06:14:53 -0000 1.13 +++ sys/arch/riscv/riscv/genassym.cf 18 Oct 2022 06:47:36 -0000 @@ -188,6 +188,7 @@ define VM_KERNEL_DTB_BASE VM_KERNEL_DTB_ define VM_KERNEL_DTB_SIZE VM_KERNEL_DTB_SIZE define VM_KERNEL_IO_BASE VM_KERNEL_IO_BASE define VM_KERNEL_IO_BASE VM_KERNEL_IO_BASE +define RISCV_DIRECTMAP_START RISCV_DIRECTMAP_START define USPACE USPACE ifdef XSEGSHIFT Index: sys/arch/riscv/riscv/locore.S =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/riscv/locore.S,v retrieving revision 1.39 diff -u -p -r1.39 locore.S --- sys/arch/riscv/riscv/locore.S 16 Oct 2022 06:14:53 -0000 1.39 +++ sys/arch/riscv/riscv/locore.S 18 Oct 2022 06:47:37 -0000 @@ -120,7 +120,6 @@ ENTRY_NP(start) sub s8, s8, t0 - PTR_LA s5, _C_LABEL(lwp0uspace) PTR_LA s6, _C_LABEL(bootstk) /* @@ -140,6 +139,7 @@ ENTRY_NP(start) #ifdef notyet PTR_LA s4, _C_LABEL(l3_pte) #endif + PTR_LA s5, _C_LABEL(l1_ptedm) // s5 = last page table for DM (RV64 only) #endif PTR_LA s7, _C_LABEL(mmutables_end) @@ -147,7 +147,7 @@ ENTRY_NP(start) // s2 L1 PDE (SV32:4MiB megapages, SV{39,48}: 2MiB megapages) // s3 L2 PDE (_LP64 SV39 only) // s4 L3 PDE (_LP64 SV48 only) - // s5 lwp0uspace + // s5 L1 PDE (directmap) // s6 bootstk // s7 end of memory to clear @@ -160,10 +160,10 @@ ENTRY_NP(start) VPRINTS("l3: ") VPRINTXNL(s4) #endif + VPRINTS("l1(dm): ") + VPRINTXNL(s5) #endif - VPRINTS("uspace: ") - VPRINTXNL(s5) VPRINTS("bootstk: ") VPRINTXNL(s6) @@ -211,6 +211,12 @@ ENTRY_NP(start) VPRINTS("l2pde: ") VPRINTX(t1) + + + + + + #else #ifdef notyet REG_S t0, 0(s4) @@ -224,9 +230,26 @@ ENTRY_NP(start) VPRINTS(": ") VPRINTXNL(t0) - VPRINTS("\n\r") #endif // _LP64 +#ifdef _LP64 + srli t0, s5, (PGSHIFT - PTE_PPN_SHIFT) + or t0, t0, s7 // Assumes s5[11:0] == 0 + + li t1, ((RISCV_DIRECTMAP_START >> XSEGSHIFT) & (NPDEPG - 1)) * SZREG + add t1, t1, s3 + + REG_S t0, 0(t1) + + VPRINTS("l2pde/dm:") + VPRINTX(t1) + + VPRINTS(": ") + VPRINTXNL(t0) +#endif + + VPRINTS("\n\r") + // kernel VA li t1, ((VM_MIN_KERNEL_ADDRESS >> SEGSHIFT) & (NPDEPG - 1)) * SZREG add s9, s2, t1 @@ -569,6 +592,8 @@ l2_pte: l3_pte: .space PAGE_SIZE #endif +l1_ptedm: + .space PAGE_SIZE #endif mmutables_end: Index: sys/arch/riscv/riscv/pmap_machdep.c =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/riscv/pmap_machdep.c,v retrieving revision 1.13 diff -u -p -r1.13 pmap_machdep.c --- sys/arch/riscv/riscv/pmap_machdep.c 16 Oct 2022 08:43:44 -0000 1.13 +++ sys/arch/riscv/riscv/pmap_machdep.c 18 Oct 2022 06:47:37 -0000 @@ -33,6 +33,7 @@ #include "opt_riscv_debug.h" #define __PMAP_PRIVATE +#define __PTE_PRIVATE #include __RCSID("$NetBSD: pmap_machdep.c,v 1.13 2022/10/16 08:43:44 skrll Exp $"); @@ -192,7 +193,7 @@ pmap_md_pdetab_init(struct pmap *pmap) void pmap_bootstrap(vaddr_t vstart, vaddr_t vend) { - extern pd_entry_t l1_pte[512]; + extern pd_entry_t l1_pte[PAGE_SIZE / sizeof(pd_entry_t)]; pmap_t pm = pmap_kernel(); pmap_bootstrap_common(); @@ -220,7 +221,6 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t v kcpuset_set(pm->pm_active, cpu_number()); #endif - VPRINTF("nkmempages "); /* * Compute the number of pages kmem_arena will have. This will also * be called by uvm_km_bootstrap later, but that doesn't matter @@ -258,7 +258,9 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t v pmap_limits.virtual_start = vstart; pmap_limits.virtual_end = vend; - VPRINTF("\nlimits: %" PRIxVADDR " - %" PRIxVADDR "\n", vstart, vend); + VPRINTF("limits: %" PRIxVADDR " - %" PRIxVADDR "\n", vstart, vend); + pmap_curmaxkvaddr = vstart; /* XXXXXXXXXX surely this is wrong?? */ + VPRINTF("initial pmap_curmaxkvaddr = %"PRIxVADDR"\n", pmap_curmaxkvaddr); /* * Initialize the pools. Index: sys/arch/riscv/riscv/riscv_machdep.c =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/riscv/riscv_machdep.c,v retrieving revision 1.23 diff -u -p -r1.23 riscv_machdep.c --- sys/arch/riscv/riscv/riscv_machdep.c 18 Oct 2022 04:24:54 -0000 1.23 +++ sys/arch/riscv/riscv/riscv_machdep.c 18 Oct 2022 06:47:37 -0000 @@ -35,6 +35,7 @@ #include __RCSID("$NetBSD: riscv_machdep.c,v 1.23 2022/10/18 04:24:54 skrll Exp $"); +#define __PTE_PRIVATE #include #include @@ -385,6 +386,53 @@ cpu_startup(void) printf("avail memory = %s\n", pbuf); } + +static void +cpu_kernel_vm_init(paddr_t memory_start, paddr_t memory_end) +{ + extern char __kernel_text[]; + extern char _end[]; + + vaddr_t kernstart = trunc_page((vaddr_t)__kernel_text); + vaddr_t kernend = round_page((vaddr_t)_end); + paddr_t kernstart_phys = KERN_VTOPHYS(kernstart); + paddr_t kernend_phys = KERN_VTOPHYS(kernend); + + VPRINTF("%s: kernel phys start %#" PRIxPADDR " end %#" PRIxPADDR "\n", + __func__, kernstart_phys, kernend_phys); + fdt_memory_remove_range(kernstart_phys, + kernend_phys - kernstart_phys); + + /* + * Now we have APs started the pages used for stacks and L1PT can + * be given to uvm + */ + extern char const __start__init_memory[]; + extern char const __stop__init_memory[] __weak; + if (__start__init_memory != __stop__init_memory) { + const paddr_t spa = KERN_VTOPHYS((vaddr_t)__start__init_memory); + const paddr_t epa = KERN_VTOPHYS((vaddr_t)__stop__init_memory); + + VPRINTF("%s: init phys start %#" PRIxPADDR + " end %#" PRIxPADDR "\n", __func__, spa, epa); + fdt_memory_remove_range(spa, epa - spa); + } + +#ifdef _LP64 + paddr_t pa = memory_start & ~XSEGOFSET; + pmap_direct_base = RISCV_DIRECTMAP_START - pa; + + /* Allocate gigapages covering all physical memory in the direct map. */ + for (size_t i = pl2_i(RISCV_DIRECTMAP_START); + i < pl2_i(RISCV_DIRECTMAP_END) && pa < memory_end; + i++, pa += L2_SIZE) { + l2_pte[i] = PA_TO_PTE(pa) | PTE_KERN | PTE_RW; + VPRINTF("dm: %p : %#" PRIxPADDR "\n", &l2_pte[i], l2_pte[i]); + } +#endif + pt_dump(printf); +} + static void riscv_init_lwp0_uarea(void) { @@ -411,6 +459,20 @@ riscv_print_memory(const struct fdt_memo static void +riscv_add_memory(const struct fdt_memory *m, void *arg) +{ + paddr_t first = atop(m->start); + paddr_t last = atop(m->end); + int freelist = VM_FREELIST_DEFAULT; + + VPRINTF("adding %#16" PRIxPADDR " - %#16" PRIxPADDR" to freelist %d\n", + m->start, m->end, freelist); + uvm_page_physload(first, last, first, last, freelist); + physmem += last - first; +} + + +static void parse_bi_bootargs(char *args) { int howto; @@ -510,7 +572,7 @@ init_riscv(register_t hartid, paddr_t dt PRIx64 ")\n", __func__, memory_start, memory_end, memory_size); /* Perform PT build and VM init */ - //cpu_kernel_vm_init(); + cpu_kernel_vm_init(memory_start, memory_end); VPRINTF("bootargs: %s\n", bootargs); @@ -531,7 +593,7 @@ init_riscv(register_t hartid, paddr_t dt kernelvmstart = kernend_mega; -#define DPRINTF(v) VPRINTF("%24s = 0x%16lx\n", #v, (unsigned long)v); +#define DPRINTF(v) VPRINTF("%24s = 0x%016lx\n", #v, (unsigned long)v); VPRINTF("------------------------------------------\n"); DPRINTF(kern_vtopdiff); @@ -546,6 +608,7 @@ init_riscv(register_t hartid, paddr_t dt DPRINTF(kernend); DPRINTF(kernend_mega); DPRINTF(VM_MAX_KERNEL_ADDRESS); + DPRINTF(pmap_direct_base); VPRINTF("------------------------------------------\n"); #undef DPRINTF @@ -555,20 +618,111 @@ init_riscv(register_t hartid, paddr_t dt kernelvmstart = VM_KERNEL_VM_BASE; /* - * msgbuf is allocated from the bottom of any one of memory blocks - * to avoid corruption due to bootloader or changing kernel layout. + * msgbuf is allocated from the top of the last biggest memory block. */ paddr_t msgbufaddr = 0; - KASSERT(msgbufaddr != 0); /* no space for msgbuf */ #ifdef _LP64 - initmsgbuf((void *)RISCV_PA_TO_KVA(msgbufaddr), MSGBUFSIZE); + /* XXX check all ranges for last one with a big enough hole */ + msgbufaddr = memory_end - MSGBUFSIZE; + KASSERT(msgbufaddr != 0); /* no space for msgbuf */ + fdt_memory_remove_range(msgbufaddr, msgbufaddr + MSGBUFSIZE); + msgbufaddr = RISCV_PA_TO_KVA(msgbufaddr); + VPRINTF("msgbufaddr = %#lx\n", msgbufaddr); + initmsgbuf((void *)msgbufaddr, MSGBUFSIZE); #endif + KASSERT(msgbufaddr != 0); /* no space for msgbuf */ uvm_md_init(); + physmem = 0; + fdt_memory_foreach(riscv_add_memory, NULL); + pmap_bootstrap(kernelvmstart, VM_MAX_KERNEL_ADDRESS); /* Finish setting up lwp0 on our end before we call main() */ riscv_init_lwp0_uarea(); } + + +#ifdef _LP64 +static void +pte_bits(void (*pr)(const char *, ...), pt_entry_t pte) +{ + (*pr)("%c%c%c%c%c%c%c%c", + (pte & PTE_D) ? 'D' : '.', + (pte & PTE_A) ? 'A' : '.', + (pte & PTE_G) ? 'G' : '.', + (pte & PTE_U) ? 'U' : '.', + (pte & PTE_X) ? 'X' : '.', + (pte & PTE_W) ? 'W' : '.', + (pte & PTE_R) ? 'R' : '.', + (pte & PTE_V) ? 'V' : '.'); +} +#endif + +/* + * XXX All this has gotten ugly and we haven't got to L0 ptes yet... + */ +void +pt_dump(void (*pr)(const char *, ...)) +{ +#ifdef _LP64 + const register_t satp = riscvreg_satp_read(); + const paddr_t satp_pa = __SHIFTOUT(satp, SATP_PPN) << PAGE_SHIFT; + pt_entry_t *l2, *l1, entry; + vaddr_t va_l2, va_l1; + + (*pr)("satp = 0x%" PRIxREGISTER "\n", satp); + l2 = (void *)PMAP_DIRECT_MAP(satp_pa); + (*pr)("l2 @ pa %#" PRIxREGISTER "\n", satp_pa); + for (size_t i = 0; i < PAGE_SIZE / sizeof(pt_entry_t); i++) { + entry = l2[i]; + va_l2 = i << L2_SHIFT; + size_t topbit = 63; + switch (__SHIFTOUT(satp, SATP_MODE)) { + case SATP_MODE_SV39: + topbit = 39 - 1; + break; + case SATP_MODE_SV48: + topbit = 48 - 1; + break; + } + if (va_l2 & __BIT(topbit)) { + va_l2 |= __BITS(63, topbit + 1); + } + if (entry != 0) { + paddr_t pa = __SHIFTOUT(entry, PTE_PPN2) << L2_SHIFT; + if (PTE_IS_LEAF(entry)) { + (*pr)("l2 %3zu va 0x%016lx pa 0x%010lx - ", i, va_l2, pa); + pte_bits(pr, entry); + (*pr)("\n"); + } else { + pa = __SHIFTOUT(entry, PTE_PPN) << PAGE_SHIFT; + + (*pr)("l2 %3zu va 0x%016lx -> 0x%010lx - ", i, va_l2, pa); + pte_bits(pr, entry); + (*pr)("\n"); + + l1 = (void *)PMAP_DIRECT_MAP(pa); + (*pr)(" l1 @ pa %#"PRIxREGISTER"\n", pa); + for (size_t j = 0; j < PAGE_SIZE / sizeof(pt_entry_t); j++) { + entry = l1[j]; + va_l1 = va_l2 + (j << L1_SHIFT); + if (entry != 0) { + pa = __SHIFTOUT(entry, PTE_PPN1) << L1_SHIFT; + if (PTE_IS_LEAF(entry)) { + (*pr)(" l1 %3zu va 0x%016lx pa 0x%010lx - ", j, va_l1, pa); + pte_bits(pr, entry); + (*pr)("\n"); + } else { + (*pr)(" l1 %3zu va 0x%016lx -> 0x%010lx - ", i, va_l1, pa); + /* XXX ... descend */ + } + } + } + } + } + } +#endif +} Index: sys/arch/riscv/riscv/trap.c =================================================================== RCS file: /cvsroot/src/sys/arch/riscv/riscv/trap.c,v retrieving revision 1.19 diff -u -p -r1.19 trap.c --- sys/arch/riscv/riscv/trap.c 15 Oct 2022 06:41:43 -0000 1.19 +++ sys/arch/riscv/riscv/trap.c 18 Oct 2022 06:47:37 -0000 @@ -47,6 +47,7 @@ __RCSID("$NetBSD: trap.c,v 1.19 2022/10/ #include #include +#include #include #define MACHINE_ECALL_TRAP_MASK (__BIT(CAUSE_MACHINE_ECALL)) @@ -396,6 +397,12 @@ cpu_trap(struct trapframe *tf, register_ bool ok = true; ksiginfo_t ksi; +printf("cpu_trap()\n"); +pt_dump(printf); +printf(" status %#"PRIxREGISTER"\n", status); +printf(" cause %#"PRIxREGISTER"\n", cause); +printf(" tval %#"PRIxREGISTER"\n", tval); + if (__predict_true(fault_mask & FAULT_TRAP_MASK)) { #ifndef _LP64 #if 0