From 5b27c9a2891c57ffedd49e1650274acb13364a41 Mon Sep 17 00:00:00 2001 From: Nick Hudson Date: Sun, 17 Oct 2021 11:46:59 +0100 Subject: [PATCH 1/7] Rework Arm (32bit and 64bit) AP startup so that cpu_hatch doesn't sleep. The AP initialisation code in cpu_init_secondary_processor will read and initialise the required system registers and state for the BP to attach and report. The VFP detection handler on <= armv7 relies on the global undefined handler being in place until the BP attaches vfp. Rework the interrupt handler code so this new sequence, Thankfully, this removes a bunch of code for bcm2836mp Tested on RPI3, Apple M1, QEMU Fixes PR port-arm/56264: diagnostic assertion "l->l_stat == LSONPROC" failed on RPI3 --- sys/arch/aarch64/aarch64/aarch64_machdep.c | 2 +- sys/arch/aarch64/aarch64/cpu.c | 99 +++++++------ sys/arch/aarch64/aarch64/cpufunc.c | 107 ++++++-------- sys/arch/aarch64/aarch64/db_machdep.c | 19 ++- sys/arch/aarch64/aarch64/locore.S | 56 +++----- sys/arch/aarch64/include/cpu.h | 34 ++++- sys/arch/aarch64/include/cpufunc.h | 33 +---- sys/arch/aarch64/include/db_machdep.h | 3 +- sys/arch/arm/apple/apple_intc.c | 87 ++++++----- sys/arch/arm/arm/cpu_subr.c | 7 +- sys/arch/arm/arm/undefined.c | 36 +++++ sys/arch/arm/arm32/arm32_boot.c | 23 +-- sys/arch/arm/arm32/arm32_machdep.c | 12 +- sys/arch/arm/arm32/cpu.c | 52 ++++--- sys/arch/arm/broadcom/bcm2835_intr.c | 110 +++----------- sys/arch/arm/cortex/gicv3_its.c | 16 ++- sys/arch/arm/cortex/gicv3_its.h | 3 + sys/arch/arm/cortex/gtmr.c | 1 + sys/arch/arm/include/cpu.h | 14 +- sys/arch/arm/include/locore.h | 1 + sys/arch/arm/include/undefined.h | 5 +- sys/arch/arm/pic/pic.c | 2 +- sys/arch/arm/vfp/vfp_init.c | 160 +++++++++------------ 23 files changed, 423 insertions(+), 459 deletions(-) diff --git a/sys/arch/aarch64/aarch64/aarch64_machdep.c b/sys/arch/aarch64/aarch64/aarch64_machdep.c index 14f1d3a54e6f..e7c1911b4a75 100644 --- a/sys/arch/aarch64/aarch64/aarch64_machdep.c +++ b/sys/arch/aarch64/aarch64/aarch64_machdep.c @@ -359,7 +359,7 @@ initarm_common(vaddr_t kvm_base, vsize_t kvm_size, VM_MAX_KERNEL_ADDRESS); #ifdef DDB - db_machdep_init(); + db_machdep_cpu_init(); #endif uvm_md_init(); diff --git a/sys/arch/aarch64/aarch64/cpu.c b/sys/arch/aarch64/aarch64/cpu.c index f29c403f900d..6ab581fa95b6 100644 --- a/sys/arch/aarch64/aarch64/cpu.c +++ b/sys/arch/aarch64/aarch64/cpu.c @@ -72,12 +72,13 @@ __KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.66 2021/10/30 10:47:03 skrll Exp $"); #endif void cpu_attach(device_t, cpuid_t); +void cpu_setup_id(struct cpu_info *); + static void identify_aarch64_model(uint32_t, char *, size_t); static void cpu_identify(device_t self, struct cpu_info *); static void cpu_identify1(device_t self, struct cpu_info *); static void cpu_identify2(device_t self, struct cpu_info *); static void cpu_init_counter(struct cpu_info *); -static void cpu_setup_id(struct cpu_info *); static void cpu_setup_sysctl(device_t, struct cpu_info *); static void cpu_setup_rng(device_t, struct cpu_info *); static void cpu_setup_aes(device_t, struct cpu_info *); @@ -109,7 +110,6 @@ cpu_attach(device_t dv, cpuid_t id) if (unit == 0) { ci = curcpu(); ci->ci_cpuid = id; - cpu_setup_id(ci); } else { #ifdef MULTIPROCESSOR if ((boothowto & RB_MD1) != 0) { @@ -150,26 +150,31 @@ cpu_attach(device_t dv, cpuid_t id) arm_cpu_do_topology(ci); cpu_identify(dv, ci); + cpu_setup_sysctl(dv, ci); + #ifdef MULTIPROCESSOR if (unit != 0) { mi_cpu_attach(ci); pmap_tlb_info_attach(&pmap_tlb0_info, ci); - return; + aarch64_parsecacheinfo(ci); } #endif /* MULTIPROCESSOR */ - set_cpufuncs(); fpu_attach(ci); cpu_identify1(dv, ci); - - /* aarch64_getcacheinfo(0) was called by locore.S */ - aarch64_printcacheinfo(dv); + aarch64_printcacheinfo(dv, ci); cpu_identify2(dv, ci); + if (unit != 0) { + return; + } + + db_machdep_init(ci); + cpu_init_counter(ci); - cpu_setup_sysctl(dv, ci); + /* These currently only check the BP. */ cpu_setup_rng(dv, ci); cpu_setup_aes(dv, ci); cpu_setup_chacha(dv, ci); @@ -251,10 +256,9 @@ cpu_identify(device_t self, struct cpu_info *ci) static void cpu_identify1(device_t self, struct cpu_info *ci) { - uint64_t ctr, clidr, sctlr; /* for cache */ + struct aarch64_sysctl_cpu_id *id = &ci->ci_id; + uint64_t sctlr = ci->ci_sctlr_el1; - /* SCTLR - System Control Register */ - sctlr = reg_sctlr_el1_read(); if (sctlr & SCTLR_I) aprint_verbose_dev(self, "IC enabled"); else @@ -288,8 +292,8 @@ cpu_identify1(device_t self, struct cpu_info *ci) /* * CTR - Cache Type Register */ - ctr = reg_ctr_el0_read(); - clidr = reg_clidr_el1_read(); + const uint64_t ctr = id->ac_ctr; + const uint64_t clidr = id->ac_clidr; aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B," " Exclusives Reservation Granule %" PRIu64 "B\n", __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4, @@ -313,22 +317,14 @@ cpu_identify1(device_t self, struct cpu_info *ci) static void cpu_identify2(device_t self, struct cpu_info *ci) { - struct aarch64_sysctl_cpu_id *id = &ci->ci_id; - uint64_t dfr0; - - if (!CPU_IS_PRIMARY(ci)) { - cpu_setup_id(ci); - cpu_setup_sysctl(self, ci); - } - - dfr0 = reg_id_aa64dfr0_el1_read(); + struct aarch64_sysctl_cpu_id * const id = &ci->ci_id; aprint_debug_dev(self, "midr=0x%" PRIx32 " mpidr=0x%" PRIx32 "\n", - (uint32_t)ci->ci_id.ac_midr, (uint32_t)ci->ci_id.ac_mpidr); + (uint32_t)id->ac_midr, (uint32_t)id->ac_mpidr); aprint_verbose_dev(self, "revID=0x%" PRIx64, id->ac_revidr); /* ID_AA64DFR0_EL1 */ - switch (__SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER)) { + switch (__SHIFTOUT(id->ac_aa64dfr0, ID_AA64DFR0_EL1_PMUVER)) { case ID_AA64DFR0_EL1_PMUVER_V3: aprint_verbose(", PMCv3"); break; @@ -501,13 +497,16 @@ cpu_init_counter(struct cpu_info *ci) } /* - * Fill in this CPUs id data. Must be called from hatched cpus. + * Fill in this CPUs id data. Must be called on all cpus. */ -static void +void __noasan cpu_setup_id(struct cpu_info *ci) { struct aarch64_sysctl_cpu_id *id = &ci->ci_id; + /* SCTLR - System Control Register */ + ci->ci_sctlr_el1 = reg_sctlr_el1_read(); + memset(id, 0, sizeof *id); id->ac_midr = reg_midr_el1_read(); @@ -611,10 +610,6 @@ cpu_setup_rng(device_t dv, struct cpu_info *ci) { struct aarch64_sysctl_cpu_id *id = &ci->ci_id; - /* Probably shared between cores. */ - if (!CPU_IS_PRIMARY(ci)) - return; - /* Verify that it is supported. */ switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) { case ID_AA64ISAR0_EL1_RNDR_RNDRRS: @@ -676,25 +671,47 @@ cpu_setup_chacha(device_t dv, struct cpu_info *ci) } #ifdef MULTIPROCESSOR +/* + * Initialise a secondary processor. + * + * printf isn't available as kmutex(9) relies on curcpu which isn't setup yet. + * + */ +void __noasan +cpu_init_secondary_processor(int cpuindex) +{ + struct cpu_info * ci = &cpu_info_store[cpuindex]; + struct aarch64_sysctl_cpu_id *id = &ci->ci_id; + + aarch64_setcpufuncs(ci); + + /* Sets ci->ci_{sctlr,midr,mpidr}, etc */ + cpu_setup_id(ci); + + arm_cpu_topology_set(ci, id->ac_mpidr); + aarch64_getcacheinfo(ci); + + cpu_set_hatched(cpuindex); + + /* + * return to assembly to wait for cpu_boot_secondary_processors + */ +} + + +/* + * When we are called, the MMU and caches are on and we are running on the stack + * of the idlelwp for this cpu. + */ void cpu_hatch(struct cpu_info *ci) { KASSERT(curcpu() == ci); KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); - mutex_enter(&cpu_hatch_lock); - - set_cpufuncs(); - fpu_attach(ci); - - cpu_identify1(ci->ci_dev, ci); - aarch64_getcacheinfo(device_unit(ci->ci_dev)); - aarch64_printcacheinfo(ci->ci_dev); - cpu_identify2(ci->ci_dev, ci); #ifdef DDB - db_machdep_init(); + db_machdep_cpu_init(); #endif - mutex_exit(&cpu_hatch_lock); cpu_init_counter(ci); diff --git a/sys/arch/aarch64/aarch64/cpufunc.c b/sys/arch/aarch64/aarch64/cpufunc.c index 30db2d96841f..08f4590efbee 100644 --- a/sys/arch/aarch64/aarch64/cpufunc.c +++ b/sys/arch/aarch64/aarch64/cpufunc.c @@ -53,13 +53,7 @@ u_int aarch64_cache_prefer_mask; int aarch64_pan_enabled __read_mostly; int aarch64_pac_enabled __read_mostly; -/* cache info per cluster. the same cluster has the same cache configuration? */ -#define MAXCPUPACKAGES MAXCPUS /* maximum of ci->ci_package_id */ -static struct aarch64_cache_info *aarch64_cacheinfo[MAXCPUPACKAGES]; -static struct aarch64_cache_info aarch64_cacheinfo0[MAX_CACHE_LEVEL]; - - -static void +static void __noasan extract_cacheunit(int level, bool insn, int cachetype, struct aarch64_cache_info *cacheinfo) { @@ -101,35 +95,14 @@ extract_cacheunit(int level, bool insn, int cachetype, cunit->cache_size = cunit->cache_way_size * cunit->cache_ways; } -void -aarch64_getcacheinfo(int unit) + +/* Must be called on each processor */ +void __noasan +aarch64_getcacheinfo(struct cpu_info *ci) { - struct cpu_info * const ci = curcpu(); + struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; uint32_t clidr, ctr; - u_int vindexsize; int level, cachetype; - struct aarch64_cache_info *cinfo = NULL; - - if (cputype == 0) - cputype = aarch64_cpuid(); - - /* already extract about this cluster? */ - KASSERT(ci->ci_package_id < MAXCPUPACKAGES); - cinfo = aarch64_cacheinfo[ci->ci_package_id]; - if (cinfo != NULL) { - ci->ci_cacheinfo = cinfo; - return; - } - - /* Need static buffer for the boot CPU */ - if (unit == 0) - cinfo = aarch64_cacheinfo0; - else - cinfo = kmem_zalloc(sizeof(struct aarch64_cache_info) - * MAX_CACHE_LEVEL, KM_SLEEP); - aarch64_cacheinfo[ci->ci_package_id] = cinfo; - ci->ci_cacheinfo = cinfo; - /* * CTR - Cache Type Register @@ -150,19 +123,6 @@ aarch64_getcacheinfo(int unit) break; } - /* remember maximum alignment */ - if (arm_dcache_maxline < __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE)) { - arm_dcache_maxline = __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE); - arm_dcache_align = sizeof(int) << arm_dcache_maxline; - arm_dcache_align_mask = arm_dcache_align - 1; - } - -#ifdef MULTIPROCESSOR - if (coherency_unit < arm_dcache_align) - panic("coherency_unit %ld < %d; increase COHERENCY_UNIT", - coherency_unit, arm_dcache_align); -#endif - /* * CLIDR - Cache Level ID Register * CSSELR - Cache Size Selection Register @@ -213,6 +173,29 @@ aarch64_getcacheinfo(int unit) */ cachetype = CACHE_TYPE_PIPT; } +} + + +void +aarch64_parsecacheinfo(struct cpu_info *ci) +{ + struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; + struct aarch64_sysctl_cpu_id *id = &ci->ci_id; + const uint32_t ctr = id->ac_ctr; + u_int vindexsize; + + /* remember maximum alignment */ + if (arm_dcache_maxline < __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE)) { + arm_dcache_maxline = __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE); + arm_dcache_align = sizeof(int) << arm_dcache_maxline; + arm_dcache_align_mask = arm_dcache_align - 1; + } + +#ifdef MULTIPROCESSOR + if (coherency_unit < arm_dcache_align) + panic("coherency_unit %ld < %d; increase COHERENCY_UNIT", + coherency_unit, arm_dcache_align); +#endif /* calculate L1 icache virtual index size */ if ((cinfo[0].icache.cache_type == CACHE_TYPE_VIVT || @@ -232,6 +215,7 @@ aarch64_getcacheinfo(int unit) if (vindexsize > aarch64_cache_vindexsize) { aarch64_cache_vindexsize = vindexsize; aarch64_cache_prefer_mask = vindexsize - 1; + if (uvm.page_init_done) uvm_page_recolor(vindexsize / PAGE_SIZE); } @@ -311,13 +295,11 @@ prt_cache(device_t self, struct aarch64_cache_info *cinfo, int level) } void -aarch64_printcacheinfo(device_t dev) +aarch64_printcacheinfo(device_t dev, struct cpu_info *ci) { - struct aarch64_cache_info *cinfo; + struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; int level; - cinfo = curcpu()->ci_cacheinfo; - for (level = 0; level < MAX_CACHE_LEVEL; level++) if (prt_cache(dev, cinfo, level) < 0) break; @@ -382,11 +364,10 @@ ln_dcache_inv_all(int level, struct aarch64_cache_unit *cunit) void aarch64_dcache_wbinv_all(void) { - struct aarch64_cache_info *cinfo; + struct cpu_info * const ci = curcpu(); + struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; int level; - cinfo = curcpu()->ci_cacheinfo; - for (level = 0; level < MAX_CACHE_LEVEL; level++) { if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) break; @@ -400,11 +381,10 @@ aarch64_dcache_wbinv_all(void) void aarch64_dcache_inv_all(void) { - struct aarch64_cache_info *cinfo; + struct cpu_info * const ci = curcpu(); + struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; int level; - cinfo = curcpu()->ci_cacheinfo; - for (level = 0; level < MAX_CACHE_LEVEL; level++) { if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) break; @@ -418,11 +398,10 @@ aarch64_dcache_inv_all(void) void aarch64_dcache_wb_all(void) { - struct aarch64_cache_info *cinfo; + struct cpu_info * const ci = curcpu(); + struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; int level; - cinfo = curcpu()->ci_cacheinfo; - for (level = 0; level < MAX_CACHE_LEVEL; level++) { if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) break; @@ -436,7 +415,15 @@ aarch64_dcache_wb_all(void) int set_cpufuncs(void) { - struct cpu_info * const ci = curcpu(); + // This is only called from the BP + + return aarch64_setcpufuncs(&cpu_info_store[0]); +} + + +int +aarch64_setcpufuncs(struct cpu_info *ci) +{ const uint64_t ctr = reg_ctr_el0_read(); const uint64_t clidr = reg_clidr_el1_read(); diff --git a/sys/arch/aarch64/aarch64/db_machdep.c b/sys/arch/aarch64/aarch64/db_machdep.c index 547d6d29d6cd..96ac9628d57e 100644 --- a/sys/arch/aarch64/aarch64/db_machdep.c +++ b/sys/arch/aarch64/aarch64/db_machdep.c @@ -819,7 +819,7 @@ db_md_breakwatchpoints_reload(void) } void -db_machdep_init(void) +db_machdep_cpu_init(void) { uint64_t dfr, mdscr; int i, cpu_max_breakpoint, cpu_max_watchpoint; @@ -842,16 +842,27 @@ db_machdep_init(void) mdscr |= MDSCR_MDE | MDSCR_KDE; reg_mdscr_el1_write(mdscr); reg_oslar_el1_write(0); +} - /* num of {watch,break}point may be different depending on the core */ - membar_consumer(); +void +db_machdep_init(struct cpu_info * const ci) +{ + struct aarch64_sysctl_cpu_id * const id = &ci->ci_id; + const uint64_t dfr = id->ac_aa64dfr0; + const u_int cpu_max_breakpoint = __SHIFTOUT(dfr, ID_AA64DFR0_EL1_BRPS); + const u_int cpu_max_watchpoint = __SHIFTOUT(dfr, ID_AA64DFR0_EL1_WRPS); + + /* + * num of {watch,break}point may be different depending on the + * core. + */ if (max_breakpoint > cpu_max_breakpoint) max_breakpoint = cpu_max_breakpoint; if (max_watchpoint > cpu_max_watchpoint) max_watchpoint = cpu_max_watchpoint; - membar_producer(); } + static void show_breakpoints(void) { diff --git a/sys/arch/aarch64/aarch64/locore.S b/sys/arch/aarch64/aarch64/locore.S index 04c7c9c7d16b..e935a8c0c9b9 100644 --- a/sys/arch/aarch64/aarch64/locore.S +++ b/sys/arch/aarch64/aarch64/locore.S @@ -194,15 +194,22 @@ vstart: msr sctlr_el1, x0 1: + adrl x19, cpu_info_store /* curcpu (&cpu_info_store[0] */ + + mov x0, x19 + bl cpu_setup_id + /* set topology information */ - adrl x0, cpu_info_store /* curcpu */ + mov x0, x19 mrs x1, mpidr_el1 mov x2, #0 bl arm_cpu_topology_set - /* get cache configuration */ - mov x0, xzr + /* get and parse the cache configuration */ + mov x0, x19 bl aarch64_getcacheinfo + mov x0, x19 + bl aarch64_parsecacheinfo #ifdef KASAN adrl x0, lwp0uspace @@ -453,7 +460,6 @@ ENTRY_NP(cpu_mpstart) CPU_DPRINTREG("CurrentEL = ", x20) #endif /* LOCORE_EL2 */ - bl mmu_disable bl init_sysregs @@ -498,46 +504,14 @@ mp_vstart: msr tpidr_el0, xzr msr tpidrro_el0, xzr - mov x0, #CPU_INFO_SIZE - mul x0, x27, x0 - adrl x1, _C_LABEL(cpu_info_store) - add x0, x0, x1 /* x0 = &cpu_info_store[cpuindex] */ - - /* temporarily set tpidr_el1 to curcpu until the idle lwp is setup */ - msr tpidr_el1, x0 /* tpidr_el1 = curcpu = x0 */ - - /* fill curcpu()->ci_{midr,mpidr} */ - mrs x1, midr_el1 - str x1, [x0, #CI_MIDR] /* curcpu()->ci_cpuid = midr_el1 */ - mrs x1, mpidr_el1 - str x1, [x0, #CI_MPIDR] /* curcpu()->ci_mpidr = mpidr_el1 */ - - /* set topology information */ - mov x2, #0 - bl arm_cpu_topology_set - - /* x28 = &arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)] */ - adrl x0, _C_LABEL(arm_cpu_hatched) - // Appease clang - mov x1, x27, lsr #6 - orr x1, xzr, x27, lsr #6 - add x28, x0, x1, lsl #3 + mov x0, x27 + bl cpu_init_secondary_processor /* x29 = __BIT(cpuindex % (sizeof(u_long) * NBBY)) */ mov x0, #1 and x2, x27, #63 lsl x29, x0, x2 - /* - * atomic_or_ulong(&arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)], - * _BIT(cpuindex % ((sizeof(u_long) * NBBY) - * to inform the boot processor. - */ - mov x0, x28 - mov x1, x29 - bl _C_LABEL(atomic_or_ulong) /* hatched! */ - dsb sy - sev - /* x28 = &arm_cpu_mbox[cpuindex / (sizeof(u_long) * NBBY)] */ adrl x0, _C_LABEL(arm_cpu_mbox) // Appease clang - mov x1, x27, lsr #6 @@ -554,11 +528,15 @@ mp_vstart: b 1b 9: + mov x0, #CPU_INFO_SIZE + mul x0, x27, x0 + adrl x1, _C_LABEL(cpu_info_store) + add x0, x0, x1 /* x0 = &cpu_info_store[cpuindex] */ + /* * set curlwp (tpidr_el1 and curcpu()->ci_curlwp) now we know the * idle lwp from curcpu()->ci_idlelwp */ - mrs x0, tpidr_el1 /* curcpu (temporarily) */ ldr x1, [x0, #CI_IDLELWP] /* x0 = curcpu()->ci_idlelwp */ msr tpidr_el1, x1 /* tpidr_el1 = curlwp = x1 */ str x1, [x0, #CI_CURLWP] /* curlwp is idlelwp */ diff --git a/sys/arch/aarch64/include/cpu.h b/sys/arch/aarch64/include/cpu.h index a5c6889c02fc..1be7c3ffff1b 100644 --- a/sys/arch/aarch64/include/cpu.h +++ b/sys/arch/aarch64/include/cpu.h @@ -72,6 +72,32 @@ struct aarch64_cpufuncs { void (*cf_icache_sync_range)(vaddr_t, vsize_t); }; +#define MAX_CACHE_LEVEL 8 /* ARMv8 has maximum 8 level cache */ + +struct aarch64_cache_unit { + u_int cache_type; +#define CACHE_TYPE_VPIPT 0 /* VMID-aware PIPT */ +#define CACHE_TYPE_VIVT 1 /* ASID-tagged VIVT */ +#define CACHE_TYPE_VIPT 2 +#define CACHE_TYPE_PIPT 3 + u_int cache_line_size; + u_int cache_ways; + u_int cache_sets; + u_int cache_way_size; + u_int cache_size; +}; + +struct aarch64_cache_info { + u_int cacheable; +#define CACHE_CACHEABLE_NONE 0 +#define CACHE_CACHEABLE_ICACHE 1 /* instruction cache only */ +#define CACHE_CACHEABLE_DCACHE 2 /* data cache only */ +#define CACHE_CACHEABLE_IDCACHE 3 /* instruction and data caches */ +#define CACHE_CACHEABLE_UNIFIED 4 /* unified cache */ + struct aarch64_cache_unit icache; + struct aarch64_cache_unit dcache; +}; + struct cpu_info { struct cpu_data ci_data; device_t ci_dev; @@ -134,9 +160,15 @@ struct cpu_info { /* ACPI */ uint32_t ci_acpiid; /* ACPI Processor Unique ID */ + /* cached system registers */ + uint64_t ci_sctlr_el1; + uint64_t ci_sctlr_el2; + + /* sysctl(9) exposed system registers */ struct aarch64_sysctl_cpu_id ci_id; - struct aarch64_cache_info *ci_cacheinfo; + /* Cache information and CPU functions */ + struct aarch64_cache_info ci_cacheinfo[MAX_CACHE_LEVEL]; struct aarch64_cpufuncs ci_cpufuncs; #if defined(GPROF) && defined(MULTIPROCESSOR) diff --git a/sys/arch/aarch64/include/cpufunc.h b/sys/arch/aarch64/include/cpufunc.h index 243538310ca4..f7340f90207c 100644 --- a/sys/arch/aarch64/include/cpufunc.h +++ b/sys/arch/aarch64/include/cpufunc.h @@ -34,31 +34,7 @@ #include #include -struct aarch64_cache_unit { - u_int cache_type; -#define CACHE_TYPE_VPIPT 0 /* VMID-aware PIPT */ -#define CACHE_TYPE_VIVT 1 /* ASID-tagged VIVT */ -#define CACHE_TYPE_VIPT 2 -#define CACHE_TYPE_PIPT 3 - u_int cache_line_size; - u_int cache_ways; - u_int cache_sets; - u_int cache_way_size; - u_int cache_size; -}; - -struct aarch64_cache_info { - u_int cacheable; -#define CACHE_CACHEABLE_NONE 0 -#define CACHE_CACHEABLE_ICACHE 1 /* instruction cache only */ -#define CACHE_CACHEABLE_DCACHE 2 /* data cache only */ -#define CACHE_CACHEABLE_IDCACHE 3 /* instruction and data caches */ -#define CACHE_CACHEABLE_UNIFIED 4 /* unified cache */ - struct aarch64_cache_unit icache; - struct aarch64_cache_unit dcache; -}; - -#define MAX_CACHE_LEVEL 8 /* ARMv8 has maximum 8 level cache */ + extern u_int aarch64_cache_vindexsize; /* cachesize/way (VIVT/VIPT) */ extern u_int aarch64_cache_prefer_mask; extern u_int cputype; /* compat arm */ @@ -71,8 +47,10 @@ void aarch64_pan_init(int); int aarch64_pac_init(int); int set_cpufuncs(void); -void aarch64_getcacheinfo(int); -void aarch64_printcacheinfo(device_t); +int aarch64_setcpufuncs(struct cpu_info *); +void aarch64_getcacheinfo(struct cpu_info *); +void aarch64_parsecacheinfo(struct cpu_info *); +void aarch64_printcacheinfo(device_t, struct cpu_info *); void aarch64_dcache_wbinv_all(void); void aarch64_dcache_inv_all(void); @@ -103,7 +81,6 @@ void aarch64_tlbi_by_va_ll(vaddr_t); /* all ASID, a VA, lastlevel */ void aarch64_tlbi_by_asid_va(int, vaddr_t); /* an ASID, a VA */ void aarch64_tlbi_by_asid_va_ll(int, vaddr_t); /* an ASID, a VA, lastlevel */ - /* misc */ #define cpu_idnum() aarch64_cpuid() diff --git a/sys/arch/aarch64/include/db_machdep.h b/sys/arch/aarch64/include/db_machdep.h index b2a8c33fca15..a6acba21f4ac 100644 --- a/sys/arch/aarch64/include/db_machdep.h +++ b/sys/arch/aarch64/include/db_machdep.h @@ -227,7 +227,8 @@ void dump_trapframe(struct trapframe *, void (*)(const char *, ...) __printflike void dump_switchframe(struct trapframe *, void (*)(const char *, ...) __printflike(1, 2)); const char *strdisasm(vaddr_t, uint64_t); -void db_machdep_init(void); +void db_machdep_cpu_init(void); +void db_machdep_init(struct cpu_info * const); /* hardware breakpoint/watchpoint functions */ void aarch64_breakpoint_set(int, vaddr_t); diff --git a/sys/arch/arm/apple/apple_intc.c b/sys/arch/arm/apple/apple_intc.c index f9ce206dda9b..3e94d21fbfe4 100644 --- a/sys/arch/arm/apple/apple_intc.c +++ b/sys/arch/arm/apple/apple_intc.c @@ -229,7 +229,6 @@ apple_intc_fdt_establish(device_t dev, u_int *specifier, int ipl, int flags, int (*func)(void *), void *arg, const char *xname) { struct apple_intc_softc * const sc = device_private(dev); - struct apple_intc_percpu * const pc = &sc->sc_pc[cpu_index(curcpu())]; /* 1st cell is the interrupt type (0=IRQ, 1=FIQ) */ const u_int type = be32toh(specifier[0]); @@ -238,10 +237,27 @@ apple_intc_fdt_establish(device_t dev, u_int *specifier, int ipl, int flags, /* 3rd cell is the interrupt flags */ const u_int mpsafe = (flags & FDT_INTR_MPSAFE) ? IST_MPSAFE : 0; - const int irq = type == 0 ? - intno : pc->pc_pic.pic_irqbase + LOCALPIC_SOURCE_TIMER; - return intr_establish_xname(irq, ipl, IST_LEVEL | mpsafe, func, arg, - xname); + + if (type == 0) + return intr_establish_xname(intno, ipl, IST_LEVEL | mpsafe, + func, arg, xname); + + /* interate over CPUs for LOCALPIC_SOURCE_TIMER */ + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + void *ih = NULL; + for (CPU_INFO_FOREACH(cii, ci)) { + const cpuid_t cpuno = cpu_index(ci); + struct apple_intc_percpu * const pc = &sc->sc_pc[cpuno]; + struct pic_softc * const pic = &pc->pc_pic; + const int irq = pic->pic_irqbase + LOCALPIC_SOURCE_TIMER; + + void *ihn = intr_establish_xname(irq, ipl, IST_LEVEL | mpsafe, + func, arg, xname); + if (cpuno == 0) + ih = ihn; + } + return ih; } static void @@ -421,36 +437,6 @@ apple_intc_ipi_handler(void *priv) } #endif /* MULTIPROCESSOR */ -static void -apple_intc_percpu_init(void *priv, struct cpu_info *ci) -{ - struct apple_intc_softc * const sc = priv; - const u_int cpuno = cpu_index(ci); - struct apple_intc_percpu * const pc = &sc->sc_pc[cpuno]; - struct pic_softc * const pic = &pc->pc_pic; - -#ifdef MULTIPROCESSOR - pic->pic_cpus = ci->ci_kcpuset; -#endif - - pic_add(pic, PIC_IRQBASE_ALLOC); - -#ifdef MULTIPROCESSOR - if (cpuno != 0) { - struct intrsource * const is = - sc->sc_pc[0].pc_pic.pic_sources[LOCALPIC_SOURCE_TIMER]; - KASSERT(is != NULL); - - intr_establish_xname(pic->pic_irqbase + LOCALPIC_SOURCE_TIMER, - is->is_ipl, is->is_type | (is->is_mpsafe ? IST_MPSAFE : 0), - is->is_func, is->is_arg, is->is_xname); - } - - intr_establish_xname(pic->pic_irqbase + LOCALPIC_SOURCE_IPI, IPL_HIGH, - IST_LEVEL | IST_MPSAFE, apple_intc_ipi_handler, pc, "ipi"); -#endif -} - static int apple_intc_match(device_t parent, cfdata_t cf, void *aux) { @@ -467,7 +453,6 @@ apple_intc_attach(device_t parent, device_t self, void *aux) const int phandle = faa->faa_phandle; bus_addr_t addr; bus_size_t size; - u_int cpuno; int error; if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { @@ -509,18 +494,30 @@ apple_intc_attach(device_t parent, device_t self, void *aux) KASSERT(ncpu != 0); sc->sc_cpuid = kmem_zalloc(sizeof(*sc->sc_cpuid) * ncpu, KM_SLEEP); sc->sc_pc = kmem_zalloc(sizeof(*sc->sc_pc) * ncpu, KM_SLEEP); - for (cpuno = 0; cpuno < ncpu; cpuno++) { - sc->sc_pc[cpuno].pc_sc = sc; - sc->sc_pc[cpuno].pc_cpuid = cpuno; - sc->sc_pc[cpuno].pc_pic.pic_ops = &apple_intc_localpicops; - sc->sc_pc[cpuno].pc_pic.pic_maxsources = 2; - snprintf(sc->sc_pc[cpuno].pc_pic.pic_name, - sizeof(sc->sc_pc[cpuno].pc_pic.pic_name), "AIC/%u", cpuno); + + CPU_INFO_ITERATOR cii; + struct cpu_info *ci; + for (CPU_INFO_FOREACH(cii, ci)) { + const cpuid_t cpuno = cpu_index(ci); + struct apple_intc_percpu * const pc = &sc->sc_pc[cpuno]; + struct pic_softc * const pic = &pc->pc_pic; + + pc->pc_sc = sc; + pc->pc_cpuid = cpuno; + + pic->pic_cpus = ci->ci_kcpuset; + pic->pic_ops = &apple_intc_localpicops; + pic->pic_maxsources = 2; + snprintf(pic->pic_name, sizeof(pic->pic_name), "AIC/%lu", cpuno); + + pic_add(pic, PIC_IRQBASE_ALLOC); + + intr_establish_xname(pic->pic_irqbase + LOCALPIC_SOURCE_IPI, + IPL_HIGH, IST_LEVEL | IST_MPSAFE, apple_intc_ipi_handler, + pc, "ipi"); } apple_intc_cpu_init(&sc->sc_pic, curcpu()); - apple_intc_percpu_init(sc, curcpu()); - arm_fdt_cpu_hatch_register(sc, apple_intc_percpu_init); } CFATTACH_DECL_NEW(apple_intc, sizeof(struct apple_intc_softc), diff --git a/sys/arch/arm/arm/cpu_subr.c b/sys/arch/arm/arm/cpu_subr.c index 67bdacd2dea2..6c998d52e8b6 100644 --- a/sys/arch/arm/arm/cpu_subr.c +++ b/sys/arch/arm/arm/cpu_subr.c @@ -70,8 +70,6 @@ volatile u_long arm_cpu_hatched[howmany(MAXCPUS, CPUINDEX_DIVISOR)] __cacheline_ volatile u_long arm_cpu_mbox[howmany(MAXCPUS, CPUINDEX_DIVISOR)] __cacheline_aligned = { 0 }; u_int arm_cpu_max = 1; -kmutex_t cpu_hatch_lock; - void cpu_boot_secondary_processors(void) { @@ -80,8 +78,6 @@ cpu_boot_secondary_processors(void) if ((boothowto & RB_MD1) != 0) return; - mutex_init(&cpu_hatch_lock, MUTEX_DEFAULT, IPL_NONE); - VPRINTF("%s: starting secondary processors\n", __func__); /* send mbox to have secondary processors do cpu_hatch() */ @@ -127,6 +123,8 @@ cpu_set_hatched(int cpuindex) const u_long bit = __BIT(cpuindex % CPUINDEX_DIVISOR); atomic_or_ulong(&arm_cpu_hatched[off], bit); + dsb(ishst); + sev(); } void @@ -138,7 +136,6 @@ cpu_clr_mbox(int cpuindex) /* Notify cpu_boot_secondary_processors that we're done */ atomic_and_ulong(&arm_cpu_mbox[off], ~bit); - membar_producer(); dsb(ishst); sev(); } diff --git a/sys/arch/arm/arm/undefined.c b/sys/arch/arm/arm/undefined.c index ba2435be9b13..4830c700815c 100644 --- a/sys/arch/arm/arm/undefined.c +++ b/sys/arch/arm/arm/undefined.c @@ -44,6 +44,7 @@ * Created : 06/01/95 */ +#include "opt_cputypes.h" #include "opt_ddb.h" #include "opt_dtrace.h" #include "opt_kgdb.h" @@ -96,6 +97,13 @@ install_coproc_handler(int coproc, undef_handler_t handler) return uh; } +void +replace_coproc_handler(int coproc, undef_handler_t handler) +{ + LIST_INIT(&undefined_handlers[coproc]); + install_coproc_handler(coproc, handler); +} + void install_coproc_handler_static(int coproc, struct undefined_handler *uh) { @@ -199,6 +207,26 @@ gdb_trapper(u_int addr, u_int insn, struct trapframe *tf, int code) return 1; } +#ifdef FPU_VFP +/* + * Used to test for a VFP. The following function is installed as a coproc10 + * handler on the undefined instruction vector and then we issue a VFP + * instruction. If ci_vfd_id is set to zero then the VFP did not handle + * the instruction so must be absent, or disabled. + */ + +static int +vfp_test(u_int address, u_int insn, trapframe_t *frame, int fault_code) +{ + struct cpu_info * const ci = curcpu(); + + frame->tf_pc += INSN_SIZE; + ci->ci_vfp_id = 0; + + return 0; +} +#endif + static struct undefined_handler cp15_uh = { .uh_handler = cp15_trapper, }; @@ -210,6 +238,11 @@ static struct undefined_handler gdb_uh_thumb = { .uh_handler = gdb_trapper, }; #endif +#ifdef FPU_VFP +struct undefined_handler vfptest_uh = { + .uh_handler = vfp_test, +}; +#endif #ifdef KDTRACE_HOOKS #include @@ -259,6 +292,9 @@ undefined_init(void) #ifdef THUMB_CODE install_coproc_handler_static(THUMB_UNKNOWN_HANDLER, &gdb_uh_thumb); #endif +#ifdef FPU_VFP + install_coproc_handler_static(VFP_COPROC, &vfptest_uh); +#endif } void diff --git a/sys/arch/arm/arm32/arm32_boot.c b/sys/arch/arm/arm32/arm32_boot.c index 184c13a93d65..946729519549 100644 --- a/sys/arch/arm/arm32/arm32_boot.c +++ b/sys/arch/arm/arm32/arm32_boot.c @@ -238,6 +238,12 @@ initarm_common(vaddr_t kvm_base, vsize_t kvm_size, VPRINTF("undefined "); undefined_init(); +#ifdef FPU_VFP + /* vfp_detect uses an undefined handler */ + VPRINTF("vfp "); + vfp_detect(curcpu()); +#endif + /* Load memory into UVM. */ VPRINTF("page "); uvm_md_init(); @@ -362,11 +368,7 @@ cpu_hatch(struct cpu_info *ci, u_int cpuindex, void (*md_cpu_init)(struct cpu_in splhigh(); VPRINTF("%s(%s): ", __func__, cpu_name(ci)); - /* mpidr/midr filled in by armv7_mpcontinuation */ - ci->ci_ctrl = armreg_sctlr_read(); - ci->ci_arm_cpuid = cpu_idnum(); - ci->ci_arm_cputype = ci->ci_arm_cpuid & CPU_ID_CPU_MASK; - ci->ci_arm_cpurev = ci->ci_arm_cpuid & CPU_ID_REVISION_MASK; + /* mpidr/midr filled in by cpu_init_secondary_processor */ /* * Make sure we have the right vector page. @@ -409,16 +411,6 @@ cpu_hatch(struct cpu_info *ci, u_int cpuindex, void (*md_cpu_init)(struct cpu_in } #endif - mutex_enter(&cpu_hatch_lock); - - aprint_naive("%s", device_xname(ci->ci_dev)); - aprint_normal("%s", device_xname(ci->ci_dev)); - identify_arm_cpu(ci->ci_dev, ci); - VPRINTF(" vfp"); - vfp_attach(ci); - - mutex_exit(&cpu_hatch_lock); - VPRINTF(" md(%p)", md_cpu_init); if (md_cpu_init != NULL) (*md_cpu_init)(ci); @@ -430,7 +422,6 @@ cpu_hatch(struct cpu_info *ci, u_int cpuindex, void (*md_cpu_init)(struct cpu_in intr_cpu_init(ci); VPRINTF(" done!\n"); - cpu_clr_mbox(cpuindex); } #endif /* MULTIPROCESSOR */ diff --git a/sys/arch/arm/arm32/arm32_machdep.c b/sys/arch/arm/arm32/arm32_machdep.c index 5fb3c59f5378..e309fc264764 100644 --- a/sys/arch/arm/arm32/arm32_machdep.c +++ b/sys/arch/arm/arm32/arm32_machdep.c @@ -783,12 +783,22 @@ cpu_init_secondary_processor(int cpuindex) VPRINTS(" ci = "); VPRINTX((int)ci); + ci->ci_ctrl = armreg_sctlr_read(); + ci->ci_arm_cpuid = cpu_idnum(); + ci->ci_arm_cputype = ci->ci_arm_cpuid & CPU_ID_CPU_MASK; + ci->ci_arm_cpurev = ci->ci_arm_cpuid & CPU_ID_REVISION_MASK; + ci->ci_midr = armreg_midr_read(); + ci->ci_actlr = armreg_auxctl_read(); + ci->ci_revidr = armreg_revidr_read(); ci->ci_mpidr = armreg_mpidr_read(); arm_cpu_topology_set(ci, ci->ci_mpidr); - VPRINTS(" hatched|="); + VPRINTS(" vfp"); + vfp_detect(ci); + + VPRINTS(" hatched |="); VPRINTX(__BIT(cpuindex)); VPRINTS("\n\r"); diff --git a/sys/arch/arm/arm32/cpu.c b/sys/arch/arm/arm32/cpu.c index 9d54db78bc4f..4f96953a48af 100644 --- a/sys/arch/arm/arm32/cpu.c +++ b/sys/arch/arm/arm32/cpu.c @@ -75,8 +75,8 @@ uint32_t arm_cpu_marker[2] __cacheline_aligned = { 0, 0 }; /* Prototypes */ void identify_arm_cpu(device_t, struct cpu_info *); +void identify_features(device_t, struct cpu_info *); void identify_cortex_caches(device_t); -void identify_features(device_t); /* * Identify the master (boot) CPU @@ -96,11 +96,17 @@ cpu_attach(device_t dv, cpuid_t id) ci->ci_ctrl = cpu_control(0, 0); /* Get the CPU ID from coprocessor 15 */ - ci->ci_cpuid = id; ci->ci_arm_cpuid = cpu_idnum(); ci->ci_arm_cputype = ci->ci_arm_cpuid & CPU_ID_CPU_MASK; ci->ci_arm_cpurev = ci->ci_arm_cpuid & CPU_ID_REVISION_MASK; + + /* + * Get other sysregs for BP. APs information is grabbed in + * cpu_init_secondary_processor. + */ + ci->ci_actlr = armreg_auxctl_read(); + ci->ci_revidr = armreg_revidr_read(); } else { #ifdef MULTIPROCESSOR if ((boothowto & RB_MD1) != 0) { @@ -182,17 +188,11 @@ cpu_attach(device_t dv, cpuid_t id) ci->ci_kfpu_spl = -1; #ifdef MULTIPROCESSOR - /* - * and we are done if this is a secondary processor. - */ if (unit != 0) { - aprint_naive("\n"); - aprint_normal("\n"); mi_cpu_attach(ci); #ifdef ARM_MMU_EXTENDED pmap_tlb_info_attach(&pmap_tlb0_info, ci); #endif - return; } #endif @@ -230,7 +230,7 @@ cpu_attach(device_t dv, cpuid_t id) } #endif - vfp_attach(ci); /* XXX SMP */ + vfp_attach(ci); } enum cpu_class { @@ -750,10 +750,12 @@ identify_arm_cpu(device_t dv, struct cpu_info *ci) aprint_normal("\n"); - if (CPU_ID_CORTEX_P(arm_cpuid) || CPU_ID_ARM11_P(arm_cpuid) || CPU_ID_MV88SV58XX_P(arm_cpuid)) { + if (CPU_ID_CORTEX_P(arm_cpuid) || + CPU_ID_ARM11_P(arm_cpuid) || + CPU_ID_MV88SV58XX_P(arm_cpuid)) { if ((arm_cpuid & CPU_ID_CPU_MASK) != CPU_ID_ARM1136JS && (arm_cpuid & CPU_ID_CPU_MASK) != CPU_ID_ARM1176JZS) { - identify_features(dv); + identify_features(dv, ci); } } @@ -832,8 +834,20 @@ extern int cpu_simd_present; extern int cpu_simdex_present; void -identify_features(device_t dv) +identify_features(device_t dv, struct cpu_info *ci) { + const int unit = device_unit(dv); + + aprint_debug_dev(dv, "sctlr: %#x\n", ci->ci_ctrl); + aprint_debug_dev(dv, "actlr: %#x\n", ci->ci_actlr); + aprint_debug_dev(dv, "revidr: %#x\n", ci->ci_revidr); +#ifdef MULTIPROCESSOR + aprint_debug_dev(dv, "mpidr: %#x\n", ci->ci_mpidr); +#endif + + if (unit != 0) + return; + cpu_instruction_set_attributes[0] = armreg_isar0_read(); cpu_instruction_set_attributes[1] = armreg_isar1_read(); cpu_instruction_set_attributes[2] = armreg_isar2_read(); @@ -859,10 +873,10 @@ identify_features(device_t dv) #if 0 if (__SHIFTOUT(cpu_memory_model_features[3], __BITS(23,20))) { /* - * Updates to the translation tables do not require a clean - * to the point of unification to ensure visibility by - * subsequent translation table walks. - */ + * Updates to the translation tables do not require a clean + * to the point of unification to ensure visibility by + * subsequent translation table walks. + */ pmap_needs_pte_sync = 0; } #endif @@ -870,12 +884,6 @@ identify_features(device_t dv) cpu_processor_features[0] = armreg_pfr0_read(); cpu_processor_features[1] = armreg_pfr1_read(); - aprint_debug_dev(dv, "sctlr: %#x\n", armreg_sctlr_read()); - aprint_debug_dev(dv, "actlr: %#x\n", armreg_auxctl_read()); - aprint_debug_dev(dv, "revidr: %#x\n", armreg_revidr_read()); -#ifdef MULTIPROCESSOR - aprint_debug_dev(dv, "mpidr: %#x\n", armreg_mpidr_read()); -#endif aprint_debug_dev(dv, "isar: [0]=%#x [1]=%#x [2]=%#x [3]=%#x, [4]=%#x, [5]=%#x\n", cpu_instruction_set_attributes[0], diff --git a/sys/arch/arm/broadcom/bcm2835_intr.c b/sys/arch/arm/broadcom/bcm2835_intr.c index d0e1193cd817..936b2fc1cf6a 100644 --- a/sys/arch/arm/broadcom/bcm2835_intr.c +++ b/sys/arch/arm/broadcom/bcm2835_intr.c @@ -60,7 +60,6 @@ __KERNEL_RCSID(0, "$NetBSD: bcm2835_intr.c,v 1.41 2021/09/12 03:58:52 nat Exp $" #include static void bcm2835_irq_handler(void *); -static void bcm2836mp_intr_init(void *, struct cpu_info *); static void bcm2835_pic_unblock_irqs(struct pic_softc *, size_t, uint32_t); static void bcm2835_pic_block_irqs(struct pic_softc *, size_t, uint32_t); @@ -77,6 +76,7 @@ static void bcm2836mp_pic_source_name(struct pic_softc *, int, char *, size_t); #ifdef MULTIPROCESSOR int bcm2836mp_ipi_handler(void *); +static void bcm2836mp_intr_init(struct cpu_info *); static void bcm2836mp_cpu_init(struct pic_softc *, struct cpu_info *); static void bcm2836mp_send_ipi(struct pic_softc *, const kcpuset_t *, u_long); #endif @@ -359,10 +359,10 @@ bcm2835_icu_attach(device_t parent, device_t self, void *aux) ifuncs = &bcm2836mpicu_fdt_funcs; +#if defined(MULTIPROCESSOR) /* - * XXX * Register all PICs here in order to avoid pic_add() from - * cpu_hatch(). See port-arm/56264. + * cpu_hatch(). This is the only approved method. */ CPU_INFO_ITERATOR cii; struct cpu_info *ci; @@ -372,7 +372,6 @@ bcm2835_icu_attach(device_t parent, device_t self, void *aux) KASSERT(cpuid < BCM2836_NCPUS); -#if defined(MULTIPROCESSOR) pic->pic_cpus = ci->ci_kcpuset; /* * Append "#n" to avoid duplication of .pic_name[] @@ -381,14 +380,12 @@ bcm2835_icu_attach(device_t parent, device_t self, void *aux) char suffix[sizeof("#00000")]; snprintf(suffix, sizeof(suffix), "#%lu", cpuid); strlcat(pic->pic_name, suffix, sizeof(pic->pic_name)); -#endif bcm2836mp_int_base[cpuid] = pic_add(pic, PIC_IRQBASE_ALLOC); + bcm2836mp_intr_init(ci); } - - bcm2836mp_intr_init(self, curcpu()); - arm_fdt_cpu_hatch_register(self, bcm2836mp_intr_init); +#endif } else { if (bcml1icu_sc == NULL) arm_fdt_irq_set_handler(bcm2835_irq_handler); @@ -889,30 +886,18 @@ bcm2836mp_ipi_handler(void *priv) } #endif +#if defined(MULTIPROCESSOR) static void -bcm2836mp_intr_init(void *priv, struct cpu_info *ci) +bcm2836mp_intr_init(struct cpu_info *ci) { -#if defined(MULTIPROCESSOR) const cpuid_t cpuid = ci->ci_core_id; KASSERT(cpuid < BCM2836_NCPUS); intr_establish(BCM2836_INT_MAILBOX0_CPUN(cpuid), IPL_HIGH, IST_LEVEL | IST_MPSAFE, bcm2836mp_ipi_handler, ci); - - struct bcm2836mp_interrupt *bip; - TAILQ_FOREACH(bip, &bcm2836mp_interrupts, bi_next) { - if (bip->bi_done) - continue; - - const int irq = BCM2836_INT_BASECPUN(cpuid) + bip->bi_irq; - void *ih = intr_establish(irq, bip->bi_ipl, - IST_LEVEL | bip->bi_flags, bip->bi_func, bip->bi_arg); - - bip->bi_ihs[cpuid] = ih; - } -#endif } +#endif static int bcm2836mp_icu_fdt_decode_irq(u_int *specifier) @@ -928,96 +913,39 @@ bcm2836mp_icu_fdt_establish(device_t dev, u_int *specifier, int ipl, int flags, int (*func)(void *), void *arg, const char *xname) { int iflags = (flags & FDT_INTR_MPSAFE) ? IST_MPSAFE : 0; - struct bcm2836mp_interrupt *bip; - void *ih; int irq = bcm2836mp_icu_fdt_decode_irq(specifier); if (irq == -1) return NULL; - TAILQ_FOREACH(bip, &bcm2836mp_interrupts, bi_next) { - if (irq == bip->bi_irq) - return NULL; - } - - bip = kmem_alloc(sizeof(*bip), KM_SLEEP); - if (bip == NULL) - return NULL; - - bip->bi_done = false; - bip->bi_irq = irq; - bip->bi_ipl = ipl; - bip->bi_flags = IST_LEVEL | iflags; - bip->bi_func = func; - bip->bi_arg = arg; - - /* - * If we're not cold and the BPs have been started then we can - * register the interrupt for all CPUs now, e.g. PMU - */ - if (!cold) { - for (cpuid_t cpuid = 0; cpuid < BCM2836_NCPUS; cpuid++) { - ih = intr_establish_xname( - BCM2836_INT_BASECPUN(cpuid) + irq, ipl, - IST_LEVEL | iflags, func, arg, xname); - if (!ih) { - kmem_free(bip, sizeof(*bip)); - return NULL; + void *ihs[BCM2836_NCPUS]; + for (cpuid_t cpuid = 0; cpuid < BCM2836_NCPUS; cpuid++) { + ihs[cpuid] = intr_establish_xname(BCM2836_INT_BASECPUN(cpuid) + irq, ipl, + IST_LEVEL | iflags, func, arg, xname); + if (!ihs[cpuid]) { + for (cpuid_t undo = 0; undo < cpuid; undo++) { + intr_disestablish(ihs[undo]); } - bip->bi_ihs[cpuid] = ih; - + return NULL; } - bip->bi_done = true; - ih = bip->bi_ihs[0]; - goto done; - } - /* - * Otherwise we can only establish the interrupt for the BP and - * delay until bcm2836mp_intr_init is called for each AP, e.g. - * gtmr - */ - ih = intr_establish_xname(BCM2836_INT_BASECPUN(0) + irq, ipl, - IST_LEVEL | iflags, func, arg, xname); - if (!ih) { - kmem_free(bip, sizeof(*bip)); - return NULL; } - bip->bi_ihs[0] = ih; - for (cpuid_t cpuid = 1; cpuid < BCM2836_NCPUS; cpuid++) - bip->bi_ihs[cpuid] = NULL; - -done: - TAILQ_INSERT_TAIL(&bcm2836mp_interrupts, bip, bi_next); - /* * Return the intr_establish handle for cpu 0 for API compatibility. * Any cpu would do here as these sources don't support set_affinity * when the handle is used in interrupt_distribute(9) */ - return ih; + return ihs[0]; } static void bcm2836mp_icu_fdt_disestablish(device_t dev, void *ih) { - struct bcm2836mp_interrupt *bip; - - TAILQ_FOREACH(bip, &bcm2836mp_interrupts, bi_next) { - if (bip->bi_ihs[0] == ih) - break; - } - - if (bip == NULL) - return; - +#if 0 for (cpuid_t cpuid = 0; cpuid < BCM2836_NCPUS; cpuid++) intr_disestablish(bip->bi_ihs[cpuid]); - - TAILQ_REMOVE(&bcm2836mp_interrupts, bip, bi_next); - - kmem_free(bip, sizeof(*bip)); +#endif } static bool diff --git a/sys/arch/arm/cortex/gicv3_its.c b/sys/arch/arm/cortex/gicv3_its.c index 865ef18456b7..46f8aefc9293 100644 --- a/sys/arch/arm/cortex/gicv3_its.c +++ b/sys/arch/arm/cortex/gicv3_its.c @@ -496,8 +496,9 @@ gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count, gicv3_its_msi_enable(its, lpi, *count); /* - * Record target PE + * Record devid and target PE */ + its->its_devid[lpi - its->its_pic->pic_irqbase] = devid; its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; /* @@ -565,8 +566,9 @@ gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count, gicv3_its_msix_enable(its, lpi, msix_vec, bst, bsh); /* - * Record target PE + * Record devid and target PE */ + its->its_devid[lpi - its->its_pic->pic_irqbase] = devid; its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; /* @@ -601,7 +603,7 @@ gicv3_its_msi_intr_establish(struct arm_pci_msi *msi, /* Invalidate LPI configuration tables */ pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; KASSERT(pa != NULL); - const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); + const uint32_t devid = its->its_devid[lpi - its->its_pic->pic_irqbase]; gits_command_inv(its, devid, lpi - its->its_pic->pic_irqbase); return intrh; @@ -623,6 +625,7 @@ gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, gicv3_its_msi_disable(its, lpi); gicv3_its_msi_free_lpi(its, lpi); its->its_targets[lpi - its->its_pic->pic_irqbase] = NULL; + its->its_devid[lpi - its->its_pic->pic_irqbase] = 0; struct intrsource * const is = its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase]; if (is != NULL) @@ -784,6 +787,7 @@ gicv3_its_cpu_init(void *priv, struct cpu_info *ci) /* * Map collection ID of this CPU's index to this CPU's redistributor. */ + mutex_enter(its->its_lock); gits_command_mapc(its, cpu_index(ci), rdbase, true); gits_command_invall(its, cpu_index(ci)); gits_wait(its); @@ -797,10 +801,12 @@ gicv3_its_cpu_init(void *priv, struct cpu_info *ci) pa = its->its_pa[irq]; KASSERT(pa != NULL); - const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); + const uint32_t devid = its->its_devid[irq]; gits_command_movi(its, devid, irq, cpu_index(ci)); gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); } + gits_wait(its); + mutex_exit(its->its_lock); its->its_cpuonline[cpu_index(ci)] = true; } @@ -865,6 +871,7 @@ gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh, KASSERT(its->its_pic->pic_maxsources > 0); its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP); its->its_targets = kmem_zalloc(sizeof(struct cpu_info *) * its->its_pic->pic_maxsources, KM_SLEEP); + its->its_devid = kmem_zalloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP); its->its_gic = sc; its->its_rdbase = kmem_zalloc(sizeof(*its->its_rdbase) * ncpu, KM_SLEEP); its->its_cpuonline = kmem_zalloc(sizeof(*its->its_cpuonline) * ncpu, KM_SLEEP); @@ -874,6 +881,7 @@ gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh, its->its_cb.priv = its; LIST_INIT(&its->its_devices); LIST_INSERT_HEAD(&sc->sc_lpi_callbacks, &its->its_cb, list); + its->its_lock = mutex_obj_alloc(MUTEX_SPIN, IPL_NONE); gicv3_its_command_init(sc, its); gicv3_its_table_init(sc, its); diff --git a/sys/arch/arm/cortex/gicv3_its.h b/sys/arch/arm/cortex/gicv3_its.h index d27702fd4c75..b931f05287fd 100644 --- a/sys/arch/arm/cortex/gicv3_its.h +++ b/sys/arch/arm/cortex/gicv3_its.h @@ -61,6 +61,7 @@ struct gicv3_its { struct pic_softc *its_pic; struct pci_attach_args **its_pa; struct cpu_info **its_targets; + uint32_t *its_devid; LIST_HEAD(, gicv3_its_device) its_devices; @@ -68,6 +69,8 @@ struct gicv3_its { struct gicv3_dma its_tab[8]; /* ITS tables */ struct arm_pci_msi its_msi; + + kmutex_t *its_lock; }; int gicv3_its_init(struct gicv3_softc *, bus_space_handle_t, uint64_t, uint32_t); diff --git a/sys/arch/arm/cortex/gtmr.c b/sys/arch/arm/cortex/gtmr.c index 6db073530078..48c611d8d2cd 100644 --- a/sys/arch/arm/cortex/gtmr.c +++ b/sys/arch/arm/cortex/gtmr.c @@ -247,6 +247,7 @@ gtmr_init_cpu_clock(struct cpu_info *ci) KASSERT(ci == curcpu()); + /* XXX hmm... called from cpu_hatch which hasn't lowered ipl yet */ int s = splsched(); /* diff --git a/sys/arch/arm/include/cpu.h b/sys/arch/arm/include/cpu.h index a2482ba2ccd3..2103f0742f57 100644 --- a/sys/arch/arm/include/cpu.h +++ b/sys/arch/arm/include/cpu.h @@ -56,8 +56,8 @@ typedef unsigned long mpidr_t; #ifdef MULTIPROCESSOR extern u_int arm_cpu_max; extern mpidr_t cpu_mpidr[]; -extern kmutex_t cpu_hatch_lock; +void cpu_init_secondary_processor(int); void cpu_boot_secondary_processors(void); void cpu_mpstart(void); bool cpu_hatched_p(u_int); @@ -155,6 +155,8 @@ static inline void cpu_dosoftints(void); #include #include #include + +#include #include struct cpu_info { @@ -219,10 +221,14 @@ struct cpu_info { struct evcnt ci_vfp_evs[3]; uint32_t ci_midr; + uint32_t ci_actlr; + uint32_t ci_revidr; uint32_t ci_mpidr; + uint32_t ci_mvfr[2]; + uint32_t ci_capacity_dmips_mhz; - struct arm_cache_info * + struct arm_cache_info ci_cacheinfo; #if defined(GPROF) && defined(MULTIPROCESSOR) @@ -297,10 +303,6 @@ extern struct cpu_info *cpu_info[]; cii = 0, __USE(cii), ci = curcpu(); ci != NULL; ci = NULL #endif -#if defined(MULTIPROCESSOR) -void cpu_init_secondary_processor(int); -#endif - #define LWP0_CPU_INFO (&cpu_info_store[0]) static inline int diff --git a/sys/arch/arm/include/locore.h b/sys/arch/arm/include/locore.h index 4c9d9198f179..56b9e3b123ce 100644 --- a/sys/arch/arm/include/locore.h +++ b/sys/arch/arm/include/locore.h @@ -264,6 +264,7 @@ int badaddr_read(void *, size_t, void *); void swi_handler(trapframe_t *); /* vfp_init.c */ +void vfp_detect(struct cpu_info *); void vfp_attach(struct cpu_info *); void vfp_discardcontext(lwp_t *, bool); void vfp_savecontext(lwp_t *); diff --git a/sys/arch/arm/include/undefined.h b/sys/arch/arm/include/undefined.h index 29f49647b96a..0c494aa9f7e5 100644 --- a/sys/arch/arm/include/undefined.h +++ b/sys/arch/arm/include/undefined.h @@ -78,11 +78,12 @@ enum arm_coprocs { /* Prototypes for undefined.c */ void *install_coproc_handler(int, undef_handler_t); +void replace_coproc_handler(int, undef_handler_t); void remove_coproc_handler(void *); void undefined_init(void); /* - * XXX Stuff below here is for use before malloc() is available. Most code + * Stuff below here is for use before kmem(9) is available. Most code * shouldn't use it. */ @@ -93,7 +94,7 @@ struct undefined_handler { /* * Handlers installed using install_coproc_handler_static shouldn't be - * removed. + * removed. We special case the 'test vfp existance' handler. */ void install_coproc_handler_static(int, struct undefined_handler *); diff --git a/sys/arch/arm/pic/pic.c b/sys/arch/arm/pic/pic.c index 2896f5149e2c..800327850c8d 100644 --- a/sys/arch/arm/pic/pic.c +++ b/sys/arch/arm/pic/pic.c @@ -790,7 +790,7 @@ pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, (*pic->pic_ops->pic_establish_irq)(pic, is); unblock: - if (cold || !is->is_mpsafe) { + if (!mp_online || !is->is_mpsafe) { (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, __BIT(is->is_irq & 0x1f)); } else { diff --git a/sys/arch/arm/vfp/vfp_init.c b/sys/arch/arm/vfp/vfp_init.c index e6568a4404ed..00db49ec27a8 100644 --- a/sys/arch/arm/vfp/vfp_init.c +++ b/sys/arch/arm/vfp/vfp_init.c @@ -169,24 +169,6 @@ uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM; /* default to run fast */ uint32_t vfp_fpscr_default = (VFP_FPSCR_DN | VFP_FPSCR_FZ | VFP_FPSCR_RN); -/* - * Used to test for a VFP. The following function is installed as a coproc10 - * handler on the undefined instruction vector and then we issue a VFP - * instruction. If undefined_test is non zero then the VFP did not handle - * the instruction so must be absent, or disabled. - */ - -static int undefined_test; - -static int -vfp_test(u_int address, u_int insn, trapframe_t *frame, int fault_code) -{ - - frame->tf_pc += INSN_SIZE; - ++undefined_test; - return 0; -} - #else /* determine what bits can be changed */ uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM|VFP_FPSCR_RMODE; @@ -238,6 +220,12 @@ vfp_fpscr_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) } #ifndef FPU_VFP +void +vfp_detect(struct cpu_info *ci) +{ + ci->ci_vfp_id = 0; + return; +} /* * If we don't want VFP support, we still need to handle emulating VFP FPSCR * instructions. @@ -246,7 +234,7 @@ void vfp_attach(struct cpu_info *ci) { if (CPU_IS_PRIMARY(ci)) { - install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); + replace_coproc_handler(VFP_COPROC, vfp_fpscr_handler); } evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, "vfp fpscr traps"); @@ -254,9 +242,8 @@ vfp_attach(struct cpu_info *ci) #else void -vfp_attach(struct cpu_info *ci) +vfp_detect(struct cpu_info *ci) { - const char *model = NULL; if (CPU_ID_ARM11_P(ci->ci_arm_cpuid) || CPU_ID_MV88SV58XX_P(ci->ci_arm_cpuid) @@ -265,14 +252,7 @@ vfp_attach(struct cpu_info *ci) const uint32_t nsacr = armreg_nsacr_read(); const uint32_t nsacr_vfp = __BITS(VFP_COPROC,VFP_COPROC2); if ((nsacr & nsacr_vfp) != nsacr_vfp) { - aprint_normal_dev(ci->ci_dev, - "VFP access denied (NSACR=%#x)\n", nsacr); - if (CPU_IS_PRIMARY(ci)) - install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); - ci->ci_vfp_id = 0; - evcnt_attach_dynamic(&ci->ci_vfp_evs[0], - EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, - "vfp fpscr traps"); + ci->ci_fp_id = 0; return; } #endif @@ -296,36 +276,32 @@ vfp_attach(struct cpu_info *ci) bool vfp_p = __SHIFTOUT(cpacr, cpacr_vfp2) == CPACR_ALL && __SHIFTOUT(cpacr, cpacr_vfp) == CPACR_ALL; if (!vfp_p) { - aprint_normal_dev(ci->ci_dev, - "VFP access denied (CPACR=%#x)\n", cpacr); - if (CPU_IS_PRIMARY(ci)) - install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); ci->ci_vfp_id = 0; - evcnt_attach_dynamic(&ci->ci_vfp_evs[0], - EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, - "vfp fpscr traps"); return; } } - void *uh = install_coproc_handler(VFP_COPROC, vfp_test); - - undefined_test = 0; + /* borrow the ci_vfd_id field for VFP detection */ + ci->ci_vfp_id = -1; const uint32_t fpsid = armreg_fpsid_read(); - - remove_coproc_handler(uh); - - if (undefined_test != 0) { - aprint_normal_dev(ci->ci_dev, "No VFP detected\n"); - if (CPU_IS_PRIMARY(ci)) - install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); - ci->ci_vfp_id = 0; + if (ci->ci_vfp_id == 0) { return; } ci->ci_vfp_id = fpsid; - switch (fpsid & ~ VFP_FPSID_REV_MSK) { + + ci->ci_mvfr[0] = armreg_mvfr0_read(); + ci->ci_mvfr[1] = armreg_mvfr1_read(); + +} + +void +vfp_attach(struct cpu_info *ci) +{ + const char *model = NULL; + + switch (ci->ci_vfp_id & ~ VFP_FPSID_REV_MSK) { case FPU_VFP10_ARM10E: model = "VFP10 R1"; break; @@ -354,9 +330,9 @@ vfp_attach(struct cpu_info *ci) break; default: aprint_normal_dev(ci->ci_dev, "unrecognized VFP version %#x\n", - fpsid); + ci->ci_vfp_id); if (CPU_IS_PRIMARY(ci)) - install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); + replace_coproc_handler(VFP_COPROC, vfp_fpscr_handler); vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM |VFP_FPSCR_RMODE; vfp_fpscr_default = 0; @@ -364,48 +340,43 @@ vfp_attach(struct cpu_info *ci) } cpu_fpu_present = 1; - cpu_media_and_vfp_features[0] = armreg_mvfr0_read(); - cpu_media_and_vfp_features[1] = armreg_mvfr1_read(); - if (fpsid != 0) { - uint32_t f0 = armreg_mvfr0_read(); - uint32_t f1 = armreg_mvfr1_read(); - aprint_normal("vfp%d at %s: %s%s%s%s%s\n", - device_unit(ci->ci_dev), - device_xname(ci->ci_dev), - model, - ((f0 & ARM_MVFR0_ROUNDING_MASK) ? ", rounding" : ""), - ((f0 & ARM_MVFR0_EXCEPT_MASK) ? ", exceptions" : ""), - ((f1 & ARM_MVFR1_D_NAN_MASK) ? ", NaN propagation" : ""), - ((f1 & ARM_MVFR1_FTZ_MASK) ? ", denormals" : "")); - aprint_debug("vfp%d: mvfr: [0]=%#x [1]=%#x\n", - device_unit(ci->ci_dev), f0, f1); - if (CPU_IS_PRIMARY(ci)) { - if (f0 & ARM_MVFR0_ROUNDING_MASK) { - vfp_fpscr_changable |= VFP_FPSCR_RMODE; - } - if (f1 & ARM_MVFR0_EXCEPT_MASK) { - vfp_fpscr_changable |= VFP_FPSCR_ESUM; - } - // If hardware supports propagation of NaNs, select it. - if (f1 & ARM_MVFR1_D_NAN_MASK) { - vfp_fpscr_default &= ~VFP_FPSCR_DN; - vfp_fpscr_changable |= VFP_FPSCR_DN; - } - // If hardware supports denormalized numbers, use it. - if (cpu_media_and_vfp_features[1] & ARM_MVFR1_FTZ_MASK) { - vfp_fpscr_default &= ~VFP_FPSCR_FZ; - vfp_fpscr_changable |= VFP_FPSCR_FZ; - } - } - } - evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL, - ci->ci_cpuname, "vfp coproc use"); - evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL, - ci->ci_cpuname, "vfp coproc re-use"); - evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL, - ci->ci_cpuname, "vfp coproc fault"); + + const uint32_t f0 = ci->ci_mvfr[0]; + const uint32_t f1 = ci->ci_mvfr[1]; + aprint_normal("vfp%d at %s: %s%s%s%s%s\n", + device_unit(ci->ci_dev), + device_xname(ci->ci_dev), + model, + ((f0 & ARM_MVFR0_ROUNDING_MASK) ? ", rounding" : ""), + ((f0 & ARM_MVFR0_EXCEPT_MASK) ? ", exceptions" : ""), + ((f1 & ARM_MVFR1_D_NAN_MASK) ? ", NaN propagation" : ""), + ((f1 & ARM_MVFR1_FTZ_MASK) ? ", denormals" : "")); + + aprint_debug("vfp%d: mvfr: [0]=%#x [1]=%#x\n", + device_unit(ci->ci_dev), f0, f1); + if (CPU_IS_PRIMARY(ci)) { - install_coproc_handler(VFP_COPROC, vfp_handler); + cpu_media_and_vfp_features[0] = f0; + cpu_media_and_vfp_features[1] = f1; + + if (f0 & ARM_MVFR0_ROUNDING_MASK) { + vfp_fpscr_changable |= VFP_FPSCR_RMODE; + } + if (f1 & ARM_MVFR0_EXCEPT_MASK) { + vfp_fpscr_changable |= VFP_FPSCR_ESUM; + } + // If hardware supports propagation of NaNs, select it. + if (f1 & ARM_MVFR1_D_NAN_MASK) { + vfp_fpscr_default &= ~VFP_FPSCR_DN; + vfp_fpscr_changable |= VFP_FPSCR_DN; + } + // If hardware supports denormalized numbers, use it. + if (f1 & ARM_MVFR1_FTZ_MASK) { + vfp_fpscr_default &= ~VFP_FPSCR_FZ; + vfp_fpscr_changable |= VFP_FPSCR_FZ; + } + + replace_coproc_handler(VFP_COPROC, vfp_handler); install_coproc_handler(VFP_COPROC2, vfp_handler); #ifdef CPU_CORTEX if (cpu_neon_present) { @@ -416,6 +387,13 @@ vfp_attach(struct cpu_info *ci) } #endif } + + evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL, + ci->ci_cpuname, "vfp coproc use"); + evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL, + ci->ci_cpuname, "vfp coproc re-use"); + evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL, + ci->ci_cpuname, "vfp coproc fault"); } /* The real handler for VFP bounces. */ -- 2.25.1