Improve SMP support:

o  Allocate a VHPT per CPU. The VHPT is a hash table that the CPU
   uses to look up translations it can't find in the TLB. As such,
   the VHPT serves as a level 1 cache (the TLB being a level 0 cache)
   and best results are obtained when it's not shared between CPUs.
   The collision chain (i.e. the hash bucket) is shared between CPUs,
   as all buckets together constitute our collection of PTEs. To
   achieve this, the collision chain does not point to the first PTE
   in the list anymore, but to a hash bucket head structure. The
   head structure contains the pointer to the first PTE in the list,
   as well as a mutex to lock the bucket. Thus, each bucket is locked
   independently of each other. With at least 1024 buckets in the VHPT,
   this provides for sufficiently finei-grained locking to make the
   ssolution scalable to large SMP machines.
o  Add synchronisation to the lazy FP context switching. We do this
   with a seperate per-thread lock. On SMP machines the lazy high FP
   context switching without synchronisation caused inconsistent
   state, which resulted in a panic. Since the use of the high FP
   registers is not common, it's possible that races exist. The ia64
   package build has proven to be a good stress test, so this will
   get plenty of exercise in the near future.
o  Don't use the local ID of the processor we want to send the IPI to
   as the argument to ipi_send(). use the struct pcpu pointer instead.
   The reason for this is that IPI delivery is unreliable. It has been
   observed that sending an IPI to a CPU causes it to receive a stray
   external interrupt. As such, we need a way to make the delivery
   reliable. The intended solution is to queue requests in the target
   CPU's per-CPU structure and use a single IPI to inform the CPU that
   there's a new entry in the queue. If that IPI gets lost, the CPU
   can check it's queue at any convenient time (such as for each
   clock interrupt). This also allows us to send requests to a CPU
   without interrupting it, if such would be beneficial.

With these changes SMP is almost working. There are still some random
process crashes and the machine can hang due to having the IPI lost
that deals with the high FP context switch.

The overhead of introducing the hash bucket head structure results
in a performance degradation of about 1% for UP (extra pointer
indirection). This is surprisingly small and is offset by gaining
reasonably/good scalable SMP support.
This commit is contained in:
marcel 2005-08-06 20:28:19 +00:00
parent ff7f5068f0
commit c96864a4b2
9 changed files with 321 additions and 323 deletions

View File

@ -644,6 +644,7 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
add r20=24,r18 // collision chain add r20=24,r18 // collision chain
;; ;;
ld8 r21=[r21] // check VHPT tag ld8 r21=[r21] // check VHPT tag
ld8 r20=[r20] // bucket head
;; ;;
cmp.ne p15,p0=r21,r19 cmp.ne p15,p0=r21,r19
(p15) br.dpnt.few 1f (p15) br.dpnt.few 1f
@ -722,6 +723,7 @@ IVT_ENTRY(Data_TLB, 0x0800)
add r20=24,r18 // collision chain add r20=24,r18 // collision chain
;; ;;
ld8 r21=[r21] // check VHPT tag ld8 r21=[r21] // check VHPT tag
ld8 r20=[r20] // bucket head
;; ;;
cmp.ne p15,p0=r21,r19 cmp.ne p15,p0=r21,r19
(p15) br.dpnt.few 1f (p15) br.dpnt.few 1f
@ -937,6 +939,8 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
ttag r19=r16 ttag r19=r16
add r20=24,r18 // collision chain add r20=24,r18 // collision chain
;; ;;
ld8 r20=[r20] // bucket head
;;
ld8 r20=[r20] // first entry ld8 r20=[r20] // first entry
;; ;;
rsm psr.dt // turn off data translations rsm psr.dt // turn off data translations
@ -1003,6 +1007,8 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400)
ttag r19=r16 ttag r19=r16
add r20=24,r18 // collision chain add r20=24,r18 // collision chain
;; ;;
ld8 r20=[r20] // bucket head
;;
ld8 r20=[r20] // first entry ld8 r20=[r20] // first entry
;; ;;
rsm psr.dt // turn off data translations rsm psr.dt // turn off data translations
@ -1069,6 +1075,8 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
ttag r19=r16 ttag r19=r16
add r20=24,r18 // collision chain add r20=24,r18 // collision chain
;; ;;
ld8 r20=[r20] // bucket head
;;
ld8 r20=[r20] // first entry ld8 r20=[r20] // first entry
;; ;;
rsm psr.dt // turn off data translations rsm psr.dt // turn off data translations

View File

@ -147,6 +147,8 @@ interrupt(u_int64_t vector, struct trapframe *tf)
if (vector == 0) { if (vector == 0) {
vector = ib->ib_inta; vector = ib->ib_inta;
printf("ExtINT interrupt: vector=%ld\n", vector); printf("ExtINT interrupt: vector=%ld\n", vector);
if (vector == 15)
goto stray;
} }
if (vector == CLOCK_VECTOR) {/* clock interrupt */ if (vector == CLOCK_VECTOR) {/* clock interrupt */
@ -207,9 +209,11 @@ interrupt(u_int64_t vector, struct trapframe *tf)
} else if (vector == ipi_vector[IPI_HIGH_FP]) { } else if (vector == ipi_vector[IPI_HIGH_FP]) {
struct thread *thr = PCPU_GET(fpcurthread); struct thread *thr = PCPU_GET(fpcurthread);
if (thr != NULL) { if (thr != NULL) {
mtx_lock(&thr->td_md.md_highfp_mtx);
save_high_fp(&thr->td_pcb->pcb_high_fp); save_high_fp(&thr->td_pcb->pcb_high_fp);
thr->td_pcb->pcb_fpcpu = NULL; thr->td_pcb->pcb_fpcpu = NULL;
PCPU_SET(fpcurthread, NULL); PCPU_SET(fpcurthread, NULL);
mtx_unlock(&thr->td_md.md_highfp_mtx);
} }
} else if (vector == ipi_vector[IPI_RENDEZVOUS]) { } else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
rdvs[PCPU_GET(cpuid)]++; rdvs[PCPU_GET(cpuid)]++;
@ -239,6 +243,7 @@ interrupt(u_int64_t vector, struct trapframe *tf)
ia64_dispatch_intr(tf, vector); ia64_dispatch_intr(tf, vector);
} }
stray:
atomic_subtract_int(&td->td_intr_nesting_level, 1); atomic_subtract_int(&td->td_intr_nesting_level, 1);
return (TRAPF_USERMODE(tf)); return (TRAPF_USERMODE(tf));
} }

View File

@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
#include <sys/ptrace.h> #include <sys/ptrace.h>
#include <sys/random.h> #include <sys/random.h>
#include <sys/reboot.h> #include <sys/reboot.h>
#include <sys/sched.h>
#include <sys/signalvar.h> #include <sys/signalvar.h>
#include <sys/syscall.h> #include <sys/syscall.h>
#include <sys/sysctl.h> #include <sys/sysctl.h>
@ -764,8 +765,8 @@ ia64_init(void)
*/ */
proc0kstack = (vm_offset_t)kstack; proc0kstack = (vm_offset_t)kstack;
thread0.td_kstack = proc0kstack; thread0.td_kstack = proc0kstack;
thread0.td_pcb = (struct pcb *) thread0.td_kstack_pages = KSTACK_PAGES;
(thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
/* /*
* Setup the global data for the bootstrap cpu. * Setup the global data for the bootstrap cpu.
*/ */
@ -774,6 +775,8 @@ ia64_init(void)
pcpu_init(pcpup, 0, PAGE_SIZE); pcpu_init(pcpup, 0, PAGE_SIZE);
PCPU_SET(curthread, &thread0); PCPU_SET(curthread, &thread0);
mutex_init();
/* /*
* Initialize the rest of proc 0's PCB. * Initialize the rest of proc 0's PCB.
* *
@ -781,14 +784,11 @@ ia64_init(void)
* and make proc0's trapframe pointer point to it for sanity. * and make proc0's trapframe pointer point to it for sanity.
* Initialise proc0's backing store to start after u area. * Initialise proc0's backing store to start after u area.
*/ */
thread0.td_frame = (struct trapframe *)thread0.td_pcb - 1; cpu_thread_setup(&thread0);
thread0.td_frame->tf_length = sizeof(struct trapframe);
thread0.td_frame->tf_flags = FRAME_SYSCALL; thread0.td_frame->tf_flags = FRAME_SYSCALL;
thread0.td_pcb->pcb_special.sp = thread0.td_pcb->pcb_special.sp =
(u_int64_t)thread0.td_frame - 16; (u_int64_t)thread0.td_frame - 16;
thread0.td_pcb->pcb_special.bspstore = (u_int64_t)proc0kstack; thread0.td_pcb->pcb_special.bspstore = thread0.td_kstack;
mutex_init();
/* /*
* Initialize the virtual memory system. * Initialize the virtual memory system.
@ -1428,7 +1428,6 @@ set_fpregs(struct thread *td, struct fpreg *fpregs)
/* /*
* High FP register functions. * High FP register functions.
* XXX no synchronization yet.
*/ */
int int
@ -1438,13 +1437,17 @@ ia64_highfp_drop(struct thread *td)
struct pcpu *cpu; struct pcpu *cpu;
struct thread *thr; struct thread *thr;
mtx_lock(&td->td_md.md_highfp_mtx);
pcb = td->td_pcb; pcb = td->td_pcb;
cpu = pcb->pcb_fpcpu; cpu = pcb->pcb_fpcpu;
if (cpu == NULL) if (cpu == NULL) {
mtx_unlock(&td->td_md.md_highfp_mtx);
return (0); return (0);
}
pcb->pcb_fpcpu = NULL; pcb->pcb_fpcpu = NULL;
thr = cpu->pc_fpcurthread; thr = cpu->pc_fpcurthread;
cpu->pc_fpcurthread = NULL; cpu->pc_fpcurthread = NULL;
mtx_unlock(&td->td_md.md_highfp_mtx);
/* Post-mortem sanity checking. */ /* Post-mortem sanity checking. */
KASSERT(thr == td, ("Inconsistent high FP state")); KASSERT(thr == td, ("Inconsistent high FP state"));
@ -1462,22 +1465,36 @@ ia64_highfp_save(struct thread *td)
if ((td->td_frame->tf_special.psr & IA64_PSR_MFH) == 0) if ((td->td_frame->tf_special.psr & IA64_PSR_MFH) == 0)
return (ia64_highfp_drop(td)); return (ia64_highfp_drop(td));
mtx_lock(&td->td_md.md_highfp_mtx);
pcb = td->td_pcb; pcb = td->td_pcb;
cpu = pcb->pcb_fpcpu; cpu = pcb->pcb_fpcpu;
if (cpu == NULL) if (cpu == NULL) {
mtx_unlock(&td->td_md.md_highfp_mtx);
return (0); return (0);
}
#ifdef SMP #ifdef SMP
if (td == curthread)
sched_pin();
if (cpu != pcpup) { if (cpu != pcpup) {
ipi_send(cpu->pc_lid, IPI_HIGH_FP); mtx_unlock(&td->td_md.md_highfp_mtx);
while (pcb->pcb_fpcpu != cpu) ipi_send(cpu, IPI_HIGH_FP);
if (td == curthread)
sched_unpin();
while (pcb->pcb_fpcpu == cpu)
DELAY(100); DELAY(100);
return (1); return (1);
} else {
save_high_fp(&pcb->pcb_high_fp);
if (td == curthread)
sched_unpin();
} }
#endif #else
save_high_fp(&pcb->pcb_high_fp); save_high_fp(&pcb->pcb_high_fp);
#endif
pcb->pcb_fpcpu = NULL; pcb->pcb_fpcpu = NULL;
thr = cpu->pc_fpcurthread; thr = cpu->pc_fpcurthread;
cpu->pc_fpcurthread = NULL; cpu->pc_fpcurthread = NULL;
mtx_unlock(&td->td_md.md_highfp_mtx);
/* Post-mortem sanity cxhecking. */ /* Post-mortem sanity cxhecking. */
KASSERT(thr == td, ("Inconsistent high FP state")); KASSERT(thr == td, ("Inconsistent high FP state"));

View File

@ -64,8 +64,9 @@ MALLOC_DECLARE(M_PMAP);
void ia64_ap_startup(void); void ia64_ap_startup(void);
extern vm_offset_t vhpt_base, vhpt_size; extern uint64_t vhpt_base[];
extern u_int64_t ia64_lapic_address; extern size_t vhpt_size;
extern uint64_t ia64_lapic_address;
#define LID_SAPIC_ID(x) ((int)((x) >> 24) & 0xff) #define LID_SAPIC_ID(x) ((int)((x) >> 24) & 0xff)
#define LID_SAPIC_EID(x) ((int)((x) >> 16) & 0xff) #define LID_SAPIC_EID(x) ((int)((x) >> 16) & 0xff)
@ -74,9 +75,10 @@ extern u_int64_t ia64_lapic_address;
int mp_ipi_test = 0; int mp_ipi_test = 0;
/* Variables used by os_boot_rendez */ /* Variables used by os_boot_rendez and ia64_ap_startup */
void *ap_stack;
struct pcpu *ap_pcpu; struct pcpu *ap_pcpu;
void *ap_stack;
uint64_t ap_vhpt;
volatile int ap_delay; volatile int ap_delay;
volatile int ap_awake; volatile int ap_awake;
volatile int ap_spin; volatile int ap_spin;
@ -86,15 +88,16 @@ static void cpu_mp_unleash(void *);
void void
ia64_ap_startup(void) ia64_ap_startup(void)
{ {
ap_awake = 1;
ap_delay = 0;
__asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
"r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1));
pcpup = ap_pcpu; pcpup = ap_pcpu;
ia64_set_k4((intptr_t)pcpup); ia64_set_k4((intptr_t)pcpup);
__asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
"r" (ap_vhpt + (1<<8) + (vhpt_size<<2) + 1));
ap_awake = 1;
ap_delay = 0;
map_pal_code(); map_pal_code();
map_gateway_page(); map_gateway_page();
@ -102,7 +105,7 @@ ia64_ap_startup(void)
/* Wait until it's time for us to be unleashed */ /* Wait until it's time for us to be unleashed */
while (ap_spin) while (ap_spin)
/* spin */; DELAY(0);
__asm __volatile("ssm psr.i;; srlz.d;;"); __asm __volatile("ssm psr.i;; srlz.d;;");
@ -119,7 +122,7 @@ ia64_ap_startup(void)
ap_awake++; ap_awake++;
while (!smp_started) while (!smp_started)
/* spin */; DELAY(0);
CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid)); CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid));
@ -242,16 +245,17 @@ cpu_mp_start()
pc->pc_current_pmap = kernel_pmap; pc->pc_current_pmap = kernel_pmap;
pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask; pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask;
if (pc->pc_cpuid > 0) { if (pc->pc_cpuid > 0) {
ap_pcpu = pc;
ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP, ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP,
M_WAITOK); M_WAITOK);
ap_pcpu = pc; ap_vhpt = vhpt_base[pc->pc_cpuid];
ap_delay = 2000; ap_delay = 2000;
ap_awake = 0; ap_awake = 0;
if (bootverbose) if (bootverbose)
printf("SMP: waking up cpu%d\n", pc->pc_cpuid); printf("SMP: waking up cpu%d\n", pc->pc_cpuid);
ipi_send(pc->pc_lid, IPI_AP_WAKEUP); ipi_send(pc, IPI_AP_WAKEUP);
do { do {
DELAY(1000); DELAY(1000);
@ -292,7 +296,7 @@ cpu_mp_unleash(void *dummy)
ap_spin = 0; ap_spin = 0;
while (ap_awake != smp_cpus) while (ap_awake != smp_cpus)
/* spin */; DELAY(0);
if (smp_cpus != cpus || cpus != mp_ncpus) { if (smp_cpus != cpus || cpus != mp_ncpus) {
printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n", printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n",
@ -307,13 +311,13 @@ cpu_mp_unleash(void *dummy)
* send an IPI to a set of cpus. * send an IPI to a set of cpus.
*/ */
void void
ipi_selected(u_int64_t cpus, int ipi) ipi_selected(cpumask_t cpus, int ipi)
{ {
struct pcpu *pc; struct pcpu *pc;
SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
if (cpus & pc->pc_cpumask) if (cpus & pc->pc_cpumask)
ipi_send(pc->pc_lid, ipi); ipi_send(pc, ipi);
} }
} }
@ -326,7 +330,7 @@ ipi_all(int ipi)
struct pcpu *pc; struct pcpu *pc;
SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
ipi_send(pc->pc_lid, ipi); ipi_send(pc, ipi);
} }
} }
@ -340,7 +344,7 @@ ipi_all_but_self(int ipi)
SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
if (pc != pcpup) if (pc != pcpup)
ipi_send(pc->pc_lid, ipi); ipi_send(pc, ipi);
} }
} }
@ -351,7 +355,7 @@ void
ipi_self(int ipi) ipi_self(int ipi)
{ {
ipi_send(ia64_get_lid(), ipi); ipi_send(pcpup, ipi);
} }
/* /*
@ -360,17 +364,17 @@ ipi_self(int ipi)
* fields are used here. * fields are used here.
*/ */
void void
ipi_send(u_int64_t lid, int ipi) ipi_send(struct pcpu *cpu, int ipi)
{ {
volatile u_int64_t *pipi; volatile uint64_t *pipi;
u_int64_t vector; uint64_t vector;
pipi = __MEMIO_ADDR(ia64_lapic_address | pipi = __MEMIO_ADDR(ia64_lapic_address |
((lid & LID_SAPIC_MASK) >> 12)); ((cpu->pc_lid & LID_SAPIC_MASK) >> 12));
vector = (u_int64_t)(ipi_vector[ipi] & 0xff); vector = (uint64_t)(ipi_vector[ipi] & 0xff);
*pipi = vector;
CTR3(KTR_SMP, "ipi_send(%p, %ld), cpuid=%d", pipi, vector, CTR3(KTR_SMP, "ipi_send(%p, %ld), cpuid=%d", pipi, vector,
PCPU_GET(cpuid)); PCPU_GET(cpuid));
*pipi = vector;
} }
SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);

View File

@ -155,8 +155,15 @@ struct pmap kernel_pmap_store;
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
vm_offset_t vhpt_base, vhpt_size; struct ia64_bucket {
struct mtx pmap_vhptmutex; uint64_t chain;
struct mtx mutex;
u_int length;
};
struct ia64_bucket *vhpt_bucket;
uint64_t vhpt_base[MAXCPU];
size_t vhpt_size;
/* /*
* Kernel virtual memory management. * Kernel virtual memory management.
@ -177,6 +184,7 @@ static uint64_t pmap_ptc_e_count1 = 3;
static uint64_t pmap_ptc_e_count2 = 2; static uint64_t pmap_ptc_e_count2 = 2;
static uint64_t pmap_ptc_e_stride1 = 0x2000; static uint64_t pmap_ptc_e_stride1 = 0x2000;
static uint64_t pmap_ptc_e_stride2 = 0x100000000; static uint64_t pmap_ptc_e_stride2 = 0x100000000;
struct mtx pmap_ptcmutex;
/* /*
* Data for the RID allocator * Data for the RID allocator
@ -204,14 +212,11 @@ static uma_zone_t ptezone;
* VHPT instrumentation. * VHPT instrumentation.
*/ */
static int pmap_vhpt_inserts; static int pmap_vhpt_inserts;
static int pmap_vhpt_collisions;
static int pmap_vhpt_resident; static int pmap_vhpt_resident;
SYSCTL_DECL(_vm_stats); SYSCTL_DECL(_vm_stats);
SYSCTL_NODE(_vm_stats, OID_AUTO, vhpt, CTLFLAG_RD, 0, ""); SYSCTL_NODE(_vm_stats, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, inserts, CTLFLAG_RD, SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
&pmap_vhpt_inserts, 0, ""); &pmap_vhpt_inserts, 0, "");
SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, collisions, CTLFLAG_RD,
&pmap_vhpt_collisions, 0, "");
SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, resident, CTLFLAG_RD, SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, resident, CTLFLAG_RD,
&pmap_vhpt_resident, 0, ""); &pmap_vhpt_resident, 0, "");
@ -257,8 +262,11 @@ pmap_steal_memory(vm_size_t size)
void void
pmap_bootstrap() pmap_bootstrap()
{ {
int i, j, count, ridbits;
struct ia64_pal_result res; struct ia64_pal_result res;
struct ia64_lpte *pte;
vm_offset_t base, limit;
size_t size;
int i, j, count, ridbits;
/* /*
* Query the PAL Code to find the loop parameters for the * Query the PAL Code to find the loop parameters for the
@ -280,6 +288,7 @@ pmap_bootstrap()
pmap_ptc_e_count2, pmap_ptc_e_count2,
pmap_ptc_e_stride1, pmap_ptc_e_stride1,
pmap_ptc_e_stride2); pmap_ptc_e_stride2);
mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN);
/* /*
* Setup RIDs. RIDs 0..7 are reserved for the kernel. * Setup RIDs. RIDs 0..7 are reserved for the kernel.
@ -335,7 +344,8 @@ pmap_bootstrap()
kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS - kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS -
VM_GATEWAY_SIZE; VM_GATEWAY_SIZE;
for (i = 0; phys_avail[i+2]; i+= 2) ; for (i = 0; phys_avail[i+2]; i+= 2)
;
count = i+2; count = i+2;
/* /*
@ -345,19 +355,20 @@ pmap_bootstrap()
* size and aligned to a natural boundary). * size and aligned to a natural boundary).
*/ */
vhpt_size = 15; vhpt_size = 15;
while ((1<<vhpt_size) < Maxmem * 32) size = 1UL << vhpt_size;
while (size < Maxmem * 32) {
vhpt_size++; vhpt_size++;
size <<= 1;
}
vhpt_base = 0; vhpt_base[0] = 0;
while (!vhpt_base) { base = limit = 0;
vm_offset_t mask; while (vhpt_base[0] == 0) {
if (bootverbose) if (bootverbose)
printf("Trying VHPT size 0x%lx\n", (1L<<vhpt_size)); printf("Trying VHPT size 0x%lx\n", size);
mask = (1L << vhpt_size) - 1;
for (i = 0; i < count; i += 2) { for (i = 0; i < count; i += 2) {
vm_offset_t base, limit; base = (phys_avail[i] + size - 1) & ~(size - 1);
base = (phys_avail[i] + mask) & ~mask; limit = base + MAXCPU * size;
limit = base + (1L << vhpt_size);
if (limit <= phys_avail[i+1]) if (limit <= phys_avail[i+1])
/* /*
* VHPT can fit in this region * VHPT can fit in this region
@ -365,46 +376,53 @@ pmap_bootstrap()
break; break;
} }
if (!phys_avail[i]) { if (!phys_avail[i]) {
/* /* Can't fit, try next smaller size. */
* Can't fit, try next smaller size.
*/
vhpt_size--; vhpt_size--;
} else { size >>= 1;
vhpt_base = (phys_avail[i] + mask) & ~mask; } else
} vhpt_base[0] = IA64_PHYS_TO_RR7(base);
} }
if (vhpt_size < 15) if (vhpt_size < 15)
panic("Can't find space for VHPT"); panic("Can't find space for VHPT");
if (bootverbose) if (bootverbose)
printf("Putting VHPT at %p\n", (void *) vhpt_base); printf("Putting VHPT at 0x%lx\n", base);
if (vhpt_base != phys_avail[i]) {
/* if (base != phys_avail[i]) {
* Split this region. /* Split this region. */
*/
if (bootverbose) if (bootverbose)
printf("Splitting [%p-%p]\n", printf("Splitting [%p-%p]\n", (void *)phys_avail[i],
(void *) phys_avail[i], (void *)phys_avail[i+1]);
(void *) phys_avail[i+1]);
for (j = count; j > i; j -= 2) { for (j = count; j > i; j -= 2) {
phys_avail[j] = phys_avail[j-2]; phys_avail[j] = phys_avail[j-2];
phys_avail[j+1] = phys_avail[j-2+1]; phys_avail[j+1] = phys_avail[j-2+1];
} }
phys_avail[count+2] = 0; phys_avail[i+1] = base;
phys_avail[count+3] = 0; phys_avail[i+2] = limit;
phys_avail[i+1] = vhpt_base; } else
phys_avail[i+2] = vhpt_base + (1L << vhpt_size); phys_avail[i] = limit;
} else {
phys_avail[i] = vhpt_base + (1L << vhpt_size); count = size / sizeof(struct ia64_lpte);
vhpt_bucket = (void *)pmap_steal_memory(count * sizeof(struct ia64_bucket));
pte = (struct ia64_lpte *)vhpt_base[0];
for (i = 0; i < count; i++) {
pte[i].pte = 0;
pte[i].itir = 0;
pte[i].tag = 1UL << 63; /* Invalid tag */
pte[i].chain = (uintptr_t)(vhpt_bucket + i);
/* Stolen memory is zeroed! */
mtx_init(&vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
MTX_SPIN);
} }
vhpt_base = IA64_PHYS_TO_RR7(vhpt_base); for (i = 1; i < MAXCPU; i++) {
bzero((void *) vhpt_base, (1L << vhpt_size)); vhpt_base[i] = vhpt_base[i - 1] + size;
bcopy((void *)vhpt_base[i - 1], (void *)vhpt_base[i], size);
}
mtx_init(&pmap_vhptmutex, "VHPT collision chain lock", NULL, MTX_DEF); __asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
"r" (vhpt_base[0] + (1<<8) + (vhpt_size<<2) + 1));
__asm __volatile("mov cr.pta=%0;; srlz.i;;"
:: "r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1));
virtual_avail = VM_MIN_KERNEL_ADDRESS; virtual_avail = VM_MIN_KERNEL_ADDRESS;
virtual_end = VM_MAX_KERNEL_ADDRESS; virtual_end = VM_MAX_KERNEL_ADDRESS;
@ -494,12 +512,73 @@ pmap_init2()
* Manipulate TLBs for a pmap * Manipulate TLBs for a pmap
***************************************************/ ***************************************************/
#if 0
static __inline void
pmap_invalidate_page_locally(void *arg)
{
vm_offset_t va = (uintptr_t)arg;
struct ia64_lpte *pte;
pte = (struct ia64_lpte *)ia64_thash(va);
if (pte->tag == ia64_ttag(va))
pte->tag = 1UL << 63;
ia64_ptc_l(va, PAGE_SHIFT << 2);
}
#ifdef SMP
static void
pmap_invalidate_page_1(void *arg)
{
void **args = arg;
pmap_t oldpmap;
critical_enter();
oldpmap = pmap_install(args[0]);
pmap_invalidate_page_locally(args[1]);
pmap_install(oldpmap);
critical_exit();
}
#endif
static void static void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{ {
KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
("invalidating TLB for non-current pmap")); ("invalidating TLB for non-current pmap"));
ia64_ptc_g(va, PAGE_SHIFT << 2);
#ifdef SMP
if (mp_ncpus > 1) {
void *args[2];
args[0] = pmap;
args[1] = (void *)va;
smp_rendezvous(NULL, pmap_invalidate_page_1, NULL, args);
} else
#endif
pmap_invalidate_page_locally((void *)va);
}
#endif /* 0 */
static void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
struct ia64_lpte *pte;
int i, vhpt_ofs;
KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
("invalidating TLB for non-current pmap"));
vhpt_ofs = ia64_thash(va) - vhpt_base[PCPU_GET(cpuid)];
critical_enter();
for (i = 0; i < MAXCPU; i++) {
pte = (struct ia64_lpte *)(vhpt_base[i] + vhpt_ofs);
if (pte->tag == ia64_ttag(va))
pte->tag = 1UL << 63;
}
critical_exit();
mtx_lock_spin(&pmap_ptcmutex);
ia64_ptc_ga(va, PAGE_SHIFT << 2);
mtx_unlock_spin(&pmap_ptcmutex);
} }
static void static void
@ -507,9 +586,8 @@ pmap_invalidate_all_1(void *arg)
{ {
uint64_t addr; uint64_t addr;
int i, j; int i, j;
register_t psr;
psr = intr_disable(); critical_enter();
addr = pmap_ptc_e_base; addr = pmap_ptc_e_base;
for (i = 0; i < pmap_ptc_e_count1; i++) { for (i = 0; i < pmap_ptc_e_count1; i++) {
for (j = 0; j < pmap_ptc_e_count2; j++) { for (j = 0; j < pmap_ptc_e_count2; j++) {
@ -518,21 +596,22 @@ pmap_invalidate_all_1(void *arg)
} }
addr += pmap_ptc_e_stride1; addr += pmap_ptc_e_stride1;
} }
intr_restore(psr); critical_exit();
} }
static void static void
pmap_invalidate_all(pmap_t pmap) pmap_invalidate_all(pmap_t pmap)
{ {
KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
("invalidating TLB for non-current pmap")); ("invalidating TLB for non-current pmap"));
#ifdef SMP #ifdef SMP
smp_rendezvous(0, pmap_invalidate_all_1, 0, 0); if (mp_ncpus > 1)
#else smp_rendezvous(NULL, pmap_invalidate_all_1, NULL, NULL);
pmap_invalidate_all_1(0); else
#endif #endif
pmap_invalidate_all_1(NULL);
} }
static uint32_t static uint32_t
@ -582,47 +661,6 @@ pmap_free_rid(uint32_t rid)
mtx_unlock(&pmap_ridmutex); mtx_unlock(&pmap_ridmutex);
} }
/***************************************************
* Low level helper routines.....
***************************************************/
/*
* Install a pte into the VHPT
*/
static PMAP_INLINE void
pmap_install_pte(struct ia64_lpte *vhpte, struct ia64_lpte *pte)
{
uint64_t *vhp, *p;
vhp = (uint64_t *)vhpte;
p = (uint64_t *)pte;
critical_enter();
/* Invalidate the tag so the VHPT walker will not match this entry. */
vhp[2] = 1UL << 63;
ia64_mf();
vhp[0] = p[0];
vhp[1] = p[1];
ia64_mf();
/* Install a proper tag now that we're done. */
vhp[2] = p[2];
ia64_mf();
critical_exit();
}
/*
* Compare essential parts of pte.
*/
static PMAP_INLINE int
pmap_equal_pte(struct ia64_lpte *pte1, struct ia64_lpte *pte2)
{
return *(uint64_t *) pte1 == *(uint64_t *) pte2;
}
/* /*
* this routine defines the region(s) of memory that should * this routine defines the region(s) of memory that should
* not be tested for the modified bit. * not be tested for the modified bit.
@ -759,50 +797,23 @@ get_pv_entry(void)
static void static void
pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va) pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
{ {
struct ia64_bucket *bckt;
struct ia64_lpte *vhpte; struct ia64_lpte *vhpte;
pmap_vhpt_inserts++; pmap_vhpt_inserts++;
pmap_vhpt_resident++; pmap_vhpt_resident++;
vhpte = (struct ia64_lpte *) ia64_thash(va);
if (vhpte->chain)
pmap_vhpt_collisions++;
mtx_lock(&pmap_vhptmutex);
pte->chain = vhpte->chain;
ia64_mf();
vhpte->chain = ia64_tpa((vm_offset_t)pte);
ia64_mf();
if (!pmap_lpte_present(vhpte) && pmap_lpte_present(pte)) {
ia64_ptc_g(va, PAGE_SHIFT << 2);
pmap_install_pte(vhpte, pte);
}
mtx_unlock(&pmap_vhptmutex);
}
/*
* Update VHPT after a pte has changed.
*/
static void
pmap_update_vhpt(struct ia64_lpte *pte, vm_offset_t va)
{
struct ia64_lpte *vhpte;
vhpte = (struct ia64_lpte *)ia64_thash(va); vhpte = (struct ia64_lpte *)ia64_thash(va);
bckt = (struct ia64_bucket *)vhpte->chain;
mtx_lock(&pmap_vhptmutex); mtx_lock_spin(&bckt->mutex);
pte->chain = bckt->chain;
ia64_mf();
bckt->chain = ia64_tpa((vm_offset_t)pte);
ia64_mf();
if ((!pmap_lpte_present(vhpte) || vhpte->tag == pte->tag) && bckt->length++;
pmap_lpte_present(pte)) { mtx_unlock_spin(&bckt->mutex);
ia64_ptc_g(va, PAGE_SHIFT << 2);
pmap_install_pte(vhpte, pte);
}
mtx_unlock(&pmap_vhptmutex);
} }
/* /*
@ -812,55 +823,37 @@ pmap_update_vhpt(struct ia64_lpte *pte, vm_offset_t va)
static int static int
pmap_remove_vhpt(vm_offset_t va) pmap_remove_vhpt(vm_offset_t va)
{ {
struct ia64_bucket *bckt;
struct ia64_lpte *pte; struct ia64_lpte *pte;
struct ia64_lpte *lpte; struct ia64_lpte *lpte;
struct ia64_lpte *vhpte; struct ia64_lpte *vhpte;
uint64_t tag; uint64_t tag;
tag = ia64_ttag(va);
vhpte = (struct ia64_lpte *)ia64_thash(va); vhpte = (struct ia64_lpte *)ia64_thash(va);
bckt = (struct ia64_bucket *)vhpte->chain;
/* mtx_lock_spin(&bckt->mutex);
* If the VHPTE is invalid, there can't be a collision chain. lpte = NULL;
*/ pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(bckt->chain);
if (!pmap_lpte_present(vhpte)) { while (pte != NULL && pte->tag != tag) {
KASSERT(!vhpte->chain, ("bad vhpte")); lpte = pte;
pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->chain);
}
if (pte == NULL) {
mtx_unlock_spin(&bckt->mutex);
return (ENOENT); return (ENOENT);
} }
lpte = vhpte;
tag = ia64_ttag(va);
mtx_lock(&pmap_vhptmutex);
pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(vhpte->chain);
KASSERT(pte != NULL, ("foo"));
while (pte->tag != tag) {
lpte = pte;
if (pte->chain == 0) {
mtx_unlock(&pmap_vhptmutex);
return (ENOENT);
}
pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->chain);
}
/* Snip this pv_entry out of the collision chain. */ /* Snip this pv_entry out of the collision chain. */
lpte->chain = pte->chain; if (lpte == NULL)
bckt->chain = pte->chain;
else
lpte->chain = pte->chain;
ia64_mf(); ia64_mf();
/* bckt->length--;
* If the VHPTE matches as well, change it to map the first mtx_unlock_spin(&bckt->mutex);
* element from the chain if there is one.
*/
if (vhpte->tag == tag) {
if (vhpte->chain) {
pte = (void*)IA64_PHYS_TO_RR7(vhpte->chain);
pmap_install_pte(vhpte, pte);
} else
pmap_clear_present(vhpte);
}
mtx_unlock(&pmap_vhptmutex);
pmap_vhpt_resident--; pmap_vhpt_resident--;
return (0); return (0);
} }
@ -871,18 +864,21 @@ pmap_remove_vhpt(vm_offset_t va)
static struct ia64_lpte * static struct ia64_lpte *
pmap_find_vhpt(vm_offset_t va) pmap_find_vhpt(vm_offset_t va)
{ {
struct ia64_bucket *bckt;
struct ia64_lpte *pte; struct ia64_lpte *pte;
uint64_t tag; uint64_t tag;
tag = ia64_ttag(va); tag = ia64_ttag(va);
pte = (struct ia64_lpte *)ia64_thash(va); pte = (struct ia64_lpte *)ia64_thash(va);
if (pte->chain == 0) bckt = (struct ia64_bucket *)pte->chain;
if (bckt->chain == 0)
return (NULL); return (NULL);
pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->chain);
pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(bckt->chain);
while (pte->tag != tag) { while (pte->tag != tag) {
if (pte->chain == 0) if (pte->chain == 0)
return (NULL); return (NULL);
pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->chain ); pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->chain);
} }
return (pte); return (pte);
} }
@ -1028,12 +1024,9 @@ pmap_find_pte(vm_offset_t va)
return pmap_find_kpte(va); return pmap_find_kpte(va);
pte = pmap_find_vhpt(va); pte = pmap_find_vhpt(va);
if (!pte) { if (!pte)
pte = uma_zalloc(ptezone, M_NOWAIT); pte = uma_zalloc(ptezone, M_NOWAIT | M_ZERO);
if (pte != NULL) return (pte);
pmap_clear_present(pte);
}
return pte;
} }
/* /*
@ -1078,7 +1071,6 @@ static void
pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa, pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
boolean_t wired, boolean_t managed) boolean_t wired, boolean_t managed)
{ {
int wasvalid = pmap_lpte_present(pte);
pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK; pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK;
pte->pte |= PTE_PRESENT | PTE_MA_WB; pte->pte |= PTE_PRESENT | PTE_MA_WB;
@ -1089,25 +1081,6 @@ pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
pte->itir = PAGE_SHIFT << 2; pte->itir = PAGE_SHIFT << 2;
pte->tag = ia64_ttag(va); pte->tag = ia64_ttag(va);
if (wasvalid) {
pmap_update_vhpt(pte, va);
} else {
pmap_enter_vhpt(pte, va);
}
}
/*
* If a pte contains a valid mapping, clear it and update the VHPT.
*/
static void
pmap_clear_pte(struct ia64_lpte *pte, vm_offset_t va)
{
if (pmap_lpte_present(pte)) {
pmap_remove_vhpt(va);
ia64_ptc_g(va, PAGE_SHIFT << 2);
pmap_clear_present(pte);
}
} }
/* /*
@ -1129,12 +1102,10 @@ pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
*/ */
error = pmap_remove_vhpt(va); error = pmap_remove_vhpt(va);
if (error) if (error)
return error; return (error);
/* if (freepte)
* Make sure pmap_set_pte() knows it isn't in the VHPT. pmap_invalidate_page(pmap, va);
*/
pmap_clear_present(pte);
if (pmap_lpte_wired(pte)) if (pmap_lpte_wired(pte))
pmap->pm_stats.wired_count -= 1; pmap->pm_stats.wired_count -= 1;
@ -1150,12 +1121,13 @@ pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va,
if (freepte) if (freepte)
pmap_free_pte(pte, va); pmap_free_pte(pte, va);
return pmap_remove_entry(pmap, m, va, pv); error = pmap_remove_entry(pmap, m, va, pv);
} else { } else {
if (freepte) if (freepte)
pmap_free_pte(pte, va); pmap_free_pte(pte, va);
return 0;
} }
return (error);
} }
/* /*
@ -1199,18 +1171,18 @@ pmap_kextract(vm_offset_t va)
void void
pmap_qenter(vm_offset_t va, vm_page_t *m, int count) pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
{ {
int i;
struct ia64_lpte *pte; struct ia64_lpte *pte;
int i;
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
vm_offset_t tva = va + i * PAGE_SIZE; pte = pmap_find_kpte(va);
int wasvalid; if (pmap_lpte_present(pte))
pte = pmap_find_kpte(tva); pmap_invalidate_page(kernel_pmap, va);
wasvalid = pmap_lpte_present(pte); else
pmap_enter_vhpt(pte, va);
pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL); pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
pmap_set_pte(pte, tva, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE); pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
if (wasvalid) va += PAGE_SIZE;
ia64_ptc_g(tva, PAGE_SHIFT << 2);
} }
} }
@ -1221,12 +1193,16 @@ pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
void void
pmap_qremove(vm_offset_t va, int count) pmap_qremove(vm_offset_t va, int count)
{ {
int i;
struct ia64_lpte *pte; struct ia64_lpte *pte;
int i;
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
pte = pmap_find_kpte(va); pte = pmap_find_kpte(va);
pmap_clear_pte(pte, va); if (pmap_lpte_present(pte)) {
pmap_remove_vhpt(va);
pmap_invalidate_page(kernel_pmap, va);
pmap_clear_present(pte);
}
va += PAGE_SIZE; va += PAGE_SIZE;
} }
} }
@ -1239,14 +1215,14 @@ void
pmap_kenter(vm_offset_t va, vm_offset_t pa) pmap_kenter(vm_offset_t va, vm_offset_t pa)
{ {
struct ia64_lpte *pte; struct ia64_lpte *pte;
int wasvalid;
pte = pmap_find_kpte(va); pte = pmap_find_kpte(va);
wasvalid = pmap_lpte_present(pte); if (pmap_lpte_present(pte))
pmap_invalidate_page(kernel_pmap, va);
else
pmap_enter_vhpt(pte, va);
pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL); pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
pmap_set_pte(pte, va, pa, FALSE, FALSE); pmap_set_pte(pte, va, pa, FALSE, FALSE);
if (wasvalid)
ia64_ptc_g(va, PAGE_SHIFT << 2);
} }
/* /*
@ -1258,7 +1234,11 @@ pmap_kremove(vm_offset_t va)
struct ia64_lpte *pte; struct ia64_lpte *pte;
pte = pmap_find_kpte(va); pte = pmap_find_kpte(va);
pmap_clear_pte(pte, va); if (pmap_lpte_present(pte)) {
pmap_remove_vhpt(va);
pmap_invalidate_page(kernel_pmap, va);
pmap_clear_present(pte);
}
} }
/* /*
@ -1291,10 +1271,8 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va)
("removing page for non-current pmap")); ("removing page for non-current pmap"));
pte = pmap_find_vhpt(va); pte = pmap_find_vhpt(va);
if (pte) { if (pte)
pmap_remove_pte(pmap, pte, va, 0, 1); pmap_remove_pte(pmap, pte, va, 0, 1);
pmap_invalidate_page(pmap, va);
}
return; return;
} }
@ -1336,17 +1314,14 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
pte = pmap_find_vhpt(va); pte = pmap_find_vhpt(va);
KASSERT(pte != NULL, ("pte")); KASSERT(pte != NULL, ("pte"));
pmap_remove_pte(pmap, pte, va, pv, 1); pmap_remove_pte(pmap, pte, va, pv, 1);
pmap_invalidate_page(pmap, va);
} }
} }
} else { } else {
for (va = sva; va < eva; va = va += PAGE_SIZE) { for (va = sva; va < eva; va = va += PAGE_SIZE) {
pte = pmap_find_vhpt(va); pte = pmap_find_vhpt(va);
if (pte) { if (pte)
pmap_remove_pte(pmap, pte, va, 0, 1); pmap_remove_pte(pmap, pte, va, 0, 1);
pmap_invalidate_page(pmap, va);
}
} }
} }
out: out:
@ -1399,7 +1374,6 @@ pmap_remove_all(vm_page_t m)
if (pmap_lpte_ppn(pte) != VM_PAGE_TO_PHYS(m)) if (pmap_lpte_ppn(pte) != VM_PAGE_TO_PHYS(m))
panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m)); panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m));
pmap_remove_pte(pmap, pte, va, pv, 1); pmap_remove_pte(pmap, pte, va, pv, 1);
pmap_invalidate_page(pmap, va);
pmap_install(oldpmap); pmap_install(oldpmap);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
} }
@ -1459,7 +1433,6 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
} }
} }
pmap_pte_prot(pmap, pte, prot); pmap_pte_prot(pmap, pte, prot);
pmap_update_vhpt(pte, sva);
pmap_invalidate_page(pmap, sva); pmap_invalidate_page(pmap, sva);
} }
@ -1516,14 +1489,18 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
oldpmap = pmap_install(pmap); oldpmap = pmap_install(pmap);
} }
origpte = *pte; origpte = *pte;
opa = (pmap_lpte_present(&origpte)) ? pmap_lpte_ppn(&origpte) : 0UL; if (!pmap_lpte_present(pte)) {
opa = ~0UL;
pmap_enter_vhpt(pte, va);
} else
opa = pmap_lpte_ppn(pte);
managed = FALSE; managed = FALSE;
pa = VM_PAGE_TO_PHYS(m); pa = VM_PAGE_TO_PHYS(m);
/* /*
* Mapping has not changed, must be protection or wiring change. * Mapping has not changed, must be protection or wiring change.
*/ */
if (pmap_lpte_present(&origpte) && (opa == pa)) { if (opa == pa) {
/* /*
* Wiring change, just update stats. We don't worry about * Wiring change, just update stats. We don't worry about
* wiring PT pages as they remain resident as long as there * wiring PT pages as they remain resident as long as there
@ -1545,17 +1522,17 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
pmap_track_modified(va)) pmap_track_modified(va))
vm_page_dirty(m); vm_page_dirty(m);
pmap_invalidate_page(pmap, va);
goto validate; goto validate;
} }
/* /*
* Mapping has changed, invalidate old range and fall * Mapping has changed, invalidate old range and fall
* through to handle validating new mapping. * through to handle validating new mapping.
*/ */
if (opa) { if (opa != ~0UL) {
int error; pmap_remove_pte(pmap, pte, va, 0, 0);
error = pmap_remove_pte(pmap, pte, va, 0, 0); pmap_enter_vhpt(pte, va);
if (error)
panic("pmap_enter: pte vanished, va: 0x%lx", va);
} }
/* /*
@ -1582,13 +1559,6 @@ validate:
pmap_pte_prot(pmap, pte, prot); pmap_pte_prot(pmap, pte, prot);
pmap_set_pte(pte, va, pa, wired, managed); pmap_set_pte(pte, va, pa, wired, managed);
/*
* if the mapping or permission bits are different, we need
* to invalidate the page.
*/
if (!pmap_equal_pte(&origpte, pte))
pmap_invalidate_page(pmap, va);
vm_page_unlock_queues(); vm_page_unlock_queues();
pmap_install(oldpmap); pmap_install(oldpmap);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
@ -1629,30 +1599,24 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
oldpmap = pmap_install(pmap); oldpmap = pmap_install(pmap);
} }
if (pmap_lpte_present(pte))
goto reinstall;
managed = FALSE;
/* if (!pmap_lpte_present(pte)) {
* Enter on the PV list since its part of our managed memory. /* Enter on the PV list if its managed. */
*/ if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { pmap_insert_entry(pmap, va, m);
pmap_insert_entry(pmap, va, m); managed = TRUE;
managed = TRUE; } else
managed = FALSE;
/* Increment counters. */
pmap->pm_stats.resident_count++;
/* Initialise with R/O protection and enter into VHPT. */
pmap_enter_vhpt(pte, va);
pmap_pte_prot(pmap, pte, VM_PROT_READ);
pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
} }
/*
* Increment counters
*/
pmap->pm_stats.resident_count++;
/*
* Initialise PTE with read-only protection and enter into VHPT.
*/
pmap_pte_prot(pmap, pte, VM_PROT_READ);
pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
reinstall:
pmap_install(oldpmap); pmap_install(oldpmap);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
return (NULL); return (NULL);
@ -1817,7 +1781,6 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
return (FALSE); return (FALSE);
} }
#define PMAP_REMOVE_PAGES_CURPROC_ONLY
/* /*
* Remove all pages from specified address space * Remove all pages from specified address space
* this aids process exit speeds. Also, this code * this aids process exit speeds. Also, this code
@ -1829,37 +1792,33 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
void void
pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{ {
pmap_t oldpmap;
pv_entry_t pv, npv; pv_entry_t pv, npv;
#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
printf("warning: pmap_remove_pages called with non-current pmap\n"); printf("warning: pmap_remove_pages called with non-current pmap\n");
return; return;
} }
#endif
vm_page_lock_queues(); vm_page_lock_queues();
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
for (pv = TAILQ_FIRST(&pmap->pm_pvlist); oldpmap = pmap_install(pmap);
pv;
pv = npv) { for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
struct ia64_lpte *pte; struct ia64_lpte *pte;
npv = TAILQ_NEXT(pv, pv_plist); npv = TAILQ_NEXT(pv, pv_plist);
if (pv->pv_va >= eva || pv->pv_va < sva) { if (pv->pv_va >= eva || pv->pv_va < sva)
continue; continue;
}
pte = pmap_find_vhpt(pv->pv_va); pte = pmap_find_vhpt(pv->pv_va);
KASSERT(pte != NULL, ("pte")); KASSERT(pte != NULL, ("pte"));
if (pmap_lpte_wired(pte)) if (!pmap_lpte_wired(pte))
continue; pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1);
pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1);
} }
pmap_invalidate_all(pmap); pmap_install(oldpmap);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
vm_page_unlock_queues(); vm_page_unlock_queues();
} }
@ -1888,7 +1847,6 @@ pmap_page_protect(vm_page_t m, vm_prot_t prot)
pte = pmap_find_vhpt(pv->pv_va); pte = pmap_find_vhpt(pv->pv_va);
KASSERT(pte != NULL, ("pte")); KASSERT(pte != NULL, ("pte"));
pmap_pte_prot(pmap, pte, prot); pmap_pte_prot(pmap, pte, prot);
pmap_update_vhpt(pte, pv->pv_va);
pmap_invalidate_page(pmap, pv->pv_va); pmap_invalidate_page(pmap, pv->pv_va);
pmap_install(oldpmap); pmap_install(oldpmap);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
@ -1930,7 +1888,6 @@ pmap_ts_referenced(vm_page_t m)
if (pmap_lpte_accessed(pte)) { if (pmap_lpte_accessed(pte)) {
count++; count++;
pmap_clear_accessed(pte); pmap_clear_accessed(pte);
pmap_update_vhpt(pte, pv->pv_va);
pmap_invalidate_page(pv->pv_pmap, pv->pv_va); pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
} }
pmap_install(oldpmap); pmap_install(oldpmap);
@ -2038,7 +1995,6 @@ pmap_clear_modify(vm_page_t m)
KASSERT(pte != NULL, ("pte")); KASSERT(pte != NULL, ("pte"));
if (pmap_lpte_dirty(pte)) { if (pmap_lpte_dirty(pte)) {
pmap_clear_dirty(pte); pmap_clear_dirty(pte);
pmap_update_vhpt(pte, pv->pv_va);
pmap_invalidate_page(pv->pv_pmap, pv->pv_va); pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
} }
pmap_install(oldpmap); pmap_install(oldpmap);
@ -2068,7 +2024,6 @@ pmap_clear_reference(vm_page_t m)
KASSERT(pte != NULL, ("pte")); KASSERT(pte != NULL, ("pte"));
if (pmap_lpte_accessed(pte)) { if (pmap_lpte_accessed(pte)) {
pmap_clear_accessed(pte); pmap_clear_accessed(pte);
pmap_update_vhpt(pte, pv->pv_va);
pmap_invalidate_page(pv->pv_pmap, pv->pv_va); pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
} }
pmap_install(oldpmap); pmap_install(oldpmap);

View File

@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
#include <sys/exec.h> #include <sys/exec.h>
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/mutex.h> #include <sys/mutex.h>
#include <sys/sched.h>
#include <sys/smp.h> #include <sys/smp.h>
#include <sys/vmmeter.h> #include <sys/vmmeter.h>
#include <sys/sysent.h> #include <sys/sysent.h>
@ -629,7 +630,7 @@ trap(int vector, struct trapframe *tf)
if (!user) if (!user)
trap_panic(vector, tf); trap_panic(vector, tf);
critical_enter(); sched_pin();
thr = PCPU_GET(fpcurthread); thr = PCPU_GET(fpcurthread);
if (thr == td) { if (thr == td) {
/* /*
@ -643,26 +644,29 @@ trap(int vector, struct trapframe *tf)
*/ */
printf("XXX: bogusly disabled high FP regs\n"); printf("XXX: bogusly disabled high FP regs\n");
tf->tf_special.psr &= ~IA64_PSR_DFH; tf->tf_special.psr &= ~IA64_PSR_DFH;
critical_exit(); sched_unpin();
goto out; goto out;
} else if (thr != NULL) { } else if (thr != NULL) {
mtx_lock(&thr->td_md.md_highfp_mtx);
pcb = thr->td_pcb; pcb = thr->td_pcb;
save_high_fp(&pcb->pcb_high_fp); save_high_fp(&pcb->pcb_high_fp);
pcb->pcb_fpcpu = NULL; pcb->pcb_fpcpu = NULL;
PCPU_SET(fpcurthread, NULL); PCPU_SET(fpcurthread, NULL);
mtx_unlock(&thr->td_md.md_highfp_mtx);
thr = NULL; thr = NULL;
} }
mtx_lock(&td->td_md.md_highfp_mtx);
pcb = td->td_pcb; pcb = td->td_pcb;
pcpu = pcb->pcb_fpcpu; pcpu = pcb->pcb_fpcpu;
#ifdef SMP #ifdef SMP
if (pcpu != NULL) { if (pcpu != NULL) {
ipi_send(pcpu->pc_lid, IPI_HIGH_FP); mtx_unlock(&td->td_md.md_highfp_mtx);
critical_exit(); ipi_send(pcpu, IPI_HIGH_FP);
while (pcb->pcb_fpcpu != pcpu) while (pcb->pcb_fpcpu == pcpu)
DELAY(100); DELAY(100);
critical_enter(); mtx_lock(&td->td_md.md_highfp_mtx);
pcpu = pcb->pcb_fpcpu; pcpu = pcb->pcb_fpcpu;
thr = PCPU_GET(fpcurthread); thr = PCPU_GET(fpcurthread);
} }
@ -676,7 +680,8 @@ trap(int vector, struct trapframe *tf)
tf->tf_special.psr &= ~IA64_PSR_DFH; tf->tf_special.psr &= ~IA64_PSR_DFH;
} }
critical_exit(); mtx_unlock(&td->td_md.md_highfp_mtx);
sched_unpin();
goto out; goto out;
} }

View File

@ -118,6 +118,7 @@ cpu_thread_setup(struct thread *td)
sp -= sizeof(struct trapframe); sp -= sizeof(struct trapframe);
td->td_frame = (struct trapframe *)sp; td->td_frame = (struct trapframe *)sp;
td->td_frame->tf_length = sizeof(struct trapframe); td->td_frame->tf_length = sizeof(struct trapframe);
mtx_init(&td->td_md.md_highfp_mtx, "High FP lock", NULL, MTX_DEF);
} }
void void

View File

@ -30,8 +30,9 @@
#define _MACHINE_PROC_H_ #define _MACHINE_PROC_H_
struct mdthread { struct mdthread {
struct mtx md_highfp_mtx;
int md_spinlock_count; /* (k) */ int md_spinlock_count; /* (k) */
register_t md_saved_intr; /* (k) */ int md_saved_intr; /* (k) */
}; };
struct mdproc { struct mdproc {

View File

@ -28,13 +28,15 @@
#ifndef LOCORE #ifndef LOCORE
struct pcpu;
extern int ipi_vector[]; extern int ipi_vector[];
void ipi_all(int ipi); void ipi_all(int ipi);
void ipi_all_but_self(int ipi); void ipi_all_but_self(int ipi);
void ipi_selected(u_int64_t cpus, int ipi); void ipi_selected(cpumask_t cpus, int ipi);
void ipi_self(int ipi); void ipi_self(int ipi);
void ipi_send(u_int64_t lid, int ipi); void ipi_send(struct pcpu *, int ipi);
#endif /* !LOCORE */ #endif /* !LOCORE */
#endif /* _KERNEL */ #endif /* _KERNEL */