d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Eliminate all TLB shootdowns by pmap_pte_quick(): By temporarily pinning

Browse Source 
the thread that calls pmap_pte_quick() and by virtue of the page queues
lock being held, we can manage PADDR1/PMAP1 as a CPU private mapping.

The most common effect of this change is to reduce the overhead of the page
daemon on multiprocessors.

In collaboration with:  tegge
This commit is contained in:
Alan Cox 2004-02-01 20:14:00 +00:00
parent 4e3c587d23
commit 8600a4122a
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=125308
1 changed files with 52 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
sys/i386/i386/pmap.c
View File

@ -235,6 +235,21 @@ extern pt_entry_t *SMPpt;
#endif
static pt_entry_t *PMAP1 = 0, *PMAP2;
static pt_entry_t *PADDR1 = 0, *PADDR2;
#ifdef SMP
static int PMAP1cpu;
static int PMAP1changedcpu;
SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD,
&PMAP1changedcpu, 0,
"Number of times pmap_pte_quick changed CPU with same PMAP1");
#endif
static int PMAP1changed;
SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD,
&PMAP1changed, 0,
"Number of times pmap_pte_quick changed PMAP1");
static int PMAP1unchanged;
SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD,
&PMAP1unchanged, 0,
"Number of times pmap_pte_quick didn't change PMAP1");
static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
static pv_entry_t get_pv_entry(void);
@ -752,6 +767,16 @@ pmap_pte(pmap_t pmap, vm_offset_t va)
return (0);
}
static __inline void
invlcaddr(void *caddr)
{
#ifdef I386_CPU
invltlb();
#else
invlpg((u_int)caddr);
#endif
}
/*
* Super fast pmap_pte routine best used when scanning
* the pv lists. This eliminates many coarse-grained
@ -760,7 +785,7 @@ pmap_pte(pmap_t pmap, vm_offset_t va)
* to do an entire invltlb for checking a single mapping.
*
* If the given pmap is not the current pmap, vm_page_queue_mtx
* must be held.
* must be held and curthread pinned to a CPU.
*/
static pt_entry_t *
pmap_pte_quick(pmap_t pmap, vm_offset_t va)
@ -776,11 +801,24 @@ pmap_pte_quick(pmap_t pmap, vm_offset_t va)
if (pmap_is_current(pmap))
return (vtopte(va));
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
newpf = *pde & PG_FRAME;
if ((*PMAP1 & PG_FRAME) != newpf) {
*PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR1);
}
#ifdef SMP
PMAP1cpu = PCPU_GET(cpuid);
#endif
invlcaddr(PADDR1);
PMAP1changed++;
} else
#ifdef SMP
if (PMAP1cpu != PCPU_GET(cpuid)) {
PMAP1cpu = PCPU_GET(cpuid);
invlcaddr(PADDR1);
PMAP1changedcpu++;
} else
#endif
PMAP1unchanged++;
return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
}
return (0);
@ -1711,6 +1749,7 @@ pmap_remove_all(vm_page_t m)
#endif
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
s = splvm();
sched_pin();
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pv->pv_pmap->pm_stats.resident_count--;
pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
@ -1742,6 +1781,7 @@ pmap_remove_all(vm_page_t m)
free_pv_entry(pv);
}
vm_page_flag_clear(m, PG_WRITEABLE);
sched_unpin();
splx(s);
}
@ -2357,16 +2397,6 @@ pagezero(void *page)
bzero(page, PAGE_SIZE);
}
static __inline void
invlcaddr(void *caddr)
{
#ifdef I386_CPU
invltlb();
#else
invlpg((u_int)caddr);
#endif
}
/*
* pmap_zero_page zeros the specified hardware page by mapping
* the page into KVM and using bzero to clear its contents.
@ -2524,6 +2554,7 @@ pmap_remove_pages(pmap, sva, eva)
#endif
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
s = splvm();
sched_pin();
for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
if (pv->pv_va >= eva || pv->pv_va < sva) {
@ -2583,6 +2614,7 @@ pmap_remove_pages(pmap, sva, eva)
pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem);
free_pv_entry(pv);
}
sched_unpin();
splx(s);
pmap_invalidate_all(pmap);
}
@ -2604,6 +2636,7 @@ pmap_is_modified(vm_page_t m)
return FALSE;
s = splvm();
sched_pin();
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
/*
@ -2621,10 +2654,12 @@ pmap_is_modified(vm_page_t m)
#endif
pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
if (*pte & PG_M) {
sched_unpin();
splx(s);
return TRUE;
}
}
sched_unpin();
splx(s);
return (FALSE);
}
@ -2663,6 +2698,7 @@ pmap_clear_ptes(vm_page_t m, int bit)
return;
s = splvm();
sched_pin();
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
/*
* Loop over all current mappings setting/clearing as appropos If
@ -2700,6 +2736,7 @@ pmap_clear_ptes(vm_page_t m, int bit)
}
if (bit == PG_RW)
vm_page_flag_clear(m, PG_WRITEABLE);
sched_unpin();
splx(s);
}
@ -2745,6 +2782,7 @@ pmap_ts_referenced(vm_page_t m)
return (rtval);
s = splvm();
sched_pin();
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
@ -2773,6 +2811,7 @@ pmap_ts_referenced(vm_page_t m)
}
} while ((pv = pvn) != NULL && pv != pvf);
}
sched_unpin();
splx(s);
return (rtval);