d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make pmap_invalidate_cache_range() available for consumption on amd64.

Browse Source 
Add pmap_invalidate_cache_pages() method on x86. It flushes the CPU
cache for the set of pages, which are not neccessary mapped. Since its
supposed use is to prepare the move of the pages ownership to a device
that does not snoop all CPU accesses to the main memory (read GPU in
GMCH), do not rely on CPU self-snoop feature.

amd64 implementation takes advantage of the direct map. On i386,
extract the helper pmap_flush_page() from pmap_page_set_memattr(), and
use it to make a temporary mapping of the flushed page.

Reviewed by:	alc
Sponsored by:	The FreeBSD Foundation
MFC after:	3 weeks
This commit is contained in:
Konstantin Belousov 2011-04-18 21:24:42 +00:00
parent 5221106c04
commit 3136faa59d
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=220803
4 changed files with 78 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
sys/amd64/amd64/pmap.c
View File

@ -239,7 +239,6 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
@ -1105,7 +1104,9 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
}
#endif /* !SMP */
static void
#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
{
@ -1117,7 +1118,7 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
if (cpu_feature & CPUID_SS)
; /* If "Self Snoop" is supported, do nothing. */
else if ((cpu_feature & CPUID_CLFSH) != 0 &&
eva - sva < 2 * 1024 * 1024) {
eva - sva < PMAP_CLFLUSH_THRESHOLD) {
/*
* Otherwise, do per-cache line flush. Use the mfence
@ -1141,6 +1142,34 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
}
}
/*
* Remove the specified set of pages from the data and instruction caches.
*
* In contrast to pmap_invalidate_cache_range(), this function does not
* rely on the CPU's self-snoop feature, because it is intended for use
* when moving pages into a different cache domain.
*/
void
pmap_invalidate_cache_pages(vm_page_t *pages, int count)
{
vm_offset_t daddr, eva;
int i;
if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
(cpu_feature & CPUID_CLFSH) == 0)
pmap_invalidate_cache();
else {
mfence();
for (i = 0; i < count; i++) {
daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
eva = daddr + PAGE_SIZE;
for (; daddr < eva; daddr += cpu_clflush_line_size)
clflush(daddr);
}
mfence();
}
}
/*
* Are we current address space or kernel?
*/

2
sys/amd64/include/pmap.h
View File

@ -328,6 +328,8 @@ void pmap_invalidate_page(pmap_t, vm_offset_t);
void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
void pmap_invalidate_all(pmap_t);
void pmap_invalidate_cache(void);
void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
#endif /* _KERNEL */

52
sys/i386/i386/pmap.c
View File

@ -297,6 +297,7 @@ static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static void pmap_flush_page(vm_page_t m);
static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
@ -1136,6 +1137,8 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
}
#endif /* !SMP */
#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
{
@ -1148,7 +1151,7 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
if (cpu_feature & CPUID_SS)
; /* If "Self Snoop" is supported, do nothing. */
else if ((cpu_feature & CPUID_CLFSH) != 0 &&
eva - sva < 2 * 1024 * 1024) {
eva - sva < PMAP_CLFLUSH_THRESHOLD) {
/*
* Otherwise, do per-cache line flush. Use the mfence
@ -1172,6 +1175,20 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
}
}
void
pmap_invalidate_cache_pages(vm_page_t *pages, int count)
{
int i;
if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
(cpu_feature & CPUID_CLFSH) == 0) {
pmap_invalidate_cache();
} else {
for (i = 0; i < count; i++)
pmap_flush_page(pages[i]);
}
}
/*
* Are we current address space or kernel? N.B. We return FALSE when
* a pmap's page table is in use because a kernel thread is borrowing
@ -4825,8 +4842,6 @@ pmap_unmapdev(vm_offset_t va, vm_size_t size)
void
pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
{
struct sysmaps *sysmaps;
vm_offset_t sva, eva;
m->md.pat_mode = ma;
if ((m->flags & PG_FICTITIOUS) != 0)
@ -4849,25 +4864,42 @@ pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
* invalidation. In the worst case, whole cache is flushed by
* pmap_invalidate_cache_range().
*/
if ((cpu_feature & (CPUID_SS|CPUID_CLFSH)) == CPUID_CLFSH) {
if ((cpu_feature & CPUID_SS) == 0)
pmap_flush_page(m);
}
static void
pmap_flush_page(vm_page_t m)
{
struct sysmaps *sysmaps;
vm_offset_t sva, eva;
if ((cpu_feature & CPUID_CLFSH) != 0) {
sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
mtx_lock(&sysmaps->lock);
if (*sysmaps->CMAP2)
panic("pmap_page_set_memattr: CMAP2 busy");
panic("pmap_flush_page: CMAP2 busy");
sched_pin();
*sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0);
invlcaddr(sysmaps->CADDR2);
sva = (vm_offset_t)sysmaps->CADDR2;
eva = sva + PAGE_SIZE;
} else
sva = eva = 0; /* gcc */
pmap_invalidate_cache_range(sva, eva);
if (sva != 0) {
/*
* Use mfence despite the ordering implied by
* mtx_{un,}lock() because clflush is not guaranteed
* to be ordered by any other instruction.
*/
mfence();
for (; sva < eva; sva += cpu_clflush_line_size)
clflush(sva);
mfence();
*sysmaps->CMAP2 = 0;
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
} else
pmap_invalidate_cache();
}
/*

3
sys/i386/include/pmap.h
View File

@ -522,7 +522,8 @@ void pmap_invalidate_page(pmap_t, vm_offset_t);
void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
void pmap_invalidate_all(pmap_t);
void pmap_invalidate_cache(void);
void pmap_invalidate_cache_range(vm_offset_t, vm_offset_t);
void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
#endif /* _KERNEL */