d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Add support for PAE and more than 4 gigs of ram on x86, dependent on the

Browse Source 
kernel opition 'options PAE'.  This will only work with device drivers which
  either use busdma, or are able to handle 64 bit physical addresses.

Thanks to Lanny Baron from FreeBSD Systems for the loan of a test machine
with 6 gigs of ram.

Sponsored by:	DARPA, Network Associates Laboratories, FreeBSD Systems
This commit is contained in:
jake 2003-03-30 05:24:52 +00:00
parent 751b8b88db
commit abd082c833
22 changed files with 436 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
sys/amd64/amd64/bios.c
View File

@ -384,12 +384,16 @@ bios16(struct bios_args *args, char *fmt, ...)
args->seg.code32.limit = 0xffff;
ptd = (pd_entry_t *)rcr3();
if (ptd == (u_int *)IdlePTD) {
#ifdef PAE
if (ptd == IdlePDPT) {
#else
if (ptd == IdlePTD) {
#endif
/*
* no page table, so create one and install it.
*/
pte = (pt_entry_t *)malloc(PAGE_SIZE, M_TEMP, M_WAITOK);
ptd = (pd_entry_t *)((u_int)ptd + KERNBASE);
ptd = (pd_entry_t *)((u_int)IdlePTD + KERNBASE);
*ptd = vtophys(pte) | PG_RW | PG_V;
} else {
/*

35
sys/amd64/amd64/locore.S
View File

@ -138,6 +138,11 @@ SMPpt: .long 0 /* relocated version */
.globl IdlePTD
IdlePTD: .long 0 /* phys addr of kernel PTD */
#ifdef PAE
.globl IdlePDPT
IdlePDPT: .long 0 /* phys addr of kernel PDPT */
#endif
#ifdef SMP
.globl KPTphys
#endif
@ -323,8 +328,16 @@ NON_GPROF_ENTRY(btext)
1:
/* Now enable paging */
#ifdef PAE
movl R(IdlePDPT), %eax
movl %eax, %cr3
movl %cr4, %eax
orl $CR4_PAE, %eax
movl %eax, %cr4
#else
movl R(IdlePTD), %eax
movl %eax,%cr3 /* load ptd addr into mmu */
#endif
movl %cr0,%eax /* get control word */
orl $CR0_PE|CR0_PG,%eax /* enable paging */
movl %eax,%cr0 /* and let's page NOW! */
@ -341,7 +354,11 @@ begin:
xorl %ebp,%ebp /* mark end of frames */
#ifdef PAE
movl IdlePDPT,%esi
#else
movl IdlePTD,%esi
#endif
movl %esi,(KSTACK_PAGES*PAGE_SIZE-PCB_SIZE+PCB_CR3)(%eax)
pushl physfree /* value of first for init386(first) */
@ -749,6 +766,11 @@ no_kernend:
movl %esi,R(KPTphys)
/* Allocate Page Table Directory */
#ifdef PAE
/* XXX only need 32 bytes (easier for now) */
ALLOCPAGES(1)
movl %esi,R(IdlePDPT)
#endif
ALLOCPAGES(NPGPTD)
movl %esi,R(IdlePTD)
@ -804,6 +826,12 @@ no_kernend:
fillkptphys(%edx)
/* Map page directory. */
#ifdef PAE
movl R(IdlePDPT), %eax
movl $1, %ecx
fillkptphys($PG_RW)
#endif
movl R(IdlePTD), %eax
movl $NPGPTD, %ecx
fillkptphys($PG_RW)
@ -889,4 +917,11 @@ no_kernend:
movl $NPGPTD,%ecx
fillkpt(R(IdlePTD), $PG_RW)
#ifdef PAE
movl R(IdlePTD), %eax
xorl %ebx, %ebx
movl $NPGPTD, %ecx
fillkpt(R(IdlePDPT), $0x0)
#endif
ret

35
sys/amd64/amd64/locore.s
View File

@ -138,6 +138,11 @@ SMPpt: .long 0 /* relocated version */
.globl IdlePTD
IdlePTD: .long 0 /* phys addr of kernel PTD */
#ifdef PAE
.globl IdlePDPT
IdlePDPT: .long 0 /* phys addr of kernel PDPT */
#endif
#ifdef SMP
.globl KPTphys
#endif
@ -323,8 +328,16 @@ NON_GPROF_ENTRY(btext)
1:
/* Now enable paging */
#ifdef PAE
movl R(IdlePDPT), %eax
movl %eax, %cr3
movl %cr4, %eax
orl $CR4_PAE, %eax
movl %eax, %cr4
#else
movl R(IdlePTD), %eax
movl %eax,%cr3 /* load ptd addr into mmu */
#endif
movl %cr0,%eax /* get control word */
orl $CR0_PE|CR0_PG,%eax /* enable paging */
movl %eax,%cr0 /* and let's page NOW! */
@ -341,7 +354,11 @@ begin:
xorl %ebp,%ebp /* mark end of frames */
#ifdef PAE
movl IdlePDPT,%esi
#else
movl IdlePTD,%esi
#endif
movl %esi,(KSTACK_PAGES*PAGE_SIZE-PCB_SIZE+PCB_CR3)(%eax)
pushl physfree /* value of first for init386(first) */
@ -749,6 +766,11 @@ no_kernend:
movl %esi,R(KPTphys)
/* Allocate Page Table Directory */
#ifdef PAE
/* XXX only need 32 bytes (easier for now) */
ALLOCPAGES(1)
movl %esi,R(IdlePDPT)
#endif
ALLOCPAGES(NPGPTD)
movl %esi,R(IdlePTD)
@ -804,6 +826,12 @@ no_kernend:
fillkptphys(%edx)
/* Map page directory. */
#ifdef PAE
movl R(IdlePDPT), %eax
movl $1, %ecx
fillkptphys($PG_RW)
#endif
movl R(IdlePTD), %eax
movl $NPGPTD, %ecx
fillkptphys($PG_RW)
@ -889,4 +917,11 @@ no_kernend:
movl $NPGPTD,%ecx
fillkpt(R(IdlePTD), $PG_RW)
#ifdef PAE
movl R(IdlePTD), %eax
xorl %ebx, %ebx
movl $NPGPTD, %ecx
fillkpt(R(IdlePDPT), $0x0)
#endif
ret

10
sys/amd64/amd64/machdep.c
View File

@ -1578,11 +1578,13 @@ int15e820:
if (smap->length == 0)
goto next_run;
#ifndef PAE
if (smap->base >= 0xffffffff) {
printf("%uK of memory above 4GB ignored\n",
(u_int)(smap->length / 1024));
goto next_run;
}
#endif
for (i = 0; i <= physmap_idx; i += 2) {
if (smap->base < physmap[i + 1]) {
@ -2071,7 +2073,11 @@ init386(first)
dblfault_tss.tss_esp2 = (int)&dblfault_stack[sizeof(dblfault_stack)];
dblfault_tss.tss_ss = dblfault_tss.tss_ss0 = dblfault_tss.tss_ss1 =
dblfault_tss.tss_ss2 = GSEL(GDATA_SEL, SEL_KPL);
#ifdef PAE
dblfault_tss.tss_cr3 = (int)IdlePDPT;
#else
dblfault_tss.tss_cr3 = (int)IdlePTD;
#endif
dblfault_tss.tss_eip = (int)dblfault_handler;
dblfault_tss.tss_eflags = PSL_KERNEL;
dblfault_tss.tss_ds = dblfault_tss.tss_es =
@ -2115,7 +2121,11 @@ init386(first)
/* setup proc 0's pcb */
thread0.td_pcb->pcb_flags = 0; /* XXXKSE */
#ifdef PAE
thread0.td_pcb->pcb_cr3 = (int)IdlePDPT;
#else
thread0.td_pcb->pcb_cr3 = (int)IdlePTD;
#endif
thread0.td_pcb->pcb_ext = 0;
thread0.td_frame = &proc0_tf;
}

12
sys/amd64/amd64/mpboot.S
View File

@ -40,6 +40,8 @@
#include "assym.s"
#define R(x) ((x)-KERNBASE)
/*
* this code MUST be enabled here and in mp_machdep.c
* it follows the very early stages of AP boot by placing values in CMOS ram.
@ -74,8 +76,16 @@
NON_GPROF_ENTRY(MPentry)
CHECKPOINT(0x36, 3)
/* Now enable paging mode */
movl IdlePTD-KERNBASE, %eax
#ifdef PAE
movl R(IdlePDPT), %eax
movl %eax, %cr3
movl %cr4, %eax
orl $CR4_PAE, %eax
movl %eax, %cr4
#else
movl R(IdlePTD), %eax
movl %eax,%cr3
#endif
movl %cr0,%eax
orl $CR0_PE|CR0_PG,%eax /* enable paging */
movl %eax,%cr0 /* let the games begin! */

60
sys/amd64/amd64/pmap.c
View File

@ -197,6 +197,10 @@ static int nkpt;
vm_offset_t kernel_vm_end;
extern u_int32_t KERNend;
#ifdef PAE
static uma_zone_t pdptzone;
#endif
/*
* Data for the pv entry allocation mechanism
*/
@ -248,7 +252,10 @@ static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex);
static vm_page_t pmap_page_lookup(vm_object_t object, vm_pindex_t pindex);
static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t);
static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
static void *pmap_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
static void *pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
#ifdef PAE
static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
#endif
static pd_entry_t pdir4mb;
@ -323,6 +330,9 @@ pmap_bootstrap(firstaddr, loadaddr)
* Initialize the kernel pmap (which is statically allocated).
*/
kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD);
#ifdef PAE
kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
#endif
kernel_pmap->pm_active = -1; /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvlist);
LIST_INIT(&allpmaps);
@ -504,12 +514,21 @@ pmap_set_opt(void)
}
static void *
pmap_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
*flags = UMA_SLAB_PRIV;
return (void *)kmem_alloc(kernel_map, bytes);
}
#ifdef PAE
static void *
pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
*flags = UMA_SLAB_PRIV;
return (contigmalloc(PAGE_SIZE, NULL, 0, 0x0ULL, 0xffffffffULL, 1, 0));
}
#endif
/*
* Initialize the pmap module.
* Called by vm_init, to initialize any structures that the pmap
@ -545,9 +564,15 @@ pmap_init(phys_start, phys_end)
initial_pvs = MINPV;
pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
uma_zone_set_allocf(pvzone, pmap_allocf);
uma_zone_set_allocf(pvzone, pmap_pv_allocf);
uma_prealloc(pvzone, initial_pvs);
#ifdef PAE
pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1, 0);
uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
#endif
/*
* Now it is safe to enable pv_table recording.
*/
@ -1241,6 +1266,9 @@ pmap_pinit0(pmap)
{
pmap->pm_pdir = (pd_entry_t *)(KERNBASE + (vm_offset_t)IdlePTD);
#ifdef PAE
pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT);
#endif
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvlist);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@ -1265,9 +1293,18 @@ pmap_pinit(pmap)
* No need to allocate page table space yet but we do need a valid
* page directory table.
*/
if (pmap->pm_pdir == NULL)
if (pmap->pm_pdir == NULL) {
pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map,
NBPTD);
#ifdef PAE
pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
KASSERT(((vm_offset_t)pmap->pm_pdpt &
((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
("pmap_pinit: pdpt misaligned"));
KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
("pmap_pinit: pdpt above 4g"));
#endif
}
/*
* allocate object for the ptes
@ -1310,6 +1347,9 @@ pmap_pinit(pmap)
for (i = 0; i < NPGPTD; i++) {
pa = VM_PAGE_TO_PHYS(ptdpg[i]);
pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M;
#ifdef PAE
pmap->pm_pdpt[i] = pa | PG_V;
#endif
}
pmap->pm_active = 0;
@ -1485,6 +1525,10 @@ pmap_release(pmap_t pmap)
vm_page_lock_queues();
for (i = 0; i < NPGPTD; i++) {
m = TAILQ_FIRST(&object->memq);
#ifdef PAE
KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
("pmap_release: got wrong ptd page"));
#endif
m->wire_count--;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_busy(m);
@ -1680,7 +1724,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
pt_entry_t oldpte;
vm_page_t m;
oldpte = atomic_readandclear_int(ptq);
oldpte = pte_load_clear(ptq);
if (oldpte & PG_W)
pmap->pm_stats.wired_count -= 1;
/*
@ -1846,7 +1890,7 @@ pmap_remove_all(vm_page_t m)
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pv->pv_pmap->pm_stats.resident_count--;
pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
tpte = atomic_readandclear_int(pte);
tpte = pte_load_clear(pte);
if (tpte & PG_W)
pv->pv_pmap->pm_stats.wired_count--;
if (tpte & PG_A)
@ -3283,7 +3327,11 @@ pmap_activate(struct thread *td)
#else
pmap->pm_active |= 1;
#endif
#ifdef PAE
cr3 = vtophys(pmap->pm_pdpt);
#else
cr3 = vtophys(pmap->pm_pdir);
#endif
/* XXXKSE this is wrong.
* pmap_activate is for the current thread on the current cpu
*/

8
sys/amd64/amd64/vm_machdep.c
View File

@ -170,7 +170,11 @@ cpu_fork(td1, p2, td2, flags)
* Set registers for trampoline to user mode. Leave space for the
* return address on stack. These are the kernel mode register values.
*/
#ifdef PAE
pcb2->pcb_cr3 = vtophys(vmspace_pmap(p2->p_vmspace)->pm_pdpt);
#else
pcb2->pcb_cr3 = vtophys(vmspace_pmap(p2->p_vmspace)->pm_pdir);
#endif
pcb2->pcb_edi = 0;
pcb2->pcb_esi = (int)fork_return; /* fork_trampoline argument */
pcb2->pcb_ebp = 0;
@ -342,7 +346,11 @@ cpu_set_upcall(struct thread *td, void *pcb)
* Set registers for trampoline to user mode. Leave space for the
* return address on stack. These are the kernel mode register values.
*/
#ifdef PAE
pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pdpt);
#else
pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pdir);
#endif
pcb2->pcb_edi = 0;
pcb2->pcb_esi = (int)fork_return; /* trampoline arg */
pcb2->pcb_ebp = 0;

12
sys/amd64/include/bus_amd64.h
View File

@ -92,15 +92,23 @@
/*
* Bus address and size types
*/
typedef u_int bus_addr_t;
typedef u_int bus_size_t;
#ifdef PAE
typedef uint64_t bus_addr_t;
#else
typedef uint32_t bus_addr_t;
#endif
typedef uint32_t bus_size_t;
#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF
#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
#define BUS_SPACE_MAXSIZE 0xFFFFFFFF
#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF
#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
#ifdef PAE
#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL
#else
#define BUS_SPACE_MAXADDR 0xFFFFFFFF
#endif
#define BUS_SPACE_UNRESTRICTED (~0)

12
sys/amd64/include/bus_at386.h
View File

@ -92,15 +92,23 @@
/*
* Bus address and size types
*/
typedef u_int bus_addr_t;
typedef u_int bus_size_t;
#ifdef PAE
typedef uint64_t bus_addr_t;
#else
typedef uint32_t bus_addr_t;
#endif
typedef uint32_t bus_size_t;
#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF
#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
#define BUS_SPACE_MAXSIZE 0xFFFFFFFF
#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF
#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
#ifdef PAE
#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL
#else
#define BUS_SPACE_MAXADDR 0xFFFFFFFF
#endif
#define BUS_SPACE_UNRESTRICTED (~0)

56
sys/amd64/include/pmap.h
View File

@ -88,8 +88,12 @@
* This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
*/
#ifndef KVA_PAGES
#ifdef PAE
#define KVA_PAGES 512
#else
#define KVA_PAGES 256
#endif
#endif
/*
* Pte related macros
@ -97,8 +101,12 @@
#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDRSHIFT)|((pti)<<PAGE_SHIFT)))
#ifndef NKPT
#ifdef PAE
#define NKPT 120 /* actual number of kernel page tables */
#else
#define NKPT 30 /* actual number of kernel page tables */
#endif
#endif
#ifndef NKPDE
#ifdef SMP
#define NKPDE (KVA_PAGES - (NPGPTD + 1)) /* number of page tables/pde's */
@ -134,12 +142,25 @@
#include <sys/queue.h>
typedef u_int32_t pd_entry_t;
typedef u_int32_t pt_entry_t;
#ifdef PAE
typedef uint64_t pdpt_entry_t;
typedef uint64_t pd_entry_t;
typedef uint64_t pt_entry_t;
#define PTESHIFT (3)
#define PDESHIFT (3)
#else
typedef uint32_t pd_entry_t;
typedef uint32_t pt_entry_t;
#define PTESHIFT (2)
#define PDESHIFT (2)
#endif
/*
* Address of current and alternate address space page table maps
* and directories.
@ -149,6 +170,9 @@ extern pt_entry_t PTmap[], APTmap[];
extern pd_entry_t PTD[], APTD[];
extern pd_entry_t PTDpde[], APTDpde[];
#ifdef PAE
extern pdpt_entry_t *IdlePDPT;
#endif
extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */
#endif
@ -183,6 +207,30 @@ pmap_kextract(vm_offset_t va)
}
#define vtophys(va) pmap_kextract(((vm_offset_t) (va)))
#ifdef PAE
static __inline pt_entry_t
pte_load_clear(pt_entry_t *pte)
{
pt_entry_t r;
r = *pte;
__asm __volatile(
"1:\n"
"\tcmpxchg8b %1\n"
"\tjnz 1b"
: "+A" (r)
: "m" (*pte), "b" (0), "c" (0));
return (r);
}
#else
#define pte_load_clear(pte) atomic_readandclear_int(pte)
#endif
#endif
/*
@ -202,6 +250,10 @@ struct pmap {
int pm_active; /* active on cpus */
struct pmap_statistics pm_stats; /* pmap statistics */
LIST_ENTRY(pmap) pm_list; /* List of all pmaps */
#ifdef PAE
pdpt_entry_t *pm_pdpt; /* KVA of page director pointer
table */
#endif
};
#define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list))

3
sys/conf/options.i386
View File

@ -33,6 +33,9 @@ APIC_IO opt_global.h
# Change KVM size. Changes things all over the kernel.
KVA_PAGES opt_global.h
# Physical address extensions and support for >4G ram. As above.
PAE opt_global.h
CLK_CALIBRATION_LOOP opt_clock.h
CLK_USE_I8254_CALIBRATION opt_clock.h
CLK_USE_TSC_CALIBRATION opt_clock.h

8
sys/i386/i386/bios.c
View File

@ -384,12 +384,16 @@ bios16(struct bios_args *args, char *fmt, ...)
args->seg.code32.limit = 0xffff;
ptd = (pd_entry_t *)rcr3();
if (ptd == (u_int *)IdlePTD) {
#ifdef PAE
if (ptd == IdlePDPT) {
#else
if (ptd == IdlePTD) {
#endif
/*
* no page table, so create one and install it.
*/
pte = (pt_entry_t *)malloc(PAGE_SIZE, M_TEMP, M_WAITOK);
ptd = (pd_entry_t *)((u_int)ptd + KERNBASE);
ptd = (pd_entry_t *)((u_int)IdlePTD + KERNBASE);
*ptd = vtophys(pte) | PG_RW | PG_V;
} else {
/*

35
sys/i386/i386/locore.s
View File

@ -138,6 +138,11 @@ SMPpt: .long 0 /* relocated version */
.globl IdlePTD
IdlePTD: .long 0 /* phys addr of kernel PTD */
#ifdef PAE
.globl IdlePDPT
IdlePDPT: .long 0 /* phys addr of kernel PDPT */
#endif
#ifdef SMP
.globl KPTphys
#endif
@ -323,8 +328,16 @@ NON_GPROF_ENTRY(btext)
1:
/* Now enable paging */
#ifdef PAE
movl R(IdlePDPT), %eax
movl %eax, %cr3
movl %cr4, %eax
orl $CR4_PAE, %eax
movl %eax, %cr4
#else
movl R(IdlePTD), %eax
movl %eax,%cr3 /* load ptd addr into mmu */
#endif
movl %cr0,%eax /* get control word */
orl $CR0_PE|CR0_PG,%eax /* enable paging */
movl %eax,%cr0 /* and let's page NOW! */
@ -341,7 +354,11 @@ begin:
xorl %ebp,%ebp /* mark end of frames */
#ifdef PAE
movl IdlePDPT,%esi
#else
movl IdlePTD,%esi
#endif
movl %esi,(KSTACK_PAGES*PAGE_SIZE-PCB_SIZE+PCB_CR3)(%eax)
pushl physfree /* value of first for init386(first) */
@ -749,6 +766,11 @@ no_kernend:
movl %esi,R(KPTphys)
/* Allocate Page Table Directory */
#ifdef PAE
/* XXX only need 32 bytes (easier for now) */
ALLOCPAGES(1)
movl %esi,R(IdlePDPT)
#endif
ALLOCPAGES(NPGPTD)
movl %esi,R(IdlePTD)
@ -804,6 +826,12 @@ no_kernend:
fillkptphys(%edx)
/* Map page directory. */
#ifdef PAE
movl R(IdlePDPT), %eax
movl $1, %ecx
fillkptphys($PG_RW)
#endif
movl R(IdlePTD), %eax
movl $NPGPTD, %ecx
fillkptphys($PG_RW)
@ -889,4 +917,11 @@ no_kernend:
movl $NPGPTD,%ecx
fillkpt(R(IdlePTD), $PG_RW)
#ifdef PAE
movl R(IdlePTD), %eax
xorl %ebx, %ebx
movl $NPGPTD, %ecx
fillkpt(R(IdlePDPT), $0x0)
#endif
ret

10
sys/i386/i386/machdep.c
View File

@ -1578,11 +1578,13 @@ int15e820:
if (smap->length == 0)
goto next_run;
#ifndef PAE
if (smap->base >= 0xffffffff) {
printf("%uK of memory above 4GB ignored\n",
(u_int)(smap->length / 1024));
goto next_run;
}
#endif
for (i = 0; i <= physmap_idx; i += 2) {
if (smap->base < physmap[i + 1]) {
@ -2071,7 +2073,11 @@ init386(first)
dblfault_tss.tss_esp2 = (int)&dblfault_stack[sizeof(dblfault_stack)];
dblfault_tss.tss_ss = dblfault_tss.tss_ss0 = dblfault_tss.tss_ss1 =
dblfault_tss.tss_ss2 = GSEL(GDATA_SEL, SEL_KPL);
#ifdef PAE
dblfault_tss.tss_cr3 = (int)IdlePDPT;
#else
dblfault_tss.tss_cr3 = (int)IdlePTD;
#endif
dblfault_tss.tss_eip = (int)dblfault_handler;
dblfault_tss.tss_eflags = PSL_KERNEL;
dblfault_tss.tss_ds = dblfault_tss.tss_es =
@ -2115,7 +2121,11 @@ init386(first)
/* setup proc 0's pcb */
thread0.td_pcb->pcb_flags = 0; /* XXXKSE */
#ifdef PAE
thread0.td_pcb->pcb_cr3 = (int)IdlePDPT;
#else
thread0.td_pcb->pcb_cr3 = (int)IdlePTD;
#endif
thread0.td_pcb->pcb_ext = 0;
thread0.td_frame = &proc0_tf;
}

12
sys/i386/i386/mpboot.s
View File

@ -40,6 +40,8 @@
#include "assym.s"
#define R(x) ((x)-KERNBASE)
/*
* this code MUST be enabled here and in mp_machdep.c
* it follows the very early stages of AP boot by placing values in CMOS ram.
@ -74,8 +76,16 @@
NON_GPROF_ENTRY(MPentry)
CHECKPOINT(0x36, 3)
/* Now enable paging mode */
movl IdlePTD-KERNBASE, %eax
#ifdef PAE
movl R(IdlePDPT), %eax
movl %eax, %cr3
movl %cr4, %eax
orl $CR4_PAE, %eax
movl %eax, %cr4
#else
movl R(IdlePTD), %eax
movl %eax,%cr3
#endif
movl %cr0,%eax
orl $CR0_PE|CR0_PG,%eax /* enable paging */
movl %eax,%cr0 /* let the games begin! */

60
sys/i386/i386/pmap.c
View File

@ -197,6 +197,10 @@ static int nkpt;
vm_offset_t kernel_vm_end;
extern u_int32_t KERNend;
#ifdef PAE
static uma_zone_t pdptzone;
#endif
/*
* Data for the pv entry allocation mechanism
*/
@ -248,7 +252,10 @@ static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex);
static vm_page_t pmap_page_lookup(vm_object_t object, vm_pindex_t pindex);
static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t);
static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
static void *pmap_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
static void *pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
#ifdef PAE
static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
#endif
static pd_entry_t pdir4mb;
@ -323,6 +330,9 @@ pmap_bootstrap(firstaddr, loadaddr)
* Initialize the kernel pmap (which is statically allocated).
*/
kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD);
#ifdef PAE
kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
#endif
kernel_pmap->pm_active = -1; /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvlist);
LIST_INIT(&allpmaps);
@ -504,12 +514,21 @@ pmap_set_opt(void)
}
static void *
pmap_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
*flags = UMA_SLAB_PRIV;
return (void *)kmem_alloc(kernel_map, bytes);
}
#ifdef PAE
static void *
pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
*flags = UMA_SLAB_PRIV;
return (contigmalloc(PAGE_SIZE, NULL, 0, 0x0ULL, 0xffffffffULL, 1, 0));
}
#endif
/*
* Initialize the pmap module.
* Called by vm_init, to initialize any structures that the pmap
@ -545,9 +564,15 @@ pmap_init(phys_start, phys_end)
initial_pvs = MINPV;
pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
uma_zone_set_allocf(pvzone, pmap_allocf);
uma_zone_set_allocf(pvzone, pmap_pv_allocf);
uma_prealloc(pvzone, initial_pvs);
#ifdef PAE
pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1, 0);
uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
#endif
/*
* Now it is safe to enable pv_table recording.
*/
@ -1241,6 +1266,9 @@ pmap_pinit0(pmap)
{
pmap->pm_pdir = (pd_entry_t *)(KERNBASE + (vm_offset_t)IdlePTD);
#ifdef PAE
pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT);
#endif
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvlist);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@ -1265,9 +1293,18 @@ pmap_pinit(pmap)
* No need to allocate page table space yet but we do need a valid
* page directory table.
*/
if (pmap->pm_pdir == NULL)
if (pmap->pm_pdir == NULL) {
pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map,
NBPTD);
#ifdef PAE
pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
KASSERT(((vm_offset_t)pmap->pm_pdpt &
((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
("pmap_pinit: pdpt misaligned"));
KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
("pmap_pinit: pdpt above 4g"));
#endif
}
/*
* allocate object for the ptes
@ -1310,6 +1347,9 @@ pmap_pinit(pmap)
for (i = 0; i < NPGPTD; i++) {
pa = VM_PAGE_TO_PHYS(ptdpg[i]);
pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M;
#ifdef PAE
pmap->pm_pdpt[i] = pa | PG_V;
#endif
}
pmap->pm_active = 0;
@ -1485,6 +1525,10 @@ pmap_release(pmap_t pmap)
vm_page_lock_queues();
for (i = 0; i < NPGPTD; i++) {
m = TAILQ_FIRST(&object->memq);
#ifdef PAE
KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
("pmap_release: got wrong ptd page"));
#endif
m->wire_count--;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_busy(m);
@ -1680,7 +1724,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
pt_entry_t oldpte;
vm_page_t m;
oldpte = atomic_readandclear_int(ptq);
oldpte = pte_load_clear(ptq);
if (oldpte & PG_W)
pmap->pm_stats.wired_count -= 1;
/*
@ -1846,7 +1890,7 @@ pmap_remove_all(vm_page_t m)
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pv->pv_pmap->pm_stats.resident_count--;
pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
tpte = atomic_readandclear_int(pte);
tpte = pte_load_clear(pte);
if (tpte & PG_W)
pv->pv_pmap->pm_stats.wired_count--;
if (tpte & PG_A)
@ -3283,7 +3327,11 @@ pmap_activate(struct thread *td)
#else
pmap->pm_active |= 1;
#endif
#ifdef PAE
cr3 = vtophys(pmap->pm_pdpt);
#else
cr3 = vtophys(pmap->pm_pdir);
#endif
/* XXXKSE this is wrong.
* pmap_activate is for the current thread on the current cpu
*/

3
sys/i386/i386/vm86bios.s
View File

@ -123,6 +123,9 @@ ENTRY(vm86_bioscall)
movl SCR_NEWPTD(%edx),%eax /* mapping for vm86 page table */
movl %eax,0(%ebx) /* ... install as PTD entry 0 */
#ifdef PAE
movl IdlePDPT,%ecx
#endif
movl %ecx,%cr3 /* new page tables */
movl SCR_VMFRAME(%edx),%esp /* switch to new stack */

8
sys/i386/i386/vm_machdep.c
View File

@ -170,7 +170,11 @@ cpu_fork(td1, p2, td2, flags)
* Set registers for trampoline to user mode. Leave space for the
* return address on stack. These are the kernel mode register values.
*/
#ifdef PAE
pcb2->pcb_cr3 = vtophys(vmspace_pmap(p2->p_vmspace)->pm_pdpt);
#else
pcb2->pcb_cr3 = vtophys(vmspace_pmap(p2->p_vmspace)->pm_pdir);
#endif
pcb2->pcb_edi = 0;
pcb2->pcb_esi = (int)fork_return; /* fork_trampoline argument */
pcb2->pcb_ebp = 0;
@ -342,7 +346,11 @@ cpu_set_upcall(struct thread *td, void *pcb)
* Set registers for trampoline to user mode. Leave space for the
* return address on stack. These are the kernel mode register values.
*/
#ifdef PAE
pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pdpt);
#else
pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pdir);
#endif
pcb2->pcb_edi = 0;
pcb2->pcb_esi = (int)fork_return; /* trampoline arg */
pcb2->pcb_ebp = 0;

4
sys/i386/include/_types.h
View File

@ -102,7 +102,11 @@ typedef __uint64_t __uint_least64_t;
typedef __uint32_t __u_register_t;
typedef __uint32_t __vm_offset_t;
typedef __int64_t __vm_ooffset_t;
#ifdef PAE
typedef __uint64_t __vm_paddr_t;
#else
typedef __uint32_t __vm_paddr_t;
#endif
typedef __uint64_t __vm_pindex_t;
typedef __uint32_t __vm_size_t;

12
sys/i386/include/bus_at386.h
View File

@ -92,15 +92,23 @@
/*
* Bus address and size types
*/
typedef u_int bus_addr_t;
typedef u_int bus_size_t;
#ifdef PAE
typedef uint64_t bus_addr_t;
#else
typedef uint32_t bus_addr_t;
#endif
typedef uint32_t bus_size_t;
#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF
#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
#define BUS_SPACE_MAXSIZE 0xFFFFFFFF
#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF
#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
#ifdef PAE
#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL
#else
#define BUS_SPACE_MAXADDR 0xFFFFFFFF
#endif
#define BUS_SPACE_UNRESTRICTED (~0)

5
sys/i386/include/param.h
View File

@ -87,8 +87,13 @@
#define PAGE_MASK (PAGE_SIZE-1)
#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
#ifdef PAE
#define NPGPTD 4
#define PDRSHIFT 21 /* LOG2(NBPDR) */
#else
#define NPGPTD 1
#define PDRSHIFT 22 /* LOG2(NBPDR) */
#endif
#define NBPTD (NPGPTD<<PAGE_SHIFT)
#define NPDEPTD (NBPTD/(sizeof (pd_entry_t)))

56
sys/i386/include/pmap.h
View File

@ -88,8 +88,12 @@
* This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
*/
#ifndef KVA_PAGES
#ifdef PAE
#define KVA_PAGES 512
#else
#define KVA_PAGES 256
#endif
#endif
/*
* Pte related macros
@ -97,8 +101,12 @@
#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDRSHIFT)|((pti)<<PAGE_SHIFT)))
#ifndef NKPT
#ifdef PAE
#define NKPT 120 /* actual number of kernel page tables */
#else
#define NKPT 30 /* actual number of kernel page tables */
#endif
#endif
#ifndef NKPDE
#ifdef SMP
#define NKPDE (KVA_PAGES - (NPGPTD + 1)) /* number of page tables/pde's */
@ -134,12 +142,25 @@
#include <sys/queue.h>
typedef u_int32_t pd_entry_t;
typedef u_int32_t pt_entry_t;
#ifdef PAE
typedef uint64_t pdpt_entry_t;
typedef uint64_t pd_entry_t;
typedef uint64_t pt_entry_t;
#define PTESHIFT (3)
#define PDESHIFT (3)
#else
typedef uint32_t pd_entry_t;
typedef uint32_t pt_entry_t;
#define PTESHIFT (2)
#define PDESHIFT (2)
#endif
/*
* Address of current and alternate address space page table maps
* and directories.
@ -149,6 +170,9 @@ extern pt_entry_t PTmap[], APTmap[];
extern pd_entry_t PTD[], APTD[];
extern pd_entry_t PTDpde[], APTDpde[];
#ifdef PAE
extern pdpt_entry_t *IdlePDPT;
#endif
extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */
#endif
@ -183,6 +207,30 @@ pmap_kextract(vm_offset_t va)
}
#define vtophys(va) pmap_kextract(((vm_offset_t) (va)))
#ifdef PAE
static __inline pt_entry_t
pte_load_clear(pt_entry_t *pte)
{
pt_entry_t r;
r = *pte;
__asm __volatile(
"1:\n"
"\tcmpxchg8b %1\n"
"\tjnz 1b"
: "+A" (r)
: "m" (*pte), "b" (0), "c" (0));
return (r);
}
#else
#define pte_load_clear(pte) atomic_readandclear_int(pte)
#endif
#endif
/*
@ -202,6 +250,10 @@ struct pmap {
int pm_active; /* active on cpus */
struct pmap_statistics pm_stats; /* pmap statistics */
LIST_ENTRY(pmap) pm_list; /* List of all pmaps */
#ifdef PAE
pdpt_entry_t *pm_pdpt; /* KVA of page director pointer
table */
#endif
};
#define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list))