2005-01-06 22:18:23 +00:00
|
|
|
/*-
|
1993-06-12 14:58:17 +00:00
|
|
|
* Copyright (c) 1991 Regents of the University of California.
|
|
|
|
|
* All rights reserved.
|
|
|
|
|
*
|
|
|
|
|
* This code is derived from software contributed to Berkeley by
|
|
|
|
|
* the Systems Programming Group of the University of Utah Computer
|
|
|
|
|
* Science Department and William Jolitz of UUNET Technologies Inc.
|
|
|
|
|
*
|
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
|
* are met:
|
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
|
* 4. Neither the name of the University nor the names of its contributors
|
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
|
* without specific prior written permission.
|
|
|
|
|
*
|
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
|
*
|
|
|
|
|
* Derived from hp300 version by Mike Hibler, this version by William
|
|
|
|
|
* Jolitz uses a recursive map [a pde points to the page directory] to
|
|
|
|
|
* map the page tables using the pagetables themselves. This is done to
|
|
|
|
|
* reduce the impact on kernel virtual memory for lots of sparse address
|
|
|
|
|
* space, and to reduce the cost of memory to each process.
|
|
|
|
|
*
|
1993-10-15 10:07:45 +00:00
|
|
|
* from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90
|
|
|
|
|
* from: @(#)pmap.h 7.4 (Berkeley) 5/12/91
|
1999-08-28 01:08:13 +00:00
|
|
|
* $FreeBSD$
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
|
|
|
|
|
1994-11-14 14:12:24 +00:00
|
|
|
#ifndef _MACHINE_PMAP_H_
|
|
|
|
|
#define _MACHINE_PMAP_H_
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1996-05-02 22:25:18 +00:00
|
|
|
/*
|
2005-12-06 21:09:01 +00:00
|
|
|
* Page-directory and page-table entries follow this format, with a few
|
1996-05-02 22:25:18 +00:00
|
|
|
* of the fields not present here and there, depending on a lot of things.
|
|
|
|
|
*/
|
|
|
|
|
/* ---- Intel Nomenclature ---- */
|
|
|
|
|
#define PG_V 0x001 /* P Valid */
|
|
|
|
|
#define PG_RW 0x002 /* R/W Read/Write */
|
|
|
|
|
#define PG_U 0x004 /* U/S User/Supervisor */
|
|
|
|
|
#define PG_NC_PWT 0x008 /* PWT Write through */
|
|
|
|
|
#define PG_NC_PCD 0x010 /* PCD Cache disable */
|
|
|
|
|
#define PG_A 0x020 /* A Accessed */
|
|
|
|
|
#define PG_M 0x040 /* D Dirty */
|
|
|
|
|
#define PG_PS 0x080 /* PS Page size (0=4k,1=4M) */
|
2006-05-01 22:07:00 +00:00
|
|
|
#define PG_PTE_PAT 0x080 /* PAT PAT index */
|
1996-05-02 22:25:18 +00:00
|
|
|
#define PG_G 0x100 /* G Global */
|
|
|
|
|
#define PG_AVAIL1 0x200 /* / Available for system */
|
|
|
|
|
#define PG_AVAIL2 0x400 /* < programmers use */
|
|
|
|
|
#define PG_AVAIL3 0x800 /* \ */
|
2006-05-01 22:07:00 +00:00
|
|
|
#define PG_PDE_PAT 0x1000 /* PAT PAT index */
|
2007-04-06 18:15:03 +00:00
|
|
|
#ifdef PAE
|
|
|
|
|
#define PG_NX (1ull<<63) /* No-execute */
|
|
|
|
|
#endif
|
1996-05-02 22:25:18 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
/* Our various interpretations of the above */
|
|
|
|
|
#define PG_W PG_AVAIL1 /* "Wired" pseudoflag */
|
1996-05-18 03:38:05 +00:00
|
|
|
#define PG_MANAGED PG_AVAIL2
|
2007-04-06 18:15:03 +00:00
|
|
|
#ifdef PAE
|
|
|
|
|
#define PG_FRAME (0x000ffffffffff000ull)
|
|
|
|
|
#define PG_PS_FRAME (0x000fffffffe00000ull)
|
|
|
|
|
#else
|
|
|
|
|
#define PG_FRAME (~PAGE_MASK)
|
|
|
|
|
#define PG_PS_FRAME (0xffc00000)
|
|
|
|
|
#endif
|
1996-05-02 22:25:18 +00:00
|
|
|
#define PG_PROT (PG_RW|PG_U) /* all protection bits . */
|
|
|
|
|
#define PG_N (PG_NC_PWT|PG_NC_PCD) /* Non-cacheable */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2009-08-31 17:42:52 +00:00
|
|
|
/* Page level cache control fields used to determine the PAT type */
|
|
|
|
|
#define PG_PDE_CACHE (PG_PDE_PAT | PG_NC_PWT | PG_NC_PCD)
|
|
|
|
|
#define PG_PTE_CACHE (PG_PTE_PAT | PG_NC_PWT | PG_NC_PCD)
|
|
|
|
|
|
MFamd64 with few changes:
1. Add support for automatic promotion of 4KB page mappings to 2MB page
mappings. Automatic promotion can be enabled by setting the tunable
"vm.pmap.pg_ps_enabled" to a non-zero value. By default, automatic
promotion is disabled. Tested by: kris
2. To date, we have assumed that the TLB will only set the PG_M bit in a
PTE if that PTE has the PG_RW bit set. However, this assumption does
not hold on recent processors from Intel. For example, consider a PTE
that has the PG_RW bit set but the PG_M bit clear. Suppose this PTE
is cached in the TLB and later the PG_RW bit is cleared in the PTE,
but the corresponding TLB entry is not (yet) invalidated.
Historically, upon a write access using this (stale) TLB entry, the
TLB would observe that the PG_RW bit had been cleared and initiate a
page fault, aborting the setting of the PG_M bit in the PTE. Now,
however, P4- and Core2-family processors will set the PG_M bit before
observing that the PG_RW bit is clear and initiating a page fault. In
other words, the write does not occur but the PG_M bit is still set.
The real impact of this difference is not that great. Specifically,
we should no longer assert that any PTE with the PG_M bit set must
also have the PG_RW bit set, and we should ignore the state of the
PG_M bit unless the PG_RW bit is set.
2008-03-27 04:34:17 +00:00
|
|
|
/*
|
|
|
|
|
* Promotion to a 2 or 4MB (PDE) page mapping requires that the corresponding
|
|
|
|
|
* 4KB (PTE) page mappings have identical settings for the following fields:
|
|
|
|
|
*/
|
|
|
|
|
#define PG_PTE_PROMOTE (PG_MANAGED | PG_W | PG_G | PG_PTE_PAT | \
|
|
|
|
|
PG_M | PG_A | PG_NC_PCD | PG_NC_PWT | PG_U | PG_RW | PG_V)
|
|
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
/*
|
|
|
|
|
* Page Protection Exception bits
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define PGEX_P 0x01 /* Protection violation vs. not present */
|
|
|
|
|
#define PGEX_W 0x02 /* during a Write cycle */
|
|
|
|
|
#define PGEX_U 0x04 /* access from User mode (UPL) */
|
2007-04-12 17:00:56 +00:00
|
|
|
#define PGEX_RSV 0x08 /* reserved PTE field is non-zero */
|
2007-04-06 18:15:03 +00:00
|
|
|
#define PGEX_I 0x10 /* during an instruction fetch */
|
1996-05-02 14:21:14 +00:00
|
|
|
|
2001-09-21 06:23:03 +00:00
|
|
|
/*
|
|
|
|
|
* Size of Kernel address space. This is the number of page table pages
|
|
|
|
|
* (4MB each) to use for the kernel. 256 pages == 1 Gigabyte.
|
|
|
|
|
* This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
|
2008-01-14 22:53:01 +00:00
|
|
|
* For PAE, the page table page unit size is 2MB. This means that 512 pages
|
|
|
|
|
* is 1 Gigabyte. Double everything. It must be a multiple of 8 for PAE.
|
2001-09-21 06:23:03 +00:00
|
|
|
*/
|
|
|
|
|
#ifndef KVA_PAGES
|
2003-03-30 05:24:52 +00:00
|
|
|
#ifdef PAE
|
|
|
|
|
#define KVA_PAGES 512
|
|
|
|
|
#else
|
2001-09-21 06:23:03 +00:00
|
|
|
#define KVA_PAGES 256
|
|
|
|
|
#endif
|
2003-03-30 05:24:52 +00:00
|
|
|
#endif
|
2001-09-21 06:23:03 +00:00
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
/*
|
|
|
|
|
* Pte related macros
|
|
|
|
|
*/
|
|
|
|
|
#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDRSHIFT)|((pti)<<PAGE_SHIFT)))
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2015-04-08 19:46:13 +00:00
|
|
|
/*
|
|
|
|
|
* The initial number of kernel page table pages that are constructed
|
|
|
|
|
* by locore must be sufficient to map vm_page_array. That number can
|
|
|
|
|
* be calculated as follows:
|
|
|
|
|
* max_phys / PAGE_SIZE * sizeof(struct vm_page) / NBPDR
|
|
|
|
|
* PAE: max_phys 16G, sizeof(vm_page) 76, NBPDR 2M, 152 page table pages.
|
|
|
|
|
* Non-PAE: max_phys 4G, sizeof(vm_page) 68, NBPDR 4M, 18 page table pages.
|
|
|
|
|
*/
|
1994-01-14 16:25:31 +00:00
|
|
|
#ifndef NKPT
|
2003-03-30 05:24:52 +00:00
|
|
|
#ifdef PAE
|
2006-11-13 20:33:54 +00:00
|
|
|
#define NKPT 240
|
2003-03-30 05:24:52 +00:00
|
|
|
#else
|
2006-11-13 20:33:54 +00:00
|
|
|
#define NKPT 30
|
These changes embody the support of the fully coherent merged VM buffer cache,
much higher filesystem I/O performance, and much better paging performance. It
represents the culmination of over 6 months of R&D.
The majority of the merged VM/cache work is by John Dyson.
The following highlights the most significant changes. Additionally, there are
(mostly minor) changes to the various filesystem modules (nfs, msdosfs, etc) to
support the new VM/buffer scheme.
vfs_bio.c:
Significant rewrite of most of vfs_bio to support the merged VM buffer cache
scheme. The scheme is almost fully compatible with the old filesystem
interface. Significant improvement in the number of opportunities for write
clustering.
vfs_cluster.c, vfs_subr.c
Upgrade and performance enhancements in vfs layer code to support merged
VM/buffer cache. Fixup of vfs_cluster to eliminate the bogus pagemove stuff.
vm_object.c:
Yet more improvements in the collapse code. Elimination of some windows that
can cause list corruption.
vm_pageout.c:
Fixed it, it really works better now. Somehow in 2.0, some "enhancements"
broke the code. This code has been reworked from the ground-up.
vm_fault.c, vm_page.c, pmap.c, vm_object.c
Support for small-block filesystems with merged VM/buffer cache scheme.
pmap.c vm_map.c
Dynamic kernel VM size, now we dont have to pre-allocate excessive numbers of
kernel PTs.
vm_glue.c
Much simpler and more effective swapping code. No more gratuitous swapping.
proc.h
Fixed the problem that the p_lock flag was not being cleared on a fork.
swap_pager.c, vnode_pager.c
Removal of old vfs_bio cruft to support the past pseudo-coherency. Now the
code doesn't need it anymore.
machdep.c
Changes to better support the parameter values for the merged VM/buffer cache
scheme.
machdep.c, kern_exec.c, vm_glue.c
Implemented a seperate submap for temporary exec string space and another one
to contain process upages. This eliminates all map fragmentation problems
that previously existed.
ffs_inode.c, ufs_inode.c, ufs_readwrite.c
Changes for merged VM/buffer cache. Add "bypass" support for sneaking in on
busy buffers.
Submitted by: John Dyson and David Greenman
1995-01-09 16:06:02 +00:00
|
|
|
#endif
|
2003-03-30 05:24:52 +00:00
|
|
|
#endif
|
2006-11-13 20:33:54 +00:00
|
|
|
|
1994-01-14 16:25:31 +00:00
|
|
|
#ifndef NKPDE
|
2003-04-07 14:27:19 +00:00
|
|
|
#define NKPDE (KVA_PAGES) /* number of page tables/pde's */
|
2001-09-21 06:23:03 +00:00
|
|
|
#endif
|
1994-01-14 16:25:31 +00:00
|
|
|
|
1993-10-12 13:58:01 +00:00
|
|
|
/*
|
|
|
|
|
* The *PTDI values control the layout of virtual memory
|
|
|
|
|
*
|
|
|
|
|
* XXX This works for now, but I am not real happy with it, I'll fix it
|
|
|
|
|
* right after I fix locore.s and the magic 28K hole
|
|
|
|
|
*/
|
2007-11-13 23:00:24 +00:00
|
|
|
#define KPTDI (NPDEPTD-NKPDE) /* start of kernel virtual pde's */
|
2003-02-23 21:20:00 +00:00
|
|
|
#define PTDPTDI (KPTDI-NPGPTD) /* ptd entry that points to ptd! */
|
First steps in rewriting locore.s, and making info useful
when the machine panics.
i386/i386/locore.s:
1) got rid of most .set directives that were being used like
#define's, and replaced them with appropriate #define's in
the appropriate header files (accessed via genassym).
2) added comments to header inclusions and global definitions,
and global variables
3) replaced some hardcoded constants with cpp defines (such as
PDESIZE and others)
4) aligned all comments to the same column to make them easier to
read
5) moved macro definitions for ENTRY, ALIGN, NOP, etc. to
/sys/i386/include/asmacros.h
6) added #ifdef BDE_DEBUGGER around all of Bruce's debugger code
7) added new global '_KERNend' to store last location+1 of kernel
8) cleaned up zeroing of bss so that only bss is zeroed
9) fix zeroing of page tables so that it really does zero them all
- not just if they follow the bss.
10) rewrote page table initialization code so that 1) works correctly
and 2) write protects the kernel text by default
11) properly initialize the kernel page directory, upages, p0stack PT,
and page tables. The previous scheme was more than a bit
screwy.
12) change allocation of virtual area of IO hole so that it is
fixed at KERNBASE + 0xa0000. The previous scheme put it
right after the kernel page tables and then later expected
it to be at KERNBASE +0xa0000
13) change multiple bogus settings of user read/write of various
areas of kernel VM - including the IO hole; we should never
be accessing the IO hole in user mode through the kernel
page tables
14) split kernel support routines such as bcopy, bzero, copyin,
copyout, etc. into a seperate file 'support.s'
15) split swtch and related routines into a seperate 'swtch.s'
16) split routines related to traps, syscalls, and interrupts
into a seperate file 'exception.s'
17) remove some unused global variables from locore that got
inserted by Garrett when he pulled them out of some .h
files.
i386/isa/icu.s:
1) clean up global variable declarations
2) move in declaration of astpending and netisr
i386/i386/pmap.c:
1) fix calculation of virtual_avail. It previously was calculated
to be right in the middle of the kernel page tables - not
a good place to start allocating kernel VM.
2) properly allocate kernel page dir/tables etc out of kernel map
- previously only took out 2 pages.
i386/i386/machdep.c:
1) modify boot() to print a warning that the system will reboot in
PANIC_REBOOT_WAIT_TIME amount of seconds, and let the user
abort with a key on the console. The machine will wait for
ever if a key is typed before the reboot. The default is
15 seconds, but can be set to 0 to mean don't wait at all,
-1 to mean wait forever, or any positive value to wait for
that many seconds.
2) print "Rebooting..." just before doing it.
kern/subr_prf.c:
1) remove PANICWAIT as it is deprecated by the change to machdep.c
i386/i386/trap.c:
1) add table of trap type strings and use it to print a real trap/
panic message rather than just a number. Lot's of work to
be done here, but this is the first step. Symbolic traceback
is in the TODO.
i386/i386/Makefile.i386:
1) add support in to build support.s, exception.s and swtch.s
...and various changes to various header files to make all of the
above happen.
1993-11-13 02:25:21 +00:00
|
|
|
|
1996-05-02 22:25:18 +00:00
|
|
|
/*
|
|
|
|
|
* XXX doesn't really belong here I guess...
|
|
|
|
|
*/
|
|
|
|
|
#define ISA_HOLE_START 0xa0000
|
|
|
|
|
#define ISA_HOLE_LENGTH (0x100000-ISA_HOLE_START)
|
|
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
#ifndef LOCORE
|
1996-09-08 16:57:53 +00:00
|
|
|
|
2004-06-20 00:33:14 +00:00
|
|
|
#include <sys/queue.h>
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
#include <sys/_cpuset.h>
|
2004-06-16 07:03:15 +00:00
|
|
|
#include <sys/_lock.h>
|
|
|
|
|
#include <sys/_mutex.h>
|
1996-09-08 16:57:53 +00:00
|
|
|
|
Sync back vmcontention branch into HEAD:
Replace the per-object resident and cached pages splay tree with a
path-compressed multi-digit radix trie.
Along with this, switch also the x86-specific handling of idle page
tables to using the radix trie.
This change is supposed to do the following:
- Allowing the acquisition of read locking for lookup operations of the
resident/cached pages collections as the per-vm_page_t splay iterators
are now removed.
- Increase the scalability of the operations on the page collections.
The radix trie does rely on the consumers locking to ensure atomicity of
its operations. In order to avoid deadlocks the bisection nodes are
pre-allocated in the UMA zone. This can be done safely because the
algorithm needs at maximum one new node per insert which means the
maximum number of the desired nodes is the number of available physical
frames themselves. However, not all the times a new bisection node is
really needed.
The radix trie implements path-compression because UFS indirect blocks
can lead to several objects with a very sparse trie, increasing the number
of levels to usually scan. It also helps in the nodes pre-fetching by
introducing the single node per-insert property.
This code is not generalized (yet) because of the possible loss of
performance by having much of the sizes in play configurable.
However, efforts to make this code more general and then reusable in
further different consumers might be really done.
The only KPI change is the removal of the function vm_page_splay() which
is now reaped.
The only KBI change, instead, is the removal of the left/right iterators
from struct vm_page, which are now reaped.
Further technical notes broken into mealpieces can be retrieved from the
svn branch:
http://svn.freebsd.org/base/user/attilio/vmcontention/
Sponsored by: EMC / Isilon storage division
In collaboration with: alc, jeff
Tested by: flo, pho, jhb, davide
Tested by: ian (arm)
Tested by: andreast (powerpc)
2013-03-18 00:25:02 +00:00
|
|
|
#include <vm/_vm_radix.h>
|
|
|
|
|
|
2003-03-30 05:24:52 +00:00
|
|
|
#ifdef PAE
|
|
|
|
|
|
|
|
|
|
typedef uint64_t pdpt_entry_t;
|
|
|
|
|
typedef uint64_t pd_entry_t;
|
|
|
|
|
typedef uint64_t pt_entry_t;
|
|
|
|
|
|
|
|
|
|
#define PTESHIFT (3)
|
|
|
|
|
#define PDESHIFT (3)
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
|
|
typedef uint32_t pd_entry_t;
|
|
|
|
|
typedef uint32_t pt_entry_t;
|
1996-05-02 14:21:14 +00:00
|
|
|
|
2003-02-23 09:45:50 +00:00
|
|
|
#define PTESHIFT (2)
|
|
|
|
|
#define PDESHIFT (2)
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2003-03-30 05:24:52 +00:00
|
|
|
#endif
|
|
|
|
|
|
1993-06-12 14:58:17 +00:00
|
|
|
/*
|
2009-03-22 18:56:26 +00:00
|
|
|
* Address of current address space page table maps and directories.
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
2003-04-03 23:44:35 +00:00
|
|
|
extern pt_entry_t PTmap[];
|
|
|
|
|
extern pd_entry_t PTD[];
|
|
|
|
|
extern pd_entry_t PTDpde[];
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2003-03-30 05:24:52 +00:00
|
|
|
#ifdef PAE
|
|
|
|
|
extern pdpt_entry_t *IdlePDPT;
|
|
|
|
|
#endif
|
2002-07-12 07:56:11 +00:00
|
|
|
extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
|
|
|
|
/*
|
2010-10-05 17:06:51 +00:00
|
|
|
* Translate a virtual address to the kernel virtual address of its page table
|
|
|
|
|
* entry (PTE). This can be used recursively. If the address of a PTE as
|
|
|
|
|
* previously returned by this macro is itself given as the argument, then the
|
|
|
|
|
* address of the page directory entry (PDE) that maps the PTE will be
|
|
|
|
|
* returned.
|
|
|
|
|
*
|
|
|
|
|
* This macro may be used before pmap_bootstrap() is called.
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
|
|
|
|
#define vtopte(va) (PTmap + i386_btop(va))
|
2010-10-05 17:06:51 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Translate a virtual address to its physical address.
|
|
|
|
|
*
|
|
|
|
|
* This macro may be used before pmap_bootstrap() is called.
|
|
|
|
|
*/
|
2005-12-06 21:09:01 +00:00
|
|
|
#define vtophys(va) pmap_kextract((vm_offset_t)(va))
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2011-01-04 14:49:54 +00:00
|
|
|
#if defined(XEN)
|
2008-08-15 20:51:31 +00:00
|
|
|
#include <sys/param.h>
|
Implement vector callback for PVHVM and unify event channel implementations
Re-structure Xen HVM support so that:
- Xen is detected and hypercalls can be performed very
early in system startup.
- Xen interrupt services are implemented using FreeBSD's native
interrupt delivery infrastructure.
- the Xen interrupt service implementation is shared between PV
and HVM guests.
- Xen interrupt handlers can optionally use a filter handler
in order to avoid the overhead of dispatch to an interrupt
thread.
- interrupt load can be distributed among all available CPUs.
- the overhead of accessing the emulated local and I/O apics
on HVM is removed for event channel port events.
- a similar optimization can eventually, and fairly easily,
be used to optimize MSI.
Early Xen detection, HVM refactoring, PVHVM interrupt infrastructure,
and misc Xen cleanups:
Sponsored by: Spectra Logic Corporation
Unification of PV & HVM interrupt infrastructure, bug fixes,
and misc Xen cleanups:
Submitted by: Roger Pau Monné
Sponsored by: Citrix Systems R&D
sys/x86/x86/local_apic.c:
sys/amd64/include/apicvar.h:
sys/i386/include/apicvar.h:
sys/amd64/amd64/apic_vector.S:
sys/i386/i386/apic_vector.s:
sys/amd64/amd64/machdep.c:
sys/i386/i386/machdep.c:
sys/i386/xen/exception.s:
sys/x86/include/segments.h:
Reserve IDT vector 0x93 for the Xen event channel upcall
interrupt handler. On Hypervisors that support the direct
vector callback feature, we can request that this vector be
called directly by an injected HVM interrupt event, instead
of a simulated PCI interrupt on the Xen platform PCI device.
This avoids all of the overhead of dealing with the emulated
I/O APIC and local APIC. It also means that the Hypervisor
can inject these events on any CPU, allowing upcalls for
different ports to be handled in parallel.
sys/amd64/amd64/mp_machdep.c:
sys/i386/i386/mp_machdep.c:
Map Xen per-vcpu area during AP startup.
sys/amd64/include/intr_machdep.h:
sys/i386/include/intr_machdep.h:
Increase the FreeBSD IRQ vector table to include space
for event channel interrupt sources.
sys/amd64/include/pcpu.h:
sys/i386/include/pcpu.h:
Remove Xen HVM per-cpu variable data. These fields are now
allocated via the dynamic per-cpu scheme. See xen_intr.c
for details.
sys/amd64/include/xen/hypercall.h:
sys/dev/xen/blkback/blkback.c:
sys/i386/include/xen/xenvar.h:
sys/i386/xen/clock.c:
sys/i386/xen/xen_machdep.c:
sys/xen/gnttab.c:
Prefer FreeBSD primatives to Linux ones in Xen support code.
sys/amd64/include/xen/xen-os.h:
sys/i386/include/xen/xen-os.h:
sys/xen/xen-os.h:
sys/dev/xen/balloon/balloon.c:
sys/dev/xen/blkback/blkback.c:
sys/dev/xen/blkfront/blkfront.c:
sys/dev/xen/console/xencons_ring.c:
sys/dev/xen/control/control.c:
sys/dev/xen/netback/netback.c:
sys/dev/xen/netfront/netfront.c:
sys/dev/xen/xenpci/xenpci.c:
sys/i386/i386/machdep.c:
sys/i386/include/pmap.h:
sys/i386/include/xen/xenfunc.h:
sys/i386/isa/npx.c:
sys/i386/xen/clock.c:
sys/i386/xen/mp_machdep.c:
sys/i386/xen/mptable.c:
sys/i386/xen/xen_clock_util.c:
sys/i386/xen/xen_machdep.c:
sys/i386/xen/xen_rtc.c:
sys/xen/evtchn/evtchn_dev.c:
sys/xen/features.c:
sys/xen/gnttab.c:
sys/xen/gnttab.h:
sys/xen/hvm.h:
sys/xen/xenbus/xenbus.c:
sys/xen/xenbus/xenbus_if.m:
sys/xen/xenbus/xenbusb_front.c:
sys/xen/xenbus/xenbusvar.h:
sys/xen/xenstore/xenstore.c:
sys/xen/xenstore/xenstore_dev.c:
sys/xen/xenstore/xenstorevar.h:
Pull common Xen OS support functions/settings into xen/xen-os.h.
sys/amd64/include/xen/xen-os.h:
sys/i386/include/xen/xen-os.h:
sys/xen/xen-os.h:
Remove constants, macros, and functions unused in FreeBSD's Xen
support.
sys/xen/xen-os.h:
sys/i386/xen/xen_machdep.c:
sys/x86/xen/hvm.c:
Introduce new functions xen_domain(), xen_pv_domain(), and
xen_hvm_domain(). These are used in favor of #ifdefs so that
FreeBSD can dynamically detect and adapt to the presence of
a hypervisor. The goal is to have an HVM optimized GENERIC,
but more is necessary before this is possible.
sys/amd64/amd64/machdep.c:
sys/dev/xen/xenpci/xenpcivar.h:
sys/dev/xen/xenpci/xenpci.c:
sys/x86/xen/hvm.c:
sys/sys/kernel.h:
Refactor magic ioport, Hypercall table and Hypervisor shared
information page setup, and move it to a dedicated HVM support
module.
HVM mode initialization is now triggered during the
SI_SUB_HYPERVISOR phase of system startup. This currently
occurs just after the kernel VM is fully setup which is
just enough infrastructure to allow the hypercall table
and shared info page to be properly mapped.
sys/xen/hvm.h:
sys/x86/xen/hvm.c:
Add definitions and a method for configuring Hypervisor event
delievery via a direct vector callback.
sys/amd64/include/xen/xen-os.h:
sys/x86/xen/hvm.c:
sys/conf/files:
sys/conf/files.amd64:
sys/conf/files.i386:
Adjust kernel build to reflect the refactoring of early
Xen startup code and Xen interrupt services.
sys/dev/xen/blkback/blkback.c:
sys/dev/xen/blkfront/blkfront.c:
sys/dev/xen/blkfront/block.h:
sys/dev/xen/control/control.c:
sys/dev/xen/evtchn/evtchn_dev.c:
sys/dev/xen/netback/netback.c:
sys/dev/xen/netfront/netfront.c:
sys/xen/xenstore/xenstore.c:
sys/xen/evtchn/evtchn_dev.c:
sys/dev/xen/console/console.c:
sys/dev/xen/console/xencons_ring.c
Adjust drivers to use new xen_intr_*() API.
sys/dev/xen/blkback/blkback.c:
Since blkback defers all event handling to a taskqueue,
convert this task queue to a "fast" taskqueue, and schedule
it via an interrupt filter. This avoids an unnecessary
ithread context switch.
sys/xen/xenstore/xenstore.c:
The xenstore driver is MPSAFE. Indicate as much when
registering its interrupt handler.
sys/xen/xenbus/xenbus.c:
sys/xen/xenbus/xenbusvar.h:
Remove unused event channel APIs.
sys/xen/evtchn.h:
Remove all kernel Xen interrupt service API definitions
from this file. It is now only used for structure and
ioctl definitions related to the event channel userland
device driver.
Update the definitions in this file to match those from
NetBSD. Implementing this interface will be necessary for
Dom0 support.
sys/xen/evtchn/evtchnvar.h:
Add a header file for implemenation internal APIs related
to managing event channels event delivery. This is used
to allow, for example, the event channel userland device
driver to access low-level routines that typical kernel
consumers of event channel services should never access.
sys/xen/interface/event_channel.h:
sys/xen/xen_intr.h:
Standardize on the evtchn_port_t type for referring to
an event channel port id. In order to prevent low-level
event channel APIs from leaking to kernel consumers who
should not have access to this data, the type is defined
twice: Once in the Xen provided event_channel.h, and again
in xen/xen_intr.h. The double declaration is protected by
__XEN_EVTCHN_PORT_DEFINED__ to ensure it is never declared
twice within a given compilation unit.
sys/xen/xen_intr.h:
sys/xen/evtchn/evtchn.c:
sys/x86/xen/xen_intr.c:
sys/dev/xen/xenpci/evtchn.c:
sys/dev/xen/xenpci/xenpcivar.h:
New implementation of Xen interrupt services. This is
similar in many respects to the i386 PV implementation with
the exception that events for bound to event channel ports
(i.e. not IPI, virtual IRQ, or physical IRQ) are further
optimized to avoid mask/unmask operations that aren't
necessary for these edge triggered events.
Stubs exist for supporting physical IRQ binding, but will
need additional work before this implementation can be
fully shared between PV and HVM.
sys/amd64/amd64/mp_machdep.c:
sys/i386/i386/mp_machdep.c:
sys/i386/xen/mp_machdep.c
sys/x86/xen/hvm.c:
Add support for placing vcpu_info into an arbritary memory
page instead of using HYPERVISOR_shared_info->vcpu_info.
This allows the creation of domains with more than 32 vcpus.
sys/i386/i386/machdep.c:
sys/i386/xen/clock.c:
sys/i386/xen/xen_machdep.c:
sys/i386/xen/exception.s:
Add support for new event channle implementation.
2013-08-29 19:52:18 +00:00
|
|
|
|
|
|
|
|
#include <xen/xen-os.h>
|
|
|
|
|
|
2008-08-15 20:51:31 +00:00
|
|
|
#include <machine/xen/xenvar.h>
|
|
|
|
|
#include <machine/xen/xenpmap.h>
|
|
|
|
|
|
|
|
|
|
extern pt_entry_t pg_nx;
|
|
|
|
|
|
|
|
|
|
#define PG_KERNEL (PG_V | PG_A | PG_RW | PG_M)
|
|
|
|
|
|
|
|
|
|
#define MACH_TO_VM_PAGE(ma) PHYS_TO_VM_PAGE(xpmap_mtop((ma)))
|
|
|
|
|
#define VM_PAGE_TO_MACH(m) xpmap_ptom(VM_PAGE_TO_PHYS((m)))
|
|
|
|
|
|
2010-11-20 20:04:29 +00:00
|
|
|
#define VTOM(va) xpmap_ptom(VTOP(va))
|
|
|
|
|
|
2008-08-15 20:51:31 +00:00
|
|
|
static __inline vm_paddr_t
|
|
|
|
|
pmap_kextract_ma(vm_offset_t va)
|
|
|
|
|
{
|
|
|
|
|
vm_paddr_t ma;
|
|
|
|
|
if ((ma = PTD[va >> PDRSHIFT]) & PG_PS) {
|
|
|
|
|
ma = (ma & ~(NBPDR - 1)) | (va & (NBPDR - 1));
|
|
|
|
|
} else {
|
|
|
|
|
ma = (*vtopte(va) & PG_FRAME) | (va & PAGE_MASK);
|
|
|
|
|
}
|
|
|
|
|
return ma;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __inline vm_paddr_t
|
|
|
|
|
pmap_kextract(vm_offset_t va)
|
|
|
|
|
{
|
|
|
|
|
return xpmap_mtop(pmap_kextract_ma(va));
|
|
|
|
|
}
|
|
|
|
|
#define vtomach(va) pmap_kextract_ma(((vm_offset_t) (va)))
|
|
|
|
|
|
|
|
|
|
vm_paddr_t pmap_extract_ma(struct pmap *pmap, vm_offset_t va);
|
|
|
|
|
|
|
|
|
|
void pmap_kenter_ma(vm_offset_t va, vm_paddr_t pa);
|
|
|
|
|
void pmap_map_readonly(struct pmap *pmap, vm_offset_t va, int len);
|
|
|
|
|
void pmap_map_readwrite(struct pmap *pmap, vm_offset_t va, int len);
|
|
|
|
|
|
|
|
|
|
static __inline pt_entry_t
|
|
|
|
|
pte_load_store(pt_entry_t *ptep, pt_entry_t v)
|
|
|
|
|
{
|
|
|
|
|
pt_entry_t r;
|
|
|
|
|
|
|
|
|
|
r = *ptep;
|
|
|
|
|
PT_SET_VA(ptep, v, TRUE);
|
|
|
|
|
return (r);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __inline pt_entry_t
|
|
|
|
|
pte_load_store_ma(pt_entry_t *ptep, pt_entry_t v)
|
|
|
|
|
{
|
|
|
|
|
pt_entry_t r;
|
|
|
|
|
|
|
|
|
|
r = *ptep;
|
|
|
|
|
PT_SET_VA_MA(ptep, v, TRUE);
|
|
|
|
|
return (r);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define pte_load_clear(ptep) pte_load_store((ptep), (pt_entry_t)0ULL)
|
|
|
|
|
|
|
|
|
|
#define pte_store(ptep, pte) pte_load_store((ptep), (pt_entry_t)pte)
|
|
|
|
|
#define pte_store_ma(ptep, pte) pte_load_store_ma((ptep), (pt_entry_t)pte)
|
|
|
|
|
#define pde_store_ma(ptep, pte) pte_load_store_ma((ptep), (pt_entry_t)pte)
|
|
|
|
|
|
|
|
|
|
#elif !defined(XEN)
|
2010-01-23 18:42:28 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* KPTmap is a linear mapping of the kernel page table. It differs from the
|
|
|
|
|
* recursive mapping in two ways: (1) it only provides access to kernel page
|
|
|
|
|
* table pages, and not user page table pages, and (2) it provides access to
|
|
|
|
|
* a kernel page table page after the corresponding virtual addresses have
|
|
|
|
|
* been promoted to a 2/4MB page mapping.
|
2010-10-05 17:06:51 +00:00
|
|
|
*
|
|
|
|
|
* KPTmap is first initialized by locore to support just NPKT page table
|
|
|
|
|
* pages. Later, it is reinitialized by pmap_bootstrap() to allow for
|
|
|
|
|
* expansion of the kernel page table.
|
2010-01-23 18:42:28 +00:00
|
|
|
*/
|
|
|
|
|
extern pt_entry_t *KPTmap;
|
|
|
|
|
|
1994-03-24 23:12:48 +00:00
|
|
|
/*
|
2010-10-05 17:06:51 +00:00
|
|
|
* Extract from the kernel page table the physical address that is mapped by
|
|
|
|
|
* the given virtual address "va".
|
|
|
|
|
*
|
|
|
|
|
* This function may be used before pmap_bootstrap() is called.
|
1994-03-24 23:12:48 +00:00
|
|
|
*/
|
2003-03-25 00:07:06 +00:00
|
|
|
static __inline vm_paddr_t
|
1994-11-14 14:12:24 +00:00
|
|
|
pmap_kextract(vm_offset_t va)
|
1994-03-24 23:12:48 +00:00
|
|
|
{
|
2003-03-25 00:07:06 +00:00
|
|
|
vm_paddr_t pa;
|
|
|
|
|
|
2003-04-08 18:22:41 +00:00
|
|
|
if ((pa = PTD[va >> PDRSHIFT]) & PG_PS) {
|
2007-04-06 18:15:03 +00:00
|
|
|
pa = (pa & PG_PS_FRAME) | (va & PDRMASK);
|
1997-07-17 04:34:03 +00:00
|
|
|
} else {
|
2010-01-23 18:42:28 +00:00
|
|
|
/*
|
|
|
|
|
* Beware of a concurrent promotion that changes the PDE at
|
|
|
|
|
* this point! For example, vtopte() must not be used to
|
|
|
|
|
* access the PTE because it would use the new PDE. It is,
|
|
|
|
|
* however, safe to use the old PDE because the page table
|
|
|
|
|
* page is preserved by the promotion.
|
|
|
|
|
*/
|
|
|
|
|
pa = KPTmap[i386_btop(va)];
|
1997-07-17 04:34:03 +00:00
|
|
|
pa = (pa & PG_FRAME) | (va & PAGE_MASK);
|
|
|
|
|
}
|
2010-01-23 18:42:28 +00:00
|
|
|
return (pa);
|
1994-03-24 23:12:48 +00:00
|
|
|
}
|
2011-01-04 14:49:54 +00:00
|
|
|
#endif
|
2008-08-18 21:35:09 +00:00
|
|
|
|
2011-01-04 14:49:54 +00:00
|
|
|
#if !defined(XEN)
|
2008-08-18 21:35:09 +00:00
|
|
|
#define PT_UPDATES_FLUSH()
|
2008-08-15 20:51:31 +00:00
|
|
|
#endif
|
1997-07-17 04:34:03 +00:00
|
|
|
|
2008-08-15 20:51:31 +00:00
|
|
|
#if defined(PAE) && !defined(XEN)
|
2003-03-30 05:24:52 +00:00
|
|
|
|
2013-08-21 22:40:29 +00:00
|
|
|
#define pde_cmpset(pdep, old, new) atomic_cmpset_64_i586(pdep, old, new)
|
|
|
|
|
#define pte_load_store(ptep, pte) atomic_swap_64_i586(ptep, pte)
|
|
|
|
|
#define pte_load_clear(ptep) atomic_swap_64_i586(ptep, 0)
|
|
|
|
|
#define pte_store(ptep, pte) atomic_store_rel_64_i586(ptep, pte)
|
2004-10-08 08:23:43 +00:00
|
|
|
|
2007-04-06 18:15:03 +00:00
|
|
|
extern pt_entry_t pg_nx;
|
|
|
|
|
|
2013-08-21 22:40:29 +00:00
|
|
|
#elif !defined(PAE) && !defined(XEN)
|
2003-03-30 05:24:52 +00:00
|
|
|
|
2013-08-21 22:40:29 +00:00
|
|
|
#define pde_cmpset(pdep, old, new) atomic_cmpset_int(pdep, old, new)
|
|
|
|
|
#define pte_load_store(ptep, pte) atomic_swap_int(ptep, pte)
|
|
|
|
|
#define pte_load_clear(ptep) atomic_swap_int(ptep, 0)
|
|
|
|
|
#define pte_store(ptep, pte) do { \
|
|
|
|
|
*(u_int *)(ptep) = (u_int)(pte); \
|
|
|
|
|
} while (0)
|
2004-10-08 08:23:43 +00:00
|
|
|
|
2003-04-28 20:35:36 +00:00
|
|
|
#endif /* PAE */
|
2003-03-30 05:24:52 +00:00
|
|
|
|
2013-08-21 22:40:29 +00:00
|
|
|
#define pte_clear(ptep) pte_store(ptep, 0)
|
2003-04-28 20:35:36 +00:00
|
|
|
|
2013-08-21 22:40:29 +00:00
|
|
|
#define pde_store(pdep, pde) pte_store(pdep, pde)
|
2003-04-28 20:35:36 +00:00
|
|
|
|
|
|
|
|
#endif /* _KERNEL */
|
1997-07-17 04:34:03 +00:00
|
|
|
|
1993-06-12 14:58:17 +00:00
|
|
|
/*
|
|
|
|
|
* Pmap stuff
|
|
|
|
|
*/
|
1996-09-08 16:57:53 +00:00
|
|
|
struct pv_entry;
|
MFamd64: shrink pv entries from 24 bytes to about 12 bytes. (336 pv entries
per page = effectively 12.19 bytes per pv entry after overheads).
Instead of using a shared UMA zone for 24 byte pv entries (two 8-byte tailq
nodes, a 4 byte pointer, and a 4 byte address), we allocate a page at a
time per process. This provides 336 pv entries per process (actually, per
pmap address space) and eliminates one of the 8-byte tailq entries since
we now can track per-process pv entries implicitly. The pointer to
the pmap can be eliminated by doing address arithmetic to find the metadata
on the page headers to find a single pointer shared by all 336 entries.
There is an 11-int bitmap for the freelist of those 336 entries.
This is mostly a mechanical conversion from amd64, except:
* i386 has to allocate kvm and map the pages, amd64 has them outside of kvm
* native word size is smaller, so bitmaps etc become 32 bit instead of 64
* no dump_add_page() etc stuff because they are in kvm always.
* various pmap internals tweaks because pmap uses direct map on amd64 but
on i386 it has to use sched_pin and temporary mappings.
Also, sysctl vm.pmap.pv_entry_max and vm.pmap.shpgperproc are now
dynamic sysctls. Like on amd64, i386 can now tune the pv entry limits
without a recompile or reboot.
This is important because of the following scenario. If you have a 1GB
file (262144 pages) mmap()ed into 50 processes, that requires 13 million
pv entries. At 24 bytes per pv entry, that is 314MB of ram and kvm, while
at 12 bytes it is 157MB. A 157MB saving is significant.
Test-run by: scottl (Thanks!)
2006-04-26 21:49:20 +00:00
|
|
|
struct pv_chunk;
|
2000-05-21 12:50:18 +00:00
|
|
|
|
|
|
|
|
struct md_page {
|
2000-05-26 02:09:24 +00:00
|
|
|
TAILQ_HEAD(,pv_entry) pv_list;
|
2009-07-12 23:31:20 +00:00
|
|
|
int pat_mode;
|
2000-05-21 12:50:18 +00:00
|
|
|
};
|
1993-06-12 14:58:17 +00:00
|
|
|
|
|
|
|
|
struct pmap {
|
2004-06-16 07:03:15 +00:00
|
|
|
struct mtx pm_mtx;
|
1993-06-12 14:58:17 +00:00
|
|
|
pd_entry_t *pm_pdir; /* KVA of page directory */
|
MFamd64: shrink pv entries from 24 bytes to about 12 bytes. (336 pv entries
per page = effectively 12.19 bytes per pv entry after overheads).
Instead of using a shared UMA zone for 24 byte pv entries (two 8-byte tailq
nodes, a 4 byte pointer, and a 4 byte address), we allocate a page at a
time per process. This provides 336 pv entries per process (actually, per
pmap address space) and eliminates one of the 8-byte tailq entries since
we now can track per-process pv entries implicitly. The pointer to
the pmap can be eliminated by doing address arithmetic to find the metadata
on the page headers to find a single pointer shared by all 336 entries.
There is an 11-int bitmap for the freelist of those 336 entries.
This is mostly a mechanical conversion from amd64, except:
* i386 has to allocate kvm and map the pages, amd64 has them outside of kvm
* native word size is smaller, so bitmaps etc become 32 bit instead of 64
* no dump_add_page() etc stuff because they are in kvm always.
* various pmap internals tweaks because pmap uses direct map on amd64 but
on i386 it has to use sched_pin and temporary mappings.
Also, sysctl vm.pmap.pv_entry_max and vm.pmap.shpgperproc are now
dynamic sysctls. Like on amd64, i386 can now tune the pv entry limits
without a recompile or reboot.
This is important because of the following scenario. If you have a 1GB
file (262144 pages) mmap()ed into 50 processes, that requires 13 million
pv entries. At 24 bytes per pv entry, that is 314MB of ram and kvm, while
at 12 bytes it is 157MB. A 157MB saving is significant.
Test-run by: scottl (Thanks!)
2006-04-26 21:49:20 +00:00
|
|
|
TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
cpuset_t pm_active; /* active on cpus */
|
1993-06-12 14:58:17 +00:00
|
|
|
struct pmap_statistics pm_stats; /* pmap statistics */
|
2000-08-16 21:24:44 +00:00
|
|
|
LIST_ENTRY(pmap) pm_list; /* List of all pmaps */
|
2003-03-30 05:24:52 +00:00
|
|
|
#ifdef PAE
|
|
|
|
|
pdpt_entry_t *pm_pdpt; /* KVA of page director pointer
|
|
|
|
|
table */
|
|
|
|
|
#endif
|
Sync back vmcontention branch into HEAD:
Replace the per-object resident and cached pages splay tree with a
path-compressed multi-digit radix trie.
Along with this, switch also the x86-specific handling of idle page
tables to using the radix trie.
This change is supposed to do the following:
- Allowing the acquisition of read locking for lookup operations of the
resident/cached pages collections as the per-vm_page_t splay iterators
are now removed.
- Increase the scalability of the operations on the page collections.
The radix trie does rely on the consumers locking to ensure atomicity of
its operations. In order to avoid deadlocks the bisection nodes are
pre-allocated in the UMA zone. This can be done safely because the
algorithm needs at maximum one new node per insert which means the
maximum number of the desired nodes is the number of available physical
frames themselves. However, not all the times a new bisection node is
really needed.
The radix trie implements path-compression because UFS indirect blocks
can lead to several objects with a very sparse trie, increasing the number
of levels to usually scan. It also helps in the nodes pre-fetching by
introducing the single node per-insert property.
This code is not generalized (yet) because of the possible loss of
performance by having much of the sizes in play configurable.
However, efforts to make this code more general and then reusable in
further different consumers might be really done.
The only KPI change is the removal of the function vm_page_splay() which
is now reaped.
The only KBI change, instead, is the removal of the left/right iterators
from struct vm_page, which are now reaped.
Further technical notes broken into mealpieces can be retrieved from the
svn branch:
http://svn.freebsd.org/base/user/attilio/vmcontention/
Sponsored by: EMC / Isilon storage division
In collaboration with: alc, jeff
Tested by: flo, pho, jhb, davide
Tested by: ian (arm)
Tested by: andreast (powerpc)
2013-03-18 00:25:02 +00:00
|
|
|
struct vm_radix pm_root; /* spare page table pages */
|
1993-06-12 14:58:17 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
typedef struct pmap *pmap_t;
|
|
|
|
|
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
2002-04-29 07:43:16 +00:00
|
|
|
extern struct pmap kernel_pmap_store;
|
|
|
|
|
#define kernel_pmap (&kernel_pmap_store)
|
2004-06-16 07:03:15 +00:00
|
|
|
|
|
|
|
|
#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)
|
|
|
|
|
#define PMAP_LOCK_ASSERT(pmap, type) \
|
|
|
|
|
mtx_assert(&(pmap)->pm_mtx, (type))
|
|
|
|
|
#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)
|
|
|
|
|
#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \
|
2004-09-29 19:20:40 +00:00
|
|
|
NULL, MTX_DEF | MTX_DUPOK)
|
2004-06-16 07:03:15 +00:00
|
|
|
#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)
|
|
|
|
|
#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)
|
|
|
|
|
#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)
|
|
|
|
|
#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)
|
1993-06-12 14:58:17 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* For each vm_page_t, there is a list of all currently valid virtual
|
2006-11-13 06:26:57 +00:00
|
|
|
* mappings of that page. An entry is a pv_entry_t, the list is pv_list.
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
|
|
|
|
typedef struct pv_entry {
|
|
|
|
|
vm_offset_t pv_va; /* virtual address for mapping */
|
2013-03-02 14:19:08 +00:00
|
|
|
TAILQ_ENTRY(pv_entry) pv_next;
|
1993-06-12 14:58:17 +00:00
|
|
|
} *pv_entry_t;
|
|
|
|
|
|
MFamd64: shrink pv entries from 24 bytes to about 12 bytes. (336 pv entries
per page = effectively 12.19 bytes per pv entry after overheads).
Instead of using a shared UMA zone for 24 byte pv entries (two 8-byte tailq
nodes, a 4 byte pointer, and a 4 byte address), we allocate a page at a
time per process. This provides 336 pv entries per process (actually, per
pmap address space) and eliminates one of the 8-byte tailq entries since
we now can track per-process pv entries implicitly. The pointer to
the pmap can be eliminated by doing address arithmetic to find the metadata
on the page headers to find a single pointer shared by all 336 entries.
There is an 11-int bitmap for the freelist of those 336 entries.
This is mostly a mechanical conversion from amd64, except:
* i386 has to allocate kvm and map the pages, amd64 has them outside of kvm
* native word size is smaller, so bitmaps etc become 32 bit instead of 64
* no dump_add_page() etc stuff because they are in kvm always.
* various pmap internals tweaks because pmap uses direct map on amd64 but
on i386 it has to use sched_pin and temporary mappings.
Also, sysctl vm.pmap.pv_entry_max and vm.pmap.shpgperproc are now
dynamic sysctls. Like on amd64, i386 can now tune the pv entry limits
without a recompile or reboot.
This is important because of the following scenario. If you have a 1GB
file (262144 pages) mmap()ed into 50 processes, that requires 13 million
pv entries. At 24 bytes per pv entry, that is 314MB of ram and kvm, while
at 12 bytes it is 157MB. A 157MB saving is significant.
Test-run by: scottl (Thanks!)
2006-04-26 21:49:20 +00:00
|
|
|
/*
|
|
|
|
|
* pv_entries are allocated in chunks per-process. This avoids the
|
|
|
|
|
* need to track per-pmap assignments.
|
|
|
|
|
*/
|
|
|
|
|
#define _NPCM 11
|
|
|
|
|
#define _NPCPV 336
|
|
|
|
|
struct pv_chunk {
|
|
|
|
|
pmap_t pc_pmap;
|
|
|
|
|
TAILQ_ENTRY(pv_chunk) pc_list;
|
|
|
|
|
uint32_t pc_map[_NPCM]; /* bitmap; 1 = free */
|
2012-05-26 06:10:25 +00:00
|
|
|
TAILQ_ENTRY(pv_chunk) pc_lru;
|
MFamd64: shrink pv entries from 24 bytes to about 12 bytes. (336 pv entries
per page = effectively 12.19 bytes per pv entry after overheads).
Instead of using a shared UMA zone for 24 byte pv entries (two 8-byte tailq
nodes, a 4 byte pointer, and a 4 byte address), we allocate a page at a
time per process. This provides 336 pv entries per process (actually, per
pmap address space) and eliminates one of the 8-byte tailq entries since
we now can track per-process pv entries implicitly. The pointer to
the pmap can be eliminated by doing address arithmetic to find the metadata
on the page headers to find a single pointer shared by all 336 entries.
There is an 11-int bitmap for the freelist of those 336 entries.
This is mostly a mechanical conversion from amd64, except:
* i386 has to allocate kvm and map the pages, amd64 has them outside of kvm
* native word size is smaller, so bitmaps etc become 32 bit instead of 64
* no dump_add_page() etc stuff because they are in kvm always.
* various pmap internals tweaks because pmap uses direct map on amd64 but
on i386 it has to use sched_pin and temporary mappings.
Also, sysctl vm.pmap.pv_entry_max and vm.pmap.shpgperproc are now
dynamic sysctls. Like on amd64, i386 can now tune the pv entry limits
without a recompile or reboot.
This is important because of the following scenario. If you have a 1GB
file (262144 pages) mmap()ed into 50 processes, that requires 13 million
pv entries. At 24 bytes per pv entry, that is 314MB of ram and kvm, while
at 12 bytes it is 157MB. A 157MB saving is significant.
Test-run by: scottl (Thanks!)
2006-04-26 21:49:20 +00:00
|
|
|
struct pv_entry pc_pventry[_NPCPV];
|
|
|
|
|
};
|
|
|
|
|
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2014-02-11 22:02:40 +00:00
|
|
|
extern caddr_t CADDR3;
|
|
|
|
|
extern pt_entry_t *CMAP3;
|
2003-03-25 00:07:06 +00:00
|
|
|
extern vm_paddr_t phys_avail[];
|
2005-06-29 22:28:46 +00:00
|
|
|
extern vm_paddr_t dump_avail[];
|
2003-10-01 23:46:08 +00:00
|
|
|
extern int pseflag;
|
|
|
|
|
extern int pgeflag;
|
1997-11-20 19:30:35 +00:00
|
|
|
extern char *ptvmmap; /* poor name! */
|
1995-03-16 18:17:34 +00:00
|
|
|
extern vm_offset_t virtual_avail;
|
|
|
|
|
extern vm_offset_t virtual_end;
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2009-07-12 23:31:20 +00:00
|
|
|
#define pmap_page_get_memattr(m) ((vm_memattr_t)(m)->md.pat_mode)
|
2012-06-16 18:56:19 +00:00
|
|
|
#define pmap_page_is_write_mapped(m) (((m)->aflags & PGA_WRITEABLE) != 0)
|
2006-08-11 19:22:57 +00:00
|
|
|
#define pmap_unmapbios(va, sz) pmap_unmapdev((va), (sz))
|
2004-06-13 03:44:11 +00:00
|
|
|
|
2010-10-05 17:06:51 +00:00
|
|
|
/*
|
|
|
|
|
* Only the following functions or macros may be used before pmap_bootstrap()
|
|
|
|
|
* is called: pmap_kenter(), pmap_kextract(), pmap_kremove(), vtophys(), and
|
|
|
|
|
* vtopte().
|
|
|
|
|
*/
|
2007-03-17 19:42:06 +00:00
|
|
|
void pmap_bootstrap(vm_paddr_t);
|
2009-07-29 08:49:58 +00:00
|
|
|
int pmap_cache_bits(int mode, boolean_t is_pde);
|
2006-08-11 19:22:57 +00:00
|
|
|
int pmap_change_attr(vm_offset_t, vm_size_t, int);
|
2006-05-01 22:07:00 +00:00
|
|
|
void pmap_init_pat(void);
|
2003-03-25 00:07:06 +00:00
|
|
|
void pmap_kenter(vm_offset_t va, vm_paddr_t pa);
|
2004-04-10 23:28:49 +00:00
|
|
|
void *pmap_kenter_temporary(vm_paddr_t pa, int i);
|
2003-03-16 04:16:03 +00:00
|
|
|
void pmap_kremove(vm_offset_t);
|
2006-08-11 19:22:57 +00:00
|
|
|
void *pmap_mapbios(vm_paddr_t, vm_size_t);
|
2003-03-25 00:07:06 +00:00
|
|
|
void *pmap_mapdev(vm_paddr_t, vm_size_t);
|
2006-08-11 19:22:57 +00:00
|
|
|
void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, int);
|
MFamd64 with few changes:
1. Add support for automatic promotion of 4KB page mappings to 2MB page
mappings. Automatic promotion can be enabled by setting the tunable
"vm.pmap.pg_ps_enabled" to a non-zero value. By default, automatic
promotion is disabled. Tested by: kris
2. To date, we have assumed that the TLB will only set the PG_M bit in a
PTE if that PTE has the PG_RW bit set. However, this assumption does
not hold on recent processors from Intel. For example, consider a PTE
that has the PG_RW bit set but the PG_M bit clear. Suppose this PTE
is cached in the TLB and later the PG_RW bit is cleared in the PTE,
but the corresponding TLB entry is not (yet) invalidated.
Historically, upon a write access using this (stale) TLB entry, the
TLB would observe that the PG_RW bit had been cleared and initiate a
page fault, aborting the setting of the PG_M bit in the PTE. Now,
however, P4- and Core2-family processors will set the PG_M bit before
observing that the PG_RW bit is clear and initiating a page fault. In
other words, the write does not occur but the PG_M bit is still set.
The real impact of this difference is not that great. Specifically,
we should no longer assert that any PTE with the PG_M bit set must
also have the PG_RW bit set, and we should ignore the state of the
PG_M bit unless the PG_RW bit is set.
2008-03-27 04:34:17 +00:00
|
|
|
boolean_t pmap_page_is_mapped(vm_page_t m);
|
2009-07-12 23:31:20 +00:00
|
|
|
void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma);
|
2002-03-20 05:48:58 +00:00
|
|
|
void pmap_unmapdev(vm_offset_t, vm_size_t);
|
2003-11-08 03:01:26 +00:00
|
|
|
pt_entry_t *pmap_pte(pmap_t, vm_offset_t) __pure2;
|
2002-07-12 07:56:11 +00:00
|
|
|
void pmap_invalidate_page(pmap_t, vm_offset_t);
|
|
|
|
|
void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
|
|
|
|
|
void pmap_invalidate_all(pmap_t);
|
2006-05-01 21:36:47 +00:00
|
|
|
void pmap_invalidate_cache(void);
|
2011-04-18 21:24:42 +00:00
|
|
|
void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
|
2014-10-08 16:48:03 +00:00
|
|
|
void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva,
|
|
|
|
|
boolean_t force);
|
1993-12-19 00:55:01 +00:00
|
|
|
|
1999-12-29 04:46:21 +00:00
|
|
|
#endif /* _KERNEL */
|
1996-10-12 20:36:15 +00:00
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
#endif /* !LOCORE */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1994-11-14 14:12:24 +00:00
|
|
|
#endif /* !_MACHINE_PMAP_H_ */
|
