2001-06-10 02:39:37 +00:00
|
|
|
/*-
|
2017-11-20 19:43:44 +00:00
|
|
|
* SPDX-License-Identifier: BSD-4-Clause
|
|
|
|
*
|
2001-06-10 02:39:37 +00:00
|
|
|
* Copyright (C) 1995, 1996 Wolfgang Solfrank.
|
|
|
|
* Copyright (C) 1995, 1996 TooLs GmbH.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 3. All advertising materials mentioning features or use of this software
|
|
|
|
* must display the following acknowledgement:
|
|
|
|
* This product includes software developed by TooLs GmbH.
|
|
|
|
* 4. The name of TooLs GmbH may not be used to endorse or promote products
|
|
|
|
* derived from this software without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
|
|
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
|
|
* IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
|
|
|
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
|
|
|
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
|
|
|
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
|
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* $NetBSD: vmparam.h,v 1.11 2000/02/11 19:25:16 thorpej Exp $
|
|
|
|
* $FreeBSD$
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _MACHINE_VMPARAM_H_
|
|
|
|
#define _MACHINE_VMPARAM_H_
|
|
|
|
|
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
2018-01-19 17:46:31 +00:00
|
|
|
#ifndef LOCORE
|
|
|
|
#include <machine/md_var.h>
|
|
|
|
#endif
|
|
|
|
|
2011-01-14 11:36:44 +00:00
|
|
|
#define USRSTACK SHAREDPAGE
|
2001-06-10 02:39:37 +00:00
|
|
|
|
|
|
|
#ifndef MAXTSIZ
|
2015-01-10 06:54:10 +00:00
|
|
|
#define MAXTSIZ (1*1024*1024*1024) /* max text size */
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef DFLDSIZ
|
2009-12-02 06:49:22 +00:00
|
|
|
#define DFLDSIZ (128*1024*1024) /* default data size */
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef MAXDSIZ
|
2017-12-20 16:49:45 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
#define MAXDSIZ (32UL*1024*1024*1024) /* max data size */
|
|
|
|
#else
|
2009-12-02 06:49:22 +00:00
|
|
|
#define MAXDSIZ (1*1024*1024*1024) /* max data size */
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif
|
2017-12-20 16:49:45 +00:00
|
|
|
#endif
|
2001-06-10 02:39:37 +00:00
|
|
|
|
|
|
|
#ifndef DFLSSIZ
|
2009-12-02 06:49:22 +00:00
|
|
|
#define DFLSSIZ (8*1024*1024) /* default stack size */
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef MAXSSIZ
|
2018-01-03 20:20:43 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
#define MAXSSIZ (512*1024*1024) /* max stack size */
|
|
|
|
#else
|
2009-12-02 06:49:22 +00:00
|
|
|
#define MAXSSIZ (64*1024*1024) /* max stack size */
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif
|
2018-01-03 20:20:43 +00:00
|
|
|
#endif
|
2001-06-10 02:39:37 +00:00
|
|
|
|
2011-12-11 17:23:03 +00:00
|
|
|
#ifdef AIM
|
2019-08-20 01:26:02 +00:00
|
|
|
#define VM_MAXUSER_ADDRESS32 0xfffff000
|
2011-12-11 17:23:03 +00:00
|
|
|
#else
|
2019-08-20 01:26:02 +00:00
|
|
|
#define VM_MAXUSER_ADDRESS32 0x7ffff000
|
2011-12-11 17:23:03 +00:00
|
|
|
#endif
|
|
|
|
|
2001-06-10 02:39:37 +00:00
|
|
|
/*
|
|
|
|
* Would like to have MAX addresses = 0, but this doesn't (currently) work
|
|
|
|
*/
|
2010-07-13 05:32:19 +00:00
|
|
|
#ifdef __powerpc64__
|
2020-05-11 02:33:37 +00:00
|
|
|
/*
|
|
|
|
* Virtual addresses of things. Derived from the page directory and
|
|
|
|
* page table indexes from pmap.h for precision.
|
|
|
|
*
|
|
|
|
* kernel map should be able to start at 0xc008000000000000 -
|
|
|
|
* but at least the functional simulator doesn't like it
|
|
|
|
*
|
|
|
|
* 0x0000000000000000 - 0x000fffffffffffff user map
|
|
|
|
* 0xc000000000000000 - 0xc007ffffffffffff direct map
|
|
|
|
* 0xc008000000000000 - 0xc00fffffffffffff kernel map
|
|
|
|
*
|
|
|
|
*/
|
2019-08-20 01:26:02 +00:00
|
|
|
#define VM_MIN_ADDRESS 0x0000000000000000
|
2020-05-11 02:33:37 +00:00
|
|
|
#define VM_MAXUSER_ADDRESS 0x000fffffc0000000
|
|
|
|
#define VM_MAX_ADDRESS 0xc00fffffffffffff
|
|
|
|
#define VM_MIN_KERNEL_ADDRESS 0xc008000000000000
|
|
|
|
#define VM_MAX_KERNEL_ADDRESS 0xc0080007ffffffff
|
2019-08-20 01:26:02 +00:00
|
|
|
#define VM_MAX_SAFE_KERNEL_ADDRESS VM_MAX_KERNEL_ADDRESS
|
2010-07-13 05:32:19 +00:00
|
|
|
#else
|
2019-08-20 01:26:02 +00:00
|
|
|
#define VM_MIN_ADDRESS 0
|
2011-12-11 17:23:03 +00:00
|
|
|
#define VM_MAXUSER_ADDRESS VM_MAXUSER_ADDRESS32
|
2019-08-20 01:26:02 +00:00
|
|
|
#define VM_MAX_ADDRESS 0xffffffff
|
2010-07-13 05:32:19 +00:00
|
|
|
#endif
|
2019-08-20 01:26:02 +00:00
|
|
|
|
2011-12-11 17:23:03 +00:00
|
|
|
#define SHAREDPAGE (VM_MAXUSER_ADDRESS - PAGE_SIZE)
|
2008-03-03 13:20:52 +00:00
|
|
|
|
2011-12-11 17:23:03 +00:00
|
|
|
#define FREEBSD32_SHAREDPAGE (VM_MAXUSER_ADDRESS32 - PAGE_SIZE)
|
2011-01-14 11:36:44 +00:00
|
|
|
#define FREEBSD32_USRSTACK FREEBSD32_SHAREDPAGE
|
2008-03-03 13:20:52 +00:00
|
|
|
|
2018-11-28 16:00:52 +00:00
|
|
|
#define KERNBASE 0x00100100 /* start of kernel virtual */
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
|
2019-08-20 01:26:02 +00:00
|
|
|
#ifdef AIM
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#ifndef __powerpc64__
|
2010-07-13 05:32:19 +00:00
|
|
|
#define VM_MIN_KERNEL_ADDRESS ((vm_offset_t)KERNEL_SR << ADDR_SR_SHFT)
|
2010-02-20 16:23:29 +00:00
|
|
|
#define VM_MAX_SAFE_KERNEL_ADDRESS (VM_MIN_KERNEL_ADDRESS + 2*SEGMENT_LENGTH -1)
|
|
|
|
#define VM_MAX_KERNEL_ADDRESS (VM_MIN_KERNEL_ADDRESS + 3*SEGMENT_LENGTH - 1)
|
2010-07-13 05:32:19 +00:00
|
|
|
#endif
|
2001-06-10 02:39:37 +00:00
|
|
|
|
2008-03-03 13:20:52 +00:00
|
|
|
/*
|
|
|
|
* Use the direct-mapped BAT registers for UMA small allocs. This
|
|
|
|
* takes pressure off the small amount of available KVA.
|
|
|
|
*/
|
|
|
|
#define UMA_MD_SMALL_ALLOC
|
|
|
|
|
2010-07-13 05:32:19 +00:00
|
|
|
#else /* Book-E */
|
2008-03-03 13:20:52 +00:00
|
|
|
|
2019-08-15 03:42:15 +00:00
|
|
|
/* Use the direct map for UMA small allocs on powerpc64. */
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
#define UMA_MD_SMALL_ALLOC
|
2019-08-20 01:26:02 +00:00
|
|
|
#else
|
|
|
|
#define VM_MIN_KERNEL_ADDRESS 0xc0000000
|
|
|
|
#define VM_MAX_KERNEL_ADDRESS 0xffffefff
|
2013-10-27 14:03:51 +00:00
|
|
|
#define VM_MAX_SAFE_KERNEL_ADDRESS VM_MAX_KERNEL_ADDRESS
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#endif
|
2008-03-03 13:20:52 +00:00
|
|
|
|
|
|
|
#endif /* AIM/E500 */
|
2004-02-11 07:27:34 +00:00
|
|
|
|
2008-03-03 13:20:52 +00:00
|
|
|
#if !defined(LOCORE)
|
2001-06-10 02:39:37 +00:00
|
|
|
struct pmap_physseg {
|
|
|
|
struct pv_entry *pvent;
|
|
|
|
char *attrs;
|
|
|
|
};
|
2008-03-03 13:20:52 +00:00
|
|
|
#endif
|
2001-06-10 02:39:37 +00:00
|
|
|
|
2020-05-11 02:33:37 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
#define VM_PHYSSEG_MAX 63 /* 1? */
|
|
|
|
#else
|
|
|
|
#define VM_PHYSSEG_MAX 16 /* 1? */
|
|
|
|
#endif
|
2019-08-16 00:45:14 +00:00
|
|
|
|
|
|
|
#define PHYS_AVAIL_SZ 256 /* Allows up to 16GB Ram on pSeries with
|
|
|
|
* logical memory block size of 64MB.
|
|
|
|
* For more Ram increase the lmb or this value.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* XXX This is non-sensical. Phys avail should hold contiguous regions. */
|
|
|
|
#define PHYS_AVAIL_ENTRIES PHYS_AVAIL_SZ
|
2001-06-10 02:39:37 +00:00
|
|
|
|
2007-05-05 19:50:28 +00:00
|
|
|
/*
|
2010-12-20 14:25:01 +00:00
|
|
|
* The physical address space is densely populated on 32-bit systems,
|
|
|
|
* but may not be on 64-bit ones.
|
2007-05-05 19:50:28 +00:00
|
|
|
*/
|
2016-01-16 21:24:12 +00:00
|
|
|
#ifdef __powerpc64__
|
2010-12-20 14:25:01 +00:00
|
|
|
#define VM_PHYSSEG_SPARSE
|
|
|
|
#else
|
2007-05-05 19:50:28 +00:00
|
|
|
#define VM_PHYSSEG_DENSE
|
2010-12-20 14:25:01 +00:00
|
|
|
#endif
|
2007-05-05 19:50:28 +00:00
|
|
|
|
Enable the new physical memory allocator.
This allocator uses a binary buddy system with a twist. First and
foremost, this allocator is required to support the implementation of
superpages. As a side effect, it enables a more robust implementation
of contigmalloc(9). Moreover, this reimplementation of
contigmalloc(9) eliminates the acquisition of Giant by
contigmalloc(..., M_NOWAIT, ...).
The twist is that this allocator tries to reduce the number of TLB
misses incurred by accesses through a direct map to small, UMA-managed
objects and page table pages. Roughly speaking, the physical pages
that are allocated for such purposes are clustered together in the
physical address space. The performance benefits vary. In the most
extreme case, a uniprocessor kernel running on an Opteron, I measured
an 18% reduction in system time during a buildworld.
This allocator does not implement page coloring. The reason is that
superpages have much the same effect. The contiguous physical memory
allocation necessary for a superpage is inherently colored.
Finally, the one caveat is that this allocator does not effectively
support prezeroed pages. I hope this is temporary. On i386, this is
a slight pessimization. However, on amd64, the beneficial effects of
the direct-map optimization outweigh the ill effects. I speculate
that this is true in general of machines with a direct map.
Approved by: re
2007-06-16 04:57:06 +00:00
|
|
|
/*
|
2015-06-08 04:59:32 +00:00
|
|
|
* Create two free page pools: VM_FREEPOOL_DEFAULT is the default pool
|
Enable the new physical memory allocator.
This allocator uses a binary buddy system with a twist. First and
foremost, this allocator is required to support the implementation of
superpages. As a side effect, it enables a more robust implementation
of contigmalloc(9). Moreover, this reimplementation of
contigmalloc(9) eliminates the acquisition of Giant by
contigmalloc(..., M_NOWAIT, ...).
The twist is that this allocator tries to reduce the number of TLB
misses incurred by accesses through a direct map to small, UMA-managed
objects and page table pages. Roughly speaking, the physical pages
that are allocated for such purposes are clustered together in the
physical address space. The performance benefits vary. In the most
extreme case, a uniprocessor kernel running on an Opteron, I measured
an 18% reduction in system time during a buildworld.
This allocator does not implement page coloring. The reason is that
superpages have much the same effect. The contiguous physical memory
allocation necessary for a superpage is inherently colored.
Finally, the one caveat is that this allocator does not effectively
support prezeroed pages. I hope this is temporary. On i386, this is
a slight pessimization. However, on amd64, the beneficial effects of
the direct-map optimization outweigh the ill effects. I speculate
that this is true in general of machines with a direct map.
Approved by: re
2007-06-16 04:57:06 +00:00
|
|
|
* from which physical pages are allocated and VM_FREEPOOL_DIRECT is
|
|
|
|
* the pool from which physical pages for small UMA objects are
|
|
|
|
* allocated.
|
|
|
|
*/
|
2015-06-08 04:59:32 +00:00
|
|
|
#define VM_NFREEPOOL 2
|
Enable the new physical memory allocator.
This allocator uses a binary buddy system with a twist. First and
foremost, this allocator is required to support the implementation of
superpages. As a side effect, it enables a more robust implementation
of contigmalloc(9). Moreover, this reimplementation of
contigmalloc(9) eliminates the acquisition of Giant by
contigmalloc(..., M_NOWAIT, ...).
The twist is that this allocator tries to reduce the number of TLB
misses incurred by accesses through a direct map to small, UMA-managed
objects and page table pages. Roughly speaking, the physical pages
that are allocated for such purposes are clustered together in the
physical address space. The performance benefits vary. In the most
extreme case, a uniprocessor kernel running on an Opteron, I measured
an 18% reduction in system time during a buildworld.
This allocator does not implement page coloring. The reason is that
superpages have much the same effect. The contiguous physical memory
allocation necessary for a superpage is inherently colored.
Finally, the one caveat is that this allocator does not effectively
support prezeroed pages. I hope this is temporary. On i386, this is
a slight pessimization. However, on amd64, the beneficial effects of
the direct-map optimization outweigh the ill effects. I speculate
that this is true in general of machines with a direct map.
Approved by: re
2007-06-16 04:57:06 +00:00
|
|
|
#define VM_FREEPOOL_DEFAULT 0
|
|
|
|
#define VM_FREEPOOL_DIRECT 1
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create one free page list.
|
|
|
|
*/
|
2001-06-10 02:39:37 +00:00
|
|
|
#define VM_NFREELIST 1
|
|
|
|
#define VM_FREELIST_DEFAULT 0
|
|
|
|
|
2020-05-11 02:33:37 +00:00
|
|
|
#ifdef __powerpc64__
|
2020-11-06 14:12:45 +00:00
|
|
|
/* The largest allocation size is 16MB. */
|
2020-05-11 02:33:37 +00:00
|
|
|
#define VM_NFREEORDER 13
|
|
|
|
#else
|
2020-11-06 14:12:45 +00:00
|
|
|
/* The largest allocation size is 4MB. */
|
Enable the new physical memory allocator.
This allocator uses a binary buddy system with a twist. First and
foremost, this allocator is required to support the implementation of
superpages. As a side effect, it enables a more robust implementation
of contigmalloc(9). Moreover, this reimplementation of
contigmalloc(9) eliminates the acquisition of Giant by
contigmalloc(..., M_NOWAIT, ...).
The twist is that this allocator tries to reduce the number of TLB
misses incurred by accesses through a direct map to small, UMA-managed
objects and page table pages. Roughly speaking, the physical pages
that are allocated for such purposes are clustered together in the
physical address space. The performance benefits vary. In the most
extreme case, a uniprocessor kernel running on an Opteron, I measured
an 18% reduction in system time during a buildworld.
This allocator does not implement page coloring. The reason is that
superpages have much the same effect. The contiguous physical memory
allocation necessary for a superpage is inherently colored.
Finally, the one caveat is that this allocator does not effectively
support prezeroed pages. I hope this is temporary. On i386, this is
a slight pessimization. However, on amd64, the beneficial effects of
the direct-map optimization outweigh the ill effects. I speculate
that this is true in general of machines with a direct map.
Approved by: re
2007-06-16 04:57:06 +00:00
|
|
|
#define VM_NFREEORDER 11
|
2020-05-11 02:33:37 +00:00
|
|
|
#endif
|
Enable the new physical memory allocator.
This allocator uses a binary buddy system with a twist. First and
foremost, this allocator is required to support the implementation of
superpages. As a side effect, it enables a more robust implementation
of contigmalloc(9). Moreover, this reimplementation of
contigmalloc(9) eliminates the acquisition of Giant by
contigmalloc(..., M_NOWAIT, ...).
The twist is that this allocator tries to reduce the number of TLB
misses incurred by accesses through a direct map to small, UMA-managed
objects and page table pages. Roughly speaking, the physical pages
that are allocated for such purposes are clustered together in the
physical address space. The performance benefits vary. In the most
extreme case, a uniprocessor kernel running on an Opteron, I measured
an 18% reduction in system time during a buildworld.
This allocator does not implement page coloring. The reason is that
superpages have much the same effect. The contiguous physical memory
allocation necessary for a superpage is inherently colored.
Finally, the one caveat is that this allocator does not effectively
support prezeroed pages. I hope this is temporary. On i386, this is
a slight pessimization. However, on amd64, the beneficial effects of
the direct-map optimization outweigh the ill effects. I speculate
that this is true in general of machines with a direct map.
Approved by: re
2007-06-16 04:57:06 +00:00
|
|
|
|
2020-05-11 02:33:37 +00:00
|
|
|
#ifndef VM_NRESERVLEVEL
|
|
|
|
#ifdef __powerpc64__
|
2020-11-06 14:12:45 +00:00
|
|
|
/* Enable superpage reservations: 1 level. */
|
2020-05-11 02:33:37 +00:00
|
|
|
#define VM_NRESERVLEVEL 1
|
|
|
|
#else
|
2020-11-06 14:12:45 +00:00
|
|
|
/* Disable superpage reservations. */
|
2007-12-27 16:45:39 +00:00
|
|
|
#define VM_NRESERVLEVEL 0
|
|
|
|
#endif
|
2020-05-11 02:33:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef VM_LEVEL_0_ORDER
|
2020-11-06 14:12:45 +00:00
|
|
|
/* Level 0 reservations consist of 512 (RPT) or 4096 (HPT) pages. */
|
|
|
|
#define VM_LEVEL_0_ORDER vm_level_0_order
|
|
|
|
#ifndef __ASSEMBLER__
|
|
|
|
extern int vm_level_0_order;
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef VM_LEVEL_0_ORDER_MAX
|
|
|
|
#define VM_LEVEL_0_ORDER_MAX 12
|
2020-05-11 02:33:37 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
#ifdef SMP
|
|
|
|
#define PA_LOCK_COUNT 256
|
|
|
|
#endif
|
|
|
|
#endif
|
2007-12-27 16:45:39 +00:00
|
|
|
|
2001-06-10 02:39:37 +00:00
|
|
|
#ifndef VM_INITIAL_PAGEIN
|
|
|
|
#define VM_INITIAL_PAGEIN 16
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef SGROWSIZ
|
|
|
|
#define SGROWSIZ (128UL*1024) /* amount to grow stack */
|
|
|
|
#endif
|
|
|
|
|
As of r257209, all architectures have defined VM_KMEM_SIZE_SCALE. In other
words, every architecture is now auto-sizing the kmem arena. This revision
changes kmeminit() so that the definition of VM_KMEM_SIZE_SCALE becomes
mandatory and the definition of VM_KMEM_SIZE becomes optional.
Replace or eliminate all existing definitions of VM_KMEM_SIZE. With
auto-sizing enabled, VM_KMEM_SIZE effectively became an alternate spelling
for VM_KMEM_SIZE_MIN on most architectures. Use VM_KMEM_SIZE_MIN for
clarity.
Change kmeminit() so that the effect of defining VM_KMEM_SIZE is similar to
that of setting the tunable vm.kmem_size. Whereas the macros
VM_KMEM_SIZE_{MAX,MIN,SCALE} have had the same effect as the tunables
vm.kmem_size_{max,min,scale}, the effects of VM_KMEM_SIZE and vm.kmem_size
have been distinct. In particular, whereas VM_KMEM_SIZE was overridden by
VM_KMEM_SIZE_{MAX,MIN,SCALE} and vm.kmem_size_{max,min,scale}, vm.kmem_size
was not. Remedy this inconsistency. Now, VM_KMEM_SIZE can be used to set
the size of the kmem arena at compile-time without that value being
overridden by auto-sizing.
Update the nearby comments to reflect the kmem submap being replaced by the
kmem arena. Stop duplicating the auto-sizing formula in every machine-
dependent vmparam.h and place it in kmeminit() where auto-sizing takes
place.
Reviewed by: kib (an earlier version)
Sponsored by: EMC / Isilon Storage Division
2013-11-08 16:25:00 +00:00
|
|
|
/*
|
|
|
|
* How many physical pages per kmem arena virtual page.
|
|
|
|
*/
|
|
|
|
#ifndef VM_KMEM_SIZE_SCALE
|
|
|
|
#define VM_KMEM_SIZE_SCALE (3)
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif
|
|
|
|
|
2013-10-27 14:03:51 +00:00
|
|
|
/*
|
As of r257209, all architectures have defined VM_KMEM_SIZE_SCALE. In other
words, every architecture is now auto-sizing the kmem arena. This revision
changes kmeminit() so that the definition of VM_KMEM_SIZE_SCALE becomes
mandatory and the definition of VM_KMEM_SIZE becomes optional.
Replace or eliminate all existing definitions of VM_KMEM_SIZE. With
auto-sizing enabled, VM_KMEM_SIZE effectively became an alternate spelling
for VM_KMEM_SIZE_MIN on most architectures. Use VM_KMEM_SIZE_MIN for
clarity.
Change kmeminit() so that the effect of defining VM_KMEM_SIZE is similar to
that of setting the tunable vm.kmem_size. Whereas the macros
VM_KMEM_SIZE_{MAX,MIN,SCALE} have had the same effect as the tunables
vm.kmem_size_{max,min,scale}, the effects of VM_KMEM_SIZE and vm.kmem_size
have been distinct. In particular, whereas VM_KMEM_SIZE was overridden by
VM_KMEM_SIZE_{MAX,MIN,SCALE} and vm.kmem_size_{max,min,scale}, vm.kmem_size
was not. Remedy this inconsistency. Now, VM_KMEM_SIZE can be used to set
the size of the kmem arena at compile-time without that value being
overridden by auto-sizing.
Update the nearby comments to reflect the kmem submap being replaced by the
kmem arena. Stop duplicating the auto-sizing formula in every machine-
dependent vmparam.h and place it in kmeminit() where auto-sizing takes
place.
Reviewed by: kib (an earlier version)
Sponsored by: EMC / Isilon Storage Division
2013-11-08 16:25:00 +00:00
|
|
|
* Optional floor (in bytes) on the size of the kmem arena.
|
2013-10-27 14:03:51 +00:00
|
|
|
*/
|
As of r257209, all architectures have defined VM_KMEM_SIZE_SCALE. In other
words, every architecture is now auto-sizing the kmem arena. This revision
changes kmeminit() so that the definition of VM_KMEM_SIZE_SCALE becomes
mandatory and the definition of VM_KMEM_SIZE becomes optional.
Replace or eliminate all existing definitions of VM_KMEM_SIZE. With
auto-sizing enabled, VM_KMEM_SIZE effectively became an alternate spelling
for VM_KMEM_SIZE_MIN on most architectures. Use VM_KMEM_SIZE_MIN for
clarity.
Change kmeminit() so that the effect of defining VM_KMEM_SIZE is similar to
that of setting the tunable vm.kmem_size. Whereas the macros
VM_KMEM_SIZE_{MAX,MIN,SCALE} have had the same effect as the tunables
vm.kmem_size_{max,min,scale}, the effects of VM_KMEM_SIZE and vm.kmem_size
have been distinct. In particular, whereas VM_KMEM_SIZE was overridden by
VM_KMEM_SIZE_{MAX,MIN,SCALE} and vm.kmem_size_{max,min,scale}, vm.kmem_size
was not. Remedy this inconsistency. Now, VM_KMEM_SIZE can be used to set
the size of the kmem arena at compile-time without that value being
overridden by auto-sizing.
Update the nearby comments to reflect the kmem submap being replaced by the
kmem arena. Stop duplicating the auto-sizing formula in every machine-
dependent vmparam.h and place it in kmeminit() where auto-sizing takes
place.
Reviewed by: kib (an earlier version)
Sponsored by: EMC / Isilon Storage Division
2013-11-08 16:25:00 +00:00
|
|
|
#ifndef VM_KMEM_SIZE_MIN
|
|
|
|
#define VM_KMEM_SIZE_MIN (12 * 1024 * 1024)
|
2010-07-13 05:32:19 +00:00
|
|
|
#endif
|
|
|
|
|
2013-10-27 14:03:51 +00:00
|
|
|
/*
|
As of r257209, all architectures have defined VM_KMEM_SIZE_SCALE. In other
words, every architecture is now auto-sizing the kmem arena. This revision
changes kmeminit() so that the definition of VM_KMEM_SIZE_SCALE becomes
mandatory and the definition of VM_KMEM_SIZE becomes optional.
Replace or eliminate all existing definitions of VM_KMEM_SIZE. With
auto-sizing enabled, VM_KMEM_SIZE effectively became an alternate spelling
for VM_KMEM_SIZE_MIN on most architectures. Use VM_KMEM_SIZE_MIN for
clarity.
Change kmeminit() so that the effect of defining VM_KMEM_SIZE is similar to
that of setting the tunable vm.kmem_size. Whereas the macros
VM_KMEM_SIZE_{MAX,MIN,SCALE} have had the same effect as the tunables
vm.kmem_size_{max,min,scale}, the effects of VM_KMEM_SIZE and vm.kmem_size
have been distinct. In particular, whereas VM_KMEM_SIZE was overridden by
VM_KMEM_SIZE_{MAX,MIN,SCALE} and vm.kmem_size_{max,min,scale}, vm.kmem_size
was not. Remedy this inconsistency. Now, VM_KMEM_SIZE can be used to set
the size of the kmem arena at compile-time without that value being
overridden by auto-sizing.
Update the nearby comments to reflect the kmem submap being replaced by the
kmem arena. Stop duplicating the auto-sizing formula in every machine-
dependent vmparam.h and place it in kmeminit() where auto-sizing takes
place.
Reviewed by: kib (an earlier version)
Sponsored by: EMC / Isilon Storage Division
2013-11-08 16:25:00 +00:00
|
|
|
* Optional ceiling (in bytes) on the size of the kmem arena: 40% of the
|
|
|
|
* usable KVA space.
|
2013-10-27 14:03:51 +00:00
|
|
|
*/
|
2010-07-13 05:32:19 +00:00
|
|
|
#ifndef VM_KMEM_SIZE_MAX
|
2013-10-27 14:03:51 +00:00
|
|
|
#define VM_KMEM_SIZE_MAX ((VM_MAX_SAFE_KERNEL_ADDRESS - \
|
|
|
|
VM_MIN_KERNEL_ADDRESS + 1) * 2 / 5)
|
2010-07-13 05:32:19 +00:00
|
|
|
#endif
|
|
|
|
|
2020-05-11 02:33:37 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
#define ZERO_REGION_SIZE (2 * 1024 * 1024) /* 2MB */
|
|
|
|
#else
|
2011-05-13 19:35:01 +00:00
|
|
|
#define ZERO_REGION_SIZE (64 * 1024) /* 64KB */
|
2020-05-11 02:33:37 +00:00
|
|
|
#endif
|
|
|
|
|
2014-08-05 09:44:10 +00:00
|
|
|
/*
|
|
|
|
* On 32-bit OEA, the only purpose for which sf_buf is used is to implement
|
|
|
|
* an opaque pointer required by the machine-independent parts of the kernel.
|
|
|
|
* That pointer references the vm_page that is "mapped" by the sf_buf. The
|
|
|
|
* actual mapping is provided by the direct virtual-to-physical mapping.
|
|
|
|
*
|
|
|
|
* On OEA64 and Book-E, we need to do something a little more complicated. Use
|
|
|
|
* the runtime-detected hw_direct_map to pick between the two cases. Our
|
|
|
|
* friends in vm_machdep.c will do the same to ensure nothing gets confused.
|
|
|
|
*/
|
|
|
|
#define SFBUF
|
|
|
|
#define SFBUF_NOMD
|
2018-01-13 23:14:53 +00:00
|
|
|
|
|
|
|
/*
|
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
2018-01-19 17:46:31 +00:00
|
|
|
* We (usually) have a direct map of all physical memory, so provide
|
2018-03-07 17:08:07 +00:00
|
|
|
* a macro to use to get the kernel VA address for a given PA. Check the
|
|
|
|
* value of PMAP_HAS_PMAP before using.
|
2018-01-13 23:14:53 +00:00
|
|
|
*/
|
2018-03-07 17:08:07 +00:00
|
|
|
#ifndef LOCORE
|
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
2018-01-19 17:46:31 +00:00
|
|
|
#ifdef __powerpc64__
|
2018-03-07 17:08:07 +00:00
|
|
|
#define DMAP_BASE_ADDRESS 0xc000000000000000UL
|
2020-05-11 02:33:37 +00:00
|
|
|
#define DMAP_MIN_ADDRESS DMAP_BASE_ADDRESS
|
|
|
|
#define DMAP_MAX_ADDRESS 0xc007ffffffffffffUL
|
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
2018-01-19 17:46:31 +00:00
|
|
|
#else
|
|
|
|
#define DMAP_BASE_ADDRESS 0x00000000UL
|
2018-03-07 17:08:07 +00:00
|
|
|
#define DMAP_MAX_ADDRESS 0xbfffffffUL
|
|
|
|
#endif
|
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
2018-01-19 17:46:31 +00:00
|
|
|
#endif
|
|
|
|
|
powerpc/pmap: NUMA-ize vm_page_array on powerpc
Summary:
This matches r351198 from amd64. This only applies to AIM64 and Book-E.
On AIM64 it short-circuits with one domain, to behave similar to
existing. Otherwise it will allocate 16MB huge pages to hold the page
array, across all NUMA domains. On the first domain it will shift the
page array base up, to "upper-align" the page array in that domain, so
as to reduce the number of pages from the next domain appearing in this
domain. After the first domain, subsequent domains will be allocated in
full 16MB pages, until the final domain, which can be short. This means
some inner domains may have pages accounted in earlier domains.
On Book-E the page array is setup at MMU bootstrap time so that it's
always mapped in TLB1, on both 32-bit and 64-bit. This reduces the TLB0
overhead for touching the vm_page_array, which reduces up to one TLB
miss per array access.
Since page_range (vm_page_startup()) is no longer used on Book-E but is on
32-bit AIM, mark the variable as potentially unused, rather than using a
nasty #if defined() list.
Reviewed by: luporl
Differential Revision: https://reviews.freebsd.org/D21449
2019-12-07 03:34:03 +00:00
|
|
|
#if defined(__powerpc64__) || defined(BOOKE)
|
|
|
|
/*
|
|
|
|
* powerpc64 and Book-E will provide their own page array allocators.
|
|
|
|
*
|
|
|
|
* On AIM, this will allocate a single virtual array, with pages from the
|
|
|
|
* correct memory domains.
|
|
|
|
* On Book-E this will let us put the array in TLB1, removing the need for TLB
|
|
|
|
* thrashing.
|
|
|
|
*
|
|
|
|
* VM_MIN_KERNEL_ADDRESS is just a dummy. It will get set by the MMU driver.
|
|
|
|
*/
|
|
|
|
#define PA_MIN_ADDRESS VM_MIN_KERNEL_ADDRESS
|
|
|
|
#define PMAP_HAS_PAGE_ARRAY 1
|
|
|
|
#endif
|
|
|
|
|
2020-09-21 22:20:37 +00:00
|
|
|
#if defined(__powerpc64__)
|
|
|
|
/*
|
|
|
|
* Need a page dump array for minidump.
|
|
|
|
*/
|
|
|
|
#define MINIDUMP_PAGE_TRACKING 1
|
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* No minidump with 32-bit powerpc.
|
|
|
|
*/
|
|
|
|
#define MINIDUMP_PAGE_TRACKING 0
|
|
|
|
#endif
|
|
|
|
|
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
2018-01-19 17:46:31 +00:00
|
|
|
#define PMAP_HAS_DMAP (hw_direct_map)
|
|
|
|
#define PHYS_TO_DMAP(x) ({ \
|
|
|
|
KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \
|
|
|
|
(x) | DMAP_BASE_ADDRESS; })
|
|
|
|
#define DMAP_TO_PHYS(x) ({ \
|
|
|
|
KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \
|
|
|
|
(x) &~ DMAP_BASE_ADDRESS; })
|
|
|
|
|
2020-09-23 19:34:21 +00:00
|
|
|
/*
|
|
|
|
* No non-transparent large page support in the pmap.
|
|
|
|
*/
|
|
|
|
#define PMAP_HAS_LARGEPAGES 0
|
|
|
|
|
2001-06-10 02:39:37 +00:00
|
|
|
#endif /* _MACHINE_VMPARAM_H_ */
|