2008-03-03 17:17:00 +00:00
|
|
|
/*-
|
2017-11-27 15:09:59 +00:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
|
|
*
|
2009-05-21 11:43:37 +00:00
|
|
|
* Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
|
2008-03-03 17:17:00 +00:00
|
|
|
* Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
|
|
|
|
* NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
|
|
|
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
|
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
|
|
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
|
|
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
|
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* Some hw specific parts of this pmap were derived or influenced
|
|
|
|
* by NetBSD's ibm4xx pmap module. More generic code is shared with
|
|
|
|
* a few other pmap modules from the FreeBSD tree.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* VM layout notes:
|
|
|
|
*
|
|
|
|
* Kernel and user threads run within one common virtual address space
|
|
|
|
* defined by AS=0.
|
|
|
|
*
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
* 32-bit pmap:
|
2008-03-03 17:17:00 +00:00
|
|
|
* Virtual address space layout:
|
|
|
|
* -----------------------------
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
* 0x0000_0000 - 0x7fff_ffff : user process
|
|
|
|
* 0x8000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.)
|
2009-01-13 16:15:49 +00:00
|
|
|
* 0xc000_0000 - 0xc0ff_ffff : kernel reserved
|
2009-04-04 22:01:43 +00:00
|
|
|
* 0xc000_0000 - data_end : kernel code+data, env, metadata etc.
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
* 0xc100_0000 - 0xffff_ffff : KVA
|
2009-01-13 16:15:49 +00:00
|
|
|
* 0xc100_0000 - 0xc100_3fff : reserved for page zero/copy
|
|
|
|
* 0xc100_4000 - 0xc200_3fff : reserved for ptbl bufs
|
|
|
|
* 0xc200_4000 - 0xc200_8fff : guard page + kstack0
|
|
|
|
* 0xc200_9000 - 0xfeef_ffff : actual free KVA space
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
*
|
|
|
|
* 64-bit pmap:
|
|
|
|
* Virtual address space layout:
|
|
|
|
* -----------------------------
|
|
|
|
* 0x0000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : user process
|
|
|
|
* 0x0000_0000_0000_0000 - 0x8fff_ffff_ffff_ffff : text, data, heap, maps, libraries
|
|
|
|
* 0x9000_0000_0000_0000 - 0xafff_ffff_ffff_ffff : mmio region
|
|
|
|
* 0xb000_0000_0000_0000 - 0xbfff_ffff_ffff_ffff : stack
|
|
|
|
* 0xc000_0000_0000_0000 - 0xcfff_ffff_ffff_ffff : kernel reserved
|
|
|
|
* 0xc000_0000_0000_0000 - endkernel-1 : kernel code & data
|
|
|
|
* endkernel - msgbufp-1 : flat device tree
|
2019-08-06 03:16:06 +00:00
|
|
|
* msgbufp - kernel_pdir-1 : message buffer
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
* kernel_pdir - kernel_pp2d-1 : kernel page directory
|
|
|
|
* kernel_pp2d - . : kernel pointers to page directory
|
|
|
|
* pmap_zero_copy_min - crashdumpmap-1 : reserved for page zero/copy
|
|
|
|
* crashdumpmap - ptbl_buf_pool_vabase-1 : reserved for ptbl bufs
|
|
|
|
* ptbl_buf_pool_vabase - virtual_avail-1 : user page directories and page tables
|
|
|
|
* virtual_avail - 0xcfff_ffff_ffff_ffff : actual free KVA space
|
|
|
|
* 0xd000_0000_0000_0000 - 0xdfff_ffff_ffff_ffff : coprocessor region
|
|
|
|
* 0xe000_0000_0000_0000 - 0xefff_ffff_ffff_ffff : mmio region
|
|
|
|
* 0xf000_0000_0000_0000 - 0xffff_ffff_ffff_ffff : direct map
|
|
|
|
* 0xf000_0000_0000_0000 - +Maxmem : physmem map
|
|
|
|
* - 0xffff_ffff_ffff_ffff : device direct map
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2018-10-22 00:21:27 +00:00
|
|
|
#include "opt_ddb.h"
|
2015-07-16 10:46:52 +00:00
|
|
|
#include "opt_kstack_pages.h"
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <sys/param.h>
|
2015-01-07 01:01:39 +00:00
|
|
|
#include <sys/conf.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <sys/malloc.h>
|
2009-01-13 15:41:58 +00:00
|
|
|
#include <sys/ktr.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <sys/proc.h>
|
|
|
|
#include <sys/user.h>
|
|
|
|
#include <sys/queue.h>
|
|
|
|
#include <sys/systm.h>
|
|
|
|
#include <sys/kernel.h>
|
2015-01-07 01:01:39 +00:00
|
|
|
#include <sys/kerneldump.h>
|
2011-08-02 15:35:43 +00:00
|
|
|
#include <sys/linker.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <sys/msgbuf.h>
|
|
|
|
#include <sys/lock.h>
|
|
|
|
#include <sys/mutex.h>
|
2012-11-03 23:22:49 +00:00
|
|
|
#include <sys/rwlock.h>
|
2011-05-17 22:03:01 +00:00
|
|
|
#include <sys/sched.h>
|
2009-05-21 11:43:37 +00:00
|
|
|
#include <sys/smp.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <sys/vmmeter.h>
|
|
|
|
|
|
|
|
#include <vm/vm.h>
|
2020-10-14 22:51:40 +00:00
|
|
|
#include <vm/vm_param.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <vm/vm_page.h>
|
|
|
|
#include <vm/vm_kern.h>
|
|
|
|
#include <vm/vm_pageout.h>
|
|
|
|
#include <vm/vm_extern.h>
|
|
|
|
#include <vm/vm_object.h>
|
|
|
|
#include <vm/vm_map.h>
|
|
|
|
#include <vm/vm_pager.h>
|
2018-03-20 01:07:22 +00:00
|
|
|
#include <vm/vm_phys.h>
|
|
|
|
#include <vm/vm_pagequeue.h>
|
2020-10-14 22:51:40 +00:00
|
|
|
#include <vm/vm_dumpset.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <vm/uma.h>
|
|
|
|
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#include <machine/_inttypes.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
#include <machine/cpu.h>
|
|
|
|
#include <machine/pcb.h>
|
2009-05-14 00:34:26 +00:00
|
|
|
#include <machine/platform.h>
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
#include <machine/tlb.h>
|
|
|
|
#include <machine/spr.h>
|
|
|
|
#include <machine/md_var.h>
|
|
|
|
#include <machine/mmuvar.h>
|
|
|
|
#include <machine/pmap.h>
|
|
|
|
#include <machine/pte.h>
|
|
|
|
|
2018-10-22 00:21:27 +00:00
|
|
|
#include <ddb/ddb.h>
|
|
|
|
|
2016-04-10 15:48:09 +00:00
|
|
|
#define SPARSE_MAPDEV
|
2020-04-11 00:17:55 +00:00
|
|
|
|
|
|
|
/* Use power-of-two mappings in mmu_booke_mapdev(), to save entries. */
|
|
|
|
#define POW2_MAPPINGS
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
#ifdef DEBUG
|
|
|
|
#define debugf(fmt, args...) printf(fmt, ##args)
|
|
|
|
#else
|
|
|
|
#define debugf(fmt, args...)
|
|
|
|
#endif
|
|
|
|
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
#define PRI0ptrX "016lx"
|
|
|
|
#else
|
|
|
|
#define PRI0ptrX "08x"
|
|
|
|
#endif
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
#define TODO panic("%s: not implemented", __func__);
|
|
|
|
|
2009-04-04 22:01:43 +00:00
|
|
|
extern unsigned char _etext[];
|
|
|
|
extern unsigned char _end[];
|
|
|
|
|
2011-08-02 15:35:43 +00:00
|
|
|
extern uint32_t *bootinfo;
|
|
|
|
|
|
|
|
vm_paddr_t kernload;
|
2009-04-04 22:01:43 +00:00
|
|
|
vm_offset_t kernstart;
|
|
|
|
vm_size_t kernsize;
|
|
|
|
|
|
|
|
/* Message buffer and tables. */
|
|
|
|
static vm_offset_t data_start;
|
|
|
|
static vm_size_t data_end;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-05-14 00:34:26 +00:00
|
|
|
/* Phys/avail memory regions. */
|
|
|
|
static struct mem_region *availmem_regions;
|
|
|
|
static int availmem_regions_sz;
|
|
|
|
static struct mem_region *physmem_regions;
|
|
|
|
static int physmem_regions_sz;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2019-08-25 20:11:35 +00:00
|
|
|
#ifndef __powerpc64__
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Reserved KVA space and mutex for mmu_booke_zero_page. */
|
|
|
|
static vm_offset_t zero_page_va;
|
|
|
|
static struct mtx zero_page_mutex;
|
|
|
|
|
|
|
|
/* Reserved KVA space and mutex for mmu_booke_copy_page. */
|
|
|
|
static vm_offset_t copy_page_src_va;
|
|
|
|
static vm_offset_t copy_page_dst_va;
|
|
|
|
static struct mtx copy_page_mutex;
|
2019-08-25 20:11:35 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
static struct mtx tlbivax_mutex;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/**************************************************************************/
|
|
|
|
/* PMAP */
|
|
|
|
/**************************************************************************/
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
static int mmu_booke_enter_locked(pmap_t, vm_offset_t, vm_page_t,
|
2014-08-08 17:12:03 +00:00
|
|
|
vm_prot_t, u_int flags, int8_t psind);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
unsigned int kptbl_min; /* Index of the first kernel ptbl. */
|
2019-08-12 03:03:56 +00:00
|
|
|
static uma_zone_t ptbl_root_zone;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If user pmap is processed with mmu_booke_remove and the resident count
|
|
|
|
* drops to 0, there are no more pages to remove, so we need not continue.
|
|
|
|
*/
|
|
|
|
#define PMAP_REMOVE_DONE(pmap) \
|
|
|
|
((pmap) != kernel_pmap && (pmap)->pm_stats.resident_count == 0)
|
|
|
|
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#if defined(COMPAT_FREEBSD32) || !defined(__powerpc64__)
|
2015-03-01 21:47:38 +00:00
|
|
|
extern int elf32_nxstack;
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#endif
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/**************************************************************************/
|
|
|
|
/* TLB and TID handling */
|
|
|
|
/**************************************************************************/
|
|
|
|
|
|
|
|
/* Translation ID busy table */
|
2009-01-13 15:41:58 +00:00
|
|
|
static volatile pmap_t tidbusy[MAXCPU][TID_MAX + 1];
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/*
|
2009-01-13 15:41:58 +00:00
|
|
|
* TLB0 capabilities (entry, way numbers etc.). These can vary between e500
|
|
|
|
* core revisions and should be read from h/w registers during early config.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
2009-01-13 15:41:58 +00:00
|
|
|
uint32_t tlb0_entries;
|
|
|
|
uint32_t tlb0_ways;
|
|
|
|
uint32_t tlb0_entries_per_way;
|
2015-08-28 03:03:09 +00:00
|
|
|
uint32_t tlb1_entries;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
#define TLB0_ENTRIES (tlb0_entries)
|
|
|
|
#define TLB0_WAYS (tlb0_ways)
|
|
|
|
#define TLB0_ENTRIES_PER_WAY (tlb0_entries_per_way)
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2015-08-28 03:03:09 +00:00
|
|
|
#define TLB1_ENTRIES (tlb1_entries)
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
static tlbtid_t tid_alloc(struct pmap *);
|
|
|
|
|
2018-10-22 00:21:27 +00:00
|
|
|
#ifdef DDB
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
static void tlb_print_entry(int, uint32_t, uint64_t, uint32_t, uint32_t);
|
|
|
|
#else
|
2009-01-13 15:41:58 +00:00
|
|
|
static void tlb_print_entry(int, uint32_t, uint32_t, uint32_t, uint32_t);
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#endif
|
2018-10-22 00:21:27 +00:00
|
|
|
#endif
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
static void tlb1_read_entry(tlb_entry_t *, unsigned int);
|
|
|
|
static void tlb1_write_entry(tlb_entry_t *, unsigned int);
|
2008-03-03 17:17:00 +00:00
|
|
|
static int tlb1_iomapped(int, vm_paddr_t, vm_size_t, vm_offset_t *);
|
2019-11-04 00:35:40 +00:00
|
|
|
static vm_size_t tlb1_mapin_region(vm_offset_t, vm_paddr_t, vm_size_t, int);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
static __inline uint32_t tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma);
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
static vm_size_t tsize2size(unsigned int);
|
|
|
|
static unsigned int size2tsize(vm_size_t);
|
2019-11-04 00:35:40 +00:00
|
|
|
static unsigned long ilog2(unsigned long);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
static void set_mas4_defaults(void);
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
static inline void tlb0_flush_entry(vm_offset_t);
|
2008-03-03 17:17:00 +00:00
|
|
|
static inline unsigned int tlb0_tableidx(vm_offset_t, unsigned int);
|
|
|
|
|
|
|
|
/**************************************************************************/
|
|
|
|
/* Page table management */
|
|
|
|
/**************************************************************************/
|
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
static struct rwlock_padalign pvh_global_lock;
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Data for the pv entry allocation mechanism */
|
|
|
|
static uma_zone_t pvzone;
|
|
|
|
static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
|
|
|
|
|
|
|
|
#define PV_ENTRY_ZONE_MIN 2048 /* min pv entries in uma zone */
|
|
|
|
|
|
|
|
#ifndef PMAP_SHPGPERPROC
|
|
|
|
#define PMAP_SHPGPERPROC 200
|
|
|
|
#endif
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
|
|
|
|
static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t);
|
|
|
|
static int pte_remove(pmap_t, vm_offset_t, uint8_t);
|
|
|
|
static pte_t *pte_find(pmap_t, vm_offset_t);
|
2020-04-11 00:12:34 +00:00
|
|
|
static void kernel_pte_alloc(vm_offset_t, vm_offset_t);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
static pv_entry_t pv_alloc(void);
|
2008-03-03 17:17:00 +00:00
|
|
|
static void pv_free(pv_entry_t);
|
|
|
|
static void pv_insert(pmap_t, vm_offset_t, vm_page_t);
|
|
|
|
static void pv_remove(pmap_t, vm_offset_t, vm_page_t);
|
|
|
|
|
2015-08-04 19:46:13 +00:00
|
|
|
static void booke_pmap_init_qpages(void);
|
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
static inline void tlb_miss_lock(void);
|
|
|
|
static inline void tlb_miss_unlock(void);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2015-03-28 21:39:42 +00:00
|
|
|
#ifdef SMP
|
2016-04-19 01:48:18 +00:00
|
|
|
extern tlb_entry_t __boot_tlb1[];
|
2009-05-21 11:43:37 +00:00
|
|
|
void pmap_bootstrap_ap(volatile uint32_t *);
|
2015-03-28 21:39:42 +00:00
|
|
|
#endif
|
2009-05-21 11:43:37 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Kernel MMU interface
|
|
|
|
*/
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_clear_modify(vm_page_t);
|
|
|
|
static void mmu_booke_copy(pmap_t, pmap_t, vm_offset_t,
|
2009-06-13 08:57:04 +00:00
|
|
|
vm_size_t, vm_offset_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_copy_page(vm_page_t, vm_page_t);
|
|
|
|
static void mmu_booke_copy_pages(vm_page_t *,
|
Add pmap function pmap_copy_pages(), which copies the content of the
pages around, taking array of vm_page_t both for source and
destination. Starting offsets and total transfer size are specified.
The function implements optimal algorithm for copying using the
platform-specific optimizations. For instance, on the architectures
were the direct map is available, no transient mappings are created,
for i386 the per-cpu ephemeral page frame is used. The code was
typically borrowed from the pmap_copy_page() for the same
architecture.
Only i386/amd64, powerpc aim and arm/arm-v6 implementations were
tested at the time of commit. High-level code, not committed yet to
the tree, ensures that the use of the function is only allowed after
explicit enablement.
For sparc64, the existing code has known issues and a stab is added
instead, to allow the kernel linking.
Sponsored by: The FreeBSD Foundation
Tested by: pho (i386, amd64), scottl (amd64), ian (arm and arm-v6)
MFC after: 2 weeks
2013-03-14 20:18:12 +00:00
|
|
|
vm_offset_t, vm_page_t *, vm_offset_t, int);
|
2020-05-27 01:24:12 +00:00
|
|
|
static int mmu_booke_enter(pmap_t, vm_offset_t, vm_page_t,
|
2014-08-08 17:12:03 +00:00
|
|
|
vm_prot_t, u_int flags, int8_t psind);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_enter_object(pmap_t, vm_offset_t, vm_offset_t,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_page_t, vm_prot_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_enter_quick(pmap_t, vm_offset_t, vm_page_t,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_prot_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static vm_paddr_t mmu_booke_extract(pmap_t, vm_offset_t);
|
|
|
|
static vm_page_t mmu_booke_extract_and_hold(pmap_t, vm_offset_t,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_prot_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_init(void);
|
|
|
|
static boolean_t mmu_booke_is_modified(vm_page_t);
|
|
|
|
static boolean_t mmu_booke_is_prefaultable(pmap_t, vm_offset_t);
|
|
|
|
static boolean_t mmu_booke_is_referenced(vm_page_t);
|
|
|
|
static int mmu_booke_ts_referenced(vm_page_t);
|
|
|
|
static vm_offset_t mmu_booke_map(vm_offset_t *, vm_paddr_t, vm_paddr_t,
|
2008-03-03 17:17:00 +00:00
|
|
|
int);
|
2020-05-27 01:24:12 +00:00
|
|
|
static int mmu_booke_mincore(pmap_t, vm_offset_t,
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
vm_paddr_t *);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_object_init_pt(pmap_t, vm_offset_t,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_object_t, vm_pindex_t, vm_size_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static boolean_t mmu_booke_page_exists_quick(pmap_t, vm_page_t);
|
|
|
|
static void mmu_booke_page_init(vm_page_t);
|
|
|
|
static int mmu_booke_page_wired_mappings(vm_page_t);
|
|
|
|
static int mmu_booke_pinit(pmap_t);
|
|
|
|
static void mmu_booke_pinit0(pmap_t);
|
|
|
|
static void mmu_booke_protect(pmap_t, vm_offset_t, vm_offset_t,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_prot_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_qenter(vm_offset_t, vm_page_t *, int);
|
|
|
|
static void mmu_booke_qremove(vm_offset_t, int);
|
|
|
|
static void mmu_booke_release(pmap_t);
|
|
|
|
static void mmu_booke_remove(pmap_t, vm_offset_t, vm_offset_t);
|
|
|
|
static void mmu_booke_remove_all(vm_page_t);
|
|
|
|
static void mmu_booke_remove_write(vm_page_t);
|
|
|
|
static void mmu_booke_unwire(pmap_t, vm_offset_t, vm_offset_t);
|
|
|
|
static void mmu_booke_zero_page(vm_page_t);
|
|
|
|
static void mmu_booke_zero_page_area(vm_page_t, int, int);
|
|
|
|
static void mmu_booke_activate(struct thread *);
|
|
|
|
static void mmu_booke_deactivate(struct thread *);
|
|
|
|
static void mmu_booke_bootstrap(vm_offset_t, vm_offset_t);
|
|
|
|
static void *mmu_booke_mapdev(vm_paddr_t, vm_size_t);
|
|
|
|
static void *mmu_booke_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
|
|
|
|
static void mmu_booke_unmapdev(vm_offset_t, vm_size_t);
|
|
|
|
static vm_paddr_t mmu_booke_kextract(vm_offset_t);
|
|
|
|
static void mmu_booke_kenter(vm_offset_t, vm_paddr_t);
|
|
|
|
static void mmu_booke_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t);
|
|
|
|
static void mmu_booke_kremove(vm_offset_t);
|
|
|
|
static boolean_t mmu_booke_dev_direct_mapped(vm_paddr_t, vm_size_t);
|
|
|
|
static void mmu_booke_sync_icache(pmap_t, vm_offset_t,
|
2009-10-21 18:38:02 +00:00
|
|
|
vm_size_t);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_dumpsys_map(vm_paddr_t pa, size_t,
|
2015-01-07 01:01:39 +00:00
|
|
|
void **);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_dumpsys_unmap(vm_paddr_t pa, size_t,
|
2015-01-07 01:01:39 +00:00
|
|
|
void *);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_scan_init(void);
|
|
|
|
static vm_offset_t mmu_booke_quick_enter_page(vm_page_t m);
|
|
|
|
static void mmu_booke_quick_remove_page(vm_offset_t addr);
|
|
|
|
static int mmu_booke_change_attr(vm_offset_t addr,
|
2016-02-27 20:39:36 +00:00
|
|
|
vm_size_t sz, vm_memattr_t mode);
|
2020-05-27 01:24:12 +00:00
|
|
|
static int mmu_booke_decode_kernel_ptr(vm_offset_t addr,
|
2018-01-29 04:33:41 +00:00
|
|
|
int *is_user, vm_offset_t *decoded_addr);
|
2020-05-27 01:24:12 +00:00
|
|
|
static void mmu_booke_page_array_startup(long);
|
|
|
|
static boolean_t mmu_booke_page_is_mapped(vm_page_t m);
|
2021-03-27 03:10:46 +00:00
|
|
|
static bool mmu_booke_ps_enabled(pmap_t pmap);
|
2018-01-15 06:46:33 +00:00
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
static struct pmap_funcs mmu_booke_methods = {
|
2008-03-03 17:17:00 +00:00
|
|
|
/* pmap dispatcher interface */
|
2020-05-27 01:24:12 +00:00
|
|
|
.clear_modify = mmu_booke_clear_modify,
|
|
|
|
.copy = mmu_booke_copy,
|
|
|
|
.copy_page = mmu_booke_copy_page,
|
|
|
|
.copy_pages = mmu_booke_copy_pages,
|
|
|
|
.enter = mmu_booke_enter,
|
|
|
|
.enter_object = mmu_booke_enter_object,
|
|
|
|
.enter_quick = mmu_booke_enter_quick,
|
|
|
|
.extract = mmu_booke_extract,
|
|
|
|
.extract_and_hold = mmu_booke_extract_and_hold,
|
|
|
|
.init = mmu_booke_init,
|
|
|
|
.is_modified = mmu_booke_is_modified,
|
|
|
|
.is_prefaultable = mmu_booke_is_prefaultable,
|
|
|
|
.is_referenced = mmu_booke_is_referenced,
|
|
|
|
.ts_referenced = mmu_booke_ts_referenced,
|
|
|
|
.map = mmu_booke_map,
|
|
|
|
.mincore = mmu_booke_mincore,
|
|
|
|
.object_init_pt = mmu_booke_object_init_pt,
|
|
|
|
.page_exists_quick = mmu_booke_page_exists_quick,
|
|
|
|
.page_init = mmu_booke_page_init,
|
|
|
|
.page_wired_mappings = mmu_booke_page_wired_mappings,
|
|
|
|
.pinit = mmu_booke_pinit,
|
|
|
|
.pinit0 = mmu_booke_pinit0,
|
|
|
|
.protect = mmu_booke_protect,
|
|
|
|
.qenter = mmu_booke_qenter,
|
|
|
|
.qremove = mmu_booke_qremove,
|
|
|
|
.release = mmu_booke_release,
|
|
|
|
.remove = mmu_booke_remove,
|
|
|
|
.remove_all = mmu_booke_remove_all,
|
|
|
|
.remove_write = mmu_booke_remove_write,
|
|
|
|
.sync_icache = mmu_booke_sync_icache,
|
|
|
|
.unwire = mmu_booke_unwire,
|
|
|
|
.zero_page = mmu_booke_zero_page,
|
|
|
|
.zero_page_area = mmu_booke_zero_page_area,
|
|
|
|
.activate = mmu_booke_activate,
|
|
|
|
.deactivate = mmu_booke_deactivate,
|
|
|
|
.quick_enter_page = mmu_booke_quick_enter_page,
|
|
|
|
.quick_remove_page = mmu_booke_quick_remove_page,
|
|
|
|
.page_array_startup = mmu_booke_page_array_startup,
|
|
|
|
.page_is_mapped = mmu_booke_page_is_mapped,
|
2021-03-27 03:10:46 +00:00
|
|
|
.ps_enabled = mmu_booke_ps_enabled,
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Internal interfaces */
|
2020-05-27 01:24:12 +00:00
|
|
|
.bootstrap = mmu_booke_bootstrap,
|
|
|
|
.dev_direct_mapped = mmu_booke_dev_direct_mapped,
|
|
|
|
.mapdev = mmu_booke_mapdev,
|
|
|
|
.mapdev_attr = mmu_booke_mapdev_attr,
|
|
|
|
.kenter = mmu_booke_kenter,
|
|
|
|
.kenter_attr = mmu_booke_kenter_attr,
|
|
|
|
.kextract = mmu_booke_kextract,
|
|
|
|
.kremove = mmu_booke_kremove,
|
|
|
|
.unmapdev = mmu_booke_unmapdev,
|
|
|
|
.change_attr = mmu_booke_change_attr,
|
|
|
|
.decode_kernel_ptr = mmu_booke_decode_kernel_ptr,
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-04-04 22:01:43 +00:00
|
|
|
/* dumpsys() support */
|
2020-05-27 01:24:12 +00:00
|
|
|
.dumpsys_map_chunk = mmu_booke_dumpsys_map,
|
|
|
|
.dumpsys_unmap_chunk = mmu_booke_dumpsys_unmap,
|
|
|
|
.dumpsys_pa_init = mmu_booke_scan_init,
|
2008-03-03 17:17:00 +00:00
|
|
|
};
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
MMU_DEF(booke_mmu, MMU_TYPE_BOOKE, mmu_booke_methods);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
#include "pmap_64.c"
|
|
|
|
#else
|
|
|
|
#include "pmap_32.c"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static vm_offset_t tlb1_map_base = VM_MAPDEV_BASE;
|
|
|
|
|
2013-10-26 14:52:55 +00:00
|
|
|
static __inline uint32_t
|
2015-07-04 19:00:38 +00:00
|
|
|
tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
|
2013-10-26 14:52:55 +00:00
|
|
|
{
|
|
|
|
uint32_t attrib;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (ma != VM_MEMATTR_DEFAULT) {
|
|
|
|
switch (ma) {
|
|
|
|
case VM_MEMATTR_UNCACHEABLE:
|
2016-02-11 13:15:37 +00:00
|
|
|
return (MAS2_I | MAS2_G);
|
2013-10-26 14:52:55 +00:00
|
|
|
case VM_MEMATTR_WRITE_COMBINING:
|
|
|
|
case VM_MEMATTR_WRITE_BACK:
|
|
|
|
case VM_MEMATTR_PREFETCHABLE:
|
2016-02-11 13:15:37 +00:00
|
|
|
return (MAS2_I);
|
2013-10-26 14:52:55 +00:00
|
|
|
case VM_MEMATTR_WRITE_THROUGH:
|
2016-02-11 13:15:37 +00:00
|
|
|
return (MAS2_W | MAS2_M);
|
2016-02-27 20:39:36 +00:00
|
|
|
case VM_MEMATTR_CACHEABLE:
|
|
|
|
return (MAS2_M);
|
2013-10-26 14:52:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Assume the page is cache inhibited and access is guarded unless
|
|
|
|
* it's in our available memory array.
|
|
|
|
*/
|
|
|
|
attrib = _TLB_ENTRY_IO;
|
|
|
|
for (i = 0; i < physmem_regions_sz; i++) {
|
|
|
|
if ((pa >= physmem_regions[i].mr_start) &&
|
|
|
|
(pa < (physmem_regions[i].mr_start +
|
|
|
|
physmem_regions[i].mr_size))) {
|
|
|
|
attrib = _TLB_ENTRY_MEM;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return (attrib);
|
|
|
|
}
|
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
static inline void
|
|
|
|
tlb_miss_lock(void)
|
|
|
|
{
|
|
|
|
#ifdef SMP
|
|
|
|
struct pcpu *pc;
|
|
|
|
|
|
|
|
if (!smp_started)
|
|
|
|
return;
|
|
|
|
|
2011-05-31 15:11:43 +00:00
|
|
|
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
|
2009-05-21 11:43:37 +00:00
|
|
|
if (pc != pcpup) {
|
|
|
|
CTR3(KTR_PMAP, "%s: tlb miss LOCK of CPU=%d, "
|
2018-02-17 20:59:12 +00:00
|
|
|
"tlb_lock=%p", __func__, pc->pc_cpuid, pc->pc_booke.tlb_lock);
|
2009-05-21 11:43:37 +00:00
|
|
|
|
|
|
|
KASSERT((pc->pc_cpuid != PCPU_GET(cpuid)),
|
|
|
|
("tlb_miss_lock: tried to lock self"));
|
|
|
|
|
2018-02-17 20:59:12 +00:00
|
|
|
tlb_lock(pc->pc_booke.tlb_lock);
|
2009-05-21 11:43:37 +00:00
|
|
|
|
|
|
|
CTR1(KTR_PMAP, "%s: locked", __func__);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
tlb_miss_unlock(void)
|
|
|
|
{
|
|
|
|
#ifdef SMP
|
|
|
|
struct pcpu *pc;
|
|
|
|
|
|
|
|
if (!smp_started)
|
|
|
|
return;
|
|
|
|
|
2011-05-31 15:11:43 +00:00
|
|
|
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
|
2009-05-21 11:43:37 +00:00
|
|
|
if (pc != pcpup) {
|
|
|
|
CTR2(KTR_PMAP, "%s: tlb miss UNLOCK of CPU=%d",
|
|
|
|
__func__, pc->pc_cpuid);
|
|
|
|
|
2018-02-17 20:59:12 +00:00
|
|
|
tlb_unlock(pc->pc_booke.tlb_lock);
|
2009-05-21 11:43:37 +00:00
|
|
|
|
|
|
|
CTR1(KTR_PMAP, "%s: unlocked", __func__);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Return number of entries in TLB0. */
|
|
|
|
static __inline void
|
|
|
|
tlb0_get_tlbconf(void)
|
|
|
|
{
|
|
|
|
uint32_t tlb0_cfg;
|
|
|
|
|
|
|
|
tlb0_cfg = mfspr(SPR_TLB0CFG);
|
2009-01-13 15:41:58 +00:00
|
|
|
tlb0_entries = tlb0_cfg & TLBCFG_NENTRY_MASK;
|
|
|
|
tlb0_ways = (tlb0_cfg & TLBCFG_ASSOC_MASK) >> TLBCFG_ASSOC_SHIFT;
|
|
|
|
tlb0_entries_per_way = tlb0_entries / tlb0_ways;
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* Return number of entries in TLB1. */
|
|
|
|
static __inline void
|
|
|
|
tlb1_get_tlbconf(void)
|
|
|
|
{
|
|
|
|
uint32_t tlb1_cfg;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
tlb1_cfg = mfspr(SPR_TLB1CFG);
|
|
|
|
tlb1_entries = tlb1_cfg & TLBCFG_NENTRY_MASK;
|
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/**************************************************************************/
|
|
|
|
/* Page table related */
|
|
|
|
/**************************************************************************/
|
2019-05-08 16:05:18 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* Allocate pv_entry structure. */
|
|
|
|
pv_entry_t
|
|
|
|
pv_alloc(void)
|
|
|
|
{
|
|
|
|
pv_entry_t pv;
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
pv_entry_count++;
|
|
|
|
if (pv_entry_count > pv_entry_high_water)
|
|
|
|
pagedaemon_wakeup(0); /* XXX powerpc NUMA */
|
|
|
|
pv = uma_zalloc(pvzone, M_NOWAIT);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
return (pv);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* Free pv_entry structure. */
|
|
|
|
static __inline void
|
|
|
|
pv_free(pv_entry_t pve)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
pv_entry_count--;
|
|
|
|
uma_zfree(pvzone, pve);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* Allocate and initialize pv_entry structure. */
|
|
|
|
static void
|
|
|
|
pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2020-03-10 03:30:11 +00:00
|
|
|
pv_entry_t pve;
|
|
|
|
|
|
|
|
//int su = (pmap == kernel_pmap);
|
|
|
|
//debugf("pv_insert: s (su = %d pmap = 0x%08x va = 0x%08x m = 0x%08x)\n", su,
|
|
|
|
// (u_int32_t)pmap, va, (u_int32_t)m);
|
|
|
|
|
|
|
|
pve = pv_alloc();
|
|
|
|
if (pve == NULL)
|
|
|
|
panic("pv_insert: no pv entries!");
|
|
|
|
|
|
|
|
pve->pv_pmap = pmap;
|
|
|
|
pve->pv_va = va;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* add to pv_list */
|
|
|
|
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
|
|
rw_assert(&pvh_global_lock, RA_WLOCKED);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
TAILQ_INSERT_TAIL(&m->md.pv_list, pve, pv_link);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
//debugf("pv_insert: e\n");
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* Destroy pv entry. */
|
2016-01-19 03:07:25 +00:00
|
|
|
static void
|
2020-03-10 03:30:11 +00:00
|
|
|
pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m)
|
2016-01-19 03:07:25 +00:00
|
|
|
{
|
2020-03-10 03:30:11 +00:00
|
|
|
pv_entry_t pve;
|
|
|
|
|
|
|
|
//int su = (pmap == kernel_pmap);
|
|
|
|
//debugf("pv_remove: s (su = %d pmap = 0x%08x va = 0x%08x)\n", su, (u_int32_t)pmap, va);
|
2016-01-19 03:07:25 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
|
|
rw_assert(&pvh_global_lock, RA_WLOCKED);
|
2016-01-19 03:07:25 +00:00
|
|
|
|
2020-03-10 03:30:11 +00:00
|
|
|
/* find pv entry */
|
|
|
|
TAILQ_FOREACH(pve, &m->md.pv_list, pv_link) {
|
|
|
|
if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {
|
|
|
|
/* remove from pv_list */
|
|
|
|
TAILQ_REMOVE(&m->md.pv_list, pve, pv_link);
|
|
|
|
if (TAILQ_EMPTY(&m->md.pv_list))
|
|
|
|
vm_page_aflag_clear(m, PGA_WRITEABLE);
|
|
|
|
|
|
|
|
/* free pv entry struct */
|
|
|
|
pv_free(pve);
|
|
|
|
break;
|
|
|
|
}
|
2016-01-19 03:07:25 +00:00
|
|
|
}
|
2020-03-10 03:30:11 +00:00
|
|
|
|
|
|
|
//debugf("pv_remove: e\n");
|
2016-01-19 03:07:25 +00:00
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/**************************************************************************/
|
|
|
|
/* PMAP related */
|
|
|
|
/**************************************************************************/
|
|
|
|
|
|
|
|
/*
|
2011-05-28 04:10:44 +00:00
|
|
|
* This is called during booke_init, before the system is really initialized.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2015-12-24 04:30:15 +00:00
|
|
|
vm_paddr_t phys_kernelend;
|
2008-03-03 17:17:00 +00:00
|
|
|
struct mem_region *mp, *mp1;
|
|
|
|
int cnt, i, j;
|
2015-12-24 04:30:15 +00:00
|
|
|
vm_paddr_t s, e, sz;
|
|
|
|
vm_paddr_t physsz, hwphyssz;
|
2008-03-03 17:17:00 +00:00
|
|
|
u_int phys_avail_count;
|
2015-12-24 04:30:15 +00:00
|
|
|
vm_size_t kstack0_sz;
|
2008-08-26 17:07:37 +00:00
|
|
|
vm_paddr_t kstack0_phys;
|
2020-04-11 00:12:34 +00:00
|
|
|
vm_offset_t kstack0;
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
void *dpcpu;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
debugf("mmu_booke_bootstrap: entered\n");
|
|
|
|
|
2015-03-01 21:23:23 +00:00
|
|
|
/* Set interesting system properties */
|
2018-04-03 00:45:38 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
hw_direct_map = 1;
|
|
|
|
#else
|
2015-03-01 21:23:23 +00:00
|
|
|
hw_direct_map = 0;
|
2018-04-03 00:45:38 +00:00
|
|
|
#endif
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#if defined(COMPAT_FREEBSD32) || !defined(__powerpc64__)
|
2015-03-01 21:23:23 +00:00
|
|
|
elf32_nxstack = 1;
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#endif
|
2015-03-01 21:23:23 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/* Initialize invalidation mutex */
|
|
|
|
mtx_init(&tlbivax_mutex, "tlbivax", NULL, MTX_SPIN);
|
|
|
|
|
|
|
|
/* Read TLB0 size and associativity. */
|
|
|
|
tlb0_get_tlbconf();
|
|
|
|
|
2011-08-02 15:35:43 +00:00
|
|
|
/*
|
|
|
|
* Align kernel start and end address (kernel image).
|
|
|
|
* Note that kernel end does not necessarily relate to kernsize.
|
|
|
|
* kernsize is the size of the kernel that is actually mapped.
|
|
|
|
*/
|
2009-04-04 22:01:43 +00:00
|
|
|
data_start = round_page(kernelend);
|
|
|
|
data_end = data_start;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2011-08-02 15:35:43 +00:00
|
|
|
/* Allocate the dynamic per-cpu area. */
|
|
|
|
dpcpu = (void *)data_end;
|
|
|
|
data_end += DPCPU_SIZE;
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Allocate space for the message buffer. */
|
2009-04-04 22:01:43 +00:00
|
|
|
msgbufp = (struct msgbuf *)data_end;
|
2011-01-21 10:26:26 +00:00
|
|
|
data_end += msgbufsize;
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
debugf(" msgbufp at 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n",
|
|
|
|
(uintptr_t)msgbufp, data_end);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-04-04 22:01:43 +00:00
|
|
|
data_end = round_page(data_end);
|
2020-04-11 00:12:34 +00:00
|
|
|
data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end));
|
2008-03-03 17:17:00 +00:00
|
|
|
|
powerpc/pmap: NUMA-ize vm_page_array on powerpc
Summary:
This matches r351198 from amd64. This only applies to AIM64 and Book-E.
On AIM64 it short-circuits with one domain, to behave similar to
existing. Otherwise it will allocate 16MB huge pages to hold the page
array, across all NUMA domains. On the first domain it will shift the
page array base up, to "upper-align" the page array in that domain, so
as to reduce the number of pages from the next domain appearing in this
domain. After the first domain, subsequent domains will be allocated in
full 16MB pages, until the final domain, which can be short. This means
some inner domains may have pages accounted in earlier domains.
On Book-E the page array is setup at MMU bootstrap time so that it's
always mapped in TLB1, on both 32-bit and 64-bit. This reduces the TLB0
overhead for touching the vm_page_array, which reduces up to one TLB
miss per array access.
Since page_range (vm_page_startup()) is no longer used on Book-E but is on
32-bit AIM, mark the variable as potentially unused, rather than using a
nasty #if defined() list.
Reviewed by: luporl
Differential Revision: https://reviews.freebsd.org/D21449
2019-12-07 03:34:03 +00:00
|
|
|
/* Retrieve phys/avail mem regions */
|
|
|
|
mem_regions(&physmem_regions, &physmem_regions_sz,
|
|
|
|
&availmem_regions, &availmem_regions_sz);
|
|
|
|
|
|
|
|
if (PHYS_AVAIL_ENTRIES < availmem_regions_sz)
|
|
|
|
panic("mmu_booke_bootstrap: phys_avail too small");
|
|
|
|
|
|
|
|
data_end = round_page(data_end);
|
|
|
|
vm_page_array = (vm_page_t)data_end;
|
|
|
|
/*
|
|
|
|
* Get a rough idea (upper bound) on the size of the page array. The
|
|
|
|
* vm_page_array will not handle any more pages than we have in the
|
|
|
|
* avail_regions array, and most likely much less.
|
|
|
|
*/
|
|
|
|
sz = 0;
|
|
|
|
for (mp = availmem_regions; mp->mr_size; mp++) {
|
|
|
|
sz += mp->mr_size;
|
|
|
|
}
|
|
|
|
sz = (round_page(sz) / (PAGE_SIZE + sizeof(struct vm_page)));
|
|
|
|
data_end += round_page(sz * sizeof(struct vm_page));
|
|
|
|
|
2019-11-04 00:35:40 +00:00
|
|
|
/* Pre-round up to 1MB. This wastes some space, but saves TLB entries */
|
|
|
|
data_end = roundup2(data_end, 1 << 20);
|
powerpc/pmap: NUMA-ize vm_page_array on powerpc
Summary:
This matches r351198 from amd64. This only applies to AIM64 and Book-E.
On AIM64 it short-circuits with one domain, to behave similar to
existing. Otherwise it will allocate 16MB huge pages to hold the page
array, across all NUMA domains. On the first domain it will shift the
page array base up, to "upper-align" the page array in that domain, so
as to reduce the number of pages from the next domain appearing in this
domain. After the first domain, subsequent domains will be allocated in
full 16MB pages, until the final domain, which can be short. This means
some inner domains may have pages accounted in earlier domains.
On Book-E the page array is setup at MMU bootstrap time so that it's
always mapped in TLB1, on both 32-bit and 64-bit. This reduces the TLB0
overhead for touching the vm_page_array, which reduces up to one TLB
miss per array access.
Since page_range (vm_page_startup()) is no longer used on Book-E but is on
32-bit AIM, mark the variable as potentially unused, rather than using a
nasty #if defined() list.
Reviewed by: luporl
Differential Revision: https://reviews.freebsd.org/D21449
2019-12-07 03:34:03 +00:00
|
|
|
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
debugf(" data_end: 0x%"PRI0ptrX"\n", data_end);
|
2019-12-04 03:51:30 +00:00
|
|
|
debugf(" kernstart: %#zx\n", kernstart);
|
|
|
|
debugf(" kernsize: %#zx\n", kernsize);
|
2019-11-04 00:35:40 +00:00
|
|
|
|
2011-08-02 15:35:43 +00:00
|
|
|
if (data_end - kernstart > kernsize) {
|
|
|
|
kernsize += tlb1_mapin_region(kernstart + kernsize,
|
2019-11-04 00:35:40 +00:00
|
|
|
kernload + kernsize, (data_end - kernstart) - kernsize,
|
|
|
|
_TLB_ENTRY_MEM);
|
2011-08-02 15:35:43 +00:00
|
|
|
}
|
|
|
|
data_end = kernstart + kernsize;
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
debugf(" updated data_end: 0x%"PRI0ptrX"\n", data_end);
|
2009-04-04 22:01:43 +00:00
|
|
|
|
2008-08-28 07:38:08 +00:00
|
|
|
/*
|
|
|
|
* Clear the structures - note we can only do it safely after the
|
2009-01-13 15:41:58 +00:00
|
|
|
* possible additional TLB1 translations are in place (above) so that
|
2009-04-04 22:01:43 +00:00
|
|
|
* all range up to the currently calculated 'data_end' is covered.
|
2008-08-28 07:38:08 +00:00
|
|
|
*/
|
2020-04-11 00:12:34 +00:00
|
|
|
bzero((void *)data_start, data_end - data_start);
|
2011-08-02 15:35:43 +00:00
|
|
|
dpcpu_init(dpcpu, 0);
|
2008-08-28 07:38:08 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*******************************************************/
|
|
|
|
/* Set the start and end of kva. */
|
|
|
|
/*******************************************************/
|
2009-04-04 22:01:43 +00:00
|
|
|
virtual_avail = round_page(data_end);
|
2008-03-03 17:17:00 +00:00
|
|
|
virtual_end = VM_MAX_KERNEL_ADDRESS;
|
|
|
|
|
2019-08-25 20:11:35 +00:00
|
|
|
#ifndef __powerpc64__
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Allocate KVA space for page zero/copy operations. */
|
|
|
|
zero_page_va = virtual_avail;
|
|
|
|
virtual_avail += PAGE_SIZE;
|
|
|
|
copy_page_src_va = virtual_avail;
|
|
|
|
virtual_avail += PAGE_SIZE;
|
|
|
|
copy_page_dst_va = virtual_avail;
|
|
|
|
virtual_avail += PAGE_SIZE;
|
2018-05-28 00:19:08 +00:00
|
|
|
debugf("zero_page_va = 0x%"PRI0ptrX"\n", zero_page_va);
|
2018-11-28 04:02:26 +00:00
|
|
|
debugf("copy_page_src_va = 0x%"PRI0ptrX"\n", copy_page_src_va);
|
|
|
|
debugf("copy_page_dst_va = 0x%"PRI0ptrX"\n", copy_page_dst_va);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Initialize page zero/copy mutexes. */
|
|
|
|
mtx_init(&zero_page_mutex, "mmu_booke_zero_page", NULL, MTX_DEF);
|
|
|
|
mtx_init(©_page_mutex, "mmu_booke_copy_page", NULL, MTX_DEF);
|
|
|
|
|
|
|
|
/* Allocate KVA space for ptbl bufs. */
|
|
|
|
ptbl_buf_pool_vabase = virtual_avail;
|
|
|
|
virtual_avail += PTBL_BUFS * PTBL_PAGES * PAGE_SIZE;
|
2018-11-28 04:02:26 +00:00
|
|
|
debugf("ptbl_buf_pool_vabase = 0x%"PRI0ptrX" end = 0x%"PRI0ptrX"\n",
|
2009-01-13 15:41:58 +00:00
|
|
|
ptbl_buf_pool_vabase, virtual_avail);
|
2019-08-06 03:16:06 +00:00
|
|
|
#endif
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Calculate corresponding physical addresses for the kernel region. */
|
2009-04-04 22:01:43 +00:00
|
|
|
phys_kernelend = kernload + kernsize;
|
2008-03-03 17:17:00 +00:00
|
|
|
debugf("kernel image and allocated data:\n");
|
2019-11-02 21:15:56 +00:00
|
|
|
debugf(" kernload = 0x%09jx\n", (uintmax_t)kernload);
|
2018-11-28 04:02:26 +00:00
|
|
|
debugf(" kernstart = 0x%"PRI0ptrX"\n", kernstart);
|
|
|
|
debugf(" kernsize = 0x%"PRI0ptrX"\n", kernsize);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/*
|
2009-01-13 16:15:49 +00:00
|
|
|
* Remove kernel physical address range from avail regions list. Page
|
|
|
|
* align all regions. Non-page aligned memory isn't very interesting
|
|
|
|
* to us. Also, sort the entries for ascending addresses.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
2009-05-14 00:34:26 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
sz = 0;
|
|
|
|
cnt = availmem_regions_sz;
|
|
|
|
debugf("processing avail regions:\n");
|
|
|
|
for (mp = availmem_regions; mp->mr_size; mp++) {
|
|
|
|
s = mp->mr_start;
|
|
|
|
e = mp->mr_start + mp->mr_size;
|
2015-12-24 04:30:15 +00:00
|
|
|
debugf(" %09jx-%09jx -> ", (uintmax_t)s, (uintmax_t)e);
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Check whether this region holds all of the kernel. */
|
|
|
|
if (s < kernload && e > phys_kernelend) {
|
|
|
|
availmem_regions[cnt].mr_start = phys_kernelend;
|
|
|
|
availmem_regions[cnt++].mr_size = e - phys_kernelend;
|
|
|
|
e = kernload;
|
|
|
|
}
|
|
|
|
/* Look whether this regions starts within the kernel. */
|
|
|
|
if (s >= kernload && s < phys_kernelend) {
|
|
|
|
if (e <= phys_kernelend)
|
|
|
|
goto empty;
|
|
|
|
s = phys_kernelend;
|
|
|
|
}
|
|
|
|
/* Now look whether this region ends within the kernel. */
|
|
|
|
if (e > kernload && e <= phys_kernelend) {
|
|
|
|
if (s >= kernload)
|
|
|
|
goto empty;
|
|
|
|
e = kernload;
|
|
|
|
}
|
|
|
|
/* Now page align the start and size of the region. */
|
|
|
|
s = round_page(s);
|
|
|
|
e = trunc_page(e);
|
|
|
|
if (e < s)
|
|
|
|
e = s;
|
|
|
|
sz = e - s;
|
2015-12-24 04:30:15 +00:00
|
|
|
debugf("%09jx-%09jx = %jx\n",
|
|
|
|
(uintmax_t)s, (uintmax_t)e, (uintmax_t)sz);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Check whether some memory is left here. */
|
|
|
|
if (sz == 0) {
|
|
|
|
empty:
|
|
|
|
memmove(mp, mp + 1,
|
|
|
|
(cnt - (mp - availmem_regions)) * sizeof(*mp));
|
|
|
|
cnt--;
|
|
|
|
mp--;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Do an insertion sort. */
|
|
|
|
for (mp1 = availmem_regions; mp1 < mp; mp1++)
|
|
|
|
if (s < mp1->mr_start)
|
|
|
|
break;
|
|
|
|
if (mp1 < mp) {
|
|
|
|
memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1);
|
|
|
|
mp1->mr_start = s;
|
|
|
|
mp1->mr_size = sz;
|
|
|
|
} else {
|
|
|
|
mp->mr_start = s;
|
|
|
|
mp->mr_size = sz;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
availmem_regions_sz = cnt;
|
|
|
|
|
2008-08-26 17:07:37 +00:00
|
|
|
/*******************************************************/
|
|
|
|
/* Steal physical memory for kernel stack from the end */
|
|
|
|
/* of the first avail region */
|
|
|
|
/*******************************************************/
|
2015-08-10 17:18:21 +00:00
|
|
|
kstack0_sz = kstack_pages * PAGE_SIZE;
|
2008-08-26 17:07:37 +00:00
|
|
|
kstack0_phys = availmem_regions[0].mr_start +
|
|
|
|
availmem_regions[0].mr_size;
|
|
|
|
kstack0_phys -= kstack0_sz;
|
|
|
|
availmem_regions[0].mr_size -= kstack0_sz;
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*******************************************************/
|
|
|
|
/* Fill in phys_avail table, based on availmem_regions */
|
|
|
|
/*******************************************************/
|
|
|
|
phys_avail_count = 0;
|
|
|
|
physsz = 0;
|
|
|
|
hwphyssz = 0;
|
|
|
|
TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
|
|
|
|
|
|
|
|
debugf("fill in phys_avail:\n");
|
|
|
|
for (i = 0, j = 0; i < availmem_regions_sz; i++, j += 2) {
|
2015-08-28 03:03:09 +00:00
|
|
|
debugf(" region: 0x%jx - 0x%jx (0x%jx)\n",
|
2015-12-24 04:30:15 +00:00
|
|
|
(uintmax_t)availmem_regions[i].mr_start,
|
|
|
|
(uintmax_t)availmem_regions[i].mr_start +
|
2009-01-13 16:15:49 +00:00
|
|
|
availmem_regions[i].mr_size,
|
2015-12-24 04:30:15 +00:00
|
|
|
(uintmax_t)availmem_regions[i].mr_size);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2008-08-28 07:38:08 +00:00
|
|
|
if (hwphyssz != 0 &&
|
|
|
|
(physsz + availmem_regions[i].mr_size) >= hwphyssz) {
|
2008-03-03 17:17:00 +00:00
|
|
|
debugf(" hw.physmem adjust\n");
|
|
|
|
if (physsz < hwphyssz) {
|
|
|
|
phys_avail[j] = availmem_regions[i].mr_start;
|
2008-08-28 07:38:08 +00:00
|
|
|
phys_avail[j + 1] =
|
|
|
|
availmem_regions[i].mr_start +
|
2008-03-03 17:17:00 +00:00
|
|
|
hwphyssz - physsz;
|
|
|
|
physsz = hwphyssz;
|
|
|
|
phys_avail_count++;
|
2019-10-23 00:31:19 +00:00
|
|
|
dump_avail[j] = phys_avail[j];
|
|
|
|
dump_avail[j + 1] = phys_avail[j + 1];
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
phys_avail[j] = availmem_regions[i].mr_start;
|
|
|
|
phys_avail[j + 1] = availmem_regions[i].mr_start +
|
|
|
|
availmem_regions[i].mr_size;
|
|
|
|
phys_avail_count++;
|
|
|
|
physsz += availmem_regions[i].mr_size;
|
2019-10-23 00:31:19 +00:00
|
|
|
dump_avail[j] = phys_avail[j];
|
|
|
|
dump_avail[j + 1] = phys_avail[j + 1];
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
physmem = btoc(physsz);
|
|
|
|
|
|
|
|
/* Calculate the last available physical address. */
|
|
|
|
for (i = 0; phys_avail[i + 2] != 0; i += 2)
|
|
|
|
;
|
|
|
|
Maxmem = powerpc_btop(phys_avail[i + 1]);
|
|
|
|
|
|
|
|
debugf("Maxmem = 0x%08lx\n", Maxmem);
|
|
|
|
debugf("phys_avail_count = %d\n", phys_avail_count);
|
2016-01-26 03:52:14 +00:00
|
|
|
debugf("physsz = 0x%09jx physmem = %jd (0x%09jx)\n",
|
|
|
|
(uintmax_t)physsz, (uintmax_t)physmem, (uintmax_t)physmem);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2018-04-03 00:45:38 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
/*
|
|
|
|
* Map the physical memory contiguously in TLB1.
|
|
|
|
* Round so it fits into a single mapping.
|
|
|
|
*/
|
|
|
|
tlb1_mapin_region(DMAP_BASE_ADDRESS, 0,
|
2019-11-04 00:35:40 +00:00
|
|
|
phys_avail[i + 1], _TLB_ENTRY_MEM);
|
2018-04-03 00:45:38 +00:00
|
|
|
#endif
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*******************************************************/
|
|
|
|
/* Initialize (statically allocated) kernel pmap. */
|
|
|
|
/*******************************************************/
|
|
|
|
PMAP_LOCK_INIT(kernel_pmap);
|
|
|
|
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
debugf("kernel_pmap = 0x%"PRI0ptrX"\n", (uintptr_t)kernel_pmap);
|
2020-04-11 00:12:34 +00:00
|
|
|
kernel_pte_alloc(virtual_avail, kernstart);
|
2009-01-13 15:41:58 +00:00
|
|
|
for (i = 0; i < MAXCPU; i++) {
|
|
|
|
kernel_pmap->pm_tid[i] = TID_KERNEL;
|
|
|
|
|
|
|
|
/* Initialize each CPU's tidbusy entry 0 with kernel_pmap */
|
2015-08-22 07:20:03 +00:00
|
|
|
tidbusy[i][TID_KERNEL] = kernel_pmap;
|
2009-01-13 15:41:58 +00:00
|
|
|
}
|
2009-06-05 09:09:46 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/* Mark kernel_pmap active on all CPUs */
|
2011-05-09 16:16:15 +00:00
|
|
|
CPU_FILL(&kernel_pmap->pm_active);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
/*
|
|
|
|
* Initialize the global pv list lock.
|
|
|
|
*/
|
|
|
|
rw_init(&pvh_global_lock, "pmap pv global");
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*******************************************************/
|
|
|
|
/* Final setup */
|
|
|
|
/*******************************************************/
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2008-08-26 17:07:37 +00:00
|
|
|
/* Enter kstack0 into kernel map, provide guard page */
|
|
|
|
kstack0 = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
|
|
|
|
thread0.td_kstack = kstack0;
|
2015-08-10 17:18:21 +00:00
|
|
|
thread0.td_kstack_pages = kstack_pages;
|
2008-08-26 17:07:37 +00:00
|
|
|
|
2019-11-02 21:15:56 +00:00
|
|
|
debugf("kstack_sz = 0x%08jx\n", (uintmax_t)kstack0_sz);
|
|
|
|
debugf("kstack0_phys at 0x%09jx - 0x%09jx\n",
|
|
|
|
(uintmax_t)kstack0_phys, (uintmax_t)kstack0_phys + kstack0_sz);
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
debugf("kstack0 at 0x%"PRI0ptrX" - 0x%"PRI0ptrX"\n",
|
|
|
|
kstack0, kstack0 + kstack0_sz);
|
2020-09-01 21:20:08 +00:00
|
|
|
|
2008-08-26 17:07:37 +00:00
|
|
|
virtual_avail += KSTACK_GUARD_PAGES * PAGE_SIZE + kstack0_sz;
|
2015-08-10 17:18:21 +00:00
|
|
|
for (i = 0; i < kstack_pages; i++) {
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kenter(kstack0, kstack0_phys);
|
2008-08-26 17:07:37 +00:00
|
|
|
kstack0 += PAGE_SIZE;
|
|
|
|
kstack0_phys += PAGE_SIZE;
|
|
|
|
}
|
2015-03-01 21:23:23 +00:00
|
|
|
|
|
|
|
pmap_bootstrapped = 1;
|
2020-09-01 21:20:08 +00:00
|
|
|
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
debugf("virtual_avail = %"PRI0ptrX"\n", virtual_avail);
|
|
|
|
debugf("virtual_end = %"PRI0ptrX"\n", virtual_end);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
debugf("mmu_booke_bootstrap: exit\n");
|
|
|
|
}
|
|
|
|
|
2015-03-28 21:39:42 +00:00
|
|
|
#ifdef SMP
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
void
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb1_ap_prep(void)
|
|
|
|
{
|
|
|
|
tlb_entry_t *e, tmp;
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
/* Prepare TLB1 image for AP processors */
|
|
|
|
e = __boot_tlb1;
|
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&tmp, i);
|
|
|
|
|
|
|
|
if ((tmp.mas1 & MAS1_VALID) && (tmp.mas2 & _TLB_ENTRY_SHARED))
|
|
|
|
memcpy(e++, &tmp, sizeof(tmp));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
void
|
|
|
|
pmap_bootstrap_ap(volatile uint32_t *trcp __unused)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Finish TLB1 configuration: the BSP already set up its TLB1 and we
|
2016-04-19 01:48:18 +00:00
|
|
|
* have the snapshot of its contents in the s/w __boot_tlb1[] table
|
|
|
|
* created by tlb1_ap_prep(), so use these values directly to
|
|
|
|
* (re)program AP's TLB1 hardware.
|
|
|
|
*
|
|
|
|
* Start at index 1 because index 0 has the kernel map.
|
2009-05-21 11:43:37 +00:00
|
|
|
*/
|
2016-04-19 01:48:18 +00:00
|
|
|
for (i = 1; i < TLB1_ENTRIES; i++) {
|
|
|
|
if (__boot_tlb1[i].mas1 & MAS1_VALID)
|
|
|
|
tlb1_write_entry(&__boot_tlb1[i], i);
|
2009-05-21 11:43:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
set_mas4_defaults();
|
|
|
|
}
|
2015-03-28 21:39:42 +00:00
|
|
|
#endif
|
2009-05-21 11:43:37 +00:00
|
|
|
|
2015-08-04 19:46:13 +00:00
|
|
|
static void
|
|
|
|
booke_pmap_init_qpages(void)
|
|
|
|
{
|
|
|
|
struct pcpu *pc;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
CPU_FOREACH(i) {
|
|
|
|
pc = pcpu_find(i);
|
|
|
|
pc->pc_qmap_addr = kva_alloc(PAGE_SIZE);
|
|
|
|
if (pc->pc_qmap_addr == 0)
|
|
|
|
panic("pmap_init_qpages: unable to allocate KVA");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, booke_pmap_init_qpages, NULL);
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Get the physical page address for the given pmap/virtual address.
|
|
|
|
*/
|
|
|
|
static vm_paddr_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_extract(pmap_t pmap, vm_offset_t va)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
vm_paddr_t pa;
|
|
|
|
|
|
|
|
PMAP_LOCK(pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
pa = pte_vatopa(pmap, va);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_UNLOCK(pmap);
|
|
|
|
|
|
|
|
return (pa);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Extract the physical page address associated with the given
|
|
|
|
* kernel virtual address.
|
|
|
|
*/
|
|
|
|
static vm_paddr_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kextract(vm_offset_t va)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
2017-11-29 01:26:07 +00:00
|
|
|
vm_paddr_t p = 0;
|
2013-10-26 16:49:41 +00:00
|
|
|
int i;
|
|
|
|
|
2019-08-15 03:42:15 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
if (va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS)
|
|
|
|
return (DMAP_TO_PHYS(va));
|
|
|
|
#endif
|
|
|
|
|
2017-11-29 01:26:07 +00:00
|
|
|
if (va >= VM_MIN_KERNEL_ADDRESS && va <= VM_MAX_KERNEL_ADDRESS)
|
2020-05-27 01:24:12 +00:00
|
|
|
p = pte_vatopa(kernel_pmap, va);
|
2020-09-01 21:20:08 +00:00
|
|
|
|
2017-11-21 03:12:16 +00:00
|
|
|
if (p == 0) {
|
|
|
|
/* Check TLB1 mappings */
|
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
|
|
|
continue;
|
|
|
|
if (va >= e.virt && va < e.virt + e.size)
|
|
|
|
return (e.phys + (va - e.virt));
|
|
|
|
}
|
2013-10-26 16:49:41 +00:00
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2017-11-21 03:12:16 +00:00
|
|
|
return (p);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize the pmap module.
|
|
|
|
* Called by vm_init, to initialize any structures that the pmap
|
|
|
|
* system needs to map virtual memory.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_init()
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
int shpgperproc = PMAP_SHPGPERPROC;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize the address space (zone) for the pv entries. Set a
|
|
|
|
* high water mark so that the system can recover from excessive
|
|
|
|
* numbers of pv entries.
|
|
|
|
*/
|
|
|
|
pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL,
|
|
|
|
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
|
|
|
|
|
|
|
|
TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
|
2014-03-22 10:26:09 +00:00
|
|
|
pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
|
|
|
|
pv_entry_high_water = 9 * (pv_entry_max / 10);
|
|
|
|
|
2013-02-26 23:35:27 +00:00
|
|
|
uma_zone_reserve_kva(pvzone, pv_entry_max);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Pre-fill pvzone with initial number of pv entries. */
|
|
|
|
uma_prealloc(pvzone, PV_ENTRY_ZONE_MIN);
|
|
|
|
|
2019-08-12 03:03:56 +00:00
|
|
|
/* Create a UMA zone for page table roots. */
|
|
|
|
ptbl_root_zone = uma_zcreate("pmap root", PMAP_ROOT_SIZE,
|
|
|
|
NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, UMA_ZONE_VM);
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Initialize ptbl allocation. */
|
|
|
|
ptbl_init();
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Map a list of wired pages into kernel virtual address space. This is
|
|
|
|
* intended for temporary mappings which do not need page modification or
|
|
|
|
* references recorded. Existing mappings in the region are overwritten.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_qenter(vm_offset_t sva, vm_page_t *m, int count)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
vm_offset_t va;
|
|
|
|
|
|
|
|
va = sva;
|
|
|
|
while (count-- > 0) {
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kenter(va, VM_PAGE_TO_PHYS(*m));
|
2008-03-03 17:17:00 +00:00
|
|
|
va += PAGE_SIZE;
|
|
|
|
m++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove page mappings from kernel virtual address space. Intended for
|
|
|
|
* temporary mappings entered by mmu_booke_qenter.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_qremove(vm_offset_t sva, int count)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
vm_offset_t va;
|
|
|
|
|
|
|
|
va = sva;
|
|
|
|
while (count-- > 0) {
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kremove(va);
|
2008-03-03 17:17:00 +00:00
|
|
|
va += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Map a wired page into kernel virtual address space.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kenter(vm_offset_t va, vm_paddr_t pa)
|
2013-10-26 14:52:55 +00:00
|
|
|
{
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kenter_attr(va, pa, VM_MEMATTR_DEFAULT);
|
2013-10-26 14:52:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2009-01-13 16:15:49 +00:00
|
|
|
uint32_t flags;
|
2008-03-03 17:17:00 +00:00
|
|
|
pte_t *pte;
|
|
|
|
|
2009-01-13 16:15:49 +00:00
|
|
|
KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) &&
|
|
|
|
(va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_kenter: invalid va"));
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2013-10-26 14:52:55 +00:00
|
|
|
flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID;
|
2016-02-11 13:15:37 +00:00
|
|
|
flags |= tlb_calc_wimg(pa, ma) << PTE_MAS2_SHIFT;
|
|
|
|
flags |= PTE_PS_4KB;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(kernel_pmap, va);
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
KASSERT((pte != NULL), ("mmu_booke_kenter: invalid va. NULL PTE"));
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2020-09-01 21:20:08 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
if (PTE_ISVALID(pte)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR1(KTR_PMAP, "%s: replacing entry!", __func__);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Flush entry from TLB0 */
|
2009-01-13 15:41:58 +00:00
|
|
|
tlb0_flush_entry(va);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte = PTE_RPN_FROM_PA(pa) | flags;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
//debugf("mmu_booke_kenter: pdir_idx = %d ptbl_idx = %d va=0x%08x "
|
|
|
|
// "pa=0x%08x rpn=0x%08x flags=0x%08x\n",
|
|
|
|
// pdir_idx, ptbl_idx, va, pa, pte->rpn, pte->flags);
|
|
|
|
|
|
|
|
/* Flush the real memory from the instruction cache. */
|
2016-02-11 13:15:37 +00:00
|
|
|
if ((flags & (PTE_I | PTE_G)) == 0)
|
2008-03-03 17:17:00 +00:00
|
|
|
__syncicache((void *)va, PAGE_SIZE);
|
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove a page from kernel page table.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kremove(vm_offset_t va)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
|
2018-11-28 04:02:26 +00:00
|
|
|
CTR2(KTR_PMAP,"%s: s (va = 0x%"PRI0ptrX")\n", __func__, va);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) &&
|
|
|
|
(va <= VM_MAX_KERNEL_ADDRESS)),
|
2008-03-03 17:17:00 +00:00
|
|
|
("mmu_booke_kremove: invalid va"));
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(kernel_pmap, va);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
if (!PTE_ISVALID(pte)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR1(KTR_PMAP, "%s: invalid pte", __func__);
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/* Invalidate entry in TLB0, update PTE. */
|
|
|
|
tlb0_flush_entry(va);
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte = 0;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2018-01-29 04:33:41 +00:00
|
|
|
/*
|
|
|
|
* Figure out where a given kernel pointer (usually in a fault) points
|
|
|
|
* to from the VM's perspective, potentially remapping into userland's
|
|
|
|
* address space.
|
|
|
|
*/
|
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_decode_kernel_ptr(vm_offset_t addr, int *is_user,
|
2018-01-29 04:33:41 +00:00
|
|
|
vm_offset_t *decoded_addr)
|
|
|
|
{
|
|
|
|
|
2019-08-20 01:26:02 +00:00
|
|
|
if (trunc_page(addr) <= VM_MAXUSER_ADDRESS)
|
2018-01-29 04:33:41 +00:00
|
|
|
*is_user = 1;
|
|
|
|
else
|
|
|
|
*is_user = 0;
|
|
|
|
|
|
|
|
*decoded_addr = addr;
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2020-05-11 02:33:37 +00:00
|
|
|
static boolean_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_page_is_mapped(vm_page_t m)
|
2020-05-11 02:33:37 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
return (!TAILQ_EMPTY(&(m)->md.pv_list));
|
|
|
|
}
|
|
|
|
|
2021-03-27 03:10:46 +00:00
|
|
|
static bool
|
|
|
|
mmu_booke_ps_enabled(pmap_t pmap __unused)
|
|
|
|
{
|
|
|
|
return (false);
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Initialize pmap associated with process 0.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_pinit0(pmap_t pmap)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2009-01-13 16:15:49 +00:00
|
|
|
|
2013-08-22 18:12:24 +00:00
|
|
|
PMAP_LOCK_INIT(pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_pinit(pmap);
|
2008-03-03 17:17:00 +00:00
|
|
|
PCPU_SET(curpmap, pmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Insert the given physical page at the specified virtual address in the
|
|
|
|
* target physical map with the protection requested. If specified the page
|
|
|
|
* will be wired down.
|
|
|
|
*/
|
2014-08-08 17:12:03 +00:00
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
|
2014-08-08 17:12:03 +00:00
|
|
|
vm_prot_t prot, u_int flags, int8_t psind)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2014-08-08 17:12:03 +00:00
|
|
|
int error;
|
2009-01-13 16:15:49 +00:00
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_LOCK(pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
error = mmu_booke_enter_locked(pmap, va, m, prot, flags, psind);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_UNLOCK(pmap);
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2014-08-08 17:12:03 +00:00
|
|
|
return (error);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2014-08-08 17:12:03 +00:00
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
|
2014-08-08 17:12:03 +00:00
|
|
|
vm_prot_t prot, u_int pmap_flags, int8_t psind __unused)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
vm_paddr_t pa;
|
2020-04-11 00:12:34 +00:00
|
|
|
pte_t flags;
|
2014-08-08 17:12:03 +00:00
|
|
|
int error, su, sync;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
pa = VM_PAGE_TO_PHYS(m);
|
|
|
|
su = (pmap == kernel_pmap);
|
|
|
|
sync = 0;
|
|
|
|
|
|
|
|
//debugf("mmu_booke_enter_locked: s (pmap=0x%08x su=%d tid=%d m=0x%08x va=0x%08x "
|
2014-08-08 17:12:03 +00:00
|
|
|
// "pa=0x%08x prot=0x%08x flags=%#x)\n",
|
2008-03-03 17:17:00 +00:00
|
|
|
// (u_int32_t)pmap, su, pmap->pm_tid,
|
2014-08-08 17:12:03 +00:00
|
|
|
// (u_int32_t)m, va, pa, prot, flags);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
if (su) {
|
2009-01-13 16:15:49 +00:00
|
|
|
KASSERT(((va >= virtual_avail) &&
|
|
|
|
(va <= VM_MAX_KERNEL_ADDRESS)),
|
|
|
|
("mmu_booke_enter_locked: kernel pmap, non kernel va"));
|
2008-03-03 17:17:00 +00:00
|
|
|
} else {
|
|
|
|
KASSERT((va <= VM_MAXUSER_ADDRESS),
|
2009-01-13 16:15:49 +00:00
|
|
|
("mmu_booke_enter_locked: user pmap, non user va"));
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
2019-10-16 07:09:15 +00:00
|
|
|
if ((m->oflags & VPO_UNMANAGED) == 0) {
|
|
|
|
if ((pmap_flags & PMAP_ENTER_QUICK_LOCKED) == 0)
|
|
|
|
VM_PAGE_OBJECT_BUSY_ASSERT(m);
|
|
|
|
else
|
|
|
|
VM_OBJECT_ASSERT_LOCKED(m->object);
|
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there is an existing mapping, and the physical address has not
|
|
|
|
* changed, must be protection or wiring change.
|
|
|
|
*/
|
2020-05-27 01:24:12 +00:00
|
|
|
if (((pte = pte_find(pmap, va)) != NULL) &&
|
2008-03-03 17:17:00 +00:00
|
|
|
(PTE_ISVALID(pte)) && (PTE_PA(pte) == pa)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Before actually updating pte->flags we calculate and
|
|
|
|
* prepare its new value in a helper var.
|
|
|
|
*/
|
2016-02-11 13:15:37 +00:00
|
|
|
flags = *pte;
|
2009-01-13 15:41:58 +00:00
|
|
|
flags &= ~(PTE_UW | PTE_UX | PTE_SW | PTE_SX | PTE_MODIFIED);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Wiring change, just update stats. */
|
2014-08-08 17:12:03 +00:00
|
|
|
if ((pmap_flags & PMAP_ENTER_WIRED) != 0) {
|
2008-03-03 17:17:00 +00:00
|
|
|
if (!PTE_ISWIRED(pte)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
flags |= PTE_WIRED;
|
2008-03-03 17:17:00 +00:00
|
|
|
pmap->pm_stats.wired_count++;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (PTE_ISWIRED(pte)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
flags &= ~PTE_WIRED;
|
2008-03-03 17:17:00 +00:00
|
|
|
pmap->pm_stats.wired_count--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (prot & VM_PROT_WRITE) {
|
|
|
|
/* Add write permissions. */
|
2009-01-13 15:41:58 +00:00
|
|
|
flags |= PTE_SW;
|
2008-03-03 17:17:00 +00:00
|
|
|
if (!su)
|
2009-01-13 15:41:58 +00:00
|
|
|
flags |= PTE_UW;
|
2009-05-26 06:24:50 +00:00
|
|
|
|
2010-06-05 18:20:09 +00:00
|
|
|
if ((flags & PTE_MANAGED) != 0)
|
2011-09-06 10:30:11 +00:00
|
|
|
vm_page_aflag_set(m, PGA_WRITEABLE);
|
2008-03-03 17:17:00 +00:00
|
|
|
} else {
|
|
|
|
/* Handle modified pages, sense modify status. */
|
2009-01-13 15:41:58 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The PTE_MODIFIED flag could be set by underlying
|
|
|
|
* TLB misses since we last read it (above), possibly
|
|
|
|
* other CPUs could update it so we check in the PTE
|
|
|
|
* directly rather than rely on that saved local flags
|
|
|
|
* copy.
|
|
|
|
*/
|
2008-04-27 21:04:54 +00:00
|
|
|
if (PTE_ISMODIFIED(pte))
|
|
|
|
vm_page_dirty(m);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (prot & VM_PROT_EXECUTE) {
|
2009-01-13 15:41:58 +00:00
|
|
|
flags |= PTE_SX;
|
2008-03-03 17:17:00 +00:00
|
|
|
if (!su)
|
2009-01-13 15:41:58 +00:00
|
|
|
flags |= PTE_UX;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/*
|
|
|
|
* Check existing flags for execute permissions: if we
|
|
|
|
* are turning execute permissions on, icache should
|
|
|
|
* be flushed.
|
|
|
|
*/
|
2016-02-11 13:15:37 +00:00
|
|
|
if ((*pte & (PTE_UX | PTE_SX)) == 0)
|
2008-03-03 17:17:00 +00:00
|
|
|
sync++;
|
|
|
|
}
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
flags &= ~PTE_REFERENCED;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The new flags value is all calculated -- only now actually
|
|
|
|
* update the PTE.
|
|
|
|
*/
|
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2009-01-13 15:41:58 +00:00
|
|
|
|
|
|
|
tlb0_flush_entry(va);
|
2016-02-16 02:13:55 +00:00
|
|
|
*pte &= ~PTE_FLAGS_MASK;
|
|
|
|
*pte |= flags;
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
} else {
|
|
|
|
/*
|
2009-01-13 15:41:58 +00:00
|
|
|
* If there is an existing mapping, but it's for a different
|
2008-03-03 17:17:00 +00:00
|
|
|
* physical address, pte_enter() will delete the old mapping.
|
|
|
|
*/
|
|
|
|
//if ((pte != NULL) && PTE_ISVALID(pte))
|
|
|
|
// debugf("mmu_booke_enter_locked: replace\n");
|
|
|
|
//else
|
|
|
|
// debugf("mmu_booke_enter_locked: new\n");
|
|
|
|
|
|
|
|
/* Now set up the flags and install the new mapping. */
|
|
|
|
flags = (PTE_SR | PTE_VALID);
|
2009-01-13 15:41:58 +00:00
|
|
|
flags |= PTE_M;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
if (!su)
|
|
|
|
flags |= PTE_UR;
|
|
|
|
|
|
|
|
if (prot & VM_PROT_WRITE) {
|
|
|
|
flags |= PTE_SW;
|
|
|
|
if (!su)
|
|
|
|
flags |= PTE_UW;
|
2009-05-26 06:24:50 +00:00
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
if ((m->oflags & VPO_UNMANAGED) == 0)
|
2011-09-06 10:30:11 +00:00
|
|
|
vm_page_aflag_set(m, PGA_WRITEABLE);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (prot & VM_PROT_EXECUTE) {
|
|
|
|
flags |= PTE_SX;
|
|
|
|
if (!su)
|
|
|
|
flags |= PTE_UX;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If its wired update stats. */
|
2014-08-08 17:12:03 +00:00
|
|
|
if ((pmap_flags & PMAP_ENTER_WIRED) != 0)
|
2008-03-03 17:17:00 +00:00
|
|
|
flags |= PTE_WIRED;
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
error = pte_enter(pmap, m, va, flags,
|
2014-08-08 17:12:03 +00:00
|
|
|
(pmap_flags & PMAP_ENTER_NOSLEEP) != 0);
|
|
|
|
if (error != 0)
|
|
|
|
return (KERN_RESOURCE_SHORTAGE);
|
|
|
|
|
|
|
|
if ((flags & PMAP_ENTER_WIRED) != 0)
|
|
|
|
pmap->pm_stats.wired_count++;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Flush the real memory from the instruction cache. */
|
|
|
|
if (prot & VM_PROT_EXECUTE)
|
|
|
|
sync++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sync && (su || pmap == PCPU_GET(curpmap))) {
|
|
|
|
__syncicache((void *)va, PAGE_SIZE);
|
|
|
|
sync = 0;
|
|
|
|
}
|
2014-08-08 17:12:03 +00:00
|
|
|
|
|
|
|
return (KERN_SUCCESS);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Maps a sequence of resident pages belonging to the same object.
|
|
|
|
* The sequence begins with the given page m_start. This page is
|
|
|
|
* mapped at the given virtual address start. Each subsequent page is
|
|
|
|
* mapped at a virtual address that is offset from start by the same
|
|
|
|
* amount as the page is offset from m_start within the object. The
|
|
|
|
* last page in the sequence is the page with the largest offset from
|
|
|
|
* m_start that can be mapped at a virtual address less than the given
|
|
|
|
* virtual address end. Not every virtual page between start and end
|
|
|
|
* is mapped; only those for which a resident page exists with the
|
|
|
|
* corresponding offset from m_start are mapped.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_enter_object(pmap_t pmap, vm_offset_t start,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
|
|
|
|
{
|
|
|
|
vm_page_t m;
|
|
|
|
vm_pindex_t diff, psize;
|
|
|
|
|
2013-05-21 20:38:19 +00:00
|
|
|
VM_OBJECT_ASSERT_LOCKED(m_start->object);
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
psize = atop(end - start);
|
|
|
|
m = m_start;
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_LOCK(pmap);
|
|
|
|
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_enter_locked(pmap, start + ptoa(diff), m,
|
2014-08-08 17:12:03 +00:00
|
|
|
prot & (VM_PROT_READ | VM_PROT_EXECUTE),
|
2019-10-16 07:09:15 +00:00
|
|
|
PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, 0);
|
2008-03-03 17:17:00 +00:00
|
|
|
m = TAILQ_NEXT(m, listq);
|
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pmap);
|
2020-02-22 01:31:06 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_prot_t prot)
|
|
|
|
{
|
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_LOCK(pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_enter_locked(pmap, va, m,
|
2019-10-16 07:09:15 +00:00
|
|
|
prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP |
|
|
|
|
PMAP_ENTER_QUICK_LOCKED, 0);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_UNLOCK(pmap);
|
2020-02-22 01:31:06 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove the given range of addresses from the specified map.
|
|
|
|
*
|
|
|
|
* It is assumed that the start and end are properly rounded to the page size.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_remove(pmap_t pmap, vm_offset_t va, vm_offset_t endva)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
2009-01-13 16:15:49 +00:00
|
|
|
uint8_t hold_flag;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
int su = (pmap == kernel_pmap);
|
|
|
|
|
|
|
|
//debugf("mmu_booke_remove: s (su = %d pmap=0x%08x tid=%d va=0x%08x endva=0x%08x)\n",
|
|
|
|
// su, (u_int32_t)pmap, pmap->pm_tid, va, endva);
|
|
|
|
|
|
|
|
if (su) {
|
2009-01-13 16:15:49 +00:00
|
|
|
KASSERT(((va >= virtual_avail) &&
|
|
|
|
(va <= VM_MAX_KERNEL_ADDRESS)),
|
|
|
|
("mmu_booke_remove: kernel pmap, non kernel va"));
|
2008-03-03 17:17:00 +00:00
|
|
|
} else {
|
|
|
|
KASSERT((va <= VM_MAXUSER_ADDRESS),
|
2009-01-13 16:15:49 +00:00
|
|
|
("mmu_booke_remove: user pmap, non user va"));
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (PMAP_REMOVE_DONE(pmap)) {
|
|
|
|
//debugf("mmu_booke_remove: e (empty)\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
hold_flag = PTBL_HOLD_FLAG(pmap);
|
|
|
|
//debugf("mmu_booke_remove: hold_flag = %d\n", hold_flag);
|
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_LOCK(pmap);
|
|
|
|
for (; va < endva; va += PAGE_SIZE) {
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find_next(pmap, &va);
|
2020-04-11 00:16:50 +00:00
|
|
|
if ((pte == NULL) || !PTE_ISVALID(pte))
|
|
|
|
break;
|
|
|
|
if (va >= endva)
|
|
|
|
break;
|
2020-05-27 01:24:12 +00:00
|
|
|
pte_remove(pmap, va, hold_flag);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pmap);
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
//debugf("mmu_booke_remove: e\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove physical page from all pmaps in which it resides.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_remove_all(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pv_entry_t pv, pvn;
|
2009-01-13 16:15:49 +00:00
|
|
|
uint8_t hold_flag;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2020-03-30 16:32:55 +00:00
|
|
|
TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_link, pvn) {
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
|
|
|
hold_flag = PTBL_HOLD_FLAG(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
pte_remove(pv->pv_pmap, pv->pv_va, hold_flag);
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
}
|
2011-09-06 10:30:11 +00:00
|
|
|
vm_page_aflag_clear(m, PGA_WRITEABLE);
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Map a range of physical addresses into kernel virtual address space.
|
|
|
|
*/
|
|
|
|
static vm_offset_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_map(vm_offset_t *virt, vm_paddr_t pa_start,
|
2012-05-24 21:13:24 +00:00
|
|
|
vm_paddr_t pa_end, int prot)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
vm_offset_t sva = *virt;
|
|
|
|
vm_offset_t va = sva;
|
|
|
|
|
2019-08-25 20:08:48 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
/* XXX: Handle memory not starting at 0x0. */
|
|
|
|
if (pa_end < ctob(Maxmem))
|
|
|
|
return (PHYS_TO_DMAP(pa_start));
|
|
|
|
#endif
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
while (pa_start < pa_end) {
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_kenter(va, pa_start);
|
2008-03-03 17:17:00 +00:00
|
|
|
va += PAGE_SIZE;
|
|
|
|
pa_start += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
*virt = va;
|
|
|
|
|
|
|
|
return (sva);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The pmap must be activated before it's address space can be accessed in any
|
|
|
|
* way.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_activate(struct thread *td)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pmap_t pmap;
|
2011-06-16 07:27:13 +00:00
|
|
|
u_int cpuid;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
pmap = &td->td_proc->p_vmspace->vm_pmap;
|
|
|
|
|
2018-11-28 04:02:26 +00:00
|
|
|
CTR5(KTR_PMAP, "%s: s (td = %p, proc = '%s', id = %d, pmap = 0x%"PRI0ptrX")",
|
2009-01-13 15:41:58 +00:00
|
|
|
__func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!"));
|
|
|
|
|
2013-11-11 17:37:50 +00:00
|
|
|
sched_pin();
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2011-06-16 07:27:13 +00:00
|
|
|
cpuid = PCPU_GET(cpuid);
|
|
|
|
CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
|
2008-03-03 17:17:00 +00:00
|
|
|
PCPU_SET(curpmap, pmap);
|
2020-09-01 21:20:08 +00:00
|
|
|
|
2011-06-16 07:27:13 +00:00
|
|
|
if (pmap->pm_tid[cpuid] == TID_NONE)
|
2008-03-03 17:17:00 +00:00
|
|
|
tid_alloc(pmap);
|
|
|
|
|
|
|
|
/* Load PID0 register with pmap tid value. */
|
2011-06-16 07:27:13 +00:00
|
|
|
mtspr(SPR_PID0, pmap->pm_tid[cpuid]);
|
2009-01-13 15:41:58 +00:00
|
|
|
__asm __volatile("isync");
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2015-03-04 16:45:31 +00:00
|
|
|
mtspr(SPR_DBCR0, td->td_pcb->pcb_cpu.booke.dbcr0);
|
|
|
|
|
2013-11-11 17:37:50 +00:00
|
|
|
sched_unpin();
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR3(KTR_PMAP, "%s: e (tid = %d for '%s')", __func__,
|
|
|
|
pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Deactivate the specified process's address space.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_deactivate(struct thread *td)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pmap_t pmap;
|
|
|
|
|
|
|
|
pmap = &td->td_proc->p_vmspace->vm_pmap;
|
2020-09-01 21:20:08 +00:00
|
|
|
|
2018-11-28 04:02:26 +00:00
|
|
|
CTR5(KTR_PMAP, "%s: td=%p, proc = '%s', id = %d, pmap = 0x%"PRI0ptrX,
|
2009-01-13 15:41:58 +00:00
|
|
|
__func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);
|
|
|
|
|
2015-03-04 16:45:31 +00:00
|
|
|
td->td_pcb->pcb_cpu.booke.dbcr0 = mfspr(SPR_DBCR0);
|
|
|
|
|
2011-06-16 07:27:13 +00:00
|
|
|
CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmap->pm_active);
|
2008-03-03 17:17:00 +00:00
|
|
|
PCPU_SET(curpmap, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy the range specified by src_addr/len
|
|
|
|
* from the source map to the range dst_addr/len
|
|
|
|
* in the destination map.
|
|
|
|
*
|
|
|
|
* This routine is only advisory and need not do anything.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_copy(pmap_t dst_pmap, pmap_t src_pmap,
|
2009-06-13 08:57:04 +00:00
|
|
|
vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the physical protection on the specified range of this map as requested.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_prot_t prot)
|
|
|
|
{
|
|
|
|
vm_offset_t va;
|
|
|
|
vm_page_t m;
|
|
|
|
pte_t *pte;
|
|
|
|
|
|
|
|
if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_remove(pmap, sva, eva);
|
2008-03-03 17:17:00 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (prot & VM_PROT_WRITE)
|
|
|
|
return;
|
|
|
|
|
|
|
|
PMAP_LOCK(pmap);
|
|
|
|
for (va = sva; va < eva; va += PAGE_SIZE) {
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pmap, va)) != NULL) {
|
2008-03-03 17:17:00 +00:00
|
|
|
if (PTE_ISVALID(pte)) {
|
|
|
|
m = PHYS_TO_VM_PAGE(PTE_PA(pte));
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Handle modified pages. */
|
2010-04-30 15:22:52 +00:00
|
|
|
if (PTE_ISMODIFIED(pte) && PTE_ISMANAGED(pte))
|
2008-04-27 21:04:54 +00:00
|
|
|
vm_page_dirty(m);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
tlb0_flush_entry(va);
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED);
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Clear the write and modified bits in each of the given page's mappings.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_remove_write(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pv_entry_t pv;
|
|
|
|
pte_t *pte;
|
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
|
2010-05-16 23:45:10 +00:00
|
|
|
("mmu_booke_remove_write: page %p is not managed", m));
|
2019-10-15 03:51:46 +00:00
|
|
|
vm_page_assert_busied(m);
|
2010-05-16 23:45:10 +00:00
|
|
|
|
2019-10-15 03:51:46 +00:00
|
|
|
if (!pmap_page_is_write_mapped(m))
|
|
|
|
return;
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL) {
|
2008-03-03 17:17:00 +00:00
|
|
|
if (PTE_ISVALID(pte)) {
|
|
|
|
m = PHYS_TO_VM_PAGE(PTE_PA(pte));
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Handle modified pages. */
|
2008-04-27 21:04:54 +00:00
|
|
|
if (PTE_ISMODIFIED(pte))
|
|
|
|
vm_page_dirty(m);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Flush mapping from TLB0. */
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED);
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
}
|
2011-09-06 10:30:11 +00:00
|
|
|
vm_page_aflag_clear(m, PGA_WRITEABLE);
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Atomically extract and hold the physical page with the given
|
|
|
|
* pmap and virtual address pair if that mapping permits the given
|
|
|
|
* protection.
|
|
|
|
*/
|
|
|
|
static vm_page_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_extract_and_hold(pmap_t pmap, vm_offset_t va,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_prot_t prot)
|
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
vm_page_t m;
|
2009-01-13 16:15:49 +00:00
|
|
|
uint32_t pte_wbit;
|
Change synchonization rules for vm_page reference counting.
There are several mechanisms by which a vm_page reference is held,
preventing the page from being freed back to the page allocator. In
particular, holding the page's object lock is sufficient to prevent the
page from being freed; holding the busy lock or a wiring is sufficent as
well. These references are protected by the page lock, which must
therefore be acquired for many per-page operations. This results in
false sharing since the page locks are external to the vm_page
structures themselves and each lock protects multiple structures.
Transition to using an atomically updated per-page reference counter.
The object's reference is counted using a flag bit in the counter. A
second flag bit is used to atomically block new references via
pmap_extract_and_hold() while removing managed mappings of a page.
Thus, the reference count of a page is guaranteed not to increase if the
page is unbusied, unmapped, and the object's write lock is held. As
a consequence of this, the page lock no longer protects a page's
identity; operations which move pages between objects are now
synchronized solely by the objects' locks.
The vm_page_wire() and vm_page_unwire() KPIs are changed. The former
requires that either the object lock or the busy lock is held. The
latter no longer has a return value and may free the page if it releases
the last reference to that page. vm_page_unwire_noq() behaves the same
as before; the caller is responsible for checking its return value and
freeing or enqueuing the page as appropriate. vm_page_wire_mapped() is
introduced for use in pmap_extract_and_hold(). It fails if the page is
concurrently being unmapped, typically triggering a fallback to the
fault handler. vm_page_wire() no longer requires the page lock and
vm_page_unwire() now internally acquires the page lock when releasing
the last wiring of a page (since the page lock still protects a page's
queue state). In particular, synchronization details are no longer
leaked into the caller.
The change excises the page lock from several frequently executed code
paths. In particular, vm_object_terminate() no longer bounces between
page locks as it releases an object's pages, and direct I/O and
sendfile(SF_NOCACHE) completions no longer require the page lock. In
these latter cases we now get linear scalability in the common scenario
where different threads are operating on different files.
__FreeBSD_version is bumped. The DRM ports have been updated to
accomodate the KPI changes.
Reviewed by: jeff (earlier version)
Tested by: gallatin (earlier version), pho
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D20486
2019-09-09 21:32:42 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
m = NULL;
|
|
|
|
PMAP_LOCK(pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(pmap, va);
|
2008-03-03 17:17:00 +00:00
|
|
|
if ((pte != NULL) && PTE_ISVALID(pte)) {
|
|
|
|
if (pmap == kernel_pmap)
|
|
|
|
pte_wbit = PTE_SW;
|
|
|
|
else
|
|
|
|
pte_wbit = PTE_UW;
|
|
|
|
|
Change synchonization rules for vm_page reference counting.
There are several mechanisms by which a vm_page reference is held,
preventing the page from being freed back to the page allocator. In
particular, holding the page's object lock is sufficient to prevent the
page from being freed; holding the busy lock or a wiring is sufficent as
well. These references are protected by the page lock, which must
therefore be acquired for many per-page operations. This results in
false sharing since the page locks are external to the vm_page
structures themselves and each lock protects multiple structures.
Transition to using an atomically updated per-page reference counter.
The object's reference is counted using a flag bit in the counter. A
second flag bit is used to atomically block new references via
pmap_extract_and_hold() while removing managed mappings of a page.
Thus, the reference count of a page is guaranteed not to increase if the
page is unbusied, unmapped, and the object's write lock is held. As
a consequence of this, the page lock no longer protects a page's
identity; operations which move pages between objects are now
synchronized solely by the objects' locks.
The vm_page_wire() and vm_page_unwire() KPIs are changed. The former
requires that either the object lock or the busy lock is held. The
latter no longer has a return value and may free the page if it releases
the last reference to that page. vm_page_unwire_noq() behaves the same
as before; the caller is responsible for checking its return value and
freeing or enqueuing the page as appropriate. vm_page_wire_mapped() is
introduced for use in pmap_extract_and_hold(). It fails if the page is
concurrently being unmapped, typically triggering a fallback to the
fault handler. vm_page_wire() no longer requires the page lock and
vm_page_unwire() now internally acquires the page lock when releasing
the last wiring of a page (since the page lock still protects a page's
queue state). In particular, synchronization details are no longer
leaked into the caller.
The change excises the page lock from several frequently executed code
paths. In particular, vm_object_terminate() no longer bounces between
page locks as it releases an object's pages, and direct I/O and
sendfile(SF_NOCACHE) completions no longer require the page lock. In
these latter cases we now get linear scalability in the common scenario
where different threads are operating on different files.
__FreeBSD_version is bumped. The DRM ports have been updated to
accomodate the KPI changes.
Reviewed by: jeff (earlier version)
Tested by: gallatin (earlier version), pho
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D20486
2019-09-09 21:32:42 +00:00
|
|
|
if ((*pte & pte_wbit) != 0 || (prot & VM_PROT_WRITE) == 0) {
|
2008-03-03 17:17:00 +00:00
|
|
|
m = PHYS_TO_VM_PAGE(PTE_PA(pte));
|
Change synchonization rules for vm_page reference counting.
There are several mechanisms by which a vm_page reference is held,
preventing the page from being freed back to the page allocator. In
particular, holding the page's object lock is sufficient to prevent the
page from being freed; holding the busy lock or a wiring is sufficent as
well. These references are protected by the page lock, which must
therefore be acquired for many per-page operations. This results in
false sharing since the page locks are external to the vm_page
structures themselves and each lock protects multiple structures.
Transition to using an atomically updated per-page reference counter.
The object's reference is counted using a flag bit in the counter. A
second flag bit is used to atomically block new references via
pmap_extract_and_hold() while removing managed mappings of a page.
Thus, the reference count of a page is guaranteed not to increase if the
page is unbusied, unmapped, and the object's write lock is held. As
a consequence of this, the page lock no longer protects a page's
identity; operations which move pages between objects are now
synchronized solely by the objects' locks.
The vm_page_wire() and vm_page_unwire() KPIs are changed. The former
requires that either the object lock or the busy lock is held. The
latter no longer has a return value and may free the page if it releases
the last reference to that page. vm_page_unwire_noq() behaves the same
as before; the caller is responsible for checking its return value and
freeing or enqueuing the page as appropriate. vm_page_wire_mapped() is
introduced for use in pmap_extract_and_hold(). It fails if the page is
concurrently being unmapped, typically triggering a fallback to the
fault handler. vm_page_wire() no longer requires the page lock and
vm_page_unwire() now internally acquires the page lock when releasing
the last wiring of a page (since the page lock still protects a page's
queue state). In particular, synchronization details are no longer
leaked into the caller.
The change excises the page lock from several frequently executed code
paths. In particular, vm_object_terminate() no longer bounces between
page locks as it releases an object's pages, and direct I/O and
sendfile(SF_NOCACHE) completions no longer require the page lock. In
these latter cases we now get linear scalability in the common scenario
where different threads are operating on different files.
__FreeBSD_version is bumped. The DRM ports have been updated to
accomodate the KPI changes.
Reviewed by: jeff (earlier version)
Tested by: gallatin (earlier version), pho
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D20486
2019-09-09 21:32:42 +00:00
|
|
|
if (!vm_page_wire_mapped(m))
|
|
|
|
m = NULL;
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pmap);
|
|
|
|
return (m);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize a vm_page's machine-dependent fields.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_page_init(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
|
2017-11-10 04:23:58 +00:00
|
|
|
m->md.pv_tracked = 0;
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_INIT(&m->md.pv_list);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return whether or not the specified physical page was modified
|
|
|
|
* in any of physical maps.
|
|
|
|
*/
|
|
|
|
static boolean_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_is_modified(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
pv_entry_t pv;
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
boolean_t rv;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
("mmu_booke_is_modified: page %p is not managed", m));
|
|
|
|
rv = FALSE;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
/*
|
2019-10-15 03:51:46 +00:00
|
|
|
* If the page is not busied then this check is racy.
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
*/
|
2019-10-15 03:51:46 +00:00
|
|
|
if (!pmap_page_is_write_mapped(m))
|
|
|
|
return (FALSE);
|
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
PTE_ISVALID(pte)) {
|
|
|
|
if (PTE_ISMODIFIED(pte))
|
|
|
|
rv = TRUE;
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
if (rv)
|
|
|
|
break;
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
return (rv);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-01-13 16:15:49 +00:00
|
|
|
* Return whether or not the specified virtual address is eligible
|
2008-03-03 17:17:00 +00:00
|
|
|
* for prefault.
|
|
|
|
*/
|
|
|
|
static boolean_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_is_prefaultable(pmap_t pmap, vm_offset_t addr)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
return (FALSE);
|
|
|
|
}
|
|
|
|
|
Resurrect pmap_is_referenced() and use it in mincore(). Essentially,
pmap_ts_referenced() is not always appropriate for checking whether or
not pages have been referenced because it clears any reference bits
that it encounters. For example, in mincore(), clearing the reference
bits has two negative consequences. First, it throws off the activity
count calculations performed by the page daemon. Specifically, a page
on which mincore() has called pmap_ts_referenced() looks less active
to the page daemon than it should. Consequently, the page could be
deactivated prematurely by the page daemon. Arguably, this problem
could be fixed by having mincore() duplicate the activity count
calculation on the page. However, there is a second problem for which
that is not a solution. In order to clear a reference on a 4KB page,
it may be necessary to demote a 2/4MB page mapping. Thus, a mincore()
by one process can have the side effect of demoting a superpage
mapping within another process!
2010-04-24 17:32:52 +00:00
|
|
|
/*
|
|
|
|
* Return whether or not the specified physical page was referenced
|
|
|
|
* in any physical maps.
|
|
|
|
*/
|
|
|
|
static boolean_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_is_referenced(vm_page_t m)
|
Resurrect pmap_is_referenced() and use it in mincore(). Essentially,
pmap_ts_referenced() is not always appropriate for checking whether or
not pages have been referenced because it clears any reference bits
that it encounters. For example, in mincore(), clearing the reference
bits has two negative consequences. First, it throws off the activity
count calculations performed by the page daemon. Specifically, a page
on which mincore() has called pmap_ts_referenced() looks less active
to the page daemon than it should. Consequently, the page could be
deactivated prematurely by the page daemon. Arguably, this problem
could be fixed by having mincore() duplicate the activity count
calculation on the page. However, there is a second problem for which
that is not a solution. In order to clear a reference on a 4KB page,
it may be necessary to demote a 2/4MB page mapping. Thus, a mincore()
by one process can have the side effect of demoting a superpage
mapping within another process!
2010-04-24 17:32:52 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
pv_entry_t pv;
|
|
|
|
boolean_t rv;
|
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
|
2010-05-26 18:00:44 +00:00
|
|
|
("mmu_booke_is_referenced: page %p is not managed", m));
|
Resurrect pmap_is_referenced() and use it in mincore(). Essentially,
pmap_ts_referenced() is not always appropriate for checking whether or
not pages have been referenced because it clears any reference bits
that it encounters. For example, in mincore(), clearing the reference
bits has two negative consequences. First, it throws off the activity
count calculations performed by the page daemon. Specifically, a page
on which mincore() has called pmap_ts_referenced() looks less active
to the page daemon than it should. Consequently, the page could be
deactivated prematurely by the page daemon. Arguably, this problem
could be fixed by having mincore() duplicate the activity count
calculation on the page. However, there is a second problem for which
that is not a solution. In order to clear a reference on a 4KB page,
it may be necessary to demote a 2/4MB page mapping. Thus, a mincore()
by one process can have the side effect of demoting a superpage
mapping within another process!
2010-04-24 17:32:52 +00:00
|
|
|
rv = FALSE;
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
Resurrect pmap_is_referenced() and use it in mincore(). Essentially,
pmap_ts_referenced() is not always appropriate for checking whether or
not pages have been referenced because it clears any reference bits
that it encounters. For example, in mincore(), clearing the reference
bits has two negative consequences. First, it throws off the activity
count calculations performed by the page daemon. Specifically, a page
on which mincore() has called pmap_ts_referenced() looks less active
to the page daemon than it should. Consequently, the page could be
deactivated prematurely by the page daemon. Arguably, this problem
could be fixed by having mincore() duplicate the activity count
calculation on the page. However, there is a second problem for which
that is not a solution. In order to clear a reference on a 4KB page,
it may be necessary to demote a 2/4MB page mapping. Thus, a mincore()
by one process can have the side effect of demoting a superpage
mapping within another process!
2010-04-24 17:32:52 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
|
2010-05-26 18:00:44 +00:00
|
|
|
PTE_ISVALID(pte)) {
|
|
|
|
if (PTE_ISREFERENCED(pte))
|
|
|
|
rv = TRUE;
|
|
|
|
}
|
Resurrect pmap_is_referenced() and use it in mincore(). Essentially,
pmap_ts_referenced() is not always appropriate for checking whether or
not pages have been referenced because it clears any reference bits
that it encounters. For example, in mincore(), clearing the reference
bits has two negative consequences. First, it throws off the activity
count calculations performed by the page daemon. Specifically, a page
on which mincore() has called pmap_ts_referenced() looks less active
to the page daemon than it should. Consequently, the page could be
deactivated prematurely by the page daemon. Arguably, this problem
could be fixed by having mincore() duplicate the activity count
calculation on the page. However, there is a second problem for which
that is not a solution. In order to clear a reference on a 4KB page,
it may be necessary to demote a 2/4MB page mapping. Thus, a mincore()
by one process can have the side effect of demoting a superpage
mapping within another process!
2010-04-24 17:32:52 +00:00
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
if (rv)
|
|
|
|
break;
|
|
|
|
}
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
Resurrect pmap_is_referenced() and use it in mincore(). Essentially,
pmap_ts_referenced() is not always appropriate for checking whether or
not pages have been referenced because it clears any reference bits
that it encounters. For example, in mincore(), clearing the reference
bits has two negative consequences. First, it throws off the activity
count calculations performed by the page daemon. Specifically, a page
on which mincore() has called pmap_ts_referenced() looks less active
to the page daemon than it should. Consequently, the page could be
deactivated prematurely by the page daemon. Arguably, this problem
could be fixed by having mincore() duplicate the activity count
calculation on the page. However, there is a second problem for which
that is not a solution. In order to clear a reference on a 4KB page,
it may be necessary to demote a 2/4MB page mapping. Thus, a mincore()
by one process can have the side effect of demoting a superpage
mapping within another process!
2010-04-24 17:32:52 +00:00
|
|
|
return (rv);
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Clear the modify bits on the specified physical page.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_clear_modify(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
pv_entry_t pv;
|
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
("mmu_booke_clear_modify: page %p is not managed", m));
|
2019-10-15 03:51:46 +00:00
|
|
|
vm_page_assert_busied(m);
|
|
|
|
|
|
|
|
if (!pmap_page_is_write_mapped(m))
|
|
|
|
return;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
|
Roughly half of a typical pmap_mincore() implementation is machine-
independent code. Move this code into mincore(), and eliminate the
page queues lock from pmap_mincore().
Push down the page queues lock into pmap_clear_modify(),
pmap_clear_reference(), and pmap_is_modified(). Assert that these
functions are never passed an unmanaged page.
Eliminate an inaccurate comment from powerpc/powerpc/mmu_if.m:
Contrary to what the comment says, pmap_mincore() is not simply an
optimization. Without a complete pmap_mincore() implementation,
mincore() cannot return either MINCORE_MODIFIED or MINCORE_REFERENCED
because only the pmap can provide this information.
Eliminate the page queues lock from vfs_setdirty_locked_object(),
vm_pageout_clean(), vm_object_page_collect_flush(), and
vm_object_page_clean(). Generally speaking, these are all accesses
to the page's dirty field, which are synchronized by the containing
vm object's lock.
Reduce the scope of the page queues lock in vm_object_madvise() and
vm_page_dontneed().
Reviewed by: kib (an earlier version)
2010-05-24 14:26:57 +00:00
|
|
|
PTE_ISVALID(pte)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2016-02-11 13:15:37 +00:00
|
|
|
if (*pte & (PTE_SW | PTE_UW | PTE_MODIFIED)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
tlb0_flush_entry(pv->pv_va);
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte &= ~(PTE_SW | PTE_UW | PTE_MODIFIED |
|
2008-03-03 17:17:00 +00:00
|
|
|
PTE_REFERENCED);
|
|
|
|
}
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
}
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return a count of reference bits for a page, clearing those bits.
|
|
|
|
* It is not necessary for every reference bit to be cleared, but it
|
|
|
|
* is necessary that 0 only be returned when there are truly no
|
|
|
|
* reference bits set.
|
|
|
|
*
|
2016-09-10 16:49:25 +00:00
|
|
|
* As an optimization, update the page's dirty field if a modified bit is
|
|
|
|
* found while counting reference bits. This opportunistic update can be
|
|
|
|
* performed at low cost and can eliminate the need for some future calls
|
|
|
|
* to pmap_is_modified(). However, since this function stops after
|
|
|
|
* finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
|
|
|
|
* dirty pages. Those dirty pages will only be detected by a future call
|
|
|
|
* to pmap_is_modified().
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_ts_referenced(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
pv_entry_t pv;
|
|
|
|
int count;
|
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
|
2010-06-10 16:56:35 +00:00
|
|
|
("mmu_booke_ts_referenced: page %p is not managed", m));
|
2008-03-03 17:17:00 +00:00
|
|
|
count = 0;
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL &&
|
2010-06-10 16:56:35 +00:00
|
|
|
PTE_ISVALID(pte)) {
|
2016-09-10 16:49:25 +00:00
|
|
|
if (PTE_ISMODIFIED(pte))
|
|
|
|
vm_page_dirty(m);
|
2008-03-03 17:17:00 +00:00
|
|
|
if (PTE_ISREFERENCED(pte)) {
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_lock();
|
2009-01-13 15:41:58 +00:00
|
|
|
|
|
|
|
tlb0_flush_entry(pv->pv_va);
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte &= ~PTE_REFERENCED;
|
2009-01-13 15:41:58 +00:00
|
|
|
|
2009-05-21 11:43:37 +00:00
|
|
|
tlb_miss_unlock();
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2016-09-10 16:49:25 +00:00
|
|
|
if (++count >= PMAP_TS_REFERENCED_MAX) {
|
2008-03-03 17:17:00 +00:00
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
}
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
return (count);
|
|
|
|
}
|
|
|
|
|
2014-07-13 16:27:57 +00:00
|
|
|
/*
|
|
|
|
* Clear the wired attribute from the mappings for the specified range of
|
|
|
|
* addresses in the given pmap. Every valid mapping within that range must
|
|
|
|
* have the wired attribute set. In contrast, invalid mappings cannot have
|
|
|
|
* the wired attribute set, so they are ignored.
|
|
|
|
*
|
|
|
|
* The wired attribute of the page table entry is not a hardware feature, so
|
|
|
|
* there is no need to invalidate any TLB entries.
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
|
2014-07-13 16:27:57 +00:00
|
|
|
{
|
|
|
|
vm_offset_t va;
|
|
|
|
pte_t *pte;
|
|
|
|
|
|
|
|
PMAP_LOCK(pmap);
|
|
|
|
for (va = sva; va < eva; va += PAGE_SIZE) {
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pmap, va)) != NULL &&
|
2014-07-13 16:27:57 +00:00
|
|
|
PTE_ISVALID(pte)) {
|
|
|
|
if (!PTE_ISWIRED(pte))
|
|
|
|
panic("mmu_booke_unwire: pte %p isn't wired",
|
|
|
|
pte);
|
2016-02-11 13:15:37 +00:00
|
|
|
*pte &= ~PTE_WIRED;
|
2014-07-13 16:27:57 +00:00
|
|
|
pmap->pm_stats.wired_count--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
PMAP_UNLOCK(pmap);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Return true if the pmap's pv is one of the first 16 pvs linked to from this
|
|
|
|
* page. This count may be changed upwards or downwards in the future; it is
|
|
|
|
* only necessary that true be returned for a small subset of pmaps for proper
|
|
|
|
* page aging.
|
|
|
|
*/
|
|
|
|
static boolean_t
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_page_exists_quick(pmap_t pmap, vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pv_entry_t pv;
|
|
|
|
int loops;
|
2010-06-10 16:56:35 +00:00
|
|
|
boolean_t rv;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
|
2010-06-10 16:56:35 +00:00
|
|
|
("mmu_booke_page_exists_quick: page %p is not managed", m));
|
2008-03-03 17:17:00 +00:00
|
|
|
loops = 0;
|
2010-06-10 16:56:35 +00:00
|
|
|
rv = FALSE;
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
2010-06-10 16:56:35 +00:00
|
|
|
if (pv->pv_pmap == pmap) {
|
|
|
|
rv = TRUE;
|
|
|
|
break;
|
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
if (++loops >= 16)
|
|
|
|
break;
|
|
|
|
}
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2010-06-10 16:56:35 +00:00
|
|
|
return (rv);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return the number of managed mappings to the given physical page that are
|
|
|
|
* wired.
|
|
|
|
*/
|
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_page_wired_mappings(vm_page_t m)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
pv_entry_t pv;
|
|
|
|
pte_t *pte;
|
|
|
|
int count = 0;
|
|
|
|
|
2011-08-09 21:01:36 +00:00
|
|
|
if ((m->oflags & VPO_UNMANAGED) != 0)
|
2008-03-03 17:17:00 +00:00
|
|
|
return (count);
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
|
|
|
|
PMAP_LOCK(pv->pv_pmap);
|
2020-05-27 01:24:12 +00:00
|
|
|
if ((pte = pte_find(pv->pv_pmap, pv->pv_va)) != NULL)
|
2008-03-03 17:17:00 +00:00
|
|
|
if (PTE_ISVALID(pte) && PTE_ISWIRED(pte))
|
|
|
|
count++;
|
|
|
|
PMAP_UNLOCK(pv->pv_pmap);
|
|
|
|
}
|
2012-11-03 23:22:49 +00:00
|
|
|
rw_wunlock(&pvh_global_lock);
|
2008-03-03 17:17:00 +00:00
|
|
|
return (count);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
vm_offset_t va;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This currently does not work for entries that
|
|
|
|
* overlap TLB1 entries.
|
|
|
|
*/
|
2016-04-19 01:48:18 +00:00
|
|
|
for (i = 0; i < TLB1_ENTRIES; i ++) {
|
2008-03-03 17:17:00 +00:00
|
|
|
if (tlb1_iomapped(i, pa, size, &va) == 0)
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (EFAULT);
|
|
|
|
}
|
|
|
|
|
2015-01-07 01:01:39 +00:00
|
|
|
void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_dumpsys_map(vm_paddr_t pa, size_t sz, void **va)
|
2009-04-04 22:01:43 +00:00
|
|
|
{
|
2015-01-07 01:01:39 +00:00
|
|
|
vm_paddr_t ppa;
|
|
|
|
vm_offset_t ofs;
|
2009-04-04 22:01:43 +00:00
|
|
|
vm_size_t gran;
|
|
|
|
|
|
|
|
/* Minidumps are based on virtual memory addresses. */
|
2015-01-07 01:01:39 +00:00
|
|
|
if (do_minidump) {
|
2015-08-22 07:20:03 +00:00
|
|
|
*va = (void *)(vm_offset_t)pa;
|
2015-01-07 01:01:39 +00:00
|
|
|
return;
|
2009-04-04 22:01:43 +00:00
|
|
|
}
|
2015-01-07 01:01:39 +00:00
|
|
|
|
|
|
|
/* Raw physical memory dumps don't have a virtual address. */
|
|
|
|
/* We always map a 256MB page at 256M. */
|
|
|
|
gran = 256 * 1024 * 1024;
|
2016-04-21 19:57:40 +00:00
|
|
|
ppa = rounddown2(pa, gran);
|
2015-01-07 01:01:39 +00:00
|
|
|
ofs = pa - ppa;
|
|
|
|
*va = (void *)gran;
|
|
|
|
tlb1_set_entry((vm_offset_t)va, ppa, gran, _TLB_ENTRY_IO);
|
|
|
|
|
|
|
|
if (sz > (gran - ofs))
|
|
|
|
tlb1_set_entry((vm_offset_t)(va + gran), ppa + gran, gran,
|
|
|
|
_TLB_ENTRY_IO);
|
2009-04-04 22:01:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_dumpsys_unmap(vm_paddr_t pa, size_t sz, void *va)
|
2009-04-04 22:01:43 +00:00
|
|
|
{
|
2015-01-07 01:01:39 +00:00
|
|
|
vm_paddr_t ppa;
|
|
|
|
vm_offset_t ofs;
|
|
|
|
vm_size_t gran;
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
|
|
|
int i;
|
2015-01-07 01:01:39 +00:00
|
|
|
|
|
|
|
/* Minidumps are based on virtual memory addresses. */
|
|
|
|
/* Nothing to do... */
|
|
|
|
if (do_minidump)
|
|
|
|
return;
|
2009-04-04 22:01:43 +00:00
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2009-04-04 22:01:43 +00:00
|
|
|
/* Raw physical memory dumps don't have a virtual address. */
|
2016-04-19 01:48:18 +00:00
|
|
|
i--;
|
|
|
|
e.mas1 = 0;
|
|
|
|
e.mas2 = 0;
|
|
|
|
e.mas3 = 0;
|
|
|
|
tlb1_write_entry(&e, i);
|
2015-01-07 01:01:39 +00:00
|
|
|
|
|
|
|
gran = 256 * 1024 * 1024;
|
2016-04-21 19:57:40 +00:00
|
|
|
ppa = rounddown2(pa, gran);
|
2015-01-07 01:01:39 +00:00
|
|
|
ofs = pa - ppa;
|
|
|
|
if (sz > (gran - ofs)) {
|
2016-04-19 01:48:18 +00:00
|
|
|
i--;
|
|
|
|
e.mas1 = 0;
|
|
|
|
e.mas2 = 0;
|
|
|
|
e.mas3 = 0;
|
|
|
|
tlb1_write_entry(&e, i);
|
2009-04-04 22:01:43 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-01-07 01:01:39 +00:00
|
|
|
extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1];
|
|
|
|
|
|
|
|
void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_scan_init()
|
2009-04-04 22:01:43 +00:00
|
|
|
{
|
|
|
|
vm_offset_t va;
|
2015-01-07 01:01:39 +00:00
|
|
|
pte_t *pte;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!do_minidump) {
|
|
|
|
/* Initialize phys. segments for dumpsys(). */
|
|
|
|
memset(&dump_map, 0, sizeof(dump_map));
|
|
|
|
mem_regions(&physmem_regions, &physmem_regions_sz, &availmem_regions,
|
|
|
|
&availmem_regions_sz);
|
|
|
|
for (i = 0; i < physmem_regions_sz; i++) {
|
|
|
|
dump_map[i].pa_start = physmem_regions[i].mr_start;
|
|
|
|
dump_map[i].pa_size = physmem_regions[i].mr_size;
|
2009-04-04 22:01:43 +00:00
|
|
|
}
|
2015-01-07 01:01:39 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Virtual segments for minidumps: */
|
|
|
|
memset(&dump_map, 0, sizeof(dump_map));
|
|
|
|
|
|
|
|
/* 1st: kernel .data and .bss. */
|
|
|
|
dump_map[0].pa_start = trunc_page((uintptr_t)_etext);
|
|
|
|
dump_map[0].pa_size =
|
|
|
|
round_page((uintptr_t)_end) - dump_map[0].pa_start;
|
|
|
|
|
|
|
|
/* 2nd: msgbuf and tables (see pmap_bootstrap()). */
|
|
|
|
dump_map[1].pa_start = data_start;
|
|
|
|
dump_map[1].pa_size = data_end - data_start;
|
|
|
|
|
|
|
|
/* 3rd: kernel VM. */
|
|
|
|
va = dump_map[1].pa_start + dump_map[1].pa_size;
|
|
|
|
/* Find start of next chunk (from va). */
|
|
|
|
while (va < virtual_end) {
|
|
|
|
/* Don't dump the buffer cache. */
|
|
|
|
if (va >= kmi.buffer_sva && va < kmi.buffer_eva) {
|
|
|
|
va = kmi.buffer_eva;
|
|
|
|
continue;
|
|
|
|
}
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(kernel_pmap, va);
|
2015-01-07 01:01:39 +00:00
|
|
|
if (pte != NULL && PTE_ISVALID(pte))
|
2009-04-04 22:01:43 +00:00
|
|
|
break;
|
2015-01-07 01:01:39 +00:00
|
|
|
va += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
if (va < virtual_end) {
|
|
|
|
dump_map[2].pa_start = va;
|
|
|
|
va += PAGE_SIZE;
|
|
|
|
/* Find last page in chunk. */
|
|
|
|
while (va < virtual_end) {
|
|
|
|
/* Don't run into the buffer cache. */
|
|
|
|
if (va == kmi.buffer_sva)
|
2009-04-04 22:01:43 +00:00
|
|
|
break;
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(kernel_pmap, va);
|
2015-01-07 01:01:39 +00:00
|
|
|
if (pte == NULL || !PTE_ISVALID(pte))
|
|
|
|
break;
|
|
|
|
va += PAGE_SIZE;
|
2009-04-04 22:01:43 +00:00
|
|
|
}
|
2015-01-07 01:01:39 +00:00
|
|
|
dump_map[2].pa_size = va - dump_map[2].pa_start;
|
2009-04-04 22:01:43 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Map a set of physical memory pages into the kernel virtual address space.
|
|
|
|
* Return a pointer to where it is mapped. This routine is intended to be used
|
|
|
|
* for mapping device memory, NOT real memory.
|
|
|
|
*/
|
|
|
|
static void *
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_mapdev(vm_paddr_t pa, vm_size_t size)
|
2013-10-26 14:52:55 +00:00
|
|
|
{
|
|
|
|
|
2020-05-27 01:24:12 +00:00
|
|
|
return (mmu_booke_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
|
2013-10-26 14:52:55 +00:00
|
|
|
}
|
|
|
|
|
2019-11-06 04:40:12 +00:00
|
|
|
static int
|
|
|
|
tlb1_find_pa(vm_paddr_t pa, tlb_entry_t *e)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(e, i);
|
|
|
|
if ((e->mas1 & MAS1_VALID) == 0)
|
2020-04-11 00:17:55 +00:00
|
|
|
continue;
|
|
|
|
if (e->phys == pa)
|
2019-11-06 04:40:12 +00:00
|
|
|
return (i);
|
|
|
|
}
|
|
|
|
return (-1);
|
|
|
|
}
|
|
|
|
|
2013-10-26 14:52:55 +00:00
|
|
|
static void *
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
2019-11-06 04:40:12 +00:00
|
|
|
vm_paddr_t tmppa;
|
2020-02-25 01:40:22 +00:00
|
|
|
#ifndef __powerpc64__
|
|
|
|
uintptr_t tmpva;
|
|
|
|
#endif
|
2020-04-11 00:17:55 +00:00
|
|
|
uintptr_t va, retva;
|
2008-10-25 03:36:21 +00:00
|
|
|
vm_size_t sz;
|
2013-10-26 18:18:14 +00:00
|
|
|
int i;
|
2019-11-06 04:40:12 +00:00
|
|
|
int wimge;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2012-11-03 22:02:12 +00:00
|
|
|
/*
|
2019-11-06 04:40:12 +00:00
|
|
|
* Check if this is premapped in TLB1.
|
2012-11-03 22:02:12 +00:00
|
|
|
*/
|
2019-11-06 04:40:12 +00:00
|
|
|
sz = size;
|
|
|
|
tmppa = pa;
|
|
|
|
va = ~0;
|
|
|
|
wimge = tlb_calc_wimg(pa, ma);
|
2017-11-10 04:14:48 +00:00
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
|
|
|
continue;
|
2019-11-06 04:40:12 +00:00
|
|
|
if (wimge != (e.mas2 & (MAS2_WIMGE_MASK & ~_TLB_ENTRY_SHARED)))
|
|
|
|
continue;
|
|
|
|
if (tmppa >= e.phys && tmppa < e.phys + e.size) {
|
|
|
|
va = e.virt + (pa - e.phys);
|
|
|
|
tmppa = e.phys + e.size;
|
2020-04-11 00:17:55 +00:00
|
|
|
sz -= MIN(sz, e.size - (pa - e.phys));
|
2019-11-06 04:40:12 +00:00
|
|
|
while (sz > 0 && (i = tlb1_find_pa(tmppa, &e)) != -1) {
|
|
|
|
if (wimge != (e.mas2 & (MAS2_WIMGE_MASK & ~_TLB_ENTRY_SHARED)))
|
|
|
|
break;
|
|
|
|
sz -= MIN(sz, e.size);
|
|
|
|
tmppa = e.phys + e.size;
|
|
|
|
}
|
|
|
|
if (sz != 0)
|
|
|
|
break;
|
|
|
|
return ((void *)va);
|
|
|
|
}
|
2012-11-03 22:02:12 +00:00
|
|
|
}
|
|
|
|
|
2013-10-26 18:18:14 +00:00
|
|
|
size = roundup(size, PAGE_SIZE);
|
2008-10-25 03:36:21 +00:00
|
|
|
|
2020-02-25 01:40:22 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
KASSERT(pa < VM_MAPDEV_PA_MAX,
|
|
|
|
("Unsupported physical address! %lx", pa));
|
|
|
|
va = VM_MAPDEV_BASE + pa;
|
2020-04-11 00:17:55 +00:00
|
|
|
retva = va;
|
|
|
|
#ifdef POW2_MAPPINGS
|
|
|
|
/*
|
|
|
|
* Align the mapping to a power of 2 size, taking into account that we
|
|
|
|
* may need to increase the size multiple times to satisfy the size and
|
|
|
|
* alignment requirements.
|
|
|
|
*
|
|
|
|
* This works in the general case because it's very rare (near never?)
|
|
|
|
* to have different access properties (WIMG) within a single
|
|
|
|
* power-of-two region. If a design does call for that, POW2_MAPPINGS
|
|
|
|
* can be undefined, and exact mappings will be used instead.
|
|
|
|
*/
|
|
|
|
sz = size;
|
|
|
|
size = roundup2(size, 1 << ilog2(size));
|
|
|
|
while (rounddown2(va, size) + size < va + sz)
|
|
|
|
size <<= 1;
|
|
|
|
va = rounddown2(va, size);
|
|
|
|
pa = rounddown2(pa, size);
|
|
|
|
#endif
|
2020-02-25 01:40:22 +00:00
|
|
|
#else
|
2013-10-26 20:57:26 +00:00
|
|
|
/*
|
2016-04-10 15:48:09 +00:00
|
|
|
* The device mapping area is between VM_MAXUSER_ADDRESS and
|
|
|
|
* VM_MIN_KERNEL_ADDRESS. This gives 1GB of device addressing.
|
2013-10-26 20:57:26 +00:00
|
|
|
*/
|
2016-04-10 15:48:09 +00:00
|
|
|
#ifdef SPARSE_MAPDEV
|
|
|
|
/*
|
|
|
|
* With a sparse mapdev, align to the largest starting region. This
|
|
|
|
* could feasibly be optimized for a 'best-fit' alignment, but that
|
|
|
|
* calculation could be very costly.
|
2017-11-10 04:14:48 +00:00
|
|
|
* Align to the smaller of:
|
|
|
|
* - first set bit in overlap of (pa & size mask)
|
|
|
|
* - largest size envelope
|
|
|
|
*
|
|
|
|
* It's possible the device mapping may start at a PA that's not larger
|
|
|
|
* than the size mask, so we need to offset in to maximize the TLB entry
|
|
|
|
* range and minimize the number of used TLB entries.
|
2016-04-10 15:48:09 +00:00
|
|
|
*/
|
|
|
|
do {
|
|
|
|
tmpva = tlb1_map_base;
|
2019-11-06 04:40:12 +00:00
|
|
|
sz = ffsl((~((1 << flsl(size-1)) - 1)) & pa);
|
2017-11-10 04:14:48 +00:00
|
|
|
sz = sz ? min(roundup(sz + 3, 4), flsl(size) - 1) : flsl(size) - 1;
|
|
|
|
va = roundup(tlb1_map_base, 1 << sz) | (((1 << sz) - 1) & pa);
|
2016-04-10 15:48:09 +00:00
|
|
|
} while (!atomic_cmpset_int(&tlb1_map_base, tmpva, va + size));
|
Introduce 64-bit PowerPC Book-E support
Extend the Book-E pmap to support 64-bit operation. Much of this was taken from
Juniper's Junos FreeBSD port. It uses a 3-level page table (page directory
list -- PP2D, page directory, page table), but has gaps in the page directory
list where regions will repeat, due to the design of the PP2D hash (a 20-bit gap
between the two parts of the index). In practice this may not be a problem
given the expanded address space. However, an alternative to this would be to
use a 4-level page table, like Linux, and possibly reduce the available address
space; Linux appears to use a 46-bit address space. Alternatively, a cache of
page directory pointers could be used to keep the overall design as-is, but
remove the gaps in the address space.
This includes a new kernel config for 64-bit QorIQ SoCs, based on MPC85XX, with
the following notes:
* The DPAA driver has not yet been ported to 64-bit so is not included in the
kernel config.
* This has been tested on the AmigaOne X5000, using a MD_ROOT compiled in
(total size kernel+mdroot must be under 64MB).
* This can run both 32-bit and 64-bit processes, and has even been tested to run
a 32-bit init with 64-bit children.
Many thanks to stevek and marcel for getting Juniper's FreeBSD patches open
sourced to be used here, and to stevek for reviewing, and providing some
historical contexts on quirks of the code.
Reviewed by: stevek
Obtained from: Juniper (in part)
MFC after: 2 months
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D9433
2017-03-17 21:40:14 +00:00
|
|
|
#endif
|
2020-04-11 00:17:55 +00:00
|
|
|
va = atomic_fetchadd_int(&tlb1_map_base, size);
|
|
|
|
retva = va;
|
2016-04-10 15:48:09 +00:00
|
|
|
#endif
|
2013-10-26 14:52:55 +00:00
|
|
|
|
2019-11-04 00:35:40 +00:00
|
|
|
if (tlb1_mapin_region(va, pa, size, tlb_calc_wimg(pa, ma)) != size)
|
|
|
|
return (NULL);
|
2008-10-25 03:36:21 +00:00
|
|
|
|
2020-04-11 00:17:55 +00:00
|
|
|
return ((void *)retva);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* 'Unmap' a range mapped by mmu_booke_mapdev().
|
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_unmapdev(vm_offset_t va, vm_size_t size)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2013-10-26 16:49:41 +00:00
|
|
|
#ifdef SUPPORTS_SHRINKING_TLB1
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_offset_t base, offset;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Unmap only if this is inside kernel virtual space.
|
|
|
|
*/
|
|
|
|
if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)) {
|
|
|
|
base = trunc_page(va);
|
|
|
|
offset = va & PAGE_MASK;
|
|
|
|
size = roundup(offset + size, PAGE_SIZE);
|
Revert r240317 to prevent leaking pmap entries
Subsequent to r240317, kmem_free() was replaced with kva_free() (r254025).
kva_free() releases the KVA allocation for the mapped region, but no longer
clears the pmap (pagetable) entries.
An affected pmap_unmapdev operation would leave the still-pmap'd VA space
free for allocation by other KVA consumers. However, this bug easily
avoided notice for ~7 years because most devices (1) never call
pmap_unmapdev and (2) on amd64, mostly fit within the DMAP and do not need
KVA allocations. Other affected arch are less popular: i386, MIPS, and
PowerPC. Arm64, arm32, and riscv are not affected.
Reported by: Don Morris <dgmorris AT earthlink.net>
Submitted by: Don Morris (amd64 part)
Reviewed by: kib, markj, Don (!amd64 parts)
MFC after: I don't intend to, but you might want to
Sponsored by: Dell Isilon
Differential Revision: https://reviews.freebsd.org/D25689
2020-07-16 23:29:26 +00:00
|
|
|
mmu_booke_qremove(base, atop(size));
|
2013-08-07 06:21:20 +00:00
|
|
|
kva_free(base, size);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
2013-10-26 16:49:41 +00:00
|
|
|
#endif
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-01-13 16:15:49 +00:00
|
|
|
* mmu_booke_object_init_pt preloads the ptes for a given object into the
|
|
|
|
* specified pmap. This eliminates the blast of soft faults on process startup
|
|
|
|
* and immediately after an mmap.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_object_init_pt(pmap_t pmap, vm_offset_t addr,
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_object_t object, vm_pindex_t pindex, vm_size_t size)
|
|
|
|
{
|
2009-01-13 16:15:49 +00:00
|
|
|
|
2013-03-09 02:32:23 +00:00
|
|
|
VM_OBJECT_ASSERT_WLOCKED(object);
|
2009-07-24 13:50:29 +00:00
|
|
|
KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
|
2008-03-03 17:17:00 +00:00
|
|
|
("mmu_booke_object_init_pt: non-device object"));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform the pmap work for mincore.
|
|
|
|
*/
|
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
|
2013-11-06 14:36:38 +00:00
|
|
|
/* XXX: this should be implemented at some point */
|
2008-03-03 17:17:00 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2016-02-27 20:39:36 +00:00
|
|
|
static int
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_change_attr(vm_offset_t addr, vm_size_t sz, vm_memattr_t mode)
|
2016-02-27 20:39:36 +00:00
|
|
|
{
|
|
|
|
vm_offset_t va;
|
|
|
|
pte_t *pte;
|
|
|
|
int i, j;
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
2016-02-27 20:39:36 +00:00
|
|
|
|
2019-11-06 04:40:12 +00:00
|
|
|
addr = trunc_page(addr);
|
|
|
|
|
|
|
|
/* Only allow changes to mapped kernel addresses. This includes:
|
|
|
|
* - KVA
|
|
|
|
* - DMAP (powerpc64)
|
|
|
|
* - Device mappings
|
|
|
|
*/
|
|
|
|
if (addr <= VM_MAXUSER_ADDRESS ||
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
(addr >= tlb1_map_base && addr < DMAP_BASE_ADDRESS) ||
|
|
|
|
(addr > DMAP_MAX_ADDRESS && addr < VM_MIN_KERNEL_ADDRESS) ||
|
|
|
|
#else
|
|
|
|
(addr >= tlb1_map_base && addr < VM_MIN_KERNEL_ADDRESS) ||
|
|
|
|
#endif
|
|
|
|
(addr > VM_MAX_KERNEL_ADDRESS))
|
|
|
|
return (EINVAL);
|
|
|
|
|
2016-02-27 20:39:36 +00:00
|
|
|
/* Check TLB1 mappings */
|
2016-04-19 01:48:18 +00:00
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
2016-02-27 20:39:36 +00:00
|
|
|
continue;
|
2016-04-19 01:48:18 +00:00
|
|
|
if (addr >= e.virt && addr < e.virt + e.size)
|
2016-02-27 20:39:36 +00:00
|
|
|
break;
|
|
|
|
}
|
2016-04-19 01:48:18 +00:00
|
|
|
if (i < TLB1_ENTRIES) {
|
2016-02-27 20:39:36 +00:00
|
|
|
/* Only allow full mappings to be modified for now. */
|
|
|
|
/* Validate the range. */
|
2016-04-19 01:48:18 +00:00
|
|
|
for (j = i, va = addr; va < addr + sz; va += e.size, j++) {
|
|
|
|
tlb1_read_entry(&e, j);
|
|
|
|
if (va != e.virt || (sz - (va - addr) < e.size))
|
2016-02-27 20:39:36 +00:00
|
|
|
return (EINVAL);
|
|
|
|
}
|
2016-04-19 01:48:18 +00:00
|
|
|
for (va = addr; va < addr + sz; va += e.size, i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
e.mas2 &= ~MAS2_WIMGE_MASK;
|
|
|
|
e.mas2 |= tlb_calc_wimg(e.phys, mode);
|
2016-02-27 20:39:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Write it out to the TLB. Should really re-sync with other
|
|
|
|
* cores.
|
|
|
|
*/
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb1_write_entry(&e, i);
|
2016-02-27 20:39:36 +00:00
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Not in TLB1, try through pmap */
|
|
|
|
/* First validate the range. */
|
|
|
|
for (va = addr; va < addr + sz; va += PAGE_SIZE) {
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(kernel_pmap, va);
|
2016-02-27 20:39:36 +00:00
|
|
|
if (pte == NULL || !PTE_ISVALID(pte))
|
|
|
|
return (EINVAL);
|
|
|
|
}
|
|
|
|
|
|
|
|
mtx_lock_spin(&tlbivax_mutex);
|
|
|
|
tlb_miss_lock();
|
|
|
|
for (va = addr; va < addr + sz; va += PAGE_SIZE) {
|
2020-05-27 01:24:12 +00:00
|
|
|
pte = pte_find(kernel_pmap, va);
|
2016-02-27 20:39:36 +00:00
|
|
|
*pte &= ~(PTE_MAS2_MASK << PTE_MAS2_SHIFT);
|
2016-12-04 02:15:46 +00:00
|
|
|
*pte |= tlb_calc_wimg(PTE_PA(pte), mode) << PTE_MAS2_SHIFT;
|
2016-02-27 20:39:36 +00:00
|
|
|
tlb0_flush_entry(va);
|
|
|
|
}
|
|
|
|
tlb_miss_unlock();
|
|
|
|
mtx_unlock_spin(&tlbivax_mutex);
|
|
|
|
|
2017-02-21 05:08:07 +00:00
|
|
|
return (0);
|
2016-02-27 20:39:36 +00:00
|
|
|
}
|
|
|
|
|
powerpc/pmap: NUMA-ize vm_page_array on powerpc
Summary:
This matches r351198 from amd64. This only applies to AIM64 and Book-E.
On AIM64 it short-circuits with one domain, to behave similar to
existing. Otherwise it will allocate 16MB huge pages to hold the page
array, across all NUMA domains. On the first domain it will shift the
page array base up, to "upper-align" the page array in that domain, so
as to reduce the number of pages from the next domain appearing in this
domain. After the first domain, subsequent domains will be allocated in
full 16MB pages, until the final domain, which can be short. This means
some inner domains may have pages accounted in earlier domains.
On Book-E the page array is setup at MMU bootstrap time so that it's
always mapped in TLB1, on both 32-bit and 64-bit. This reduces the TLB0
overhead for touching the vm_page_array, which reduces up to one TLB
miss per array access.
Since page_range (vm_page_startup()) is no longer used on Book-E but is on
32-bit AIM, mark the variable as potentially unused, rather than using a
nasty #if defined() list.
Reviewed by: luporl
Differential Revision: https://reviews.freebsd.org/D21449
2019-12-07 03:34:03 +00:00
|
|
|
static void
|
2020-05-27 01:24:12 +00:00
|
|
|
mmu_booke_page_array_startup(long pages)
|
powerpc/pmap: NUMA-ize vm_page_array on powerpc
Summary:
This matches r351198 from amd64. This only applies to AIM64 and Book-E.
On AIM64 it short-circuits with one domain, to behave similar to
existing. Otherwise it will allocate 16MB huge pages to hold the page
array, across all NUMA domains. On the first domain it will shift the
page array base up, to "upper-align" the page array in that domain, so
as to reduce the number of pages from the next domain appearing in this
domain. After the first domain, subsequent domains will be allocated in
full 16MB pages, until the final domain, which can be short. This means
some inner domains may have pages accounted in earlier domains.
On Book-E the page array is setup at MMU bootstrap time so that it's
always mapped in TLB1, on both 32-bit and 64-bit. This reduces the TLB0
overhead for touching the vm_page_array, which reduces up to one TLB
miss per array access.
Since page_range (vm_page_startup()) is no longer used on Book-E but is on
32-bit AIM, mark the variable as potentially unused, rather than using a
nasty #if defined() list.
Reviewed by: luporl
Differential Revision: https://reviews.freebsd.org/D21449
2019-12-07 03:34:03 +00:00
|
|
|
{
|
|
|
|
vm_page_array_size = pages;
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/**************************************************************************/
|
|
|
|
/* TID handling */
|
|
|
|
/**************************************************************************/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate a TID. If necessary, steal one from someone else.
|
|
|
|
* The new TID is flushed from the TLB before returning.
|
|
|
|
*/
|
|
|
|
static tlbtid_t
|
|
|
|
tid_alloc(pmap_t pmap)
|
|
|
|
{
|
|
|
|
tlbtid_t tid;
|
2009-01-13 15:41:58 +00:00
|
|
|
int thiscpu;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
KASSERT((pmap != kernel_pmap), ("tid_alloc: kernel pmap"));
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR2(KTR_PMAP, "%s: s (pmap = %p)", __func__, pmap);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
thiscpu = PCPU_GET(cpuid);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2018-02-17 20:59:12 +00:00
|
|
|
tid = PCPU_GET(booke.tid_next);
|
2009-01-13 15:41:58 +00:00
|
|
|
if (tid > TID_MAX)
|
|
|
|
tid = TID_MIN;
|
2018-02-17 20:59:12 +00:00
|
|
|
PCPU_SET(booke.tid_next, tid + 1);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/* If we are stealing TID then clear the relevant pmap's field */
|
|
|
|
if (tidbusy[thiscpu][tid] != NULL) {
|
|
|
|
CTR2(KTR_PMAP, "%s: warning: stealing tid %d", __func__, tid);
|
|
|
|
|
|
|
|
tidbusy[thiscpu][tid]->pm_tid[thiscpu] = TID_NONE;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/* Flush all entries from TLB0 matching this TID. */
|
2015-12-30 02:23:14 +00:00
|
|
|
tid_flush(tid);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
tidbusy[thiscpu][tid] = pmap;
|
|
|
|
pmap->pm_tid[thiscpu] = tid;
|
|
|
|
__asm __volatile("msync; isync");
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR3(KTR_PMAP, "%s: e (%02d next = %02d)", __func__, tid,
|
2018-02-17 20:59:12 +00:00
|
|
|
PCPU_GET(booke.tid_next));
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
return (tid);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**************************************************************************/
|
|
|
|
/* TLB0 handling */
|
|
|
|
/**************************************************************************/
|
|
|
|
|
|
|
|
/* Convert TLB0 va and way number to tlb0[] table index. */
|
|
|
|
static inline unsigned int
|
|
|
|
tlb0_tableidx(vm_offset_t va, unsigned int way)
|
|
|
|
{
|
|
|
|
unsigned int idx;
|
|
|
|
|
|
|
|
idx = (way * TLB0_ENTRIES_PER_WAY);
|
|
|
|
idx += (va & MAS2_TLB0_ENTRY_IDX_MASK) >> MAS2_TLB0_ENTRY_IDX_SHIFT;
|
|
|
|
return (idx);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-01-13 15:41:58 +00:00
|
|
|
* Invalidate TLB0 entry.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
2009-01-13 15:41:58 +00:00
|
|
|
static inline void
|
|
|
|
tlb0_flush_entry(vm_offset_t va)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR2(KTR_PMAP, "%s: s va=0x%08x", __func__, va);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
mtx_assert(&tlbivax_mutex, MA_OWNED);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
__asm __volatile("tlbivax 0, %0" :: "r"(va & MAS2_EPN_MASK));
|
|
|
|
__asm __volatile("isync; msync");
|
|
|
|
__asm __volatile("tlbsync; msync");
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
CTR1(KTR_PMAP, "%s: e", __func__);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**************************************************************************/
|
|
|
|
/* TLB1 handling */
|
|
|
|
/**************************************************************************/
|
2009-01-13 15:41:58 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* TLB1 mapping notes:
|
|
|
|
*
|
2013-10-26 18:18:14 +00:00
|
|
|
* TLB1[0] Kernel text and data.
|
|
|
|
* TLB1[1-15] Additional kernel text and data mappings (if required), PCI
|
2009-01-13 15:41:58 +00:00
|
|
|
* windows, other devices mappings.
|
|
|
|
*/
|
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
/*
|
|
|
|
* Read an entry from given TLB1 slot.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
tlb1_read_entry(tlb_entry_t *entry, unsigned int slot)
|
|
|
|
{
|
2017-06-27 01:57:22 +00:00
|
|
|
register_t msr;
|
2016-04-19 01:48:18 +00:00
|
|
|
uint32_t mas0;
|
|
|
|
|
|
|
|
KASSERT((entry != NULL), ("%s(): Entry is NULL!", __func__));
|
|
|
|
|
2017-06-27 01:57:22 +00:00
|
|
|
msr = mfmsr();
|
2017-06-30 02:11:32 +00:00
|
|
|
__asm __volatile("wrteei 0");
|
2017-06-27 01:57:22 +00:00
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(slot);
|
|
|
|
mtspr(SPR_MAS0, mas0);
|
|
|
|
__asm __volatile("isync; tlbre");
|
|
|
|
|
|
|
|
entry->mas1 = mfspr(SPR_MAS1);
|
|
|
|
entry->mas2 = mfspr(SPR_MAS2);
|
|
|
|
entry->mas3 = mfspr(SPR_MAS3);
|
|
|
|
|
|
|
|
switch ((mfpvr() >> 16) & 0xFFFF) {
|
|
|
|
case FSL_E500v2:
|
|
|
|
case FSL_E500mc:
|
|
|
|
case FSL_E5500:
|
2016-08-23 04:26:30 +00:00
|
|
|
case FSL_E6500:
|
2016-04-19 01:48:18 +00:00
|
|
|
entry->mas7 = mfspr(SPR_MAS7);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
entry->mas7 = 0;
|
|
|
|
break;
|
|
|
|
}
|
2019-05-22 02:43:17 +00:00
|
|
|
__asm __volatile("wrtee %0" :: "r"(msr));
|
2016-04-19 01:48:18 +00:00
|
|
|
|
|
|
|
entry->virt = entry->mas2 & MAS2_EPN_MASK;
|
|
|
|
entry->phys = ((vm_paddr_t)(entry->mas7 & MAS7_RPN) << 32) |
|
|
|
|
(entry->mas3 & MAS3_RPN);
|
|
|
|
entry->size =
|
|
|
|
tsize2size((entry->mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT);
|
|
|
|
}
|
|
|
|
|
2017-11-26 20:30:02 +00:00
|
|
|
struct tlbwrite_args {
|
|
|
|
tlb_entry_t *e;
|
|
|
|
unsigned int idx;
|
|
|
|
};
|
|
|
|
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
static uint32_t
|
|
|
|
tlb1_find_free(void)
|
|
|
|
{
|
|
|
|
tlb_entry_t e;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if ((e.mas1 & MAS1_VALID) == 0)
|
|
|
|
return (i);
|
|
|
|
}
|
|
|
|
return (-1);
|
|
|
|
}
|
|
|
|
|
2020-04-11 00:17:55 +00:00
|
|
|
static void
|
|
|
|
tlb1_purge_va_range(vm_offset_t va, vm_size_t size)
|
|
|
|
{
|
|
|
|
tlb_entry_t e;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if ((e.mas1 & MAS1_VALID) == 0)
|
|
|
|
continue;
|
|
|
|
if ((e.mas2 & MAS2_EPN_MASK) >= va &&
|
|
|
|
(e.mas2 & MAS2_EPN_MASK) < va + size) {
|
|
|
|
mtspr(SPR_MAS1, e.mas1 & ~MAS1_VALID);
|
|
|
|
__asm __volatile("isync; tlbwe; isync; msync");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
static void
|
2017-11-26 20:30:02 +00:00
|
|
|
tlb1_write_entry_int(void *arg)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2017-11-26 20:30:02 +00:00
|
|
|
struct tlbwrite_args *args = arg;
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
uint32_t idx, mas0;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
idx = args->idx;
|
|
|
|
if (idx == -1) {
|
2020-04-11 00:17:55 +00:00
|
|
|
tlb1_purge_va_range(args->e->virt, args->e->size);
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
idx = tlb1_find_free();
|
|
|
|
if (idx == -1)
|
|
|
|
panic("No free TLB1 entries!\n");
|
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Select entry */
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(idx);
|
2017-06-27 01:57:22 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
mtspr(SPR_MAS0, mas0);
|
2017-11-26 20:30:02 +00:00
|
|
|
mtspr(SPR_MAS1, args->e->mas1);
|
|
|
|
mtspr(SPR_MAS2, args->e->mas2);
|
|
|
|
mtspr(SPR_MAS3, args->e->mas3);
|
2015-08-28 03:03:09 +00:00
|
|
|
switch ((mfpvr() >> 16) & 0xFFFF) {
|
|
|
|
case FSL_E500mc:
|
|
|
|
case FSL_E5500:
|
2016-08-23 04:26:30 +00:00
|
|
|
case FSL_E6500:
|
2015-08-28 03:03:09 +00:00
|
|
|
mtspr(SPR_MAS8, 0);
|
|
|
|
/* FALLTHROUGH */
|
|
|
|
case FSL_E500v2:
|
2017-11-26 20:30:02 +00:00
|
|
|
mtspr(SPR_MAS7, args->e->mas7);
|
2015-08-28 03:03:09 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-02-16 04:38:34 +00:00
|
|
|
__asm __volatile("isync; tlbwe; isync; msync");
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2017-11-26 20:30:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
tlb1_write_entry_sync(void *arg)
|
|
|
|
{
|
|
|
|
/* Empty synchronization point for smp_rendezvous(). */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write given entry to TLB1 hardware.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
tlb1_write_entry(tlb_entry_t *e, unsigned int idx)
|
|
|
|
{
|
|
|
|
struct tlbwrite_args args;
|
|
|
|
|
|
|
|
args.e = e;
|
|
|
|
args.idx = idx;
|
|
|
|
|
|
|
|
#ifdef SMP
|
|
|
|
if ((e->mas2 & _TLB_ENTRY_SHARED) && smp_started) {
|
|
|
|
mb();
|
|
|
|
smp_rendezvous(tlb1_write_entry_sync,
|
|
|
|
tlb1_write_entry_int,
|
|
|
|
tlb1_write_entry_sync, &args);
|
|
|
|
} else
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
register_t msr;
|
|
|
|
|
|
|
|
msr = mfmsr();
|
|
|
|
__asm __volatile("wrteei 0");
|
|
|
|
tlb1_write_entry_int(&args);
|
2019-05-22 02:43:17 +00:00
|
|
|
__asm __volatile("wrtee %0" :: "r"(msr));
|
2017-11-26 20:30:02 +00:00
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert TLB TSIZE value to mapped region size.
|
|
|
|
*/
|
|
|
|
static vm_size_t
|
|
|
|
tsize2size(unsigned int tsize)
|
|
|
|
{
|
|
|
|
|
|
|
|
/*
|
|
|
|
* size = 4^tsize KB
|
|
|
|
* size = 4^tsize * 2^10 = 2^(2 * tsize - 10)
|
|
|
|
*/
|
|
|
|
|
|
|
|
return ((1 << (2 * tsize)) * 1024);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert region size (must be power of 4) to TLB TSIZE value.
|
|
|
|
*/
|
|
|
|
static unsigned int
|
|
|
|
size2tsize(vm_size_t size)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (ilog2(size) / 2 - 5);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-01-13 15:41:58 +00:00
|
|
|
* Register permanent kernel mapping in TLB1.
|
2008-03-03 17:17:00 +00:00
|
|
|
*
|
2009-01-13 15:41:58 +00:00
|
|
|
* Entries are created starting from index 0 (current free entry is
|
|
|
|
* kept in tlb1_idx) and are not supposed to be invalidated.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
Add support for the Freescale dTSEC DPAA-based ethernet controller.
Freescale's QorIQ line includes a new ethernet controller, based on their
Datapath Acceleration Architecture (DPAA). This uses a combination of a Frame
manager, Buffer manager, and Queue manager to improve performance across all
interfaces by being able to pass data directly between hardware acceleration
interfaces.
As part of this import, Freescale's Netcomm Software (ncsw) driver is imported.
This was an attempt by Freescale to create an OS-agnostic sub-driver for
managing the hardware, using shims to interface to the OS-specific APIs. This
work was abandoned, and Freescale's primary work is in the Linux driver (dual
BSD/GPL license). Hence, this was imported directly to sys/contrib, rather than
going through the vendor area. Going forward, FreeBSD-specific changes may be
made to the ncsw code, diverging from the upstream in potentially incompatible
ways. An alternative could be to import the Linux driver itself, using the
linuxKPI layer, as that would maintain parity with the vendor-maintained driver.
However, the Linux driver has not been evaluated for reliability yet, and may
have issues with the import, whereas the ncsw-based driver in this commit was
completed by Semihalf 4 years ago, and is very stable.
Other SoC modules based on DPAA, which could be added in the future:
* Security and Encryption engine (SEC4.x, SEC5.x)
* RAID engine
Additional work to be done:
* Implement polling mode
* Test vlan support
* Add support for the Pattern Matching Engine, which can do regular expression
matching on packets.
This driver has been tested on the P5020 QorIQ SoC. Others listed in the
dtsec(4) manual page are expected to work as the same DPAA engine is included in
all.
Obtained from: Semihalf
Relnotes: Yes
Sponsored by: Alex Perez/Inertial Computing
2016-02-29 03:38:00 +00:00
|
|
|
int
|
2015-07-04 19:00:38 +00:00
|
|
|
tlb1_set_entry(vm_offset_t va, vm_paddr_t pa, vm_size_t size,
|
2009-01-13 15:41:58 +00:00
|
|
|
uint32_t flags)
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
2009-01-13 15:41:58 +00:00
|
|
|
uint32_t ts, tid;
|
2013-10-26 16:49:41 +00:00
|
|
|
int tsize, index;
|
|
|
|
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
/* First try to update an existing entry. */
|
2016-04-19 01:48:18 +00:00
|
|
|
for (index = 0; index < TLB1_ENTRIES; index++) {
|
|
|
|
tlb1_read_entry(&e, index);
|
|
|
|
/* Check if we're just updating the flags, and update them. */
|
|
|
|
if (e.phys == pa && e.virt == va && e.size == size) {
|
|
|
|
e.mas2 = (va & MAS2_EPN_MASK) | flags;
|
|
|
|
tlb1_write_entry(&e, index);
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Convert size to TSIZE */
|
|
|
|
tsize = size2tsize(size);
|
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
tid = (TID_KERNEL << MAS1_TID_SHIFT) & MAS1_TID_MASK;
|
|
|
|
/* XXX TS is hard coded to 0 for now as we only use single address space */
|
|
|
|
ts = (0 << MAS1_TS_SHIFT) & MAS1_TS_MASK;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
e.phys = pa;
|
|
|
|
e.virt = va;
|
|
|
|
e.size = size;
|
|
|
|
e.mas1 = MAS1_VALID | MAS1_IPROT | ts | tid;
|
|
|
|
e.mas1 |= ((tsize << MAS1_TSIZE_SHIFT) & MAS1_TSIZE_MASK);
|
|
|
|
e.mas2 = (va & MAS2_EPN_MASK) | flags;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2009-01-13 15:41:58 +00:00
|
|
|
/* Set supervisor RWX permission bits */
|
2016-04-19 01:48:18 +00:00
|
|
|
e.mas3 = (pa & MAS3_RPN) | MAS3_SR | MAS3_SW | MAS3_SX;
|
|
|
|
e.mas7 = (pa >> 32) & MAS7_RPN;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
powerpc/booke: Fix TLB1 entry accounting
It's possible, with per-CPU mappings, for TLB1 indices to get out of sync.
This presents a problem when trying to insert an entry into TLB1 of all
CPUs. Currently that's done by assuming (hoping) that the TLBs are
perfectly synced, and inserting to the same index for all CPUs. However,
with aforementioned private mappings, this can result in overwriting
mappings on the other CPUs.
An example:
CPU0 CPU1
<setup all mappings> <idle>
3 private mappings
kick off CPU 1
initialize shared mappings (3 indices low)
Load kernel module, triggers 20 new mappings
Sync mappings at N-3
initialize 3 private mappings.
At this point, CPU 1 has all the correct mappings, while CPU 0 is missing 3
mappings that were shared across to CPU 1. When CPU 0 tries to access
memory in one of the overwritten mappings, it hangs while tripping through
the TLB miss handler. Device mappings are not stored in any page table.
This fixes by introducing a '-1' index for tlb1_write_entry_int(), so each
CPU searches for an available index private to itself.
MFC after: 3 weeks
2019-11-01 02:55:58 +00:00
|
|
|
tlb1_write_entry(&e, -1);
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2019-11-04 00:35:40 +00:00
|
|
|
* Map in contiguous RAM region into the TLB1.
|
2008-03-03 17:17:00 +00:00
|
|
|
*/
|
2019-11-04 00:35:40 +00:00
|
|
|
static vm_size_t
|
|
|
|
tlb1_mapin_region(vm_offset_t va, vm_paddr_t pa, vm_size_t size, int wimge)
|
2011-08-02 15:35:43 +00:00
|
|
|
{
|
2019-11-04 00:35:40 +00:00
|
|
|
vm_offset_t base;
|
|
|
|
vm_size_t mapped, sz, ssize;
|
2011-08-02 15:35:43 +00:00
|
|
|
|
|
|
|
mapped = 0;
|
|
|
|
base = va;
|
2019-11-04 00:35:40 +00:00
|
|
|
ssize = size;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2019-11-04 00:35:40 +00:00
|
|
|
while (size > 0) {
|
|
|
|
sz = 1UL << (ilog2(size) & ~1);
|
|
|
|
/* Align size to PA */
|
|
|
|
if (pa % sz != 0) {
|
|
|
|
do {
|
|
|
|
sz >>= 2;
|
|
|
|
} while (pa % sz != 0);
|
2011-08-02 15:35:43 +00:00
|
|
|
}
|
2019-11-04 00:35:40 +00:00
|
|
|
/* Now align from there to VA */
|
|
|
|
if (va % sz != 0) {
|
|
|
|
do {
|
|
|
|
sz >>= 2;
|
|
|
|
} while (va % sz != 0);
|
|
|
|
}
|
2020-01-18 01:22:54 +00:00
|
|
|
#ifdef __powerpc64__
|
|
|
|
/*
|
|
|
|
* Clamp TLB1 entries to 4G.
|
|
|
|
*
|
|
|
|
* While the e6500 supports up to 1TB mappings, the e5500
|
|
|
|
* only supports up to 4G mappings. (0b1011)
|
|
|
|
*
|
|
|
|
* If any e6500 machines capable of supporting a very
|
|
|
|
* large amount of memory appear in the future, we can
|
|
|
|
* revisit this.
|
|
|
|
*
|
|
|
|
* For now, though, since we have plenty of space in TLB1,
|
|
|
|
* always avoid creating entries larger than 4GB.
|
|
|
|
*/
|
|
|
|
sz = MIN(sz, 1UL << 32);
|
|
|
|
#endif
|
2019-11-04 00:35:40 +00:00
|
|
|
if (bootverbose)
|
|
|
|
printf("Wiring VA=%p to PA=%jx (size=%lx)\n",
|
|
|
|
(void *)va, (uintmax_t)pa, (long)sz);
|
|
|
|
if (tlb1_set_entry(va, pa, sz,
|
|
|
|
_TLB_ENTRY_SHARED | wimge) < 0)
|
|
|
|
return (mapped);
|
|
|
|
size -= sz;
|
|
|
|
pa += sz;
|
|
|
|
va += sz;
|
2011-08-02 15:35:43 +00:00
|
|
|
}
|
2008-03-03 17:17:00 +00:00
|
|
|
|
2011-08-02 15:35:43 +00:00
|
|
|
mapped = (va - base);
|
2018-08-19 18:54:43 +00:00
|
|
|
if (bootverbose)
|
|
|
|
printf("mapped size 0x%"PRIxPTR" (wasted space 0x%"PRIxPTR")\n",
|
2019-11-04 00:35:40 +00:00
|
|
|
mapped, mapped - ssize);
|
|
|
|
|
2011-08-02 15:35:43 +00:00
|
|
|
return (mapped);
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* TLB1 initialization routine, to be called after the very first
|
|
|
|
* assembler level setup done in locore.S.
|
|
|
|
*/
|
|
|
|
void
|
2013-10-26 18:18:14 +00:00
|
|
|
tlb1_init()
|
2008-03-03 17:17:00 +00:00
|
|
|
{
|
2019-08-20 01:26:02 +00:00
|
|
|
vm_offset_t mas2;
|
|
|
|
uint32_t mas0, mas1, mas3, mas7;
|
2011-08-02 15:35:43 +00:00
|
|
|
uint32_t tsz;
|
|
|
|
|
2015-08-28 03:03:09 +00:00
|
|
|
tlb1_get_tlbconf();
|
2011-08-02 15:35:43 +00:00
|
|
|
|
2015-08-28 03:03:09 +00:00
|
|
|
mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(0);
|
|
|
|
mtspr(SPR_MAS0, mas0);
|
|
|
|
__asm __volatile("isync; tlbre");
|
|
|
|
|
|
|
|
mas1 = mfspr(SPR_MAS1);
|
|
|
|
mas2 = mfspr(SPR_MAS2);
|
|
|
|
mas3 = mfspr(SPR_MAS3);
|
|
|
|
mas7 = mfspr(SPR_MAS7);
|
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
kernload = ((vm_paddr_t)(mas7 & MAS7_RPN) << 32) |
|
2015-08-28 03:03:09 +00:00
|
|
|
(mas3 & MAS3_RPN);
|
|
|
|
|
|
|
|
tsz = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT;
|
2016-04-19 01:48:18 +00:00
|
|
|
kernsize += (tsz > 0) ? tsize2size(tsz) : 0;
|
2019-08-20 01:26:02 +00:00
|
|
|
kernstart = trunc_page(mas2);
|
2012-07-02 21:15:56 +00:00
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Setup TLB miss defaults */
|
|
|
|
set_mas4_defaults();
|
|
|
|
}
|
|
|
|
|
2016-09-07 03:26:55 +00:00
|
|
|
/*
|
|
|
|
* pmap_early_io_unmap() should be used in short conjunction with
|
|
|
|
* pmap_early_io_map(), as in the following snippet:
|
|
|
|
*
|
|
|
|
* x = pmap_early_io_map(...);
|
|
|
|
* <do something with x>
|
|
|
|
* pmap_early_io_unmap(x, size);
|
|
|
|
*
|
|
|
|
* And avoiding more allocations between.
|
|
|
|
*/
|
2016-08-24 03:51:40 +00:00
|
|
|
void
|
|
|
|
pmap_early_io_unmap(vm_offset_t va, vm_size_t size)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
tlb_entry_t e;
|
2016-09-07 03:26:55 +00:00
|
|
|
vm_size_t isize;
|
2016-08-24 03:51:40 +00:00
|
|
|
|
2016-09-07 03:26:55 +00:00
|
|
|
size = roundup(size, PAGE_SIZE);
|
|
|
|
isize = size;
|
|
|
|
for (i = 0; i < TLB1_ENTRIES && size > 0; i++) {
|
2016-08-24 03:51:40 +00:00
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
|
|
|
continue;
|
2016-09-07 03:26:55 +00:00
|
|
|
if (va <= e.virt && (va + isize) >= (e.virt + e.size)) {
|
2016-08-24 03:51:40 +00:00
|
|
|
size -= e.size;
|
|
|
|
e.mas1 &= ~MAS1_VALID;
|
|
|
|
tlb1_write_entry(&e, i);
|
|
|
|
}
|
|
|
|
}
|
2016-09-07 03:26:55 +00:00
|
|
|
if (tlb1_map_base == va + isize)
|
|
|
|
tlb1_map_base -= isize;
|
2016-08-24 03:51:40 +00:00
|
|
|
}
|
|
|
|
|
2013-10-26 18:18:14 +00:00
|
|
|
vm_offset_t
|
|
|
|
pmap_early_io_map(vm_paddr_t pa, vm_size_t size)
|
|
|
|
{
|
|
|
|
vm_paddr_t pa_base;
|
|
|
|
vm_offset_t va, sz;
|
|
|
|
int i;
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
2013-10-26 18:18:14 +00:00
|
|
|
|
|
|
|
KASSERT(!pmap_bootstrapped, ("Do not use after PMAP is up!"));
|
2020-09-01 21:20:08 +00:00
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
tlb1_read_entry(&e, i);
|
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
2013-10-26 18:18:14 +00:00
|
|
|
continue;
|
2016-04-19 01:48:18 +00:00
|
|
|
if (pa >= e.phys && (pa + size) <=
|
|
|
|
(e.phys + e.size))
|
|
|
|
return (e.virt + (pa - e.phys));
|
2013-10-26 18:18:14 +00:00
|
|
|
}
|
|
|
|
|
2015-11-21 06:03:46 +00:00
|
|
|
pa_base = rounddown(pa, PAGE_SIZE);
|
2013-10-26 18:18:14 +00:00
|
|
|
size = roundup(size + (pa - pa_base), PAGE_SIZE);
|
2013-11-17 01:59:42 +00:00
|
|
|
tlb1_map_base = roundup2(tlb1_map_base, 1 << (ilog2(size) & ~1));
|
2013-10-26 20:57:26 +00:00
|
|
|
va = tlb1_map_base + (pa - pa_base);
|
2013-10-26 18:18:14 +00:00
|
|
|
|
|
|
|
do {
|
|
|
|
sz = 1 << (ilog2(size) & ~1);
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb1_set_entry(tlb1_map_base, pa_base, sz,
|
|
|
|
_TLB_ENTRY_SHARED | _TLB_ENTRY_IO);
|
2013-10-26 18:18:14 +00:00
|
|
|
size -= sz;
|
|
|
|
pa_base += sz;
|
2013-10-26 20:57:26 +00:00
|
|
|
tlb1_map_base += sz;
|
2013-10-26 18:18:14 +00:00
|
|
|
} while (size > 0);
|
|
|
|
|
|
|
|
return (va);
|
|
|
|
}
|
|
|
|
|
2016-11-16 05:24:42 +00:00
|
|
|
void
|
|
|
|
pmap_track_page(pmap_t pmap, vm_offset_t va)
|
|
|
|
{
|
|
|
|
vm_paddr_t pa;
|
|
|
|
vm_page_t page;
|
|
|
|
struct pv_entry *pve;
|
|
|
|
|
2016-12-05 02:27:50 +00:00
|
|
|
va = trunc_page(va);
|
2016-11-16 05:24:42 +00:00
|
|
|
pa = pmap_kextract(va);
|
2017-12-10 04:43:27 +00:00
|
|
|
page = PHYS_TO_VM_PAGE(pa);
|
2016-11-16 05:24:42 +00:00
|
|
|
|
|
|
|
rw_wlock(&pvh_global_lock);
|
|
|
|
PMAP_LOCK(pmap);
|
|
|
|
|
|
|
|
TAILQ_FOREACH(pve, &page->md.pv_list, pv_link) {
|
|
|
|
if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
page->md.pv_tracked = true;
|
|
|
|
pv_insert(pmap, va, page);
|
|
|
|
out:
|
|
|
|
PMAP_UNLOCK(pmap);
|
|
|
|
rw_wunlock(&pvh_global_lock);
|
|
|
|
}
|
|
|
|
|
2008-03-03 17:17:00 +00:00
|
|
|
/*
|
|
|
|
* Setup MAS4 defaults.
|
|
|
|
* These values are loaded to MAS0-2 on a TLB miss.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
set_mas4_defaults(void)
|
|
|
|
{
|
2009-01-13 16:15:49 +00:00
|
|
|
uint32_t mas4;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
/* Defaults: TLB0, PID0, TSIZED=4K */
|
|
|
|
mas4 = MAS4_TLBSELD0;
|
|
|
|
mas4 |= (TLB_SIZE_4K << MAS4_TSIZED_SHIFT) & MAS4_TSIZED_MASK;
|
2009-05-21 11:43:37 +00:00
|
|
|
#ifdef SMP
|
|
|
|
mas4 |= MAS4_MD;
|
|
|
|
#endif
|
2008-03-03 17:17:00 +00:00
|
|
|
mtspr(SPR_MAS4, mas4);
|
2009-01-13 16:15:49 +00:00
|
|
|
__asm __volatile("isync");
|
2008-03-03 17:17:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return 0 if the physical IO range is encompassed by one of the
|
|
|
|
* the TLB1 entries, otherwise return related error code.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
tlb1_iomapped(int i, vm_paddr_t pa, vm_size_t size, vm_offset_t *va)
|
|
|
|
{
|
2009-01-13 16:15:49 +00:00
|
|
|
uint32_t prot;
|
2008-03-03 17:17:00 +00:00
|
|
|
vm_paddr_t pa_start;
|
|
|
|
vm_paddr_t pa_end;
|
|
|
|
unsigned int entry_tsize;
|
|
|
|
vm_size_t entry_size;
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb_entry_t e;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
*va = (vm_offset_t)NULL;
|
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
tlb1_read_entry(&e, i);
|
2008-03-03 17:17:00 +00:00
|
|
|
/* Skip invalid entries */
|
2016-04-19 01:48:18 +00:00
|
|
|
if (!(e.mas1 & MAS1_VALID))
|
2008-03-03 17:17:00 +00:00
|
|
|
return (EINVAL);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The entry must be cache-inhibited, guarded, and r/w
|
|
|
|
* so it can function as an i/o page
|
|
|
|
*/
|
2016-04-19 01:48:18 +00:00
|
|
|
prot = e.mas2 & (MAS2_I | MAS2_G);
|
2008-03-03 17:17:00 +00:00
|
|
|
if (prot != (MAS2_I | MAS2_G))
|
|
|
|
return (EPERM);
|
|
|
|
|
2016-04-19 01:48:18 +00:00
|
|
|
prot = e.mas3 & (MAS3_SR | MAS3_SW);
|
2008-03-03 17:17:00 +00:00
|
|
|
if (prot != (MAS3_SR | MAS3_SW))
|
|
|
|
return (EPERM);
|
|
|
|
|
|
|
|
/* The address should be within the entry range. */
|
2016-04-19 01:48:18 +00:00
|
|
|
entry_tsize = (e.mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT;
|
2008-03-03 17:17:00 +00:00
|
|
|
KASSERT((entry_tsize), ("tlb1_iomapped: invalid entry tsize"));
|
|
|
|
|
|
|
|
entry_size = tsize2size(entry_tsize);
|
2016-04-19 01:48:18 +00:00
|
|
|
pa_start = (((vm_paddr_t)e.mas7 & MAS7_RPN) << 32) |
|
|
|
|
(e.mas3 & MAS3_RPN);
|
2015-08-22 07:20:03 +00:00
|
|
|
pa_end = pa_start + entry_size;
|
2008-03-03 17:17:00 +00:00
|
|
|
|
|
|
|
if ((pa < pa_start) || ((pa + size) > pa_end))
|
|
|
|
return (ERANGE);
|
|
|
|
|
|
|
|
/* Return virtual address of this mapping. */
|
2016-04-19 01:48:18 +00:00
|
|
|
*va = (e.mas2 & MAS2_EPN_MASK) + (pa - pa_start);
|
2008-03-03 17:17:00 +00:00
|
|
|
return (0);
|
|
|
|
}
|
2015-12-30 02:23:14 +00:00
|
|
|
|
2018-10-22 00:21:27 +00:00
|
|
|
#ifdef DDB
|
|
|
|
/* Print out contents of the MAS registers for each TLB0 entry */
|
|
|
|
static void
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
tlb_print_entry(int i, uint32_t mas1, uint64_t mas2, uint32_t mas3,
|
|
|
|
#else
|
|
|
|
tlb_print_entry(int i, uint32_t mas1, uint32_t mas2, uint32_t mas3,
|
|
|
|
#endif
|
|
|
|
uint32_t mas7)
|
|
|
|
{
|
|
|
|
int as;
|
|
|
|
char desc[3];
|
|
|
|
tlbtid_t tid;
|
|
|
|
vm_size_t size;
|
|
|
|
unsigned int tsize;
|
|
|
|
|
|
|
|
desc[2] = '\0';
|
|
|
|
if (mas1 & MAS1_VALID)
|
|
|
|
desc[0] = 'V';
|
|
|
|
else
|
|
|
|
desc[0] = ' ';
|
|
|
|
|
|
|
|
if (mas1 & MAS1_IPROT)
|
|
|
|
desc[1] = 'P';
|
|
|
|
else
|
|
|
|
desc[1] = ' ';
|
|
|
|
|
|
|
|
as = (mas1 & MAS1_TS_MASK) ? 1 : 0;
|
|
|
|
tid = MAS1_GETTID(mas1);
|
|
|
|
|
|
|
|
tsize = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT;
|
|
|
|
size = 0;
|
|
|
|
if (tsize)
|
|
|
|
size = tsize2size(tsize);
|
|
|
|
|
|
|
|
printf("%3d: (%s) [AS=%d] "
|
2019-10-19 16:09:06 +00:00
|
|
|
"sz = 0x%jx tsz = %d tid = %d mas1 = 0x%08x "
|
2018-10-22 00:21:27 +00:00
|
|
|
"mas2(va) = 0x%"PRI0ptrX" mas3(pa) = 0x%08x mas7 = 0x%08x\n",
|
2019-10-19 16:09:06 +00:00
|
|
|
i, desc, as, (uintmax_t)size, tsize, tid, mas1, mas2, mas3, mas7);
|
2018-10-22 00:21:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
DB_SHOW_COMMAND(tlb0, tlb0_print_tlbentries)
|
|
|
|
{
|
|
|
|
uint32_t mas0, mas1, mas3, mas7;
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
uint64_t mas2;
|
|
|
|
#else
|
|
|
|
uint32_t mas2;
|
|
|
|
#endif
|
|
|
|
int entryidx, way, idx;
|
|
|
|
|
|
|
|
printf("TLB0 entries:\n");
|
|
|
|
for (way = 0; way < TLB0_WAYS; way ++)
|
|
|
|
for (entryidx = 0; entryidx < TLB0_ENTRIES_PER_WAY; entryidx++) {
|
|
|
|
mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way);
|
|
|
|
mtspr(SPR_MAS0, mas0);
|
|
|
|
|
|
|
|
mas2 = entryidx << MAS2_TLB0_ENTRY_IDX_SHIFT;
|
|
|
|
mtspr(SPR_MAS2, mas2);
|
|
|
|
|
|
|
|
__asm __volatile("isync; tlbre");
|
|
|
|
|
|
|
|
mas1 = mfspr(SPR_MAS1);
|
|
|
|
mas2 = mfspr(SPR_MAS2);
|
|
|
|
mas3 = mfspr(SPR_MAS3);
|
|
|
|
mas7 = mfspr(SPR_MAS7);
|
|
|
|
|
|
|
|
idx = tlb0_tableidx(mas2, way);
|
|
|
|
tlb_print_entry(idx, mas1, mas2, mas3, mas7);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Print out contents of the MAS registers for each TLB1 entry
|
|
|
|
*/
|
|
|
|
DB_SHOW_COMMAND(tlb1, tlb1_print_tlbentries)
|
|
|
|
{
|
|
|
|
uint32_t mas0, mas1, mas3, mas7;
|
|
|
|
#ifdef __powerpc64__
|
|
|
|
uint64_t mas2;
|
|
|
|
#else
|
|
|
|
uint32_t mas2;
|
|
|
|
#endif
|
|
|
|
int i;
|
|
|
|
|
|
|
|
printf("TLB1 entries:\n");
|
|
|
|
for (i = 0; i < TLB1_ENTRIES; i++) {
|
|
|
|
mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(i);
|
|
|
|
mtspr(SPR_MAS0, mas0);
|
|
|
|
|
|
|
|
__asm __volatile("isync; tlbre");
|
|
|
|
|
|
|
|
mas1 = mfspr(SPR_MAS1);
|
|
|
|
mas2 = mfspr(SPR_MAS2);
|
|
|
|
mas3 = mfspr(SPR_MAS3);
|
|
|
|
mas7 = mfspr(SPR_MAS7);
|
|
|
|
|
|
|
|
tlb_print_entry(i, mas1, mas2, mas3, mas7);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|