freebsd-dev/sys/kern/subr_physmem.c
Kyle Evans 7771f2a0c9 kern: physmem: improve region coalescing logic
The existing logic didn't take into account newly inserted mappings
wholly contained by an existing region (or vice versa), nor did it
account for weird overlap scenarios.  The latter is probably unlikely
to happen, but the former may happen in UEFI: BootServicesData allocated
within a large chunk of ConventionalMemory.  This situation blows up vm
initialization.

While we're here, remove the "exact match" logic as it's likely wrong;
if an exact match exists with conflicting flags, for instance, then we
should probably be doing something else.  The new logic takes into
account exact matches as part of the overlapping efforts.

Reviewed by:	kib, mhorne (both earlier version)
Differential Revision:	https://reviews.freebsd.org/D32701
2021-11-03 02:32:46 -05:00

508 lines
14 KiB
C

/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2014 Ian Lepore <ian@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_acpi.h"
#include "opt_ddb.h"
/*
* Routines for describing and initializing anything related to physical memory.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/physmem.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_page.h>
#include <vm/vm_phys.h>
#include <vm/vm_dumpset.h>
#include <machine/md_var.h>
/*
* These structures are used internally to keep track of regions of physical
* ram, and regions within the physical ram that need to be excluded. An
* exclusion region can be excluded from crash dumps, from the vm pool of pages
* that can be allocated, or both, depending on the exclusion flags associated
* with the region.
*/
#ifdef DEV_ACPI
#define MAX_HWCNT 32 /* ACPI needs more regions */
#define MAX_EXCNT 32
#else
#define MAX_HWCNT 16
#define MAX_EXCNT 16
#endif
#if defined(__arm__)
#define MAX_PHYS_ADDR 0xFFFFFFFFull
#elif defined(__aarch64__) || defined(__riscv)
#define MAX_PHYS_ADDR 0xFFFFFFFFFFFFFFFFull
#endif
struct region {
vm_paddr_t addr;
vm_size_t size;
uint32_t flags;
};
static struct region hwregions[MAX_HWCNT];
static struct region exregions[MAX_EXCNT];
static size_t hwcnt;
static size_t excnt;
/*
* realmem is the total number of hardware pages, excluded or not.
* Maxmem is one greater than the last physical page number.
*/
long realmem;
long Maxmem;
/*
* Print the contents of the physical and excluded region tables using the
* provided printf-like output function (which will be either printf or
* db_printf).
*/
static void
physmem_dump_tables(int (*prfunc)(const char *, ...))
{
int flags, i;
uintmax_t addr, size;
const unsigned int mbyte = 1024 * 1024;
prfunc("Physical memory chunk(s):\n");
for (i = 0; i < hwcnt; ++i) {
addr = hwregions[i].addr;
size = hwregions[i].size;
prfunc(" 0x%08jx - 0x%08jx, %5ju MB (%7ju pages)\n", addr,
addr + size - 1, size / mbyte, size / PAGE_SIZE);
}
prfunc("Excluded memory regions:\n");
for (i = 0; i < excnt; ++i) {
addr = exregions[i].addr;
size = exregions[i].size;
flags = exregions[i].flags;
prfunc(" 0x%08jx - 0x%08jx, %5ju MB (%7ju pages) %s %s\n",
addr, addr + size - 1, size / mbyte, size / PAGE_SIZE,
(flags & EXFLAG_NOALLOC) ? "NoAlloc" : "",
(flags & EXFLAG_NODUMP) ? "NoDump" : "");
}
#ifdef DEBUG
prfunc("Avail lists:\n");
for (i = 0; phys_avail[i] != 0; ++i) {
prfunc(" phys_avail[%d] 0x%08x\n", i, phys_avail[i]);
}
for (i = 0; dump_avail[i] != 0; ++i) {
prfunc(" dump_avail[%d] 0x%08x\n", i, dump_avail[i]);
}
#endif
}
/*
* Print the contents of the static mapping table. Used for bootverbose.
*/
void
physmem_print_tables(void)
{
physmem_dump_tables(printf);
}
/*
* Walk the list of hardware regions, processing it against the list of
* exclusions that contain the given exflags, and generating an "avail list".
*
* If maxphyssz is not zero it sets upper limit, in bytes, for the total
* "avail list" size. Walk stops once the limit is reached and the last region
* is cut short if necessary.
*
* Updates the value at *pavail with the sum of all pages in all hw regions.
*
* Returns the number of pages of non-excluded memory added to the avail list.
*/
static size_t
regions_to_avail(vm_paddr_t *avail, uint32_t exflags, size_t maxavail,
uint64_t maxphyssz, long *pavail, long *prealmem)
{
size_t acnt, exi, hwi;
uint64_t end, start, xend, xstart;
long availmem, totalmem;
const struct region *exp, *hwp;
uint64_t availsz;
totalmem = 0;
availmem = 0;
availsz = 0;
acnt = 0;
for (hwi = 0, hwp = hwregions; hwi < hwcnt; ++hwi, ++hwp) {
start = hwp->addr;
end = hwp->size + start;
totalmem += atop((vm_offset_t)(end - start));
for (exi = 0, exp = exregions; exi < excnt; ++exi, ++exp) {
/*
* If the excluded region does not match given flags,
* continue checking with the next excluded region.
*/
if ((exp->flags & exflags) == 0)
continue;
xstart = exp->addr;
xend = exp->size + xstart;
/*
* If the excluded region ends before this hw region,
* continue checking with the next excluded region.
*/
if (xend <= start)
continue;
/*
* If the excluded region begins after this hw region
* we're done because both lists are sorted.
*/
if (xstart >= end)
break;
/*
* If the excluded region completely covers this hw
* region, shrink this hw region to zero size.
*/
if ((start >= xstart) && (end <= xend)) {
start = xend;
end = xend;
break;
}
/*
* If the excluded region falls wholly within this hw
* region without abutting or overlapping the beginning
* or end, create an available entry from the leading
* fragment, then adjust the start of this hw region to
* the end of the excluded region, and continue checking
* the next excluded region because another exclusion
* could affect the remainder of this hw region.
*/
if ((xstart > start) && (xend < end)) {
if ((maxphyssz != 0) &&
(availsz + xstart - start > maxphyssz)) {
xstart = maxphyssz + start - availsz;
}
if (xstart <= start)
continue;
if (acnt > 0 &&
avail[acnt - 1] == (vm_paddr_t)start) {
avail[acnt - 1] = (vm_paddr_t)xstart;
} else {
avail[acnt++] = (vm_paddr_t)start;
avail[acnt++] = (vm_paddr_t)xstart;
}
availsz += (xstart - start);
availmem += atop((vm_offset_t)(xstart - start));
start = xend;
continue;
}
/*
* We know the excluded region overlaps either the start
* or end of this hardware region (but not both), trim
* the excluded portion off the appropriate end.
*/
if (xstart <= start)
start = xend;
else
end = xstart;
}
/*
* If the trimming actions above left a non-zero size, create an
* available entry for it.
*/
if (end > start) {
if ((maxphyssz != 0) &&
(availsz + end - start > maxphyssz)) {
end = maxphyssz + start - availsz;
}
if (end <= start)
break;
if (acnt > 0 && avail[acnt - 1] == (vm_paddr_t)start) {
avail[acnt - 1] = (vm_paddr_t)end;
} else {
avail[acnt++] = (vm_paddr_t)start;
avail[acnt++] = (vm_paddr_t)end;
}
availsz += end - start;
availmem += atop((vm_offset_t)(end - start));
}
if (acnt >= maxavail)
panic("Not enough space in the dump/phys_avail arrays");
}
if (pavail != NULL)
*pavail = availmem;
if (prealmem != NULL)
*prealmem = totalmem;
return (acnt);
}
/*
* Check if the region at idx can be merged with the region above it.
*/
static size_t
merge_upper_regions(struct region *regions, size_t rcnt, size_t idx)
{
struct region *lower, *upper;
vm_paddr_t lend, uend;
size_t i, mergecnt, movecnt;
lower = &regions[idx];
lend = lower->addr + lower->size;
/*
* Continue merging in upper entries as long as we have entries to
* merge; the new block could have spanned more than one, although one
* is likely the common case.
*/
for (i = idx + 1; i < rcnt; i++) {
upper = &regions[i];
if (lend < upper->addr || lower->flags != upper->flags)
break;
uend = upper->addr + upper->size;
if (uend > lend) {
lower->size += uend - lend;
lend = lower->addr + lower->size;
}
if (uend >= lend) {
/*
* If we didn't move past the end of the upper region,
* then we don't need to bother checking for another
* merge because it would have been done already. Just
* increment i once more to maintain the invariant that
* i is one past the last entry merged.
*/
i++;
break;
}
}
/*
* We merged in the entries from [idx + 1, i); physically move the tail
* end at [i, rcnt) if we need to.
*/
mergecnt = i - (idx + 1);
if (mergecnt > 0) {
movecnt = rcnt - i;
if (movecnt == 0) {
/* Merged all the way to the end, just decrease rcnt. */
rcnt = idx + 1;
} else {
memmove(&regions[idx + 1], &regions[idx + mergecnt + 1],
movecnt * sizeof(*regions));
rcnt -= mergecnt;
}
}
return (rcnt);
}
/*
* Insertion-sort a new entry into a regions list; sorted by start address.
*/
static size_t
insert_region(struct region *regions, size_t rcnt, vm_paddr_t addr,
vm_size_t size, uint32_t flags)
{
size_t i;
vm_paddr_t nend, rend;
struct region *ep, *rp;
nend = addr + size;
ep = regions + rcnt;
for (i = 0, rp = regions; i < rcnt; ++i, ++rp) {
if (flags == rp->flags) {
rend = rp->addr + rp->size;
if (addr <= rp->addr && nend >= rp->addr) {
/*
* New mapping overlaps at the beginning, shift
* for any difference in the beginning then
* shift if the new mapping extends past.
*/
rp->size += rp->addr - addr;
rp->addr = addr;
if (nend > rend) {
rp->size += nend - rend;
rcnt = merge_upper_regions(regions,
rcnt, i);
}
return (rcnt);
} else if (addr <= rend && nend > rp->addr) {
/*
* New mapping is either entirely contained
* within or it's overlapping at the end.
*/
if (nend > rend) {
rp->size += nend - rend;
rcnt = merge_upper_regions(regions,
rcnt, i);
}
return (rcnt);
}
}
if (addr < rp->addr) {
bcopy(rp, rp + 1, (ep - rp) * sizeof(*rp));
break;
}
}
rp->addr = addr;
rp->size = size;
rp->flags = flags;
rcnt++;
return (rcnt);
}
/*
* Add a hardware memory region.
*/
void
physmem_hardware_region(uint64_t pa, uint64_t sz)
{
vm_offset_t adj;
/*
* Filter out the page at PA 0x00000000. The VM can't handle it, as
* pmap_extract() == 0 means failure.
*/
if (pa == 0) {
if (sz <= PAGE_SIZE)
return;
pa = PAGE_SIZE;
sz -= PAGE_SIZE;
} else if (pa > MAX_PHYS_ADDR) {
/* This range is past usable memory, ignore it */
return;
}
/*
* Also filter out the page at the end of the physical address space --
* if addr is non-zero and addr+size is zero we wrapped to the next byte
* beyond what vm_paddr_t can express. That leads to a NULL pointer
* deref early in startup; work around it by leaving the last page out.
*
* XXX This just in: subtract out a whole megabyte, not just 1 page.
* Reducing the size by anything less than 1MB results in the NULL
* pointer deref in _vm_map_lock_read(). Better to give up a megabyte
* than leave some folks with an unusable system while we investigate.
*/
if ((pa + sz) > (MAX_PHYS_ADDR - 1024 * 1024)) {
sz = MAX_PHYS_ADDR - pa + 1;
if (sz <= 1024 * 1024)
return;
sz -= 1024 * 1024;
}
/*
* Round the starting address up to a page boundary, and truncate the
* ending page down to a page boundary.
*/
adj = round_page(pa) - pa;
pa = round_page(pa);
sz = trunc_page(sz - adj);
if (sz > 0 && hwcnt < nitems(hwregions))
hwcnt = insert_region(hwregions, hwcnt, pa, sz, 0);
}
/*
* Add an exclusion region.
*/
void
physmem_exclude_region(vm_paddr_t pa, vm_size_t sz, uint32_t exflags)
{
vm_offset_t adj;
/*
* Truncate the starting address down to a page boundary, and round the
* ending page up to a page boundary.
*/
adj = pa - trunc_page(pa);
pa = trunc_page(pa);
sz = round_page(sz + adj);
if (excnt >= nitems(exregions))
panic("failed to exclude region %#jx-%#jx", (uintmax_t)pa,
(uintmax_t)(pa + sz));
excnt = insert_region(exregions, excnt, pa, sz, exflags);
}
size_t
physmem_avail(vm_paddr_t *avail, size_t maxavail)
{
return (regions_to_avail(avail, EXFLAG_NOALLOC, maxavail, 0, NULL, NULL));
}
/*
* Process all the regions added earlier into the global avail lists.
*
* Updates the kernel global 'physmem' with the number of physical pages
* available for use (all pages not in any exclusion region).
*
* Updates the kernel global 'Maxmem' with the page number one greater then the
* last page of physical memory in the system.
*/
void
physmem_init_kernel_globals(void)
{
size_t nextidx;
u_long hwphyssz;
hwphyssz = 0;
TUNABLE_ULONG_FETCH("hw.physmem", &hwphyssz);
regions_to_avail(dump_avail, EXFLAG_NODUMP, PHYS_AVAIL_ENTRIES,
hwphyssz, NULL, NULL);
nextidx = regions_to_avail(phys_avail, EXFLAG_NOALLOC,
PHYS_AVAIL_ENTRIES, hwphyssz, &physmem, &realmem);
if (nextidx == 0)
panic("No memory entries in phys_avail");
Maxmem = atop(phys_avail[nextidx - 1]);
}
#ifdef DDB
#include <ddb/ddb.h>
DB_SHOW_COMMAND(physmem, db_show_physmem)
{
physmem_dump_tables(db_printf);
}
#endif /* DDB */