uma: grow slabs to enforce minimum memory efficiency
Memory efficiency can be poor with awkward item sizes (e.g. 1/2 or 1 page size + epsilon). In order to achieve a minimum memory efficiency, select a slab size with a potentially larger number of pages if it yields a lower portion of waste. This may mean using page_alloc instead of uma_small_alloc, which could be more costly. Discussed with: jeff, mckusick Sponsored by: Dell EMC Isilon Differential Revision: https://reviews.freebsd.org/D23239
This commit is contained in:
Ryan Libby
parent
10c8fb47d9
commit
27ca37acb7
@ -1830,6 +1830,39 @@ slab_ipers(size_t size, int align)
|
||||
return (slab_ipers_hdr(size, rsize, UMA_SLAB_SIZE, true));
|
||||
}
|
||||
|
||||
struct keg_layout_result {
|
||||
u_int format;
|
||||
u_int slabsize;
|
||||
u_int ipers;
|
||||
u_int eff;
|
||||
};
|
||||
|
||||
static void
|
||||
keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt,
|
||||
struct keg_layout_result *kl)
|
||||
{
|
||||
u_int total;
|
||||
|
||||
kl->format = fmt;
|
||||
kl->slabsize = slabsize;
|
||||
|
||||
/* Handle INTERNAL as inline with an extra page. */
|
||||
if ((fmt & UMA_ZFLAG_INTERNAL) != 0) {
|
||||
kl->format &= ~UMA_ZFLAG_INTERNAL;
|
||||
kl->slabsize += PAGE_SIZE;
|
||||
}
|
||||
|
||||
kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize,
|
||||
(fmt & UMA_ZFLAG_OFFPAGE) == 0);
|
||||
|
||||
/* Account for memory used by an offpage slab header. */
|
||||
total = kl->slabsize;
|
||||
if ((fmt & UMA_ZFLAG_OFFPAGE) != 0)
|
||||
total += slabzone(kl->ipers)->uz_keg->uk_rsize;
|
||||
|
||||
kl->eff = UMA_FRAC_FIXPT(kl->ipers * rsize, total);
|
||||
}
|
||||
|
||||
/*
|
||||
* Determine the format of a uma keg. This determines where the slab header
|
||||
* will be placed (inline or offpage) and calculates ipers, rsize, and ppera.
|
||||
@ -1843,15 +1876,14 @@ slab_ipers(size_t size, int align)
|
||||
static void
|
||||
keg_layout(uma_keg_t keg)
|
||||
{
|
||||
struct keg_layout_result kl = {}, kl_tmp;
|
||||
u_int fmts[2];
|
||||
u_int alignsize;
|
||||
u_int eff;
|
||||
u_int eff_offpage;
|
||||
u_int format;
|
||||
u_int ipers;
|
||||
u_int ipers_offpage;
|
||||
u_int nfmt;
|
||||
u_int pages;
|
||||
u_int rsize;
|
||||
u_int slabsize;
|
||||
u_int i, j;
|
||||
|
||||
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
|
||||
(keg->uk_size <= UMA_PCPU_ALLOC_SIZE &&
|
||||
@ -1866,8 +1898,6 @@ keg_layout(uma_keg_t keg)
|
||||
PRINT_UMA_ZFLAGS));
|
||||
|
||||
alignsize = keg->uk_align + 1;
|
||||
format = 0;
|
||||
ipers = 0;
|
||||
|
||||
/*
|
||||
* Calculate the size of each allocation (rsize) according to
|
||||
@ -1877,10 +1907,7 @@ keg_layout(uma_keg_t keg)
|
||||
rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT);
|
||||
rsize = roundup2(rsize, alignsize);
|
||||
|
||||
if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) {
|
||||
slabsize = UMA_PCPU_ALLOC_SIZE;
|
||||
pages = mp_maxid + 1;
|
||||
} else if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) {
|
||||
if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) {
|
||||
/*
|
||||
* We want one item to start on every align boundary in a page.
|
||||
* To do this we will span pages. We will also extend the item
|
||||
@ -1892,23 +1919,22 @@ keg_layout(uma_keg_t keg)
|
||||
slabsize = rsize * (PAGE_SIZE / alignsize);
|
||||
slabsize = MIN(slabsize, rsize * SLAB_MAX_SETSIZE);
|
||||
slabsize = MIN(slabsize, UMA_CACHESPREAD_MAX_SIZE);
|
||||
pages = howmany(slabsize, PAGE_SIZE);
|
||||
slabsize = ptoa(pages);
|
||||
slabsize = round_page(slabsize);
|
||||
} else {
|
||||
/*
|
||||
* Choose a slab size of as many pages as it takes to represent
|
||||
* a single item. We will then try to fit as many additional
|
||||
* items into the slab as possible. At some point, we may want
|
||||
* to increase the slab size for awkward item sizes in order to
|
||||
* increase efficiency.
|
||||
* Start with a slab size of as many pages as it takes to
|
||||
* represent a single item. We will try to fit as many
|
||||
* additional items into the slab as possible.
|
||||
*/
|
||||
pages = howmany(keg->uk_size, PAGE_SIZE);
|
||||
slabsize = ptoa(pages);
|
||||
slabsize = round_page(keg->uk_size);
|
||||
}
|
||||
|
||||
/* Build a list of all of the available formats for this keg. */
|
||||
nfmt = 0;
|
||||
|
||||
/* Evaluate an inline slab layout. */
|
||||
if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0)
|
||||
ipers = slab_ipers_hdr(keg->uk_size, rsize, slabsize, true);
|
||||
fmts[nfmt++] = 0;
|
||||
|
||||
/* TODO: vm_page-embedded slab. */
|
||||
|
||||
@ -1917,65 +1943,91 @@ keg_layout(uma_keg_t keg)
|
||||
* asked to not go to the VM for buckets. If we do this we
|
||||
* may end up going to the VM for slabs which we do not
|
||||
* want to do if we're UMA_ZFLAG_CACHEONLY as a result
|
||||
* of UMA_ZONE_VM, which clearly forbids it.
|
||||
* of UMA_ZONE_VM, which clearly forbids it. In those cases,
|
||||
* evaluate a pseudo-format called INTERNAL which has an inline
|
||||
* slab header and one extra page to guarantee that it fits.
|
||||
*
|
||||
* Otherwise, see if using an OFFPAGE slab will improve our
|
||||
* efficiency.
|
||||
*/
|
||||
if ((keg->uk_flags &
|
||||
(UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)) != 0) {
|
||||
if (ipers == 0) {
|
||||
/* We need an extra page for the slab header. */
|
||||
pages++;
|
||||
slabsize = ptoa(pages);
|
||||
ipers = slab_ipers_hdr(keg->uk_size, rsize, slabsize,
|
||||
true);
|
||||
}
|
||||
goto out;
|
||||
}
|
||||
if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)) != 0)
|
||||
fmts[nfmt++] = UMA_ZFLAG_INTERNAL;
|
||||
else
|
||||
fmts[nfmt++] = UMA_ZFLAG_OFFPAGE;
|
||||
|
||||
/*
|
||||
* See if using an OFFPAGE slab will improve our efficiency.
|
||||
* Only do this if we are below our efficiency threshold.
|
||||
* Choose a slab size and format which satisfy the minimum efficiency.
|
||||
* Prefer the smallest slab size that meets the constraints.
|
||||
*
|
||||
* XXX We could try growing slabsize to limit max waste as well.
|
||||
* Historically this was not done because the VM could not
|
||||
* efficiently handle contiguous allocations.
|
||||
* Start with a minimum slab size, to accommodate CACHESPREAD. Then,
|
||||
* for small items (up to PAGE_SIZE), the iteration increment is one
|
||||
* page; and for large items, the increment is one item.
|
||||
*/
|
||||
eff = UMA_FRAC_FIXPT(ipers * rsize, slabsize);
|
||||
ipers_offpage = slab_ipers_hdr(keg->uk_size, rsize, slabsize, false);
|
||||
eff_offpage = UMA_FRAC_FIXPT(ipers_offpage * rsize,
|
||||
slabsize + slabzone(ipers_offpage)->uz_keg->uk_rsize);
|
||||
if (ipers == 0 || (eff < UMA_MIN_EFF && eff < eff_offpage)) {
|
||||
CTR5(KTR_UMA, "UMA decided we need offpage slab headers for "
|
||||
"keg: %s(%p), minimum efficiency allowed = %u%%, "
|
||||
"old efficiency = %u%%, offpage efficiency = %u%%",
|
||||
keg->uk_name, keg, UMA_FIXPT_PCT(UMA_MIN_EFF),
|
||||
UMA_FIXPT_PCT(eff), UMA_FIXPT_PCT(eff_offpage));
|
||||
format = UMA_ZFLAG_OFFPAGE;
|
||||
ipers = ipers_offpage;
|
||||
i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize);
|
||||
KASSERT(i >= 1, ("keg %s(%p) flags=0x%b slabsize=%u, rsize=%u, i=%u",
|
||||
keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize,
|
||||
rsize, i));
|
||||
for ( ; ; i++) {
|
||||
slabsize = (rsize <= PAGE_SIZE) ? ptoa(i) :
|
||||
round_page(rsize * (i - 1) + keg->uk_size);
|
||||
|
||||
for (j = 0; j < nfmt; j++) {
|
||||
/* Only if we have no viable format yet. */
|
||||
if ((fmts[j] & UMA_ZFLAG_INTERNAL) != 0 &&
|
||||
kl.ipers > 0)
|
||||
continue;
|
||||
|
||||
keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp);
|
||||
if (kl_tmp.eff <= kl.eff)
|
||||
continue;
|
||||
|
||||
kl = kl_tmp;
|
||||
|
||||
CTR6(KTR_UMA, "keg %s layout: format %#x "
|
||||
"(ipers %u * rsize %u) / slabsize %#x = %u%% eff",
|
||||
keg->uk_name, kl.format, kl.ipers, rsize,
|
||||
kl.slabsize, UMA_FIXPT_PCT(kl.eff));
|
||||
|
||||
/* Stop when we reach the minimum efficiency. */
|
||||
if (kl.eff >= UMA_MIN_EFF)
|
||||
break;
|
||||
}
|
||||
|
||||
if (kl.eff >= UMA_MIN_EFF ||
|
||||
slabsize >= SLAB_MAX_SETSIZE * rsize ||
|
||||
(keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0)
|
||||
break;
|
||||
}
|
||||
|
||||
out:
|
||||
pages = atop(kl.slabsize);
|
||||
if ((keg->uk_flags & UMA_ZONE_PCPU) != 0)
|
||||
pages *= mp_maxid + 1;
|
||||
|
||||
keg->uk_rsize = rsize;
|
||||
keg->uk_ipers = kl.ipers;
|
||||
keg->uk_ppera = pages;
|
||||
keg->uk_flags |= kl.format;
|
||||
|
||||
/*
|
||||
* How do we find the slab header if it is offpage or if not all item
|
||||
* start addresses are in the same page? We could solve the latter
|
||||
* case with vaddr alignment, but we don't.
|
||||
*/
|
||||
if ((format & UMA_ZFLAG_OFFPAGE) != 0 ||
|
||||
(ipers - 1) * rsize >= PAGE_SIZE) {
|
||||
if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 ||
|
||||
(keg->uk_ipers - 1) * rsize >= PAGE_SIZE) {
|
||||
if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0)
|
||||
format |= UMA_ZFLAG_HASH;
|
||||
keg->uk_flags |= UMA_ZFLAG_HASH;
|
||||
else
|
||||
format |= UMA_ZFLAG_VTOSLAB;
|
||||
keg->uk_flags |= UMA_ZFLAG_VTOSLAB;
|
||||
}
|
||||
keg->uk_ipers = ipers;
|
||||
keg->uk_rsize = rsize;
|
||||
keg->uk_flags |= format;
|
||||
keg->uk_ppera = pages;
|
||||
|
||||
CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u",
|
||||
__func__, keg->uk_name, keg->uk_flags, rsize, ipers, pages);
|
||||
__func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers,
|
||||
pages);
|
||||
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE,
|
||||
("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__,
|
||||
keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, ipers,
|
||||
pages));
|
||||
keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize,
|
||||
keg->uk_ipers, pages));
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user