d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Convert the slab free item list from a linked array of indices to a

Browse Source 
bitmap using sys/bitset.  This is much simpler, has lower space
   overhead and is cheaper in most cases.
 - Use a second bitmap for invariants asserts and improve the quality of
   the asserts as well as the number of erroneous conditions that we will
   catch.
 - Drastically simplify sizing code.  Special case refcnt zones since they
   will be going away.
 - Update stale comments.

Sponsored by:	EMC / Isilon Storage Division
This commit is contained in:
Jeff Roberson 2013-06-13 21:05:38 +00:00
parent 17a2737732
commit ef72505e6d
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=251709
3 changed files with 149 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

290
sys/vm/uma_core.c
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org> * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
* Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
* Copyright (c) 2004-2006 Robert N. M. Watson * Copyright (c) 2004-2006 Robert N. M. Watson
* All rights reserved. * All rights reserved.
@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/bitset.h>
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/queue.h> #include <sys/queue.h>
@ -145,8 +146,13 @@ static int booted = 0;
#define UMA_STARTUP2 2 #define UMA_STARTUP2 2
/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */ /* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */
static u_int uma_max_ipers; static const u_int uma_max_ipers = SLAB_SETSIZE;
static u_int uma_max_ipers_ref;
/*
* Only mbuf clusters use ref zones. Just provide enough references
* to support the one user. New code should not use the ref facility.
*/
static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES;
/* /*
* This is the handle used to schedule events that need to happen * This is the handle used to schedule events that need to happen
@ -208,7 +214,7 @@ static uint8_t bucket_size[BUCKET_ZONES];
/* /*
* Flags and enumerations to be passed to internal functions. * Flags and enumerations to be passed to internal functions.
*/ */
enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI }; enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */ #define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */ #define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
@ -885,18 +891,15 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
slab->us_keg = keg; slab->us_keg = keg;
slab->us_data = mem; slab->us_data = mem;
slab->us_freecount = keg->uk_ipers; slab->us_freecount = keg->uk_ipers;
slab->us_firstfree = 0;
slab->us_flags = flags; slab->us_flags = flags;
BIT_FILL(SLAB_SETSIZE, &slab->us_free);
#ifdef INVARIANTS
BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
#endif
if (keg->uk_flags & UMA_ZONE_REFCNT) { if (keg->uk_flags & UMA_ZONE_REFCNT) {
slabref = (uma_slabrefcnt_t)slab; slabref = (uma_slabrefcnt_t)slab;
for (i = 0; i < keg->uk_ipers; i++) {
slabref->us_freelist[i].us_refcnt = 0;
slabref->us_freelist[i].us_item = i+1;
}
} else {
for (i = 0; i < keg->uk_ipers; i++) for (i = 0; i < keg->uk_ipers; i++)
slab->us_freelist[i].us_item = i+1; slabref->us_refcnt[i] = 0;
} }
if (keg->uk_init != NULL) { if (keg->uk_init != NULL) {
@ -1148,31 +1151,32 @@ keg_small_init(uma_keg_t keg)
keg->uk_ppera = 1; keg->uk_ppera = 1;
} }
/*
* Calculate the size of each allocation (rsize) according to
* alignment. If the requested size is smaller than we have
* allocation bits for we round it up.
*/
rsize = keg->uk_size; rsize = keg->uk_size;
if (rsize < keg->uk_slabsize / SLAB_SETSIZE)
rsize = keg->uk_slabsize / SLAB_SETSIZE;
if (rsize & keg->uk_align) if (rsize & keg->uk_align)
rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
if (rsize < keg->uk_slabsize / 256)
rsize = keg->uk_slabsize / 256;
keg->uk_rsize = rsize; keg->uk_rsize = rsize;
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
keg->uk_rsize < sizeof(struct pcpu), keg->uk_rsize < sizeof(struct pcpu),
("%s: size %u too large", __func__, keg->uk_rsize)); ("%s: size %u too large", __func__, keg->uk_rsize));
if (keg->uk_flags & UMA_ZONE_OFFPAGE) { if (keg->uk_flags & UMA_ZONE_REFCNT)
rsize += sizeof(uint32_t);
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
shsize = 0; shsize = 0;
} else if (keg->uk_flags & UMA_ZONE_REFCNT) { else
rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */
shsize = sizeof(struct uma_slab_refcnt);
} else {
rsize += UMA_FRITM_SZ; /* Account for linkage */
shsize = sizeof(struct uma_slab); shsize = sizeof(struct uma_slab);
}
keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize; keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 256, KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
memused = keg->uk_ipers * rsize + shsize; memused = keg->uk_ipers * rsize + shsize;
@ -1189,10 +1193,18 @@ keg_small_init(uma_keg_t keg)
(keg->uk_flags & UMA_ZFLAG_CACHEONLY)) (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
return; return;
/*
* See if using an OFFPAGE slab will limit our waste. Only do
* this if it permits more items per-slab.
*
* XXX We could try growing slabsize to limit max waste as well.
* Historically this was not done because the VM could not
* efficiently handle contiguous allocations.
*/
if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) && if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
(keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) { (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize; keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 256, KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
#ifdef UMA_DEBUG #ifdef UMA_DEBUG
printf("UMA decided we need offpage slab headers for " printf("UMA decided we need offpage slab headers for "
@ -1331,34 +1343,29 @@ keg_ctor(void *mem, int size, void *udata, int flags)
keg->uk_flags &= ~UMA_ZONE_PCPU; keg->uk_flags &= ~UMA_ZONE_PCPU;
#endif #endif
/* if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
* The +UMA_FRITM_SZ added to uk_size is to account for the keg_cachespread_init(keg);
* linkage that is added to the size in keg_small_init(). If } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
* we don't account for this here then we may end up in if (keg->uk_size >
* keg_small_init() with a calculated 'ipers' of 0. (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
*/ sizeof(uint32_t)))
if (keg->uk_flags & UMA_ZONE_REFCNT) {
if (keg->uk_flags & UMA_ZONE_CACHESPREAD)
keg_cachespread_init(keg);
else if ((keg->uk_size+UMA_FRITMREF_SZ) >
(UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)))
keg_large_init(keg); keg_large_init(keg);
else else
keg_small_init(keg); keg_small_init(keg);
} else { } else {
if (keg->uk_flags & UMA_ZONE_CACHESPREAD) if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
keg_cachespread_init(keg);
else if ((keg->uk_size+UMA_FRITM_SZ) >
(UMA_SLAB_SIZE - sizeof(struct uma_slab)))
keg_large_init(keg); keg_large_init(keg);
else else
keg_small_init(keg); keg_small_init(keg);
} }
if (keg->uk_flags & UMA_ZONE_OFFPAGE) { if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
if (keg->uk_flags & UMA_ZONE_REFCNT) if (keg->uk_flags & UMA_ZONE_REFCNT) {
if (keg->uk_ipers > uma_max_ipers_ref)
panic("Too many ref items per zone: %d > %d\n",
keg->uk_ipers, uma_max_ipers_ref);
keg->uk_slabzone = slabrefzone; keg->uk_slabzone = slabrefzone;
else } else
keg->uk_slabzone = slabzone; keg->uk_slabzone = slabzone;
} }
@ -1398,25 +1405,17 @@ keg_ctor(void *mem, int size, void *udata, int flags)
u_int totsize; u_int totsize;
/* Size of the slab struct and free list */ /* Size of the slab struct and free list */
totsize = sizeof(struct uma_slab);
/* Size of the reference counts. */
if (keg->uk_flags & UMA_ZONE_REFCNT) if (keg->uk_flags & UMA_ZONE_REFCNT)
totsize = sizeof(struct uma_slab_refcnt) + totsize += keg->uk_ipers * sizeof(uint32_t);
keg->uk_ipers * UMA_FRITMREF_SZ;
else
totsize = sizeof(struct uma_slab) +
keg->uk_ipers * UMA_FRITM_SZ;
if (totsize & UMA_ALIGN_PTR) if (totsize & UMA_ALIGN_PTR)
totsize = (totsize & ~UMA_ALIGN_PTR) + totsize = (totsize & ~UMA_ALIGN_PTR) +
(UMA_ALIGN_PTR + 1); (UMA_ALIGN_PTR + 1);
keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize; keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
if (keg->uk_flags & UMA_ZONE_REFCNT)
totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt)
+ keg->uk_ipers * UMA_FRITMREF_SZ;
else
totsize = keg->uk_pgoff + sizeof(struct uma_slab)
+ keg->uk_ipers * UMA_FRITM_SZ;
/* /*
* The only way the following is possible is if with our * The only way the following is possible is if with our
* UMA_ALIGN_PTR adjustments we are now bigger than * UMA_ALIGN_PTR adjustments we are now bigger than
@ -1424,6 +1423,9 @@ keg_ctor(void *mem, int size, void *udata, int flags)
* mathematically possible for all cases, so we make * mathematically possible for all cases, so we make
* sure here anyway. * sure here anyway.
*/ */
totsize = keg->uk_pgoff + sizeof(struct uma_slab);
if (keg->uk_flags & UMA_ZONE_REFCNT)
totsize += keg->uk_ipers * sizeof(uint32_t);
if (totsize > PAGE_SIZE * keg->uk_ppera) { if (totsize > PAGE_SIZE * keg->uk_ppera) {
printf("zone %s ipers %d rsize %d size %d\n", printf("zone %s ipers %d rsize %d size %d\n",
zone->uz_name, keg->uk_ipers, keg->uk_rsize, zone->uz_name, keg->uk_ipers, keg->uk_rsize,
@ -1655,7 +1657,6 @@ uma_startup(void *bootmem, int boot_pages)
struct uma_zctor_args args; struct uma_zctor_args args;
uma_slab_t slab; uma_slab_t slab;
u_int slabsize; u_int slabsize;
u_int objsize, totsize, wsize;
int i; int i;
#ifdef UMA_DEBUG #ifdef UMA_DEBUG
@ -1663,79 +1664,6 @@ uma_startup(void *bootmem, int boot_pages)
#endif #endif
mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
/*
* Figure out the maximum number of items-per-slab we'll have if
* we're using the OFFPAGE slab header to track free items, given
* all possible object sizes and the maximum desired wastage
* (UMA_MAX_WASTE).
*
* We iterate until we find an object size for
* which the calculated wastage in keg_small_init() will be
* enough to warrant OFFPAGE. Since wastedspace versus objsize
* is an overall increasing see-saw function, we find the smallest
* objsize such that the wastage is always acceptable for objects
* with that objsize or smaller. Since a smaller objsize always
* generates a larger possible uma_max_ipers, we use this computed
* objsize to calculate the largest ipers possible. Since the
* ipers calculated for OFFPAGE slab headers is always larger than
* the ipers initially calculated in keg_small_init(), we use
* the former's equation (UMA_SLAB_SIZE / keg->uk_rsize) to
* obtain the maximum ipers possible for offpage slab headers.
*
* It should be noted that ipers versus objsize is an inversly
* proportional function which drops off rather quickly so as
* long as our UMA_MAX_WASTE is such that the objsize we calculate
* falls into the portion of the inverse relation AFTER the steep
* falloff, then uma_max_ipers shouldn't be too high (~10 on i386).
*
* Note that we have 8-bits (1 byte) to use as a freelist index
* inside the actual slab header itself and this is enough to
* accomodate us. In the worst case, a UMA_SMALLEST_UNIT sized
* object with offpage slab header would have ipers =
* UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is
* 1 greater than what our byte-integer freelist index can
* accomodate, but we know that this situation never occurs as
* for UMA_SMALLEST_UNIT-sized objects, we will never calculate
* that we need to go to offpage slab headers. Or, if we do,
* then we trap that condition below and panic in the INVARIANTS case.
*/
wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) -
(UMA_SLAB_SIZE / UMA_MAX_WASTE);
totsize = wsize;
objsize = UMA_SMALLEST_UNIT;
while (totsize >= wsize) {
totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) /
(objsize + UMA_FRITM_SZ);
totsize *= (UMA_FRITM_SZ + objsize);
objsize++;
}
if (objsize > UMA_SMALLEST_UNIT)
objsize--;
uma_max_ipers = MAX(UMA_SLAB_SIZE / objsize, 64);
wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
(UMA_SLAB_SIZE / UMA_MAX_WASTE);
totsize = wsize;
objsize = UMA_SMALLEST_UNIT;
while (totsize >= wsize) {
totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) /
(objsize + UMA_FRITMREF_SZ);
totsize *= (UMA_FRITMREF_SZ + objsize);
objsize++;
}
if (objsize > UMA_SMALLEST_UNIT)
objsize--;
uma_max_ipers_ref = MAX(UMA_SLAB_SIZE / objsize, 64);
KASSERT((uma_max_ipers_ref <= 256) && (uma_max_ipers <= 256),
("uma_startup: calculated uma_max_ipers values too large!"));
#ifdef UMA_DEBUG
printf("Calculated uma_max_ipers (for OFFPAGE) is %d\n", uma_max_ipers);
printf("Calculated uma_max_ipers_ref (for OFFPAGE) is %d\n",
uma_max_ipers_ref);
#endif
/* "manually" create the initial zone */ /* "manually" create the initial zone */
args.name = "UMA Kegs"; args.name = "UMA Kegs";
args.size = sizeof(struct uma_keg); args.size = sizeof(struct uma_keg);
@ -1783,16 +1711,9 @@ uma_startup(void *bootmem, int boot_pages)
printf("Creating slab and hash zones.\n"); printf("Creating slab and hash zones.\n");
#endif #endif
/*
* This is the max number of free list items we'll have with
* offpage slabs.
*/
slabsize = uma_max_ipers * UMA_FRITM_SZ;
slabsize += sizeof(struct uma_slab);
/* Now make a zone for slab headers */ /* Now make a zone for slab headers */
slabzone = uma_zcreate("UMA Slabs", slabzone = uma_zcreate("UMA Slabs",
slabsize, sizeof(struct uma_slab),
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
@ -1800,8 +1721,8 @@ uma_startup(void *bootmem, int boot_pages)
* We also create a zone for the bigger slabs with reference * We also create a zone for the bigger slabs with reference
* counts in them, to accomodate UMA_ZONE_REFCNT zones. * counts in them, to accomodate UMA_ZONE_REFCNT zones.
*/ */
slabsize = uma_max_ipers_ref * UMA_FRITMREF_SZ; slabsize = sizeof(struct uma_slab_refcnt);
slabsize += sizeof(struct uma_slab_refcnt); slabsize += uma_max_ipers_ref * sizeof(uint32_t);
slabrefzone = uma_zcreate("UMA RCntSlabs", slabrefzone = uma_zcreate("UMA RCntSlabs",
slabsize, slabsize,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
@ -2087,11 +2008,6 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
("uma_zalloc: Bucket pointer mangled.")); ("uma_zalloc: Bucket pointer mangled."));
cache->uc_allocs++; cache->uc_allocs++;
critical_exit(); critical_exit();
#ifdef INVARIANTS
ZONE_LOCK(zone);
uma_dbg_alloc(zone, NULL, item);
ZONE_UNLOCK(zone);
#endif
if (zone->uz_ctor != NULL) { if (zone->uz_ctor != NULL) {
if (zone->uz_ctor(item, zone->uz_size, if (zone->uz_ctor(item, zone->uz_size,
udata, flags) != 0) { udata, flags) != 0) {
@ -2101,6 +2017,9 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
return (NULL); return (NULL);
} }
} }
#ifdef INVARIANTS
uma_dbg_alloc(zone, NULL, item);
#endif
if (flags & M_ZERO) if (flags & M_ZERO)
bzero(item, zone->uz_size); bzero(item, zone->uz_size);
return (item); return (item);
@ -2403,27 +2322,18 @@ static void *
slab_alloc_item(uma_zone_t zone, uma_slab_t slab) slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
{ {
uma_keg_t keg; uma_keg_t keg;
uma_slabrefcnt_t slabref;
void *item; void *item;
uint8_t freei; uint8_t freei;
keg = slab->us_keg; keg = slab->us_keg;
mtx_assert(&keg->uk_lock, MA_OWNED); mtx_assert(&keg->uk_lock, MA_OWNED);
freei = slab->us_firstfree; freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
if (keg->uk_flags & UMA_ZONE_REFCNT) { BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
slabref = (uma_slabrefcnt_t)slab;
slab->us_firstfree = slabref->us_freelist[freei].us_item;
} else {
slab->us_firstfree = slab->us_freelist[freei].us_item;
}
item = slab->us_data + (keg->uk_rsize * freei); item = slab->us_data + (keg->uk_rsize * freei);
slab->us_freecount--; slab->us_freecount--;
keg->uk_free--; keg->uk_free--;
#ifdef INVARIANTS
uma_dbg_alloc(zone, slab, item);
#endif
/* Move this slab to the full list */ /* Move this slab to the full list */
if (slab->us_freecount == 0) { if (slab->us_freecount == 0) {
LIST_REMOVE(slab, us_link); LIST_REMOVE(slab, us_link);
@ -2602,6 +2512,9 @@ zone_alloc_item(uma_zone_t zone, void *udata, int flags)
return (NULL); return (NULL);
} }
} }
#ifdef INVARIANTS
uma_dbg_alloc(zone, slab, item);
#endif
if (flags & M_ZERO) if (flags & M_ZERO)
bzero(item, zone->uz_size); bzero(item, zone->uz_size);
@ -2636,17 +2549,15 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
return; return;
} }
#endif #endif
if (zone->uz_dtor)
zone->uz_dtor(item, zone->uz_size, udata);
#ifdef INVARIANTS #ifdef INVARIANTS
ZONE_LOCK(zone);
if (zone->uz_flags & UMA_ZONE_MALLOC) if (zone->uz_flags & UMA_ZONE_MALLOC)
uma_dbg_free(zone, udata, item); uma_dbg_free(zone, udata, item);
else else
uma_dbg_free(zone, NULL, item); uma_dbg_free(zone, NULL, item);
ZONE_UNLOCK(zone);
#endif #endif
if (zone->uz_dtor)
zone->uz_dtor(item, zone->uz_size, udata);
/* /*
* The race here is acceptable. If we miss it we'll just have to wait * The race here is acceptable. If we miss it we'll just have to wait
* a little longer for the limits to be reset. * a little longer for the limits to be reset.
@ -2807,12 +2718,19 @@ zone_free_item(uma_zone_t zone, void *item, void *udata,
enum zfreeskip skip, int flags) enum zfreeskip skip, int flags)
{ {
uma_slab_t slab; uma_slab_t slab;
uma_slabrefcnt_t slabref;
uma_keg_t keg; uma_keg_t keg;
uint8_t *mem; uint8_t *mem;
uint8_t freei; uint8_t freei;
int clearfull; int clearfull;
#ifdef INVARIANTS
if (skip == SKIP_NONE) {
if (zone->uz_flags & UMA_ZONE_MALLOC)
uma_dbg_free(zone, udata, item);
else
uma_dbg_free(zone, NULL, item);
}
#endif
if (skip < SKIP_DTOR && zone->uz_dtor) if (skip < SKIP_DTOR && zone->uz_dtor)
zone->uz_dtor(item, zone->uz_size, udata); zone->uz_dtor(item, zone->uz_size, udata);
@ -2827,7 +2745,7 @@ zone_free_item(uma_zone_t zone, void *item, void *udata,
zone->uz_frees++; zone->uz_frees++;
if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
mem = (uint8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
keg = zone_first_keg(zone); /* Must only be one. */ keg = zone_first_keg(zone); /* Must only be one. */
if (zone->uz_flags & UMA_ZONE_HASH) { if (zone->uz_flags & UMA_ZONE_HASH) {
slab = hash_sfind(&keg->uk_hash, mem); slab = hash_sfind(&keg->uk_hash, mem);
@ -2855,25 +2773,12 @@ zone_free_item(uma_zone_t zone, void *item, void *udata,
LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
} }
/* Slab management stuff */ /* Slab management. */
freei = ((unsigned long)item - (unsigned long)slab->us_data) freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
/ keg->uk_rsize; BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
#ifdef INVARIANTS
if (!skip)
uma_dbg_free(zone, slab, item);
#endif
if (keg->uk_flags & UMA_ZONE_REFCNT) {
slabref = (uma_slabrefcnt_t)slab;
slabref->us_freelist[freei].us_item = slab->us_firstfree;
} else {
slab->us_freelist[freei].us_item = slab->us_firstfree;
}
slab->us_firstfree = freei;
slab->us_freecount++; slab->us_freecount++;
/* Zone statistics */ /* Keg statistics. */
keg->uk_free++; keg->uk_free++;
clearfull = 0; clearfull = 0;
@ -2884,9 +2789,10 @@ zone_free_item(uma_zone_t zone, void *item, void *udata,
} }
/* /*
* We can handle one more allocation. Since we're clearing ZFLAG_FULL, * We can handle one more allocation. Since we're
* wake up all procs blocked on pages. This should be uncommon, so * clearing ZFLAG_FULL, wake up all procs blocked
* keeping this simple for now (rather than adding count of blocked * on pages. This should be uncommon, so keeping this
* simple for now (rather than adding count of blocked
* threads etc). * threads etc).
*/ */
wakeup(keg); wakeup(keg);
@ -2898,6 +2804,7 @@ zone_free_item(uma_zone_t zone, void *item, void *udata,
ZONE_UNLOCK(zone); ZONE_UNLOCK(zone);
} else } else
KEG_UNLOCK(keg); KEG_UNLOCK(keg);
} }
/* See uma.h */ /* See uma.h */
@ -3107,18 +3014,18 @@ uint32_t *
uma_find_refcnt(uma_zone_t zone, void *item) uma_find_refcnt(uma_zone_t zone, void *item)
{ {
uma_slabrefcnt_t slabref; uma_slabrefcnt_t slabref;
uma_slab_t slab;
uma_keg_t keg; uma_keg_t keg;
uint32_t *refcnt; uint32_t *refcnt;
int idx; int idx;
slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item & slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
(~UMA_SLAB_MASK)); slabref = (uma_slabrefcnt_t)slab;
keg = slabref->us_keg; keg = slab->us_keg;
KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT, KASSERT(keg->uk_flags & UMA_ZONE_REFCNT,
("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
idx = ((unsigned long)item - (unsigned long)slabref->us_data) idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
/ keg->uk_rsize; refcnt = &slabref->us_refcnt[idx];
refcnt = &slabref->us_freelist[idx].us_refcnt;
return refcnt; return refcnt;
} }
@ -3200,9 +3107,8 @@ uma_print_stats(void)
static void static void
slab_print(uma_slab_t slab) slab_print(uma_slab_t slab)
{ {
printf("slab: keg %p, data %p, freecount %d, firstfree %d\n", printf("slab: keg %p, data %p, freecount %d\n",
slab->us_keg, slab->us_data, slab->us_freecount, slab->us_keg, slab->us_data, slab->us_freecount);
slab->us_firstfree);
} }
static void static void

81
sys/vm/uma_dbg.c
View File

@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/bitset.h>
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/queue.h> #include <sys/queue.h>
@ -191,6 +192,7 @@ mtrash_fini(void *mem, int size)
(void)mtrash_ctor(mem, size, NULL, 0); (void)mtrash_ctor(mem, size, NULL, 0);
} }
#ifdef INVARIANTS
static uma_slab_t static uma_slab_t
uma_dbg_getslab(uma_zone_t zone, void *item) uma_dbg_getslab(uma_zone_t zone, void *item)
{ {
@ -198,15 +200,22 @@ uma_dbg_getslab(uma_zone_t zone, void *item)
uma_keg_t keg; uma_keg_t keg;
uint8_t *mem; uint8_t *mem;
mem = (uint8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
if (zone->uz_flags & UMA_ZONE_VTOSLAB) { if (zone->uz_flags & UMA_ZONE_VTOSLAB) {
slab = vtoslab((vm_offset_t)mem); slab = vtoslab((vm_offset_t)mem);
} else { } else {
/*
* It is safe to return the slab here even though the
* zone is unlocked because the item's allocation state
* essentially holds a reference.
*/
ZONE_LOCK(zone);
keg = LIST_FIRST(&zone->uz_kegs)->kl_keg; keg = LIST_FIRST(&zone->uz_kegs)->kl_keg;
if (keg->uk_flags & UMA_ZONE_HASH) if (keg->uk_flags & UMA_ZONE_HASH)
slab = hash_sfind(&keg->uk_hash, mem); slab = hash_sfind(&keg->uk_hash, mem);
else else
slab = (uma_slab_t)(mem + keg->uk_pgoff); slab = (uma_slab_t)(mem + keg->uk_pgoff);
ZONE_UNLOCK(zone);
} }
return (slab); return (slab);
@ -216,12 +225,10 @@ uma_dbg_getslab(uma_zone_t zone, void *item)
* Set up the slab's freei data such that uma_dbg_free can function. * Set up the slab's freei data such that uma_dbg_free can function.
* *
*/ */
void void
uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
{ {
uma_keg_t keg; uma_keg_t keg;
uma_slabrefcnt_t slabref;
int freei; int freei;
if (slab == NULL) { if (slab == NULL) {
@ -231,16 +238,12 @@ uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
item, zone->uz_name); item, zone->uz_name);
} }
keg = slab->us_keg; keg = slab->us_keg;
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
freei = ((unsigned long)item - (unsigned long)slab->us_data) if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
/ keg->uk_rsize; panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n",
item, zone, zone->uz_name, slab, freei);
if (keg->uk_flags & UMA_ZONE_REFCNT) { BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
slabref = (uma_slabrefcnt_t)slab;
slabref->us_freelist[freei].us_item = 255;
} else {
slab->us_freelist[freei].us_item = 255;
}
return; return;
} }
@ -250,12 +253,10 @@ uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
* and duplicate frees. * and duplicate frees.
* *
*/ */
void void
uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
{ {
uma_keg_t keg; uma_keg_t keg;
uma_slabrefcnt_t slabref;
int freei; int freei;
if (slab == NULL) { if (slab == NULL) {
@ -265,49 +266,21 @@ uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
item, zone->uz_name); item, zone->uz_name);
} }
keg = slab->us_keg; keg = slab->us_keg;
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
freei = ((unsigned long)item - (unsigned long)slab->us_data)
/ keg->uk_rsize;
if (freei >= keg->uk_ipers) if (freei >= keg->uk_ipers)
panic("zone: %s(%p) slab %p freelist %d out of range 0-%d\n", panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n",
zone->uz_name, zone, slab, freei, keg->uk_ipers-1); item, zone, zone->uz_name, slab, freei);
if (((freei * keg->uk_rsize) + slab->us_data) != item) { if (((freei * keg->uk_rsize) + slab->us_data) != item)
printf("zone: %s(%p) slab %p freed address %p unaligned.\n", panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n",
zone->uz_name, zone, slab, item); item, zone, zone->uz_name, slab, freei);
panic("should be %p\n",
(freei * keg->uk_rsize) + slab->us_data);
}
if (keg->uk_flags & UMA_ZONE_REFCNT) { if (!BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
slabref = (uma_slabrefcnt_t)slab; panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n",
if (slabref->us_freelist[freei].us_item != 255) { item, zone, zone->uz_name, slab, freei);
printf("Slab at %p, freei %d = %d.\n",
slab, freei, slabref->us_freelist[freei].us_item);
panic("Duplicate free of item %p from zone %p(%s)\n",
item, zone, zone->uz_name);
}
/* BIT_CLR_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
* When this is actually linked into the slab this will change.
* Until then the count of valid slabs will make sure we don't
* accidentally follow this and assume it's a valid index.
*/
slabref->us_freelist[freei].us_item = 0;
} else {
if (slab->us_freelist[freei].us_item != 255) {
printf("Slab at %p, freei %d = %d.\n",
slab, freei, slab->us_freelist[freei].us_item);
panic("Duplicate free of item %p from zone %p(%s)\n",
item, zone, zone->uz_name);
}
/*
* When this is actually linked into the slab this will change.
* Until then the count of valid slabs will make sure we don't
* accidentally follow this and assume it's a valid index.
*/
slab->us_freelist[freei].us_item = 0;
}
} }
#endif /* INVARIANTS */

73
sys/vm/uma_int.h
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org> * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
* Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
@ -45,12 +45,9 @@
* *
* The uma_slab_t may be embedded in a UMA_SLAB_SIZE chunk of memory or it may * The uma_slab_t may be embedded in a UMA_SLAB_SIZE chunk of memory or it may
* be allocated off the page from a special slab zone. The free list within a * be allocated off the page from a special slab zone. The free list within a
* slab is managed with a linked list of indices, which are 8 bit values. If * slab is managed with a bitmask. For item sizes that would yield more than
* UMA_SLAB_SIZE is defined to be too large I will have to switch to 16bit * 10% memory waste we potentially allocate a separate uma_slab_t if this will
* values. Currently on alpha you can get 250 or so 32 byte items and on x86 * improve the number of items per slab that will fit.
* you can get 250 or so 16byte items. For item sizes that would yield more
* than 10% memory waste we potentially allocate a separate uma_slab_t if this
* will improve the number of items per slab that will fit.
* *
* Other potential space optimizations are storing the 8bit of linkage in space * Other potential space optimizations are storing the 8bit of linkage in space
* wasted between items due to alignment problems. This may yield a much better * wasted between items due to alignment problems. This may yield a much better
@ -133,14 +130,9 @@
/* /*
* I should investigate other hashing algorithms. This should yield a low * I should investigate other hashing algorithms. This should yield a low
* number of collisions if the pages are relatively contiguous. * number of collisions if the pages are relatively contiguous.
*
* This is the same algorithm that most processor caches use.
*
* I'm shifting and masking instead of % because it should be faster.
*/ */
#define UMA_HASH(h, s) ((((unsigned long)s) >> UMA_SLAB_SHIFT) & \ #define UMA_HASH(h, s) ((((uintptr_t)s) >> UMA_SLAB_SHIFT) & (h)->uh_hashmask)
(h)->uh_hashmask)
#define UMA_HASH_INSERT(h, s, mem) \ #define UMA_HASH_INSERT(h, s, mem) \
SLIST_INSERT_HEAD(&(h)->uh_slab_hash[UMA_HASH((h), \ SLIST_INSERT_HEAD(&(h)->uh_slab_hash[UMA_HASH((h), \
@ -234,10 +226,17 @@ struct uma_keg {
}; };
typedef struct uma_keg * uma_keg_t; typedef struct uma_keg * uma_keg_t;
/* Page management structure */ /*
* Free bits per-slab.
*/
#define SLAB_SETSIZE (PAGE_SIZE / UMA_SMALLEST_UNIT)
BITSET_DEFINE(slabbits, SLAB_SETSIZE);
/* Sorry for the union, but space efficiency is important */ /*
struct uma_slab_head { * The slab structure manages a single contiguous allocation from backing
* store and subdivides it into individually allocatable items.
*/
struct uma_slab {
uma_keg_t us_keg; /* Keg we live in */ uma_keg_t us_keg; /* Keg we live in */
union { union {
LIST_ENTRY(uma_slab) _us_link; /* slabs in zone */ LIST_ENTRY(uma_slab) _us_link; /* slabs in zone */
@ -245,55 +244,31 @@ struct uma_slab_head {
} us_type; } us_type;
SLIST_ENTRY(uma_slab) us_hlink; /* Link for hash table */ SLIST_ENTRY(uma_slab) us_hlink; /* Link for hash table */
uint8_t *us_data; /* First item */ uint8_t *us_data; /* First item */
struct slabbits us_free; /* Free bitmask. */
#ifdef INVARIANTS
struct slabbits us_debugfree; /* Debug bitmask. */
#endif
uint16_t us_freecount; /* How many are free? */ uint16_t us_freecount; /* How many are free? */
uint8_t us_flags; /* Page flags see uma.h */ uint8_t us_flags; /* Page flags see uma.h */
uint8_t us_firstfree; /* First free item index */ uint8_t us_pad; /* Pad to 32bits, unused. */
}; };
/* The standard slab structure */ #define us_link us_type._us_link
struct uma_slab { #define us_size us_type._us_size
struct uma_slab_head us_head; /* slab header data */
struct {
uint8_t us_item;
} us_freelist[1]; /* actual number bigger */
};
/* /*
* The slab structure for UMA_ZONE_REFCNT zones for whose items we * The slab structure for UMA_ZONE_REFCNT zones for whose items we
* maintain reference counters in the slab for. * maintain reference counters in the slab for.
*/ */
struct uma_slab_refcnt { struct uma_slab_refcnt {
struct uma_slab_head us_head; /* slab header data */ struct uma_slab us_head; /* slab header data */
struct { uint32_t us_refcnt[0]; /* Actually larger. */
uint8_t us_item;
uint32_t us_refcnt;
} us_freelist[1]; /* actual number bigger */
}; };
#define us_keg us_head.us_keg
#define us_link us_head.us_type._us_link
#define us_size us_head.us_type._us_size
#define us_hlink us_head.us_hlink
#define us_data us_head.us_data
#define us_flags us_head.us_flags
#define us_freecount us_head.us_freecount
#define us_firstfree us_head.us_firstfree
typedef struct uma_slab * uma_slab_t; typedef struct uma_slab * uma_slab_t;
typedef struct uma_slab_refcnt * uma_slabrefcnt_t; typedef struct uma_slab_refcnt * uma_slabrefcnt_t;
typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int); typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int);
/*
* These give us the size of one free item reference within our corresponding
* uma_slab structures, so that our calculations during zone setup are correct
* regardless of what the compiler decides to do with padding the structure
* arrays within uma_slab.
*/
#define UMA_FRITM_SZ (sizeof(struct uma_slab) - sizeof(struct uma_slab_head))
#define UMA_FRITMREF_SZ (sizeof(struct uma_slab_refcnt) - \
sizeof(struct uma_slab_head))
struct uma_klink { struct uma_klink {
LIST_ENTRY(uma_klink) kl_link; LIST_ENTRY(uma_klink) kl_link;
uma_keg_t kl_keg; uma_keg_t kl_keg;