d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Add a new UMA API: uma_zcache_create(). This makes a zone without any

Browse Source 
backing memory that is only a container for per-cpu caches of arbitrary
   pointer items.  These zones have no kegs.
 - Convert the regular keg based allocator to use the new import/release
   functions.
 - Move some stats to be atomics since they would require excessive zone
   locking/unlocking with the new import/release paradigm.  Make
   zone_free_item simpler now that callers can manage more stats.
 - Check for these cache-only zones in the public APIs and debugging
   code by checking zone_first_keg() against NULL.

Sponsored by:	EMC / Isilong Storage Division
This commit is contained in:
Jeff Roberson 2013-06-17 03:43:47 +00:00
parent 40670f19e2
commit 0095a78419
3 changed files with 299 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
sys/vm/uma.h
View File

@ -123,6 +123,16 @@ typedef int (*uma_init)(void *mem, int size, int flags);
*/ */
typedef void (*uma_fini)(void *mem, int size); typedef void (*uma_fini)(void *mem, int size);
/*
* Import new memory into a cache zone.
*/
typedef int (*uma_import)(void *arg, void **store, int count, int flags);
/*
* Free memory from a cache zone.
*/
typedef void (*uma_release)(void *arg, void **store, int count);
/* /*
* What's the difference between initializing and constructing? * What's the difference between initializing and constructing?
* *
@ -215,6 +225,19 @@ uma_zone_t uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
*/ */
int uma_zsecond_add(uma_zone_t zone, uma_zone_t master); int uma_zsecond_add(uma_zone_t zone, uma_zone_t master);
/*
* Create cache-only zones.
*
* This allows uma's per-cpu cache facilities to handle arbitrary
* pointers. Consumers must specify the import and release functions to
* fill and destroy caches. UMA does not allocate any memory for these
* zones. The 'arg' parameter is passed to import/release and is caller
* specific.
*/
uma_zone_t uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor,
uma_init zinit, uma_fini zfini, uma_import zimport,
uma_release zrelease, void *arg, int flags);
/* /*
* Definitions for uma_zcreate flags * Definitions for uma_zcreate flags
* *

480
sys/vm/uma_core.c
View File

@ -131,14 +131,14 @@ static int bucketdisable = 1;
static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs); static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
/* This mutex protects the keg list */ /* This mutex protects the keg list */
static struct mtx uma_mtx; static struct mtx_padalign uma_mtx;
/* Linked list of boot time pages */ /* Linked list of boot time pages */
static LIST_HEAD(,uma_slab) uma_boot_pages = static LIST_HEAD(,uma_slab) uma_boot_pages =
LIST_HEAD_INITIALIZER(uma_boot_pages); LIST_HEAD_INITIALIZER(uma_boot_pages);
/* This mutex protects the boot time pages list */ /* This mutex protects the boot time pages list */
static struct mtx uma_boot_pages_mtx; static struct mtx_padalign uma_boot_pages_mtx;
/* Is the VM done starting up? */ /* Is the VM done starting up? */
static int booted = 0; static int booted = 0;
@ -172,6 +172,9 @@ struct uma_zctor_args {
uma_dtor dtor; uma_dtor dtor;
uma_init uminit; uma_init uminit;
uma_fini fini; uma_fini fini;
uma_import import;
uma_release release;
void *arg;
uma_keg_t keg; uma_keg_t keg;
int align; int align;
uint32_t flags; uint32_t flags;
@ -216,9 +219,6 @@ static uint8_t bucket_size[BUCKET_ZONES];
*/ */
enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }; enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
/* Prototypes.. */ /* Prototypes.. */
static void *noobj_alloc(uma_zone_t, int, uint8_t *, int); static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
@ -244,8 +244,7 @@ static void hash_free(struct uma_hash *hash);
static void uma_timeout(void *); static void uma_timeout(void *);
static void uma_startup3(void); static void uma_startup3(void);
static void *zone_alloc_item(uma_zone_t, void *, int); static void *zone_alloc_item(uma_zone_t, void *, int);
static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip, static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
int);
static void bucket_enable(void); static void bucket_enable(void);
static void bucket_init(void); static void bucket_init(void);
static uma_bucket_t bucket_alloc(int, int); static uma_bucket_t bucket_alloc(int, int);
@ -254,11 +253,14 @@ static void bucket_zone_drain(void);
static int zone_alloc_bucket(uma_zone_t zone, int flags); static int zone_alloc_bucket(uma_zone_t zone, int flags);
static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags); static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags); static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab); static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
uma_fini fini, int align, uint32_t flags); uma_fini fini, int align, uint32_t flags);
static inline void zone_relock(uma_zone_t zone, uma_keg_t keg); static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
static inline void keg_relock(uma_keg_t keg, uma_zone_t zone); static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
static void zone_release(uma_zone_t zone, void **bucket, int cnt);
void uma_print_zone(uma_zone_t); void uma_print_zone(uma_zone_t);
void uma_print_stats(void); void uma_print_stats(void);
@ -363,8 +365,7 @@ bucket_free(uma_bucket_t bucket)
struct uma_bucket_zone *ubz; struct uma_bucket_zone *ubz;
ubz = bucket_zone_lookup(bucket->ub_entries); ubz = bucket_zone_lookup(bucket->ub_entries);
zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE, zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
ZFREE_STATFREE);
} }
static void static void
@ -388,13 +389,6 @@ zone_log_warning(uma_zone_t zone)
printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
} }
static inline uma_keg_t
zone_first_keg(uma_zone_t zone)
{
return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
}
static void static void
zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t)) zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
{ {
@ -579,8 +573,7 @@ hash_free(struct uma_hash *hash)
if (hash->uh_slab_hash == NULL) if (hash->uh_slab_hash == NULL)
return; return;
if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
zone_free_item(hashzone, zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE);
else else
free(hash->uh_slab_hash, M_UMAHASH); free(hash->uh_slab_hash, M_UMAHASH);
} }
@ -599,21 +592,16 @@ hash_free(struct uma_hash *hash)
static void static void
bucket_drain(uma_zone_t zone, uma_bucket_t bucket) bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
{ {
void *item; int i;
if (bucket == NULL) if (bucket == NULL)
return; return;
while (bucket->ub_cnt > 0) { if (zone->uz_fini)
bucket->ub_cnt--; for (i = 0; i < bucket->ub_cnt; i++)
item = bucket->ub_bucket[bucket->ub_cnt]; zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
#ifdef INVARIANTS zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
bucket->ub_bucket[bucket->ub_cnt] = NULL; bucket->ub_cnt = 0;
KASSERT(item != NULL,
("bucket_drain: botched ptr, item is NULL"));
#endif
zone_free_item(zone, item, NULL, SKIP_DTOR, 0);
}
} }
/* /*
@ -767,8 +755,7 @@ finished:
obj); obj);
} }
if (keg->uk_flags & UMA_ZONE_OFFPAGE) if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab, NULL, zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
SKIP_NONE, ZFREE_STATFREE);
#ifdef UMA_DEBUG #ifdef UMA_DEBUG
printf("%s: Returning %d bytes.\n", printf("%s: Returning %d bytes.\n",
keg->uk_name, PAGE_SIZE * keg->uk_ppera); keg->uk_name, PAGE_SIZE * keg->uk_ppera);
@ -842,7 +829,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
slab = NULL; slab = NULL;
#ifdef UMA_DEBUG #ifdef UMA_DEBUG
printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name); printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
#endif #endif
allocf = keg->uk_allocf; allocf = keg->uk_allocf;
KEG_UNLOCK(keg); KEG_UNLOCK(keg);
@ -874,8 +861,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait); mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
if (mem == NULL) { if (mem == NULL) {
if (keg->uk_flags & UMA_ZONE_OFFPAGE) if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab, NULL, zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
SKIP_NONE, ZFREE_STATFREE);
KEG_LOCK(keg); KEG_LOCK(keg);
return (NULL); return (NULL);
} }
@ -929,7 +915,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
} }
if (keg->uk_flags & UMA_ZONE_OFFPAGE) if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab, zone_free_item(keg->uk_slabzone, slab,
NULL, SKIP_NONE, ZFREE_STATFREE); NULL, SKIP_NONE);
keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
flags); flags);
KEG_LOCK(keg); KEG_LOCK(keg);
@ -1483,6 +1469,24 @@ zone_ctor(void *mem, int size, void *udata, int flags)
timevalclear(&zone->uz_ratecheck); timevalclear(&zone->uz_ratecheck);
keg = arg->keg; keg = arg->keg;
/*
* This is a pure cache zone, no kegs.
*/
if (arg->import) {
zone->uz_import = arg->import;
zone->uz_release = arg->release;
zone->uz_arg = arg->arg;
zone->uz_count = BUCKET_MAX;
return (0);
}
/*
* Use the regular zone/keg/slab allocator.
*/
zone->uz_import = (uma_import)zone_import;
zone->uz_release = (uma_release)zone_release;
zone->uz_arg = zone;
if (arg->flags & UMA_ZONE_SECONDARY) { if (arg->flags & UMA_ZONE_SECONDARY) {
KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
zone->uz_init = arg->uminit; zone->uz_init = arg->uminit;
@ -1519,6 +1523,7 @@ zone_ctor(void *mem, int size, void *udata, int flags)
if (error) if (error)
return (error); return (error);
} }
/* /*
* Link in the first keg. * Link in the first keg.
*/ */
@ -1616,12 +1621,11 @@ zone_dtor(void *arg, int size, void *udata)
/* /*
* We only destroy kegs from non secondary zones. * We only destroy kegs from non secondary zones.
*/ */
if ((zone->uz_flags & UMA_ZONE_SECONDARY) == 0) { if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
mtx_lock(&uma_mtx); mtx_lock(&uma_mtx);
LIST_REMOVE(keg, uk_link); LIST_REMOVE(keg, uk_link);
mtx_unlock(&uma_mtx); mtx_unlock(&uma_mtx);
zone_free_item(kegs, keg, NULL, SKIP_NONE, zone_free_item(kegs, keg, NULL, SKIP_NONE);
ZFREE_STATFREE);
} }
} }
@ -1665,6 +1669,7 @@ uma_startup(void *bootmem, int boot_pages)
mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
/* "manually" create the initial zone */ /* "manually" create the initial zone */
memset(&args, 0, sizeof(args));
args.name = "UMA Kegs"; args.name = "UMA Kegs";
args.size = sizeof(struct uma_keg); args.size = sizeof(struct uma_keg);
args.ctor = keg_ctor; args.ctor = keg_ctor;
@ -1805,6 +1810,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
struct uma_zctor_args args; struct uma_zctor_args args;
/* This stuff is essential for the zone ctor */ /* This stuff is essential for the zone ctor */
memset(&args, 0, sizeof(args));
args.name = name; args.name = name;
args.size = size; args.size = size;
args.ctor = ctor; args.ctor = ctor;
@ -1827,6 +1833,7 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
uma_keg_t keg; uma_keg_t keg;
keg = zone_first_keg(master); keg = zone_first_keg(master);
memset(&args, 0, sizeof(args));
args.name = name; args.name = name;
args.size = keg->uk_size; args.size = keg->uk_size;
args.ctor = ctor; args.ctor = ctor;
@ -1841,6 +1848,30 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
return (zone_alloc_item(zones, &args, M_WAITOK)); return (zone_alloc_item(zones, &args, M_WAITOK));
} }
/* See uma.h */
uma_zone_t
uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
uma_fini zfini, uma_import zimport, uma_release zrelease,
void *arg, int flags)
{
struct uma_zctor_args args;
memset(&args, 0, sizeof(args));
args.name = name;
args.size = 0;
args.ctor = ctor;
args.dtor = dtor;
args.uminit = zinit;
args.fini = zfini;
args.import = zimport;
args.release = zrelease;
args.arg = arg;
args.align = 0;
args.flags = flags;
return (zone_alloc_item(zones, &args, M_WAITOK));
}
static void static void
zone_lock_pair(uma_zone_t a, uma_zone_t b) zone_lock_pair(uma_zone_t a, uma_zone_t b)
{ {
@ -1932,7 +1963,7 @@ void
uma_zdestroy(uma_zone_t zone) uma_zdestroy(uma_zone_t zone)
{ {
zone_free_item(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE); zone_free_item(zones, zone, NULL, SKIP_NONE);
} }
/* See uma.h */ /* See uma.h */
@ -2011,9 +2042,9 @@ zalloc_start:
if (zone->uz_ctor != NULL) { if (zone->uz_ctor != NULL) {
if (zone->uz_ctor(item, zone->uz_size, if (zone->uz_ctor(item, zone->uz_size,
udata, flags) != 0) { udata, flags) != 0) {
atomic_add_long(&zone->uz_fails, 1);
zone_free_item(zone, item, udata, zone_free_item(zone, item, udata,
SKIP_DTOR, ZFREE_STATFAIL | SKIP_DTOR);
ZFREE_STATFREE);
return (NULL); return (NULL);
} }
} }
@ -2069,9 +2100,9 @@ zalloc_start:
} }
/* Since we have locked the zone we may as well send back our stats */ /* Since we have locked the zone we may as well send back our stats */
zone->uz_allocs += cache->uc_allocs; atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
atomic_add_long(&zone->uz_frees, cache->uc_frees);
cache->uc_allocs = 0; cache->uc_allocs = 0;
zone->uz_frees += cache->uc_frees;
cache->uc_frees = 0; cache->uc_frees = 0;
/* Our old one is now a free bucket */ /* Our old one is now a free bucket */
@ -2319,13 +2350,12 @@ zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
} }
static void * static void *
slab_alloc_item(uma_zone_t zone, uma_slab_t slab) slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
{ {
uma_keg_t keg;
void *item; void *item;
uint8_t freei; uint8_t freei;
keg = slab->us_keg; MPASS(keg == slab->us_keg);
mtx_assert(&keg->uk_lock, MA_OWNED); mtx_assert(&keg->uk_lock, MA_OWNED);
freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1; freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
@ -2343,37 +2373,42 @@ slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
return (item); return (item);
} }
static int
zone_import(uma_zone_t zone, void **bucket, int max, int flags)
{
uma_slab_t slab;
uma_keg_t keg;
int i;
ZONE_LOCK(zone);
/* Try to keep the buckets totally full */
slab = NULL;
keg = NULL;
for (i = 0; i < max; ) {
if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
break;
keg = slab->us_keg;
while (slab->us_freecount && i < max)
bucket[i++] = slab_alloc_item(keg, slab);
/* Don't block on the next fill */
flags &= ~M_WAITOK;
flags |= M_NOWAIT;
}
if (slab != NULL)
KEG_UNLOCK(keg);
else
ZONE_UNLOCK(zone);
return i;
}
static int static int
zone_alloc_bucket(uma_zone_t zone, int flags) zone_alloc_bucket(uma_zone_t zone, int flags)
{ {
uma_bucket_t bucket; uma_bucket_t bucket;
uma_slab_t slab; int bflags;
uma_keg_t keg; int max;
int16_t saved;
int max, origflags = flags;
/*
* Try this zone's free list first so we don't allocate extra buckets.
*/
if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
KASSERT(bucket->ub_cnt == 0,
("zone_alloc_bucket: Bucket on free list is not empty."));
LIST_REMOVE(bucket, ub_link);
} else {
int bflags;
bflags = (flags & ~M_ZERO);
if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
bflags |= M_NOVM;
ZONE_UNLOCK(zone);
bucket = bucket_alloc(zone->uz_count, bflags);
ZONE_LOCK(zone);
}
if (bucket == NULL) {
return (0);
}
#ifdef SMP #ifdef SMP
/* /*
@ -2382,79 +2417,75 @@ zone_alloc_bucket(uma_zone_t zone, int flags)
* is done so that we don't allocate more memory than we really need. * is done so that we don't allocate more memory than we really need.
*/ */
if (zone->uz_fills >= mp_ncpus) if (zone->uz_fills >= mp_ncpus)
goto done; return (0);
#endif #endif
zone->uz_fills++; zone->uz_fills++;
max = zone->uz_count;
max = MIN(bucket->ub_entries, zone->uz_count);
/* Try to keep the buckets totally full */
saved = bucket->ub_cnt;
slab = NULL;
keg = NULL;
while (bucket->ub_cnt < max &&
(slab = zone->uz_slab(zone, keg, flags)) != NULL) {
keg = slab->us_keg;
while (slab->us_freecount && bucket->ub_cnt < max) {
bucket->ub_bucket[bucket->ub_cnt++] =
slab_alloc_item(zone, slab);
}
/* Don't block on the next fill */
flags |= M_NOWAIT;
}
if (slab)
zone_relock(zone, keg);
/* /*
* We unlock here because we need to call the zone's init. * Try this zone's free list first so we don't allocate extra buckets.
* It should be safe to unlock because the slab dealt with
* above is already on the appropriate list within the keg
* and the bucket we filled is not yet on any list, so we
* own it.
*/ */
if (zone->uz_init != NULL) { if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
KASSERT(bucket->ub_cnt == 0,
("zone_alloc_bucket: Bucket on free list is not empty."));
LIST_REMOVE(bucket, ub_link);
ZONE_UNLOCK(zone);
} else {
bflags = (flags & ~M_ZERO);
if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
bflags |= M_NOVM;
ZONE_UNLOCK(zone);
bucket = bucket_alloc(zone->uz_count, bflags);
if (bucket == NULL)
goto out;
}
max = MIN(bucket->ub_entries, max);
bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
max, flags);
/*
* Initialize the memory if necessary.
*/
if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
int i; int i;
ZONE_UNLOCK(zone); for (i = 0; i < bucket->ub_cnt; i++)
for (i = saved; i < bucket->ub_cnt; i++)
if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
origflags) != 0) flags) != 0)
break; break;
/* /*
* If we couldn't initialize the whole bucket, put the * If we couldn't initialize the whole bucket, put the
* rest back onto the freelist. * rest back onto the freelist.
*/ */
if (i != bucket->ub_cnt) { if (i != bucket->ub_cnt) {
int j; zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
bucket->ub_cnt - i);
for (j = i; j < bucket->ub_cnt; j++) {
zone_free_item(zone, bucket->ub_bucket[j],
NULL, SKIP_FINI, 0);
#ifdef INVARIANTS #ifdef INVARIANTS
bucket->ub_bucket[j] = NULL; bzero(&bucket->ub_bucket[i],
sizeof(void *) * (bucket->ub_cnt - i));
#endif #endif
}
bucket->ub_cnt = i; bucket->ub_cnt = i;
} }
ZONE_LOCK(zone);
} }
out:
ZONE_LOCK(zone);
zone->uz_fills--; zone->uz_fills--;
if (bucket->ub_cnt != 0) { if (bucket != NULL && bucket->ub_cnt != 0) {
LIST_INSERT_HEAD(&zone->uz_full_bucket, LIST_INSERT_HEAD(&zone->uz_full_bucket,
bucket, ub_link); bucket, ub_link);
return (1); return (1);
} }
#ifdef SMP atomic_add_long(&zone->uz_fails, 1);
done: if (bucket != NULL)
#endif bucket_free(bucket);
bucket_free(bucket);
return (0); return (0);
} }
/* /*
* Allocates an item for an internal zone * Allocates a single item from a zone.
* *
* Arguments * Arguments
* zone The zone to alloc for. * zone The zone to alloc for.
@ -2469,7 +2500,6 @@ done:
static void * static void *
zone_alloc_item(uma_zone_t zone, void *udata, int flags) zone_alloc_item(uma_zone_t zone, void *udata, int flags)
{ {
uma_slab_t slab;
void *item; void *item;
item = NULL; item = NULL;
@ -2477,20 +2507,9 @@ zone_alloc_item(uma_zone_t zone, void *udata, int flags)
#ifdef UMA_DEBUG_ALLOC #ifdef UMA_DEBUG_ALLOC
printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
#endif #endif
ZONE_LOCK(zone); if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
goto fail;
slab = zone->uz_slab(zone, NULL, flags); atomic_add_long(&zone->uz_allocs, 1);
if (slab == NULL) {
zone->uz_fails++;
ZONE_UNLOCK(zone);
return (NULL);
}
item = slab_alloc_item(zone, slab);
zone_relock(zone, slab->us_keg);
zone->uz_allocs++;
ZONE_UNLOCK(zone);
/* /*
* We have to call both the zone's init (not the keg's init) * We have to call both the zone's init (not the keg's init)
@ -2500,25 +2519,27 @@ zone_alloc_item(uma_zone_t zone, void *udata, int flags)
*/ */
if (zone->uz_init != NULL) { if (zone->uz_init != NULL) {
if (zone->uz_init(item, zone->uz_size, flags) != 0) { if (zone->uz_init(item, zone->uz_size, flags) != 0) {
zone_free_item(zone, item, udata, SKIP_FINI, zone_free_item(zone, item, udata, SKIP_FINI);
ZFREE_STATFAIL | ZFREE_STATFREE); goto fail;
return (NULL);
} }
} }
if (zone->uz_ctor != NULL) { if (zone->uz_ctor != NULL) {
if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
zone_free_item(zone, item, udata, SKIP_DTOR, zone_free_item(zone, item, udata, SKIP_DTOR);
ZFREE_STATFAIL | ZFREE_STATFREE); goto fail;
return (NULL);
} }
} }
#ifdef INVARIANTS #ifdef INVARIANTS
uma_dbg_alloc(zone, slab, item); uma_dbg_alloc(zone, NULL, item);
#endif #endif
if (flags & M_ZERO) if (flags & M_ZERO)
bzero(item, zone->uz_size); bzero(item, zone->uz_size);
return (item); return (item);
fail:
atomic_add_long(&zone->uz_fails, 1);
return (NULL);
} }
/* See uma.h */ /* See uma.h */
@ -2648,9 +2669,9 @@ zfree_start:
} }
/* Since we have locked the zone we may as well send back our stats */ /* Since we have locked the zone we may as well send back our stats */
zone->uz_allocs += cache->uc_allocs; atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
atomic_add_long(&zone->uz_frees, cache->uc_frees);
cache->uc_allocs = 0; cache->uc_allocs = 0;
zone->uz_frees += cache->uc_frees;
cache->uc_frees = 0; cache->uc_frees = 0;
bucket = cache->uc_freebucket; bucket = cache->uc_freebucket;
@ -2699,69 +2720,17 @@ zfree_start:
* If nothing else caught this, we'll just do an internal free. * If nothing else caught this, we'll just do an internal free.
*/ */
zfree_internal: zfree_internal:
zone_free_item(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE); zone_free_item(zone, item, udata, SKIP_DTOR);
return; return;
} }
/*
* Frees an item to an INTERNAL zone or allocates a free bucket
*
* Arguments:
* zone The zone to free to
* item The item we're freeing
* udata User supplied data for the dtor
* skip Skip dtors and finis
*/
static void static void
zone_free_item(uma_zone_t zone, void *item, void *udata, slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
enum zfreeskip skip, int flags)
{ {
uma_slab_t slab;
uma_keg_t keg;
uint8_t *mem;
uint8_t freei; uint8_t freei;
int clearfull;
#ifdef INVARIANTS mtx_assert(&keg->uk_lock, MA_OWNED);
if (skip == SKIP_NONE) {
if (zone->uz_flags & UMA_ZONE_MALLOC)
uma_dbg_free(zone, udata, item);
else
uma_dbg_free(zone, NULL, item);
}
#endif
if (skip < SKIP_DTOR && zone->uz_dtor)
zone->uz_dtor(item, zone->uz_size, udata);
if (skip < SKIP_FINI && zone->uz_fini)
zone->uz_fini(item, zone->uz_size);
ZONE_LOCK(zone);
if (flags & ZFREE_STATFAIL)
zone->uz_fails++;
if (flags & ZFREE_STATFREE)
zone->uz_frees++;
if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
keg = zone_first_keg(zone); /* Must only be one. */
if (zone->uz_flags & UMA_ZONE_HASH) {
slab = hash_sfind(&keg->uk_hash, mem);
} else {
mem += keg->uk_pgoff;
slab = (uma_slab_t)mem;
}
} else {
/* This prevents redundant lookups via free(). */
if ((zone->uz_flags & UMA_ZONE_MALLOC) && udata != NULL)
slab = (uma_slab_t)udata;
else
slab = vtoslab((vm_offset_t)item);
keg = slab->us_keg;
keg_relock(keg, zone);
}
MPASS(keg == slab->us_keg); MPASS(keg == slab->us_keg);
/* Do we need to remove from any lists? */ /* Do we need to remove from any lists? */
@ -2780,41 +2749,106 @@ zone_free_item(uma_zone_t zone, void *item, void *udata,
/* Keg statistics. */ /* Keg statistics. */
keg->uk_free++; keg->uk_free++;
}
static void
zone_release(uma_zone_t zone, void **bucket, int cnt)
{
void *item;
uma_slab_t slab;
uma_keg_t keg;
uint8_t *mem;
int clearfull;
int i;
clearfull = 0; clearfull = 0;
if (keg->uk_flags & UMA_ZFLAG_FULL) { ZONE_LOCK(zone);
if (keg->uk_pages < keg->uk_maxpages) { keg = zone_first_keg(zone);
keg->uk_flags &= ~UMA_ZFLAG_FULL; for (i = 0; i < cnt; i++) {
clearfull = 1; item = bucket[i];
if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
if (zone->uz_flags & UMA_ZONE_HASH) {
slab = hash_sfind(&keg->uk_hash, mem);
} else {
mem += keg->uk_pgoff;
slab = (uma_slab_t)mem;
}
} else {
slab = vtoslab((vm_offset_t)item);
if (slab->us_keg != keg) {
KEG_UNLOCK(keg);
keg = slab->us_keg;
KEG_LOCK(keg);
}
} }
slab_free_item(keg, slab, item);
if (keg->uk_flags & UMA_ZFLAG_FULL) {
if (keg->uk_pages < keg->uk_maxpages) {
keg->uk_flags &= ~UMA_ZFLAG_FULL;
clearfull = 1;
}
/* /*
* We can handle one more allocation. Since we're * We can handle one more allocation. Since we're
* clearing ZFLAG_FULL, wake up all procs blocked * clearing ZFLAG_FULL, wake up all procs blocked
* on pages. This should be uncommon, so keeping this * on pages. This should be uncommon, so keeping this
* simple for now (rather than adding count of blocked * simple for now (rather than adding count of blocked
* threads etc). * threads etc).
*/ */
wakeup(keg); wakeup(keg);
}
} }
zone_relock(zone, keg);
if (clearfull) { if (clearfull) {
zone_relock(zone, keg);
zone->uz_flags &= ~UMA_ZFLAG_FULL; zone->uz_flags &= ~UMA_ZFLAG_FULL;
wakeup(zone); wakeup(zone);
ZONE_UNLOCK(zone); }
} else ZONE_UNLOCK(zone);
KEG_UNLOCK(keg);
} }
/*
* Frees a single item to any zone.
*
* Arguments:
* zone The zone to free to
* item The item we're freeing
* udata User supplied data for the dtor
* skip Skip dtors and finis
*/
static void
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
{
#ifdef INVARIANTS
if (skip == SKIP_NONE) {
if (zone->uz_flags & UMA_ZONE_MALLOC)
uma_dbg_free(zone, udata, item);
else
uma_dbg_free(zone, NULL, item);
}
#endif
if (skip < SKIP_DTOR && zone->uz_dtor)
zone->uz_dtor(item, zone->uz_size, udata);
if (skip < SKIP_FINI && zone->uz_fini)
zone->uz_fini(item, zone->uz_size);
atomic_add_long(&zone->uz_frees, 1);
zone->uz_release(zone->uz_arg, &item, 1);
}
/* See uma.h */ /* See uma.h */
int int
uma_zone_set_max(uma_zone_t zone, int nitems) uma_zone_set_max(uma_zone_t zone, int nitems)
{ {
uma_keg_t keg; uma_keg_t keg;
ZONE_LOCK(zone);
keg = zone_first_keg(zone); keg = zone_first_keg(zone);
if (keg == NULL)
return (0);
ZONE_LOCK(zone);
keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera; keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
if (keg->uk_maxpages * keg->uk_ipers < nitems) if (keg->uk_maxpages * keg->uk_ipers < nitems)
keg->uk_maxpages += keg->uk_ppera; keg->uk_maxpages += keg->uk_ppera;
@ -2831,8 +2865,10 @@ uma_zone_get_max(uma_zone_t zone)
int nitems; int nitems;
uma_keg_t keg; uma_keg_t keg;
ZONE_LOCK(zone);
keg = zone_first_keg(zone); keg = zone_first_keg(zone);
if (keg == NULL)
return (0);
ZONE_LOCK(zone);
nitems = keg->uk_maxpages * keg->uk_ipers; nitems = keg->uk_maxpages * keg->uk_ipers;
ZONE_UNLOCK(zone); ZONE_UNLOCK(zone);
@ -2880,6 +2916,7 @@ uma_zone_set_init(uma_zone_t zone, uma_init uminit)
ZONE_LOCK(zone); ZONE_LOCK(zone);
keg = zone_first_keg(zone); keg = zone_first_keg(zone);
KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
KASSERT(keg->uk_pages == 0, KASSERT(keg->uk_pages == 0,
("uma_zone_set_init on non-empty keg")); ("uma_zone_set_init on non-empty keg"));
keg->uk_init = uminit; keg->uk_init = uminit;
@ -2894,6 +2931,7 @@ uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
ZONE_LOCK(zone); ZONE_LOCK(zone);
keg = zone_first_keg(zone); keg = zone_first_keg(zone);
KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
KASSERT(keg->uk_pages == 0, KASSERT(keg->uk_pages == 0,
("uma_zone_set_fini on non-empty keg")); ("uma_zone_set_fini on non-empty keg"));
keg->uk_fini = fini; keg->uk_fini = fini;
@ -2927,9 +2965,12 @@ uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
void void
uma_zone_set_freef(uma_zone_t zone, uma_free freef) uma_zone_set_freef(uma_zone_t zone, uma_free freef)
{ {
uma_keg_t keg;
ZONE_LOCK(zone); ZONE_LOCK(zone);
zone_first_keg(zone)->uk_freef = freef; keg = zone_first_keg(zone);
KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
keg->uk_freef = freef;
ZONE_UNLOCK(zone); ZONE_UNLOCK(zone);
} }
@ -2956,6 +2997,8 @@ uma_zone_reserve_kva(uma_zone_t zone, int count)
int pages; int pages;
keg = zone_first_keg(zone); keg = zone_first_keg(zone);
if (keg == NULL)
return (0);
pages = count / keg->uk_ipers; pages = count / keg->uk_ipers;
if (pages * keg->uk_ipers < count) if (pages * keg->uk_ipers < count)
@ -2994,6 +3037,8 @@ uma_prealloc(uma_zone_t zone, int items)
uma_keg_t keg; uma_keg_t keg;
keg = zone_first_keg(zone); keg = zone_first_keg(zone);
if (keg == NULL)
return;
ZONE_LOCK(zone); ZONE_LOCK(zone);
slabs = items / keg->uk_ipers; slabs = items / keg->uk_ipers;
if (slabs * keg->uk_ipers < items) if (slabs * keg->uk_ipers < items)
@ -3083,8 +3128,7 @@ uma_large_malloc(int size, int wait)
slab->us_flags = flags | UMA_SLAB_MALLOC; slab->us_flags = flags | UMA_SLAB_MALLOC;
slab->us_size = size; slab->us_size = size;
} else { } else {
zone_free_item(slabzone, slab, NULL, SKIP_NONE, zone_free_item(slabzone, slab, NULL, SKIP_NONE);
ZFREE_STATFAIL | ZFREE_STATFREE);
} }
return (mem); return (mem);
@ -3095,7 +3139,7 @@ uma_large_free(uma_slab_t slab)
{ {
vsetobj((vm_offset_t)slab->us_data, kmem_object); vsetobj((vm_offset_t)slab->us_data, kmem_object);
page_free(slab->us_data, slab->us_size, slab->us_flags); page_free(slab->us_data, slab->us_size, slab->us_flags);
zone_free_item(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE); zone_free_item(slabzone, slab, NULL, SKIP_NONE);
} }
void void

18
sys/vm/uma_int.h
View File

@ -296,14 +296,17 @@ struct uma_zone {
uma_ctor uz_ctor; /* Constructor for each allocation */ uma_ctor uz_ctor; /* Constructor for each allocation */
uma_dtor uz_dtor; /* Destructor */ uma_dtor uz_dtor; /* Destructor */
uma_init uz_init; /* Initializer for each item */ uma_init uz_init; /* Initializer for each item */
uma_fini uz_fini; /* Discards memory */ uma_fini uz_fini; /* Finalizer for each item. */
uma_import uz_import; /* Import new memory to cache. */
uma_release uz_release; /* Release memory from cache. */
void *uz_arg; /* Import/release argument. */
uint32_t uz_flags; /* Flags inherited from kegs */ uint32_t uz_flags; /* Flags inherited from kegs */
uint32_t uz_size; /* Size inherited from kegs */ uint32_t uz_size; /* Size inherited from kegs */
uint64_t uz_allocs UMA_ALIGN; /* Total number of allocations */ volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */
uint64_t uz_frees; /* Total number of frees */ volatile u_long uz_fails; /* Total number of alloc failures */
uint64_t uz_fails; /* Total number of alloc failures */ volatile u_long uz_frees; /* Total number of frees */
uint64_t uz_sleeps; /* Total number of alloc sleeps */ uint64_t uz_sleeps; /* Total number of alloc sleeps */
uint16_t uz_fills; /* Outstanding bucket fills */ uint16_t uz_fills; /* Outstanding bucket fills */
uint16_t uz_count; /* Highest amount of items in bucket */ uint16_t uz_count; /* Highest amount of items in bucket */
@ -333,6 +336,13 @@ struct uma_zone {
#define UMA_ZFLAG_INHERIT (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | \ #define UMA_ZFLAG_INHERIT (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | \
UMA_ZFLAG_BUCKET) UMA_ZFLAG_BUCKET)
static inline uma_keg_t
zone_first_keg(uma_zone_t zone)
{
return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
}
#undef UMA_ALIGN #undef UMA_ALIGN
#ifdef _KERNEL #ifdef _KERNEL