d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement a sysctl tree for uma zones to assist in debugging and provide

Browse Source 
more statistcs than are exported via the ABI stable vmstat interface.
Rename uz_count to uz_bucket_size because even I was confused by the
name after returning to the source years later.

Reviewed by:	rlibby
Differential Revision:	https://reviews.freebsd.org/D22554
This commit is contained in:
Jeff Roberson 2019-11-28 00:19:09 +00:00
parent 0a81b4395e
commit 20a4e15451
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=355149
2 changed files with 343 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

490
sys/vm/uma_core.c
View File

@ -119,6 +119,7 @@ static uma_zone_t hashzone;
int uma_align_cache = 64 - 1;
static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc");
/*
* Are we allowed to allocate buckets?
@ -258,8 +259,8 @@ static void zone_dtor(void *, int, void *);
static int zero_init(void *, int, int);
static void keg_small_init(uma_keg_t keg);
static void keg_large_init(uma_keg_t keg);
static void zone_foreach(void (*zfunc)(uma_zone_t));
static void zone_timeout(uma_zone_t zone);
static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *);
static void zone_timeout(uma_zone_t zone, void *);
static int hash_alloc(struct uma_hash *, u_int);
static int hash_expand(struct uma_hash *, struct uma_hash *);
static void hash_free(struct uma_hash *hash);
@ -285,10 +286,10 @@ static void uma_zero_item(void *, uma_zone_t);
static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int);
static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int);
void uma_print_zone(uma_zone_t);
void uma_print_stats(void);
static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS);
static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS);
#ifdef INVARIANTS
static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
@ -296,6 +297,7 @@ static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW, 0, "Universal Memory Allocator");
static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0,
"Memory allocation debugging");
@ -458,7 +460,7 @@ bucket_alloc(uma_zone_t zone, void *udata, int flags)
}
if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
flags |= M_NOVM;
ubz = bucket_zone_lookup(zone->uz_count);
ubz = bucket_zone_lookup(zone->uz_bucket_size);
if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
ubz++;
bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
@ -575,7 +577,7 @@ static void
uma_timeout(void *unused)
{
bucket_enable();
zone_foreach(zone_timeout);
zone_foreach(zone_timeout, NULL);
/* Reschedule this event */
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
@ -605,7 +607,7 @@ zone_domain_update_wss(uma_zone_domain_t zdom)
* Returns nothing.
*/
static void
zone_timeout(uma_zone_t zone)
zone_timeout(uma_zone_t zone, void *unused)
{
uma_keg_t keg;
u_int slabs;
@ -845,19 +847,20 @@ cache_drain(uma_zone_t zone)
}
static void
cache_shrink(uma_zone_t zone)
cache_shrink(uma_zone_t zone, void *unused)
{
if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
return;
ZONE_LOCK(zone);
zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
zone->uz_bucket_size =
(zone->uz_bucket_size_min + zone->uz_bucket_size) / 2;
ZONE_UNLOCK(zone);
}
static void
cache_drain_safe_cpu(uma_zone_t zone)
cache_drain_safe_cpu(uma_zone_t zone, void *unused)
{
uma_cache_t cache;
uma_bucket_t b1, b2, b3;
@ -920,9 +923,9 @@ pcpu_cache_drain_safe(uma_zone_t zone)
* Polite bucket sizes shrinking was not enouth, shrink aggressively.
*/
if (zone)
cache_shrink(zone);
cache_shrink(zone, NULL);
else
zone_foreach(cache_shrink);
zone_foreach(cache_shrink, NULL);
CPU_FOREACH(cpu) {
thread_lock(curthread);
@ -930,9 +933,9 @@ pcpu_cache_drain_safe(uma_zone_t zone)
thread_unlock(curthread);
if (zone)
cache_drain_safe_cpu(zone);
cache_drain_safe_cpu(zone, NULL);
else
zone_foreach(cache_drain_safe_cpu);
zone_foreach(cache_drain_safe_cpu, NULL);
}
thread_lock(curthread);
sched_unbind(curthread);
@ -991,8 +994,8 @@ bucket_cache_reclaim(uma_zone_t zone, bool drain)
* Shrink the zone bucket size to ensure that the per-CPU caches
* don't grow too large.
*/
if (zone->uz_count > zone->uz_count_min)
zone->uz_count--;
if (zone->uz_bucket_size > zone->uz_bucket_size_min)
zone->uz_bucket_size--;
}
static void
@ -1121,14 +1124,14 @@ zone_reclaim(uma_zone_t zone, int waitok, bool drain)
}
static void
zone_drain(uma_zone_t zone)
zone_drain(uma_zone_t zone, void *unused)
{
zone_reclaim(zone, M_NOWAIT, true);
}
static void
zone_trim(uma_zone_t zone)
zone_trim(uma_zone_t zone, void *unused)
{
zone_reclaim(zone, M_NOWAIT, false);
@ -1818,7 +1821,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
}
static void
zone_alloc_counters(uma_zone_t zone)
zone_alloc_counters(uma_zone_t zone, void *unused)
{
zone->uz_allocs = counter_u64_alloc(M_WAITOK);
@ -1826,6 +1829,169 @@ zone_alloc_counters(uma_zone_t zone)
zone->uz_fails = counter_u64_alloc(M_WAITOK);
}
#define UMA_MAX_DUP 999
static void
zone_alloc_sysctl(uma_zone_t zone, void *unused)
{
uma_zone_domain_t zdom;
uma_keg_t keg;
struct sysctl_oid *oid, *domainoid;
int domains, i;
static const char *nokeg = "cache zone";
char *c;
/*
* Make a sysctl safe copy of the zone name by removing
* any special characters and handling dups by appending
* an index.
*/
if (zone->uz_namecnt != 0) {
if (zone->uz_namecnt > UMA_MAX_DUP)
zone->uz_namecnt = UMA_MAX_DUP;
zone->uz_ctlname = malloc(strlen(zone->uz_name) +
sizeof(__XSTRING(UMA_MAX_DUP)) + 1 , M_UMA, M_WAITOK);
sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name,
zone->uz_namecnt);
} else
zone->uz_ctlname = strdup(zone->uz_name, M_UMA);
for (c = zone->uz_ctlname; *c != '\0'; c++)
if (strchr("./\\ -", *c) != NULL)
*c = '_';
/*
* Basic parameters at the root.
*/
zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma),
OID_AUTO, zone->uz_ctlname, CTLFLAG_RD, NULL, "");
oid = zone->uz_oid;
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"flags", CTLFLAG_RD, &zone->uz_flags, 0,
"Allocator configuration flags");
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0,
"Desired per-cpu cache size");
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0,
"Maximum allowed per-cpu cache size");
/*
* keg if present.
*/
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
"keg", CTLFLAG_RD, NULL, "");
keg = zone->uz_keg;
if ((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0) {
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"name", CTLFLAG_RD, keg->uk_name, "Keg name");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"rsize", CTLFLAG_RD, &keg->uk_rsize, 0,
"Real object size with alignment");
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"ppera", CTLFLAG_RD, &keg->uk_ppera, 0,
"pages per-slab allocation");
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"ipers", CTLFLAG_RD, &keg->uk_ipers, 0,
"items available per-slab");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"align", CTLFLAG_RD, &keg->uk_align, 0,
"item alignment mask");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"pages", CTLFLAG_RD, &keg->uk_pages, 0,
"Total pages currently allocated from VM");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"free", CTLFLAG_RD, &keg->uk_free, 0,
"items free in the slab layer");
} else
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"name", CTLFLAG_RD, nokeg, "Keg name");
/*
* Information about zone limits.
*/
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
"limit", CTLFLAG_RD, NULL, "");
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"items", CTLFLAG_RD, &zone->uz_items, 0,
"current number of cached items");
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"max_items", CTLFLAG_RD, &zone->uz_max_items, 0,
"Maximum number of cached items");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0,
"Number of threads sleeping at limit");
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0,
"Total zone limit sleeps");
/*
* Per-domain information.
*/
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
domains = vm_ndomains;
else
domains = 1;
domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
for (i = 0; i < domains; i++) {
zdom = &zone->uz_domain[i];
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, "");
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"nitems", CTLFLAG_RD, &zdom->uzd_nitems,
"number of items in this domain");
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"imax", CTLFLAG_RD, &zdom->uzd_imax,
"maximum item count in this period");
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"imin", CTLFLAG_RD, &zdom->uzd_imin,
"minimum item count in this period");
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"wss", CTLFLAG_RD, &zdom->uzd_wss,
"Working set size");
}
/*
* General statistics.
*/
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
"stats", CTLFLAG_RD, NULL, "");
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"current", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE,
zone, 1, sysctl_handle_uma_zone_cur, "I",
"Current number of allocated items");
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"allocs", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
zone, 0, sysctl_handle_uma_zone_allocs, "QU",
"Total allocation calls");
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"frees", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
zone, 0, sysctl_handle_uma_zone_frees, "QU",
"Total free calls");
SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"fails", CTLFLAG_RD, &zone->uz_fails,
"Number of allocation failures");
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"xdomain", CTLFLAG_RD, &zone->uz_xdomain, 0,
"Free calls from the wrong domain");
}
struct uma_zone_count {
const char *name;
int count;
};
static void
zone_count(uma_zone_t zone, void *arg)
{
struct uma_zone_count *cnt;
cnt = arg;
if (strcmp(zone->uz_name, cnt->name) == 0)
cnt->count++;
}
/*
* Zone header ctor. This initializes all fields, locks, etc.
*
@ -1835,6 +2001,7 @@ zone_alloc_counters(uma_zone_t zone)
static int
zone_ctor(void *mem, int size, void *udata, int flags)
{
struct uma_zone_count cnt;
struct uma_zctor_args *arg = udata;
uma_zone_t zone = mem;
uma_zone_t z;
@ -1849,9 +2016,9 @@ zone_ctor(void *mem, int size, void *udata, int flags)
zone->uz_fini = NULL;
zone->uz_sleeps = 0;
zone->uz_xdomain = 0;
zone->uz_count = 0;
zone->uz_count_min = 0;
zone->uz_count_max = BUCKET_MAX;
zone->uz_bucket_size = 0;
zone->uz_bucket_size_min = 0;
zone->uz_bucket_size_max = BUCKET_MAX;
zone->uz_flags = 0;
zone->uz_warning = NULL;
/* The domain structures follow the cpu structures. */
@ -1859,13 +2026,11 @@ zone_ctor(void *mem, int size, void *udata, int flags)
zone->uz_bkt_max = ULONG_MAX;
timevalclear(&zone->uz_ratecheck);
if (__predict_true(booted == BOOT_RUNNING))
zone_alloc_counters(zone);
else {
zone->uz_allocs = EARLY_COUNTER;
zone->uz_frees = EARLY_COUNTER;
zone->uz_fails = EARLY_COUNTER;
}
/* Count the number of duplicate names. */
cnt.name = arg->name;
cnt.count = 0;
zone_foreach(zone_count, &cnt);
zone->uz_namecnt = cnt.count;
for (i = 0; i < vm_ndomains; i++)
TAILQ_INIT(&zone->uz_domain[i].uzd_buckets);
@ -1903,6 +2068,8 @@ zone_ctor(void *mem, int size, void *udata, int flags)
keg = arg->keg;
if (arg->flags & UMA_ZONE_SECONDARY) {
KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
("Secondary zone requested UMA_ZFLAG_INTERNAL"));
KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
zone->uz_init = arg->uminit;
zone->uz_fini = arg->fini;
@ -1939,35 +2106,36 @@ zone_ctor(void *mem, int size, void *udata, int flags)
return (error);
}
/* Inherit properties from the keg. */
zone->uz_keg = keg;
zone->uz_size = keg->uk_size;
zone->uz_flags |= (keg->uk_flags &
(UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
/*
* Some internal zones don't have room allocated for the per cpu
* caches. If we're internal, bail out here.
*/
if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
("Secondary zone requested UMA_ZFLAG_INTERNAL"));
return (0);
out:
if (__predict_true(booted == BOOT_RUNNING)) {
zone_alloc_counters(zone, NULL);
zone_alloc_sysctl(zone, NULL);
} else {
zone->uz_allocs = EARLY_COUNTER;
zone->uz_frees = EARLY_COUNTER;
zone->uz_fails = EARLY_COUNTER;
}
out:
KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) !=
(UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET),
("Invalid zone flag combination"));
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) {
zone->uz_count = BUCKET_MAX;
} else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) {
zone->uz_count = BUCKET_MIN;
zone->uz_count_max = BUCKET_MIN;
} else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
zone->uz_count = 0;
if (arg->flags & UMA_ZFLAG_INTERNAL)
zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)
zone->uz_bucket_size = BUCKET_MAX;
else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0)
zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN;
else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
zone->uz_bucket_size = 0;
else
zone->uz_count = bucket_select(zone->uz_size);
zone->uz_count_min = zone->uz_count;
zone->uz_bucket_size = bucket_select(zone->uz_size);
zone->uz_bucket_size_min = zone->uz_bucket_size;
return (0);
}
@ -2013,6 +2181,8 @@ zone_dtor(void *arg, int size, void *udata)
zone = (uma_zone_t)arg;
sysctl_remove_oid(zone->uz_oid, 1, 1);
if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
cache_drain(zone);
@ -2039,6 +2209,7 @@ zone_dtor(void *arg, int size, void *udata)
counter_u64_free(zone->uz_allocs);
counter_u64_free(zone->uz_frees);
counter_u64_free(zone->uz_fails);
free(zone->uz_ctlname, M_UMA);
if (zone->uz_lockptr == &zone->uz_lock)
ZONE_LOCK_FINI(zone);
}
@ -2054,7 +2225,7 @@ zone_dtor(void *arg, int size, void *udata)
* Nothing
*/
static void
zone_foreach(void (*zfunc)(uma_zone_t))
zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg)
{
uma_keg_t keg;
uma_zone_t zone;
@ -2068,10 +2239,10 @@ zone_foreach(void (*zfunc)(uma_zone_t))
rw_rlock(&uma_rwlock);
LIST_FOREACH(keg, &uma_kegs, uk_link) {
LIST_FOREACH(zone, &keg->uk_zones, uz_link)
zfunc(zone);
zfunc(zone, arg);
}
LIST_FOREACH(zone, &uma_cachezones, uz_link)
zfunc(zone);
zfunc(zone, arg);
if (__predict_true(booted == BOOT_RUNNING))
rw_runlock(&uma_rwlock);
}
@ -2249,7 +2420,8 @@ uma_startup3(void)
uma_dbg_cnt = counter_u64_alloc(M_WAITOK);
uma_skip_cnt = counter_u64_alloc(M_WAITOK);
#endif
zone_foreach(zone_alloc_counters);
zone_foreach(zone_alloc_counters, NULL);
zone_foreach(zone_alloc_sysctl, NULL);
callout_init(&uma_callout, 1);
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
booted = BOOT_RUNNING;
@ -2665,7 +2837,7 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
critical_enter();
/* Short-circuit for zones without buckets and low memory. */
if (zone->uz_count == 0 || bucketdisable)
if (zone->uz_bucket_size == 0 || bucketdisable)
return (false);
cpu = curcpu;
@ -2702,8 +2874,8 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
* We bump the uz count when the cache size is insufficient to
* handle the working set.
*/
if (lockfail && zone->uz_count < zone->uz_count_max)
zone->uz_count++;
if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
zone->uz_bucket_size++;
/*
* Fill a bucket and attempt to use it as the alloc bucket.
@ -3005,11 +3177,11 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags)
if (zone->uz_max_items > 0) {
if (zone->uz_items >= zone->uz_max_items)
return (false);
maxbucket = MIN(zone->uz_count,
maxbucket = MIN(zone->uz_bucket_size,
zone->uz_max_items - zone->uz_items);
zone->uz_items += maxbucket;
} else
maxbucket = zone->uz_count;
maxbucket = zone->uz_bucket_size;
ZONE_UNLOCK(zone);
/* Don't wait for buckets, preserve caller's NOVM setting. */
@ -3285,8 +3457,8 @@ zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
if (ZONE_TRYLOCK(zone) == 0) {
/* Record contention to size the buckets. */
ZONE_LOCK(zone);
if (zone->uz_count < zone->uz_count_max)
zone->uz_count++;
if (zone->uz_bucket_size < zone->uz_bucket_size_max)
zone->uz_bucket_size++;
}
CTR3(KTR_UMA,
@ -3323,7 +3495,7 @@ cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item,
CRITICAL_ASSERT(curthread);
if (zone->uz_count == 0 || bucketdisable)
if (zone->uz_bucket_size == 0 || bucketdisable)
return false;
cpu = curcpu;
@ -3523,9 +3695,9 @@ uma_zone_set_max(uma_zone_t zone, int nitems)
ZONE_LOCK(zone);
ubz = bucket_zone_max(zone, nitems);
count = ubz != NULL ? ubz->ubz_entries : 0;
zone->uz_count_max = zone->uz_count = count;
if (zone->uz_count_min > zone->uz_count_max)
zone->uz_count_min = zone->uz_count_max;
zone->uz_bucket_size_max = zone->uz_bucket_size = count;
if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
zone->uz_bucket_size_min = zone->uz_bucket_size_max;
zone->uz_max_items = nitems;
ZONE_UNLOCK(zone);
@ -3549,12 +3721,12 @@ uma_zone_set_maxcache(uma_zone_t zone, int nitems)
bpcpu++;
#endif
nitems -= ubz->ubz_entries * bpcpu * mp_ncpus;
zone->uz_count_max = ubz->ubz_entries;
zone->uz_bucket_size_max = ubz->ubz_entries;
} else {
zone->uz_count_max = zone->uz_count = 0;
zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
}
if (zone->uz_count_min > zone->uz_count_max)
zone->uz_count_min = zone->uz_count_max;
if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
zone->uz_bucket_size_min = zone->uz_bucket_size_max;
zone->uz_bkt_max = nitems;
ZONE_UNLOCK(zone);
}
@ -3602,20 +3774,71 @@ uma_zone_get_cur(uma_zone_t zone)
ZONE_LOCK(zone);
nitems = counter_u64_fetch(zone->uz_allocs) -
counter_u64_fetch(zone->uz_frees);
CPU_FOREACH(i) {
/*
* See the comment in uma_vm_zone_stats() regarding the
* safety of accessing the per-cpu caches. With the zone lock
* held, it is safe, but can potentially result in stale data.
*/
nitems += zone->uz_cpu[i].uc_allocs -
zone->uz_cpu[i].uc_frees;
if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
CPU_FOREACH(i) {
/*
* See the comment in uma_vm_zone_stats() regarding
* the safety of accessing the per-cpu caches. With
* the zone lock held, it is safe, but can potentially
* result in stale data.
*/
nitems += zone->uz_cpu[i].uc_allocs -
zone->uz_cpu[i].uc_frees;
}
}
ZONE_UNLOCK(zone);
return (nitems < 0 ? 0 : nitems);
}
static uint64_t
uma_zone_get_allocs(uma_zone_t zone)
{
uint64_t nitems;
u_int i;
ZONE_LOCK(zone);
nitems = counter_u64_fetch(zone->uz_allocs);
if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
CPU_FOREACH(i) {
/*
* See the comment in uma_vm_zone_stats() regarding
* the safety of accessing the per-cpu caches. With
* the zone lock held, it is safe, but can potentially
* result in stale data.
*/
nitems += zone->uz_cpu[i].uc_allocs;
}
}
ZONE_UNLOCK(zone);
return (nitems);
}
static uint64_t
uma_zone_get_frees(uma_zone_t zone)
{
uint64_t nitems;
u_int i;
ZONE_LOCK(zone);
nitems = counter_u64_fetch(zone->uz_frees);
if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
CPU_FOREACH(i) {
/*
* See the comment in uma_vm_zone_stats() regarding
* the safety of accessing the per-cpu caches. With
* the zone lock held, it is safe, but can potentially
* result in stale data.
*/
nitems += zone->uz_cpu[i].uc_frees;
}
}
ZONE_UNLOCK(zone);
return (nitems);
}
/* See uma.h */
void
uma_zone_set_init(uma_zone_t zone, uma_init uminit)
@ -3800,14 +4023,14 @@ uma_reclaim(int req)
switch (req) {
case UMA_RECLAIM_TRIM:
zone_foreach(zone_trim);
zone_foreach(zone_trim, NULL);
break;
case UMA_RECLAIM_DRAIN:
case UMA_RECLAIM_DRAIN_CPU:
zone_foreach(zone_drain);
zone_foreach(zone_drain, NULL);
if (req == UMA_RECLAIM_DRAIN_CPU) {
pcpu_cache_drain_safe(NULL);
zone_foreach(zone_drain);
zone_foreach(zone_drain, NULL);
}
break;
default:
@ -3819,7 +4042,7 @@ uma_reclaim(int req)
* we visit again so that we can free pages that are empty once other
* zones are drained. We have to do the same for buckets.
*/
zone_drain(slabzone);
zone_drain(slabzone, NULL);
bucket_zone_drain();
sx_xunlock(&uma_reclaim_lock);
}
@ -3859,14 +4082,14 @@ uma_zone_reclaim(uma_zone_t zone, int req)
switch (req) {
case UMA_RECLAIM_TRIM:
zone_trim(zone);
zone_trim(zone, NULL);
break;
case UMA_RECLAIM_DRAIN:
zone_drain(zone);
zone_drain(zone, NULL);
break;
case UMA_RECLAIM_DRAIN_CPU:
pcpu_cache_drain_safe(zone);
zone_drain(zone);
zone_drain(zone, NULL);
break;
default:
panic("unhandled reclamation request %d", req);
@ -3977,76 +4200,6 @@ uma_avail(void)
return (uma_kmem_limit - uma_size());
}
void
uma_print_stats(void)
{
zone_foreach(uma_print_zone);
}
static void
slab_print(uma_slab_t slab)
{
printf("slab: keg %p, data %p, freecount %d\n",
slab->us_keg, slab->us_data, slab->us_freecount);
}
static void
cache_print(uma_cache_t cache)
{
printf("alloc: %p(%d), free: %p(%d), cross: %p(%d)j\n",
cache->uc_allocbucket,
cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
cache->uc_freebucket,
cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0,
cache->uc_crossbucket,
cache->uc_crossbucket?cache->uc_crossbucket->ub_cnt:0);
}
static void
uma_print_keg(uma_keg_t keg)
{
uma_domain_t dom;
uma_slab_t slab;
int i;
printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
"out %d free %d\n",
keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
keg->uk_ipers, keg->uk_ppera,
(keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
keg->uk_free);
for (i = 0; i < vm_ndomains; i++) {
dom = &keg->uk_domain[i];
printf("Part slabs:\n");
LIST_FOREACH(slab, &dom->ud_part_slab, us_link)
slab_print(slab);
printf("Free slabs:\n");
LIST_FOREACH(slab, &dom->ud_free_slab, us_link)
slab_print(slab);
printf("Full slabs:\n");
LIST_FOREACH(slab, &dom->ud_full_slab, us_link)
slab_print(slab);
}
}
void
uma_print_zone(uma_zone_t zone)
{
uma_cache_t cache;
int i;
printf("zone: %s(%p) size %d maxitems %ju flags %#x\n",
zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items,
zone->uz_flags);
if (zone->uz_lockptr != &zone->uz_lock)
uma_print_keg(zone->uz_keg);
CPU_FOREACH(i) {
cache = &zone->uz_cpu[i];
printf("CPU %d Cache:\n", i);
cache_print(cache);
}
}
#ifdef DDB
/*
* Generate statistics across both the zone and its per-cpu cache's. Return
@ -4274,13 +4427,41 @@ sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
int
sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
{
uma_zone_t zone = *(uma_zone_t *)arg1;
uma_zone_t zone;
int cur;
/*
* Some callers want to add sysctls for global zones that
* may not yet exist so they pass a pointer to a pointer.
*/
if (arg2 == 0)
zone = *(uma_zone_t *)arg1;
else
zone = arg1;
cur = uma_zone_get_cur(zone);
return (sysctl_handle_int(oidp, &cur, 0, req));
}
static int
sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS)
{
uma_zone_t zone = arg1;
uint64_t cur;
cur = uma_zone_get_allocs(zone);
return (sysctl_handle_64(oidp, &cur, 0, req));
}
static int
sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS)
{
uma_zone_t zone = arg1;
uint64_t cur;
cur = uma_zone_get_frees(zone);
return (sysctl_handle_64(oidp, &cur, 0, req));
}
#ifdef INVARIANTS
static uma_slab_t
uma_dbg_getslab(uma_zone_t zone, void *item)
@ -4500,7 +4681,8 @@ DB_SHOW_COMMAND(uma, db_show_uma)
db_printf(fmt_entry, cur_zone->uz_name,
(uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree,
(uintmax_t)allocs, (uintmax_t)sleeps,
(unsigned)cur_zone->uz_count, (intmax_t)size, xdomain);
(unsigned)cur_zone->uz_bucket_size, (intmax_t)size,
xdomain);
if (db_pager_quit)
return;
@ -4525,7 +4707,7 @@ DB_SHOW_COMMAND(umacache, db_show_umacache)
db_printf("%18s %8ju %8jd %8ld %12ju %8u\n",
z->uz_name, (uintmax_t)z->uz_size,
(intmax_t)(allocs - frees), cachefree,
(uintmax_t)allocs, z->uz_count);
(uintmax_t)allocs, z->uz_bucket_size);
if (db_pager_quit)
return;
}

11
sys/vm/uma_int.h
View File

@ -338,8 +338,8 @@ struct uma_zone {
uint64_t uz_items; /* Total items count */
uint64_t uz_max_items; /* Maximum number of items to alloc */
uint32_t uz_sleepers; /* Number of sleepers on memory */
uint16_t uz_count; /* Amount of items in full bucket */
uint16_t uz_count_max; /* Maximum amount of items there */
uint16_t uz_bucket_size; /* Number of items in full bucket */
uint16_t uz_bucket_size_max; /* Maximum number of bucket items */
/* Offset 64, used in bucket replenish. */
uma_import uz_import; /* Import new memory to cache. */
@ -364,14 +364,17 @@ struct uma_zone {
const char *uz_warning; /* Warning to print on failure */
struct timeval uz_ratecheck; /* Warnings rate-limiting */
struct task uz_maxaction; /* Task to run when at limit */
uint16_t uz_count_min; /* Minimal amount of items in bucket */
uint16_t uz_bucket_size_min; /* Min number of items in bucket */
/* Offset 256, stats. */
/* Offset 256+, stats and misc. */
counter_u64_t uz_allocs; /* Total number of allocations */
counter_u64_t uz_frees; /* Total number of frees */
counter_u64_t uz_fails; /* Total number of alloc failures */
uint64_t uz_sleeps; /* Total number of alloc sleeps */
uint64_t uz_xdomain; /* Total number of cross-domain frees */
char *uz_ctlname; /* sysctl safe name string. */
struct sysctl_oid *uz_oid; /* sysctl oid pointer. */
int uz_namecnt; /* duplicate name count. */
/*
* This HAS to be the last item because we adjust the zone size