Implement a sysctl tree for uma zones to assist in debugging and provide
more statistcs than are exported via the ABI stable vmstat interface. Rename uz_count to uz_bucket_size because even I was confused by the name after returning to the source years later. Reviewed by: rlibby Differential Revision: https://reviews.freebsd.org/D22554
This commit is contained in:
parent
0a81b4395e
commit
20a4e15451
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=355149
@ -119,6 +119,7 @@ static uma_zone_t hashzone;
|
||||
int uma_align_cache = 64 - 1;
|
||||
|
||||
static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
|
||||
static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc");
|
||||
|
||||
/*
|
||||
* Are we allowed to allocate buckets?
|
||||
@ -258,8 +259,8 @@ static void zone_dtor(void *, int, void *);
|
||||
static int zero_init(void *, int, int);
|
||||
static void keg_small_init(uma_keg_t keg);
|
||||
static void keg_large_init(uma_keg_t keg);
|
||||
static void zone_foreach(void (*zfunc)(uma_zone_t));
|
||||
static void zone_timeout(uma_zone_t zone);
|
||||
static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *);
|
||||
static void zone_timeout(uma_zone_t zone, void *);
|
||||
static int hash_alloc(struct uma_hash *, u_int);
|
||||
static int hash_expand(struct uma_hash *, struct uma_hash *);
|
||||
static void hash_free(struct uma_hash *hash);
|
||||
@ -285,10 +286,10 @@ static void uma_zero_item(void *, uma_zone_t);
|
||||
static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int);
|
||||
static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int);
|
||||
|
||||
void uma_print_zone(uma_zone_t);
|
||||
void uma_print_stats(void);
|
||||
static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
|
||||
static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
|
||||
static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS);
|
||||
static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS);
|
||||
|
||||
#ifdef INVARIANTS
|
||||
static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
|
||||
@ -296,6 +297,7 @@ static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
|
||||
static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
|
||||
static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
|
||||
|
||||
SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW, 0, "Universal Memory Allocator");
|
||||
static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0,
|
||||
"Memory allocation debugging");
|
||||
|
||||
@ -458,7 +460,7 @@ bucket_alloc(uma_zone_t zone, void *udata, int flags)
|
||||
}
|
||||
if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
|
||||
flags |= M_NOVM;
|
||||
ubz = bucket_zone_lookup(zone->uz_count);
|
||||
ubz = bucket_zone_lookup(zone->uz_bucket_size);
|
||||
if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
|
||||
ubz++;
|
||||
bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
|
||||
@ -575,7 +577,7 @@ static void
|
||||
uma_timeout(void *unused)
|
||||
{
|
||||
bucket_enable();
|
||||
zone_foreach(zone_timeout);
|
||||
zone_foreach(zone_timeout, NULL);
|
||||
|
||||
/* Reschedule this event */
|
||||
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
|
||||
@ -605,7 +607,7 @@ zone_domain_update_wss(uma_zone_domain_t zdom)
|
||||
* Returns nothing.
|
||||
*/
|
||||
static void
|
||||
zone_timeout(uma_zone_t zone)
|
||||
zone_timeout(uma_zone_t zone, void *unused)
|
||||
{
|
||||
uma_keg_t keg;
|
||||
u_int slabs;
|
||||
@ -845,19 +847,20 @@ cache_drain(uma_zone_t zone)
|
||||
}
|
||||
|
||||
static void
|
||||
cache_shrink(uma_zone_t zone)
|
||||
cache_shrink(uma_zone_t zone, void *unused)
|
||||
{
|
||||
|
||||
if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
|
||||
return;
|
||||
|
||||
ZONE_LOCK(zone);
|
||||
zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
|
||||
zone->uz_bucket_size =
|
||||
(zone->uz_bucket_size_min + zone->uz_bucket_size) / 2;
|
||||
ZONE_UNLOCK(zone);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_drain_safe_cpu(uma_zone_t zone)
|
||||
cache_drain_safe_cpu(uma_zone_t zone, void *unused)
|
||||
{
|
||||
uma_cache_t cache;
|
||||
uma_bucket_t b1, b2, b3;
|
||||
@ -920,9 +923,9 @@ pcpu_cache_drain_safe(uma_zone_t zone)
|
||||
* Polite bucket sizes shrinking was not enouth, shrink aggressively.
|
||||
*/
|
||||
if (zone)
|
||||
cache_shrink(zone);
|
||||
cache_shrink(zone, NULL);
|
||||
else
|
||||
zone_foreach(cache_shrink);
|
||||
zone_foreach(cache_shrink, NULL);
|
||||
|
||||
CPU_FOREACH(cpu) {
|
||||
thread_lock(curthread);
|
||||
@ -930,9 +933,9 @@ pcpu_cache_drain_safe(uma_zone_t zone)
|
||||
thread_unlock(curthread);
|
||||
|
||||
if (zone)
|
||||
cache_drain_safe_cpu(zone);
|
||||
cache_drain_safe_cpu(zone, NULL);
|
||||
else
|
||||
zone_foreach(cache_drain_safe_cpu);
|
||||
zone_foreach(cache_drain_safe_cpu, NULL);
|
||||
}
|
||||
thread_lock(curthread);
|
||||
sched_unbind(curthread);
|
||||
@ -991,8 +994,8 @@ bucket_cache_reclaim(uma_zone_t zone, bool drain)
|
||||
* Shrink the zone bucket size to ensure that the per-CPU caches
|
||||
* don't grow too large.
|
||||
*/
|
||||
if (zone->uz_count > zone->uz_count_min)
|
||||
zone->uz_count--;
|
||||
if (zone->uz_bucket_size > zone->uz_bucket_size_min)
|
||||
zone->uz_bucket_size--;
|
||||
}
|
||||
|
||||
static void
|
||||
@ -1121,14 +1124,14 @@ zone_reclaim(uma_zone_t zone, int waitok, bool drain)
|
||||
}
|
||||
|
||||
static void
|
||||
zone_drain(uma_zone_t zone)
|
||||
zone_drain(uma_zone_t zone, void *unused)
|
||||
{
|
||||
|
||||
zone_reclaim(zone, M_NOWAIT, true);
|
||||
}
|
||||
|
||||
static void
|
||||
zone_trim(uma_zone_t zone)
|
||||
zone_trim(uma_zone_t zone, void *unused)
|
||||
{
|
||||
|
||||
zone_reclaim(zone, M_NOWAIT, false);
|
||||
@ -1818,7 +1821,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
|
||||
}
|
||||
|
||||
static void
|
||||
zone_alloc_counters(uma_zone_t zone)
|
||||
zone_alloc_counters(uma_zone_t zone, void *unused)
|
||||
{
|
||||
|
||||
zone->uz_allocs = counter_u64_alloc(M_WAITOK);
|
||||
@ -1826,6 +1829,169 @@ zone_alloc_counters(uma_zone_t zone)
|
||||
zone->uz_fails = counter_u64_alloc(M_WAITOK);
|
||||
}
|
||||
|
||||
#define UMA_MAX_DUP 999
|
||||
static void
|
||||
zone_alloc_sysctl(uma_zone_t zone, void *unused)
|
||||
{
|
||||
uma_zone_domain_t zdom;
|
||||
uma_keg_t keg;
|
||||
struct sysctl_oid *oid, *domainoid;
|
||||
int domains, i;
|
||||
static const char *nokeg = "cache zone";
|
||||
char *c;
|
||||
|
||||
/*
|
||||
* Make a sysctl safe copy of the zone name by removing
|
||||
* any special characters and handling dups by appending
|
||||
* an index.
|
||||
*/
|
||||
if (zone->uz_namecnt != 0) {
|
||||
if (zone->uz_namecnt > UMA_MAX_DUP)
|
||||
zone->uz_namecnt = UMA_MAX_DUP;
|
||||
zone->uz_ctlname = malloc(strlen(zone->uz_name) +
|
||||
sizeof(__XSTRING(UMA_MAX_DUP)) + 1 , M_UMA, M_WAITOK);
|
||||
sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name,
|
||||
zone->uz_namecnt);
|
||||
} else
|
||||
zone->uz_ctlname = strdup(zone->uz_name, M_UMA);
|
||||
for (c = zone->uz_ctlname; *c != '\0'; c++)
|
||||
if (strchr("./\\ -", *c) != NULL)
|
||||
*c = '_';
|
||||
|
||||
/*
|
||||
* Basic parameters at the root.
|
||||
*/
|
||||
zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma),
|
||||
OID_AUTO, zone->uz_ctlname, CTLFLAG_RD, NULL, "");
|
||||
oid = zone->uz_oid;
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size");
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"flags", CTLFLAG_RD, &zone->uz_flags, 0,
|
||||
"Allocator configuration flags");
|
||||
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0,
|
||||
"Desired per-cpu cache size");
|
||||
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0,
|
||||
"Maximum allowed per-cpu cache size");
|
||||
|
||||
/*
|
||||
* keg if present.
|
||||
*/
|
||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
|
||||
"keg", CTLFLAG_RD, NULL, "");
|
||||
keg = zone->uz_keg;
|
||||
if ((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0) {
|
||||
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"name", CTLFLAG_RD, keg->uk_name, "Keg name");
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"rsize", CTLFLAG_RD, &keg->uk_rsize, 0,
|
||||
"Real object size with alignment");
|
||||
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"ppera", CTLFLAG_RD, &keg->uk_ppera, 0,
|
||||
"pages per-slab allocation");
|
||||
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"ipers", CTLFLAG_RD, &keg->uk_ipers, 0,
|
||||
"items available per-slab");
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"align", CTLFLAG_RD, &keg->uk_align, 0,
|
||||
"item alignment mask");
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"pages", CTLFLAG_RD, &keg->uk_pages, 0,
|
||||
"Total pages currently allocated from VM");
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"free", CTLFLAG_RD, &keg->uk_free, 0,
|
||||
"items free in the slab layer");
|
||||
} else
|
||||
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"name", CTLFLAG_RD, nokeg, "Keg name");
|
||||
|
||||
/*
|
||||
* Information about zone limits.
|
||||
*/
|
||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
|
||||
"limit", CTLFLAG_RD, NULL, "");
|
||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"items", CTLFLAG_RD, &zone->uz_items, 0,
|
||||
"current number of cached items");
|
||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"max_items", CTLFLAG_RD, &zone->uz_max_items, 0,
|
||||
"Maximum number of cached items");
|
||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0,
|
||||
"Number of threads sleeping at limit");
|
||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0,
|
||||
"Total zone limit sleeps");
|
||||
|
||||
/*
|
||||
* Per-domain information.
|
||||
*/
|
||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
|
||||
domains = vm_ndomains;
|
||||
else
|
||||
domains = 1;
|
||||
domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
|
||||
OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
|
||||
for (i = 0; i < domains; i++) {
|
||||
zdom = &zone->uz_domain[i];
|
||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
|
||||
OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, "");
|
||||
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"nitems", CTLFLAG_RD, &zdom->uzd_nitems,
|
||||
"number of items in this domain");
|
||||
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"imax", CTLFLAG_RD, &zdom->uzd_imax,
|
||||
"maximum item count in this period");
|
||||
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"imin", CTLFLAG_RD, &zdom->uzd_imin,
|
||||
"minimum item count in this period");
|
||||
SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"wss", CTLFLAG_RD, &zdom->uzd_wss,
|
||||
"Working set size");
|
||||
}
|
||||
|
||||
/*
|
||||
* General statistics.
|
||||
*/
|
||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
|
||||
"stats", CTLFLAG_RD, NULL, "");
|
||||
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"current", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE,
|
||||
zone, 1, sysctl_handle_uma_zone_cur, "I",
|
||||
"Current number of allocated items");
|
||||
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"allocs", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
|
||||
zone, 0, sysctl_handle_uma_zone_allocs, "QU",
|
||||
"Total allocation calls");
|
||||
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"frees", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
|
||||
zone, 0, sysctl_handle_uma_zone_frees, "QU",
|
||||
"Total free calls");
|
||||
SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"fails", CTLFLAG_RD, &zone->uz_fails,
|
||||
"Number of allocation failures");
|
||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
|
||||
"xdomain", CTLFLAG_RD, &zone->uz_xdomain, 0,
|
||||
"Free calls from the wrong domain");
|
||||
}
|
||||
|
||||
struct uma_zone_count {
|
||||
const char *name;
|
||||
int count;
|
||||
};
|
||||
|
||||
static void
|
||||
zone_count(uma_zone_t zone, void *arg)
|
||||
{
|
||||
struct uma_zone_count *cnt;
|
||||
|
||||
cnt = arg;
|
||||
if (strcmp(zone->uz_name, cnt->name) == 0)
|
||||
cnt->count++;
|
||||
}
|
||||
|
||||
/*
|
||||
* Zone header ctor. This initializes all fields, locks, etc.
|
||||
*
|
||||
@ -1835,6 +2001,7 @@ zone_alloc_counters(uma_zone_t zone)
|
||||
static int
|
||||
zone_ctor(void *mem, int size, void *udata, int flags)
|
||||
{
|
||||
struct uma_zone_count cnt;
|
||||
struct uma_zctor_args *arg = udata;
|
||||
uma_zone_t zone = mem;
|
||||
uma_zone_t z;
|
||||
@ -1849,9 +2016,9 @@ zone_ctor(void *mem, int size, void *udata, int flags)
|
||||
zone->uz_fini = NULL;
|
||||
zone->uz_sleeps = 0;
|
||||
zone->uz_xdomain = 0;
|
||||
zone->uz_count = 0;
|
||||
zone->uz_count_min = 0;
|
||||
zone->uz_count_max = BUCKET_MAX;
|
||||
zone->uz_bucket_size = 0;
|
||||
zone->uz_bucket_size_min = 0;
|
||||
zone->uz_bucket_size_max = BUCKET_MAX;
|
||||
zone->uz_flags = 0;
|
||||
zone->uz_warning = NULL;
|
||||
/* The domain structures follow the cpu structures. */
|
||||
@ -1859,13 +2026,11 @@ zone_ctor(void *mem, int size, void *udata, int flags)
|
||||
zone->uz_bkt_max = ULONG_MAX;
|
||||
timevalclear(&zone->uz_ratecheck);
|
||||
|
||||
if (__predict_true(booted == BOOT_RUNNING))
|
||||
zone_alloc_counters(zone);
|
||||
else {
|
||||
zone->uz_allocs = EARLY_COUNTER;
|
||||
zone->uz_frees = EARLY_COUNTER;
|
||||
zone->uz_fails = EARLY_COUNTER;
|
||||
}
|
||||
/* Count the number of duplicate names. */
|
||||
cnt.name = arg->name;
|
||||
cnt.count = 0;
|
||||
zone_foreach(zone_count, &cnt);
|
||||
zone->uz_namecnt = cnt.count;
|
||||
|
||||
for (i = 0; i < vm_ndomains; i++)
|
||||
TAILQ_INIT(&zone->uz_domain[i].uzd_buckets);
|
||||
@ -1903,6 +2068,8 @@ zone_ctor(void *mem, int size, void *udata, int flags)
|
||||
keg = arg->keg;
|
||||
|
||||
if (arg->flags & UMA_ZONE_SECONDARY) {
|
||||
KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
|
||||
("Secondary zone requested UMA_ZFLAG_INTERNAL"));
|
||||
KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
|
||||
zone->uz_init = arg->uminit;
|
||||
zone->uz_fini = arg->fini;
|
||||
@ -1939,35 +2106,36 @@ zone_ctor(void *mem, int size, void *udata, int flags)
|
||||
return (error);
|
||||
}
|
||||
|
||||
/* Inherit properties from the keg. */
|
||||
zone->uz_keg = keg;
|
||||
zone->uz_size = keg->uk_size;
|
||||
zone->uz_flags |= (keg->uk_flags &
|
||||
(UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
|
||||
|
||||
/*
|
||||
* Some internal zones don't have room allocated for the per cpu
|
||||
* caches. If we're internal, bail out here.
|
||||
*/
|
||||
if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
|
||||
KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
|
||||
("Secondary zone requested UMA_ZFLAG_INTERNAL"));
|
||||
return (0);
|
||||
out:
|
||||
if (__predict_true(booted == BOOT_RUNNING)) {
|
||||
zone_alloc_counters(zone, NULL);
|
||||
zone_alloc_sysctl(zone, NULL);
|
||||
} else {
|
||||
zone->uz_allocs = EARLY_COUNTER;
|
||||
zone->uz_frees = EARLY_COUNTER;
|
||||
zone->uz_fails = EARLY_COUNTER;
|
||||
}
|
||||
|
||||
out:
|
||||
KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) !=
|
||||
(UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET),
|
||||
("Invalid zone flag combination"));
|
||||
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) {
|
||||
zone->uz_count = BUCKET_MAX;
|
||||
} else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) {
|
||||
zone->uz_count = BUCKET_MIN;
|
||||
zone->uz_count_max = BUCKET_MIN;
|
||||
} else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
|
||||
zone->uz_count = 0;
|
||||
if (arg->flags & UMA_ZFLAG_INTERNAL)
|
||||
zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
|
||||
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)
|
||||
zone->uz_bucket_size = BUCKET_MAX;
|
||||
else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0)
|
||||
zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN;
|
||||
else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
|
||||
zone->uz_bucket_size = 0;
|
||||
else
|
||||
zone->uz_count = bucket_select(zone->uz_size);
|
||||
zone->uz_count_min = zone->uz_count;
|
||||
zone->uz_bucket_size = bucket_select(zone->uz_size);
|
||||
zone->uz_bucket_size_min = zone->uz_bucket_size;
|
||||
|
||||
return (0);
|
||||
}
|
||||
@ -2013,6 +2181,8 @@ zone_dtor(void *arg, int size, void *udata)
|
||||
|
||||
zone = (uma_zone_t)arg;
|
||||
|
||||
sysctl_remove_oid(zone->uz_oid, 1, 1);
|
||||
|
||||
if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
|
||||
cache_drain(zone);
|
||||
|
||||
@ -2039,6 +2209,7 @@ zone_dtor(void *arg, int size, void *udata)
|
||||
counter_u64_free(zone->uz_allocs);
|
||||
counter_u64_free(zone->uz_frees);
|
||||
counter_u64_free(zone->uz_fails);
|
||||
free(zone->uz_ctlname, M_UMA);
|
||||
if (zone->uz_lockptr == &zone->uz_lock)
|
||||
ZONE_LOCK_FINI(zone);
|
||||
}
|
||||
@ -2054,7 +2225,7 @@ zone_dtor(void *arg, int size, void *udata)
|
||||
* Nothing
|
||||
*/
|
||||
static void
|
||||
zone_foreach(void (*zfunc)(uma_zone_t))
|
||||
zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg)
|
||||
{
|
||||
uma_keg_t keg;
|
||||
uma_zone_t zone;
|
||||
@ -2068,10 +2239,10 @@ zone_foreach(void (*zfunc)(uma_zone_t))
|
||||
rw_rlock(&uma_rwlock);
|
||||
LIST_FOREACH(keg, &uma_kegs, uk_link) {
|
||||
LIST_FOREACH(zone, &keg->uk_zones, uz_link)
|
||||
zfunc(zone);
|
||||
zfunc(zone, arg);
|
||||
}
|
||||
LIST_FOREACH(zone, &uma_cachezones, uz_link)
|
||||
zfunc(zone);
|
||||
zfunc(zone, arg);
|
||||
if (__predict_true(booted == BOOT_RUNNING))
|
||||
rw_runlock(&uma_rwlock);
|
||||
}
|
||||
@ -2249,7 +2420,8 @@ uma_startup3(void)
|
||||
uma_dbg_cnt = counter_u64_alloc(M_WAITOK);
|
||||
uma_skip_cnt = counter_u64_alloc(M_WAITOK);
|
||||
#endif
|
||||
zone_foreach(zone_alloc_counters);
|
||||
zone_foreach(zone_alloc_counters, NULL);
|
||||
zone_foreach(zone_alloc_sysctl, NULL);
|
||||
callout_init(&uma_callout, 1);
|
||||
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
|
||||
booted = BOOT_RUNNING;
|
||||
@ -2665,7 +2837,7 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
|
||||
|
||||
critical_enter();
|
||||
/* Short-circuit for zones without buckets and low memory. */
|
||||
if (zone->uz_count == 0 || bucketdisable)
|
||||
if (zone->uz_bucket_size == 0 || bucketdisable)
|
||||
return (false);
|
||||
|
||||
cpu = curcpu;
|
||||
@ -2702,8 +2874,8 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
|
||||
* We bump the uz count when the cache size is insufficient to
|
||||
* handle the working set.
|
||||
*/
|
||||
if (lockfail && zone->uz_count < zone->uz_count_max)
|
||||
zone->uz_count++;
|
||||
if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
|
||||
zone->uz_bucket_size++;
|
||||
|
||||
/*
|
||||
* Fill a bucket and attempt to use it as the alloc bucket.
|
||||
@ -3005,11 +3177,11 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags)
|
||||
if (zone->uz_max_items > 0) {
|
||||
if (zone->uz_items >= zone->uz_max_items)
|
||||
return (false);
|
||||
maxbucket = MIN(zone->uz_count,
|
||||
maxbucket = MIN(zone->uz_bucket_size,
|
||||
zone->uz_max_items - zone->uz_items);
|
||||
zone->uz_items += maxbucket;
|
||||
} else
|
||||
maxbucket = zone->uz_count;
|
||||
maxbucket = zone->uz_bucket_size;
|
||||
ZONE_UNLOCK(zone);
|
||||
|
||||
/* Don't wait for buckets, preserve caller's NOVM setting. */
|
||||
@ -3285,8 +3457,8 @@ zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
|
||||
if (ZONE_TRYLOCK(zone) == 0) {
|
||||
/* Record contention to size the buckets. */
|
||||
ZONE_LOCK(zone);
|
||||
if (zone->uz_count < zone->uz_count_max)
|
||||
zone->uz_count++;
|
||||
if (zone->uz_bucket_size < zone->uz_bucket_size_max)
|
||||
zone->uz_bucket_size++;
|
||||
}
|
||||
|
||||
CTR3(KTR_UMA,
|
||||
@ -3323,7 +3495,7 @@ cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item,
|
||||
|
||||
CRITICAL_ASSERT(curthread);
|
||||
|
||||
if (zone->uz_count == 0 || bucketdisable)
|
||||
if (zone->uz_bucket_size == 0 || bucketdisable)
|
||||
return false;
|
||||
|
||||
cpu = curcpu;
|
||||
@ -3523,9 +3695,9 @@ uma_zone_set_max(uma_zone_t zone, int nitems)
|
||||
ZONE_LOCK(zone);
|
||||
ubz = bucket_zone_max(zone, nitems);
|
||||
count = ubz != NULL ? ubz->ubz_entries : 0;
|
||||
zone->uz_count_max = zone->uz_count = count;
|
||||
if (zone->uz_count_min > zone->uz_count_max)
|
||||
zone->uz_count_min = zone->uz_count_max;
|
||||
zone->uz_bucket_size_max = zone->uz_bucket_size = count;
|
||||
if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
|
||||
zone->uz_bucket_size_min = zone->uz_bucket_size_max;
|
||||
zone->uz_max_items = nitems;
|
||||
ZONE_UNLOCK(zone);
|
||||
|
||||
@ -3549,12 +3721,12 @@ uma_zone_set_maxcache(uma_zone_t zone, int nitems)
|
||||
bpcpu++;
|
||||
#endif
|
||||
nitems -= ubz->ubz_entries * bpcpu * mp_ncpus;
|
||||
zone->uz_count_max = ubz->ubz_entries;
|
||||
zone->uz_bucket_size_max = ubz->ubz_entries;
|
||||
} else {
|
||||
zone->uz_count_max = zone->uz_count = 0;
|
||||
zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
|
||||
}
|
||||
if (zone->uz_count_min > zone->uz_count_max)
|
||||
zone->uz_count_min = zone->uz_count_max;
|
||||
if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
|
||||
zone->uz_bucket_size_min = zone->uz_bucket_size_max;
|
||||
zone->uz_bkt_max = nitems;
|
||||
ZONE_UNLOCK(zone);
|
||||
}
|
||||
@ -3602,20 +3774,71 @@ uma_zone_get_cur(uma_zone_t zone)
|
||||
ZONE_LOCK(zone);
|
||||
nitems = counter_u64_fetch(zone->uz_allocs) -
|
||||
counter_u64_fetch(zone->uz_frees);
|
||||
CPU_FOREACH(i) {
|
||||
/*
|
||||
* See the comment in uma_vm_zone_stats() regarding the
|
||||
* safety of accessing the per-cpu caches. With the zone lock
|
||||
* held, it is safe, but can potentially result in stale data.
|
||||
*/
|
||||
nitems += zone->uz_cpu[i].uc_allocs -
|
||||
zone->uz_cpu[i].uc_frees;
|
||||
if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
|
||||
CPU_FOREACH(i) {
|
||||
/*
|
||||
* See the comment in uma_vm_zone_stats() regarding
|
||||
* the safety of accessing the per-cpu caches. With
|
||||
* the zone lock held, it is safe, but can potentially
|
||||
* result in stale data.
|
||||
*/
|
||||
nitems += zone->uz_cpu[i].uc_allocs -
|
||||
zone->uz_cpu[i].uc_frees;
|
||||
}
|
||||
}
|
||||
ZONE_UNLOCK(zone);
|
||||
|
||||
return (nitems < 0 ? 0 : nitems);
|
||||
}
|
||||
|
||||
static uint64_t
|
||||
uma_zone_get_allocs(uma_zone_t zone)
|
||||
{
|
||||
uint64_t nitems;
|
||||
u_int i;
|
||||
|
||||
ZONE_LOCK(zone);
|
||||
nitems = counter_u64_fetch(zone->uz_allocs);
|
||||
if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
|
||||
CPU_FOREACH(i) {
|
||||
/*
|
||||
* See the comment in uma_vm_zone_stats() regarding
|
||||
* the safety of accessing the per-cpu caches. With
|
||||
* the zone lock held, it is safe, but can potentially
|
||||
* result in stale data.
|
||||
*/
|
||||
nitems += zone->uz_cpu[i].uc_allocs;
|
||||
}
|
||||
}
|
||||
ZONE_UNLOCK(zone);
|
||||
|
||||
return (nitems);
|
||||
}
|
||||
|
||||
static uint64_t
|
||||
uma_zone_get_frees(uma_zone_t zone)
|
||||
{
|
||||
uint64_t nitems;
|
||||
u_int i;
|
||||
|
||||
ZONE_LOCK(zone);
|
||||
nitems = counter_u64_fetch(zone->uz_frees);
|
||||
if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
|
||||
CPU_FOREACH(i) {
|
||||
/*
|
||||
* See the comment in uma_vm_zone_stats() regarding
|
||||
* the safety of accessing the per-cpu caches. With
|
||||
* the zone lock held, it is safe, but can potentially
|
||||
* result in stale data.
|
||||
*/
|
||||
nitems += zone->uz_cpu[i].uc_frees;
|
||||
}
|
||||
}
|
||||
ZONE_UNLOCK(zone);
|
||||
|
||||
return (nitems);
|
||||
}
|
||||
|
||||
/* See uma.h */
|
||||
void
|
||||
uma_zone_set_init(uma_zone_t zone, uma_init uminit)
|
||||
@ -3800,14 +4023,14 @@ uma_reclaim(int req)
|
||||
|
||||
switch (req) {
|
||||
case UMA_RECLAIM_TRIM:
|
||||
zone_foreach(zone_trim);
|
||||
zone_foreach(zone_trim, NULL);
|
||||
break;
|
||||
case UMA_RECLAIM_DRAIN:
|
||||
case UMA_RECLAIM_DRAIN_CPU:
|
||||
zone_foreach(zone_drain);
|
||||
zone_foreach(zone_drain, NULL);
|
||||
if (req == UMA_RECLAIM_DRAIN_CPU) {
|
||||
pcpu_cache_drain_safe(NULL);
|
||||
zone_foreach(zone_drain);
|
||||
zone_foreach(zone_drain, NULL);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
@ -3819,7 +4042,7 @@ uma_reclaim(int req)
|
||||
* we visit again so that we can free pages that are empty once other
|
||||
* zones are drained. We have to do the same for buckets.
|
||||
*/
|
||||
zone_drain(slabzone);
|
||||
zone_drain(slabzone, NULL);
|
||||
bucket_zone_drain();
|
||||
sx_xunlock(&uma_reclaim_lock);
|
||||
}
|
||||
@ -3859,14 +4082,14 @@ uma_zone_reclaim(uma_zone_t zone, int req)
|
||||
|
||||
switch (req) {
|
||||
case UMA_RECLAIM_TRIM:
|
||||
zone_trim(zone);
|
||||
zone_trim(zone, NULL);
|
||||
break;
|
||||
case UMA_RECLAIM_DRAIN:
|
||||
zone_drain(zone);
|
||||
zone_drain(zone, NULL);
|
||||
break;
|
||||
case UMA_RECLAIM_DRAIN_CPU:
|
||||
pcpu_cache_drain_safe(zone);
|
||||
zone_drain(zone);
|
||||
zone_drain(zone, NULL);
|
||||
break;
|
||||
default:
|
||||
panic("unhandled reclamation request %d", req);
|
||||
@ -3977,76 +4200,6 @@ uma_avail(void)
|
||||
return (uma_kmem_limit - uma_size());
|
||||
}
|
||||
|
||||
void
|
||||
uma_print_stats(void)
|
||||
{
|
||||
zone_foreach(uma_print_zone);
|
||||
}
|
||||
|
||||
static void
|
||||
slab_print(uma_slab_t slab)
|
||||
{
|
||||
printf("slab: keg %p, data %p, freecount %d\n",
|
||||
slab->us_keg, slab->us_data, slab->us_freecount);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_print(uma_cache_t cache)
|
||||
{
|
||||
printf("alloc: %p(%d), free: %p(%d), cross: %p(%d)j\n",
|
||||
cache->uc_allocbucket,
|
||||
cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
|
||||
cache->uc_freebucket,
|
||||
cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0,
|
||||
cache->uc_crossbucket,
|
||||
cache->uc_crossbucket?cache->uc_crossbucket->ub_cnt:0);
|
||||
}
|
||||
|
||||
static void
|
||||
uma_print_keg(uma_keg_t keg)
|
||||
{
|
||||
uma_domain_t dom;
|
||||
uma_slab_t slab;
|
||||
int i;
|
||||
|
||||
printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
|
||||
"out %d free %d\n",
|
||||
keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
|
||||
keg->uk_ipers, keg->uk_ppera,
|
||||
(keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
|
||||
keg->uk_free);
|
||||
for (i = 0; i < vm_ndomains; i++) {
|
||||
dom = &keg->uk_domain[i];
|
||||
printf("Part slabs:\n");
|
||||
LIST_FOREACH(slab, &dom->ud_part_slab, us_link)
|
||||
slab_print(slab);
|
||||
printf("Free slabs:\n");
|
||||
LIST_FOREACH(slab, &dom->ud_free_slab, us_link)
|
||||
slab_print(slab);
|
||||
printf("Full slabs:\n");
|
||||
LIST_FOREACH(slab, &dom->ud_full_slab, us_link)
|
||||
slab_print(slab);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
uma_print_zone(uma_zone_t zone)
|
||||
{
|
||||
uma_cache_t cache;
|
||||
int i;
|
||||
|
||||
printf("zone: %s(%p) size %d maxitems %ju flags %#x\n",
|
||||
zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items,
|
||||
zone->uz_flags);
|
||||
if (zone->uz_lockptr != &zone->uz_lock)
|
||||
uma_print_keg(zone->uz_keg);
|
||||
CPU_FOREACH(i) {
|
||||
cache = &zone->uz_cpu[i];
|
||||
printf("CPU %d Cache:\n", i);
|
||||
cache_print(cache);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef DDB
|
||||
/*
|
||||
* Generate statistics across both the zone and its per-cpu cache's. Return
|
||||
@ -4274,13 +4427,41 @@ sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
|
||||
int
|
||||
sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
uma_zone_t zone = *(uma_zone_t *)arg1;
|
||||
uma_zone_t zone;
|
||||
int cur;
|
||||
|
||||
/*
|
||||
* Some callers want to add sysctls for global zones that
|
||||
* may not yet exist so they pass a pointer to a pointer.
|
||||
*/
|
||||
if (arg2 == 0)
|
||||
zone = *(uma_zone_t *)arg1;
|
||||
else
|
||||
zone = arg1;
|
||||
cur = uma_zone_get_cur(zone);
|
||||
return (sysctl_handle_int(oidp, &cur, 0, req));
|
||||
}
|
||||
|
||||
static int
|
||||
sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
uma_zone_t zone = arg1;
|
||||
uint64_t cur;
|
||||
|
||||
cur = uma_zone_get_allocs(zone);
|
||||
return (sysctl_handle_64(oidp, &cur, 0, req));
|
||||
}
|
||||
|
||||
static int
|
||||
sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
uma_zone_t zone = arg1;
|
||||
uint64_t cur;
|
||||
|
||||
cur = uma_zone_get_frees(zone);
|
||||
return (sysctl_handle_64(oidp, &cur, 0, req));
|
||||
}
|
||||
|
||||
#ifdef INVARIANTS
|
||||
static uma_slab_t
|
||||
uma_dbg_getslab(uma_zone_t zone, void *item)
|
||||
@ -4500,7 +4681,8 @@ DB_SHOW_COMMAND(uma, db_show_uma)
|
||||
db_printf(fmt_entry, cur_zone->uz_name,
|
||||
(uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree,
|
||||
(uintmax_t)allocs, (uintmax_t)sleeps,
|
||||
(unsigned)cur_zone->uz_count, (intmax_t)size, xdomain);
|
||||
(unsigned)cur_zone->uz_bucket_size, (intmax_t)size,
|
||||
xdomain);
|
||||
|
||||
if (db_pager_quit)
|
||||
return;
|
||||
@ -4525,7 +4707,7 @@ DB_SHOW_COMMAND(umacache, db_show_umacache)
|
||||
db_printf("%18s %8ju %8jd %8ld %12ju %8u\n",
|
||||
z->uz_name, (uintmax_t)z->uz_size,
|
||||
(intmax_t)(allocs - frees), cachefree,
|
||||
(uintmax_t)allocs, z->uz_count);
|
||||
(uintmax_t)allocs, z->uz_bucket_size);
|
||||
if (db_pager_quit)
|
||||
return;
|
||||
}
|
||||
|
@ -338,8 +338,8 @@ struct uma_zone {
|
||||
uint64_t uz_items; /* Total items count */
|
||||
uint64_t uz_max_items; /* Maximum number of items to alloc */
|
||||
uint32_t uz_sleepers; /* Number of sleepers on memory */
|
||||
uint16_t uz_count; /* Amount of items in full bucket */
|
||||
uint16_t uz_count_max; /* Maximum amount of items there */
|
||||
uint16_t uz_bucket_size; /* Number of items in full bucket */
|
||||
uint16_t uz_bucket_size_max; /* Maximum number of bucket items */
|
||||
|
||||
/* Offset 64, used in bucket replenish. */
|
||||
uma_import uz_import; /* Import new memory to cache. */
|
||||
@ -364,14 +364,17 @@ struct uma_zone {
|
||||
const char *uz_warning; /* Warning to print on failure */
|
||||
struct timeval uz_ratecheck; /* Warnings rate-limiting */
|
||||
struct task uz_maxaction; /* Task to run when at limit */
|
||||
uint16_t uz_count_min; /* Minimal amount of items in bucket */
|
||||
uint16_t uz_bucket_size_min; /* Min number of items in bucket */
|
||||
|
||||
/* Offset 256, stats. */
|
||||
/* Offset 256+, stats and misc. */
|
||||
counter_u64_t uz_allocs; /* Total number of allocations */
|
||||
counter_u64_t uz_frees; /* Total number of frees */
|
||||
counter_u64_t uz_fails; /* Total number of alloc failures */
|
||||
uint64_t uz_sleeps; /* Total number of alloc sleeps */
|
||||
uint64_t uz_xdomain; /* Total number of cross-domain frees */
|
||||
char *uz_ctlname; /* sysctl safe name string. */
|
||||
struct sysctl_oid *uz_oid; /* sysctl oid pointer. */
|
||||
int uz_namecnt; /* duplicate name count. */
|
||||
|
||||
/*
|
||||
* This HAS to be the last item because we adjust the zone size
|
||||
|
Loading…
Reference in New Issue
Block a user