From 3571902304966f5edad4bceddc7fb3170f4ff982 Mon Sep 17 00:00:00 2001 From: sbruno Date: Tue, 15 Jun 2010 19:28:37 +0000 Subject: [PATCH] Add a new column to the output of vmstat -z to indicate the number of times the system was forced to sleep when requesting a new allocation. Expand the debugger hook, db_show_uma, to display these results as well. This has proven to be very useful in out of memory situations when it is not known why systems have become sluggish or fail in odd ways. Reviewed by: rwatson alc Approved by: scottl (mentor) peter Obtained from: Yahoo Inc. --- lib/libmemstat/memstat.c | 8 ++++++++ lib/libmemstat/memstat.h | 1 + lib/libmemstat/memstat_internal.h | 1 + lib/libmemstat/memstat_uma.c | 2 ++ sys/vm/uma.h | 3 ++- sys/vm/uma_core.c | 25 ++++++++++++++++--------- sys/vm/uma_int.h | 1 + usr.bin/vmstat/vmstat.c | 9 +++++---- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/lib/libmemstat/memstat.c b/lib/libmemstat/memstat.c index 28e48138642b..1a08d3f4d462 100644 --- a/lib/libmemstat/memstat.c +++ b/lib/libmemstat/memstat.c @@ -188,6 +188,7 @@ _memstat_mt_reset_stats(struct memory_type *mtp) mtp->mt_count = 0; mtp->mt_free = 0; mtp->mt_failures = 0; + mtp->mt_sleeps = 0; mtp->mt_zonefree = 0; mtp->mt_kegfree = 0; @@ -304,6 +305,13 @@ memstat_get_failures(const struct memory_type *mtp) return (mtp->mt_failures); } +uint64_t +memstat_get_sleeps(const struct memory_type *mtp) +{ + + return (mtp->mt_sleeps); +} + void * memstat_get_caller_pointer(const struct memory_type *mtp, int index) { diff --git a/lib/libmemstat/memstat.h b/lib/libmemstat/memstat.h index aaa85702e0c5..e973f1a52db4 100644 --- a/lib/libmemstat/memstat.h +++ b/lib/libmemstat/memstat.h @@ -139,6 +139,7 @@ uint64_t memstat_get_bytes(const struct memory_type *mtp); uint64_t memstat_get_count(const struct memory_type *mtp); uint64_t memstat_get_free(const struct memory_type *mtp); uint64_t memstat_get_failures(const struct memory_type *mtp); +uint64_t memstat_get_sleeps(const struct memory_type *mtp); void *memstat_get_caller_pointer(const struct memory_type *mtp, int index); void memstat_set_caller_pointer(struct memory_type *mtp, diff --git a/lib/libmemstat/memstat_internal.h b/lib/libmemstat/memstat_internal.h index 7123518e283a..b7fdd7197cfe 100644 --- a/lib/libmemstat/memstat_internal.h +++ b/lib/libmemstat/memstat_internal.h @@ -65,6 +65,7 @@ struct memory_type { uint64_t mt_count; /* Number of current allocations. */ uint64_t mt_free; /* Number of cached free items. */ uint64_t mt_failures; /* Number of allocation failures. */ + uint64_t mt_sleeps; /* Number of allocation sleeps. */ /* * Caller-owned memory. diff --git a/lib/libmemstat/memstat_uma.c b/lib/libmemstat/memstat_uma.c index b24721fae5d9..4aae61a9bb05 100644 --- a/lib/libmemstat/memstat_uma.c +++ b/lib/libmemstat/memstat_uma.c @@ -208,6 +208,7 @@ retry: mtp->mt_numallocs = uthp->uth_allocs; mtp->mt_numfrees = uthp->uth_frees; mtp->mt_failures = uthp->uth_fails; + mtp->mt_sleeps = uthp->uth_sleeps; for (j = 0; j < maxcpus; j++) { upsp = (struct uma_percpu_stat *)p; @@ -402,6 +403,7 @@ memstat_kvm_uma(struct memory_type_list *list, void *kvm_handle) mtp->mt_numallocs = uz.uz_allocs; mtp->mt_numfrees = uz.uz_frees; mtp->mt_failures = uz.uz_fails; + mtp->mt_sleeps = uz.uz_sleeps; if (kz.uk_flags & UMA_ZFLAG_INTERNAL) goto skip_percpu; for (i = 0; i < mp_maxid + 1; i++) { diff --git a/sys/vm/uma.h b/sys/vm/uma.h index c04482458df2..2076d3bd0b61 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -600,7 +600,8 @@ struct uma_type_header { u_int64_t uth_allocs; /* Zone: number of allocations. */ u_int64_t uth_frees; /* Zone: number of frees. */ u_int64_t uth_fails; /* Zone: number of alloc failures. */ - u_int64_t _uth_reserved1[3]; /* Reserved. */ + u_int64_t uth_sleeps; /* Zone: number of alloc sleeps. */ + u_int64_t _uth_reserved1[2]; /* Reserved. */ }; struct uma_percpu_stat { diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index a3855ca62639..2dcd14fdd1a3 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -1396,6 +1396,7 @@ zone_ctor(void *mem, int size, void *udata, int flags) zone->uz_allocs = 0; zone->uz_frees = 0; zone->uz_fails = 0; + zone->uz_sleeps = 0; zone->uz_fills = zone->uz_count = 0; zone->uz_flags = 0; keg = arg->keg; @@ -2283,6 +2284,7 @@ zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags) */ if (full && !empty) { zone->uz_flags |= UMA_ZFLAG_FULL; + zone->uz_sleeps++; msleep(zone, zone->uz_lock, PVM, "zonelimit", hz/100); zone->uz_flags &= ~UMA_ZFLAG_FULL; continue; @@ -3094,13 +3096,13 @@ uma_print_zone(uma_zone_t zone) */ static void uma_zone_sumstat(uma_zone_t z, int *cachefreep, u_int64_t *allocsp, - u_int64_t *freesp) + u_int64_t *freesp, u_int64_t *sleepsp) { uma_cache_t cache; - u_int64_t allocs, frees; + u_int64_t allocs, frees, sleeps; int cachefree, cpu; - allocs = frees = 0; + allocs = frees = sleeps = 0; cachefree = 0; CPU_FOREACH(cpu) { cache = &z->uz_cpu[cpu]; @@ -3113,12 +3115,15 @@ uma_zone_sumstat(uma_zone_t z, int *cachefreep, u_int64_t *allocsp, } allocs += z->uz_allocs; frees += z->uz_frees; + sleeps += z->uz_sleeps; if (cachefreep != NULL) *cachefreep = cachefree; if (allocsp != NULL) *allocsp = allocs; if (freesp != NULL) *freesp = frees; + if (sleepsp != NULL) + *sleepsp = sleeps; } #endif /* DDB */ @@ -3226,6 +3231,7 @@ restart: uth.uth_allocs = z->uz_allocs; uth.uth_frees = z->uz_frees; uth.uth_fails = z->uz_fails; + uth.uth_sleeps = z->uz_sleeps; if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) { ZONE_UNLOCK(z); mtx_unlock(&uma_mtx); @@ -3277,32 +3283,33 @@ out: #ifdef DDB DB_SHOW_COMMAND(uma, db_show_uma) { - u_int64_t allocs, frees; + u_int64_t allocs, frees, sleeps; uma_bucket_t bucket; uma_keg_t kz; uma_zone_t z; int cachefree; - db_printf("%18s %8s %8s %8s %12s\n", "Zone", "Size", "Used", "Free", - "Requests"); + db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", + "Requests", "Sleeps"); LIST_FOREACH(kz, &uma_kegs, uk_link) { LIST_FOREACH(z, &kz->uk_zones, uz_link) { if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { allocs = z->uz_allocs; frees = z->uz_frees; + sleeps = z->uz_sleeps; cachefree = 0; } else uma_zone_sumstat(z, &cachefree, &allocs, - &frees); + &frees, &sleeps); if (!((z->uz_flags & UMA_ZONE_SECONDARY) && (LIST_FIRST(&kz->uk_zones) != z))) cachefree += kz->uk_free; LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) cachefree += bucket->ub_cnt; - db_printf("%18s %8ju %8jd %8d %12ju\n", z->uz_name, + db_printf("%18s %8ju %8jd %8d %12ju %8ju\n", z->uz_name, (uintmax_t)kz->uk_size, (intmax_t)(allocs - frees), cachefree, - (uintmax_t)allocs); + (uintmax_t)allocs, sleeps); } } } diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h index 27396ea077dc..77135933ee3f 100644 --- a/sys/vm/uma_int.h +++ b/sys/vm/uma_int.h @@ -327,6 +327,7 @@ struct uma_zone { u_int64_t uz_allocs UMA_ALIGN; /* Total number of allocations */ u_int64_t uz_frees; /* Total number of frees */ u_int64_t uz_fails; /* Total number of alloc failures */ + u_int64_t uz_sleeps; /* Total number of alloc sleeps */ uint16_t uz_fills; /* Outstanding bucket fills */ uint16_t uz_count; /* Highest value ub_ptr can have */ diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c index 4105a36ab1c7..7583142ae298 100644 --- a/usr.bin/vmstat/vmstat.c +++ b/usr.bin/vmstat/vmstat.c @@ -1294,16 +1294,17 @@ domemstat_zone(void) memstat_strerror(error)); } } - printf("%-20s %8s %8s %8s %8s %8s %8s\n\n", "ITEM", "SIZE", - "LIMIT", "USED", "FREE", "REQUESTS", "FAILURES"); + printf("%-20s %6s %6s %8s %8s %8s %4s %4s\n\n", "ITEM", "SIZE", + "LIMIT", "USED", "FREE", "REQ", "FAIL", "SLEEP"); for (mtp = memstat_mtl_first(mtlp); mtp != NULL; mtp = memstat_mtl_next(mtp)) { strlcpy(name, memstat_get_name(mtp), MEMTYPE_MAXNAME); strcat(name, ":"); - printf("%-20s %8llu, %8llu, %8llu, %8llu, %8llu, %8llu\n", name, + printf("%-20s %6llu, %6llu,%8llu,%8llu,%8llu,%4llu,%4llu\n",name, memstat_get_size(mtp), memstat_get_countlimit(mtp), memstat_get_count(mtp), memstat_get_free(mtp), - memstat_get_numallocs(mtp), memstat_get_failures(mtp)); + memstat_get_numallocs(mtp), memstat_get_failures(mtp), + memstat_get_sleeps(mtp)); } memstat_mtl_free(mtlp); printf("\n");