Add MALLOC_DEBUG_MAXZONES debug malloc(9) option to use multiple uma
zones for each malloc bucket size. The purpose is to isolate different malloc types into hash classes, so that any buffer overruns or use-after-free will usually only affect memory from malloc types in that hash class. This is purely a debugging tool; by varying the hash function and tracking which hash class was corrupted, the intersection of the hash classes from each instance will point to a single malloc type that is being misused. At this point inspection or memguard(9) can be used to catch the offending code. Add MALLOC_DEBUG_MAXZONES=8 to -current GENERIC configuration files. The suggestion to have this on by default came from Kostik Belousov on -arch. This code is based on work by Ron Steinke at Isilon Systems. Reviewed by: -arch (mostly silence) Reviewed by: zml Approved by: zml (mentor)
This commit is contained in:
parent
c954ded250
commit
d7854da193
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=210564
@ -76,6 +76,7 @@ options INVARIANTS # Enable calls of extra sanity checking
|
||||
options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS
|
||||
options WITNESS # Enable checks to detect deadlocks and cycles
|
||||
options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
|
||||
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
|
||||
# Make an SMP-capable kernel by default
|
||||
options SMP # Symmetric MultiProcessor Kernel
|
||||
|
@ -384,6 +384,20 @@ options SYSCTL_DEBUG
|
||||
# resulting kernel.
|
||||
options NO_SYSCTL_DESCR
|
||||
|
||||
#
|
||||
# MALLOC_DEBUG_MAXZONES enables multiple uma zones for malloc(9)
|
||||
# allocations that are smaller than a page. The purpose is to isolate
|
||||
# different malloc types into hash classes, so that any buffer
|
||||
# overruns or use-after-free will usually only affect memory from
|
||||
# malloc types in that hash class. This is purely a debugging tool;
|
||||
# by varying the hash function and tracking which hash class was
|
||||
# corrupted, the intersection of the hash classes from each instance
|
||||
# will point to a single malloc type that is being misused. At this
|
||||
# point inspection or memguard(9) can be used to catch the offending
|
||||
# code.
|
||||
#
|
||||
options MALLOC_DEBUG_MAXZONES=8
|
||||
|
||||
#
|
||||
# DEBUG_MEMGUARD builds and enables memguard(9), a replacement allocator
|
||||
# for the kernel used to detect modify-after-free scenarios. See the
|
||||
|
@ -586,6 +586,7 @@ VM_LEVEL_0_ORDER opt_vm.h
|
||||
NO_SWAPPING opt_vm.h
|
||||
MALLOC_MAKE_FAILURES opt_vm.h
|
||||
MALLOC_PROFILE opt_vm.h
|
||||
MALLOC_DEBUG_MAXZONES opt_vm.h
|
||||
|
||||
# The MemGuard replacement allocator used for tamper-after-free detection
|
||||
DEBUG_MEMGUARD opt_vm.h
|
||||
|
@ -76,6 +76,7 @@ options INVARIANTS # Enable calls of extra sanity checking
|
||||
options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS
|
||||
options WITNESS # Enable checks to detect deadlocks and cycles
|
||||
options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
|
||||
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
|
||||
# To make an SMP kernel, the next two lines are needed
|
||||
options SMP # Symmetric MultiProcessor Kernel
|
||||
|
@ -68,6 +68,7 @@ options UFS_GJOURNAL # Enable gjournal-based UFS journaling
|
||||
options WITNESS # Enable checks to detect deadlocks and cycles
|
||||
options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
|
||||
options _KPOSIX_PRIORITY_SCHEDULING # Posix P1003_1B RT extensions
|
||||
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
|
||||
# Various "busses"
|
||||
device firewire # FireWire bus code
|
||||
|
@ -131,6 +131,11 @@ static int kmemcount;
|
||||
#define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT)
|
||||
static uint8_t kmemsize[KMEM_ZSIZE + 1];
|
||||
|
||||
#ifndef MALLOC_DEBUG_MAXZONES
|
||||
#define MALLOC_DEBUG_MAXZONES 1
|
||||
#endif
|
||||
static int numzones = MALLOC_DEBUG_MAXZONES;
|
||||
|
||||
/*
|
||||
* Small malloc(9) memory allocations are allocated from a set of UMA buckets
|
||||
* of various sizes.
|
||||
@ -142,25 +147,25 @@ static uint8_t kmemsize[KMEM_ZSIZE + 1];
|
||||
struct {
|
||||
int kz_size;
|
||||
char *kz_name;
|
||||
uma_zone_t kz_zone;
|
||||
uma_zone_t kz_zone[MALLOC_DEBUG_MAXZONES];
|
||||
} kmemzones[] = {
|
||||
{16, "16", NULL},
|
||||
{32, "32", NULL},
|
||||
{64, "64", NULL},
|
||||
{128, "128", NULL},
|
||||
{256, "256", NULL},
|
||||
{512, "512", NULL},
|
||||
{1024, "1024", NULL},
|
||||
{2048, "2048", NULL},
|
||||
{4096, "4096", NULL},
|
||||
{16, "16", },
|
||||
{32, "32", },
|
||||
{64, "64", },
|
||||
{128, "128", },
|
||||
{256, "256", },
|
||||
{512, "512", },
|
||||
{1024, "1024", },
|
||||
{2048, "2048", },
|
||||
{4096, "4096", },
|
||||
#if PAGE_SIZE > 4096
|
||||
{8192, "8192", NULL},
|
||||
{8192, "8192", },
|
||||
#if PAGE_SIZE > 8192
|
||||
{16384, "16384", NULL},
|
||||
{16384, "16384", },
|
||||
#if PAGE_SIZE > 16384
|
||||
{32768, "32768", NULL},
|
||||
{32768, "32768", },
|
||||
#if PAGE_SIZE > 32768
|
||||
{65536, "65536", NULL},
|
||||
{65536, "65536", },
|
||||
#if PAGE_SIZE > 65536
|
||||
#error "Unsupported PAGE_SIZE"
|
||||
#endif /* 65536 */
|
||||
@ -215,14 +220,16 @@ static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
|
||||
*/
|
||||
static time_t t_malloc_fail;
|
||||
|
||||
#if defined(MALLOC_MAKE_FAILURES) || (MALLOC_DEBUG_MAXZONES > 1)
|
||||
SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
|
||||
"Kernel malloc debugging options");
|
||||
#endif
|
||||
|
||||
/*
|
||||
* malloc(9) fault injection -- cause malloc failures every (n) mallocs when
|
||||
* the caller specifies M_NOWAIT. If set to 0, no failures are caused.
|
||||
*/
|
||||
#ifdef MALLOC_MAKE_FAILURES
|
||||
SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
|
||||
"Kernel malloc debugging options");
|
||||
|
||||
static int malloc_failure_rate;
|
||||
static int malloc_nowait_count;
|
||||
static int malloc_failure_count;
|
||||
@ -233,6 +240,60 @@ SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
|
||||
&malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
|
||||
#endif
|
||||
|
||||
/*
|
||||
* malloc(9) uma zone separation -- sub-page buffer overruns in one
|
||||
* malloc type will affect only a subset of other malloc types.
|
||||
*/
|
||||
#if MALLOC_DEBUG_MAXZONES > 1
|
||||
static void
|
||||
tunable_set_numzones(void)
|
||||
{
|
||||
|
||||
TUNABLE_INT_FETCH("debug.malloc.numzones",
|
||||
&numzones);
|
||||
|
||||
/* Sanity check the number of malloc uma zones. */
|
||||
if (numzones <= 0)
|
||||
numzones = 1;
|
||||
if (numzones > MALLOC_DEBUG_MAXZONES)
|
||||
numzones = MALLOC_DEBUG_MAXZONES;
|
||||
}
|
||||
SYSINIT(numzones, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_set_numzones, NULL);
|
||||
SYSCTL_INT(_debug_malloc, OID_AUTO, numzones, CTLFLAG_RDTUN,
|
||||
&numzones, 0, "Number of malloc uma subzones");
|
||||
|
||||
/*
|
||||
* Any number that changes regularly is an okay choice for the
|
||||
* offset. Build numbers are pretty good of you have them.
|
||||
*/
|
||||
static u_int zone_offset = __FreeBSD_version;
|
||||
TUNABLE_INT("debug.malloc.zone_offset", &zone_offset);
|
||||
SYSCTL_UINT(_debug_malloc, OID_AUTO, zone_offset, CTLFLAG_RDTUN,
|
||||
&zone_offset, 0, "Separate malloc types by examining the "
|
||||
"Nth character in the malloc type short description.");
|
||||
|
||||
static u_int
|
||||
mtp_get_subzone(const char *desc)
|
||||
{
|
||||
size_t len;
|
||||
u_int val;
|
||||
|
||||
if (desc == NULL || (len = strlen(desc)) == 0)
|
||||
return (0);
|
||||
val = desc[zone_offset % len];
|
||||
return (val % numzones);
|
||||
}
|
||||
#elif MALLOC_DEBUG_MAXZONES == 0
|
||||
#error "MALLOC_DEBUG_MAXZONES must be positive."
|
||||
#else
|
||||
static inline u_int
|
||||
mtp_get_subzone(const char *desc)
|
||||
{
|
||||
|
||||
return (0);
|
||||
}
|
||||
#endif /* MALLOC_DEBUG_MAXZONES > 1 */
|
||||
|
||||
int
|
||||
malloc_last_fail(void)
|
||||
{
|
||||
@ -327,6 +388,7 @@ void *
|
||||
malloc(unsigned long size, struct malloc_type *mtp, int flags)
|
||||
{
|
||||
int indx;
|
||||
struct malloc_type_internal *mtip;
|
||||
caddr_t va;
|
||||
uma_zone_t zone;
|
||||
#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
|
||||
@ -374,10 +436,14 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
|
||||
#endif
|
||||
|
||||
if (size <= KMEM_ZMAX) {
|
||||
mtip = mtp->ks_handle;
|
||||
if (size & KMEM_ZMASK)
|
||||
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
|
||||
indx = kmemsize[size >> KMEM_ZSHIFT];
|
||||
zone = kmemzones[indx].kz_zone;
|
||||
KASSERT(mtip->mti_zone < numzones,
|
||||
("mti_zone %u out of range %d",
|
||||
mtip->mti_zone, numzones));
|
||||
zone = kmemzones[indx].kz_zone[mtip->mti_zone];
|
||||
#ifdef MALLOC_PROFILE
|
||||
krequests[size >> KMEM_ZSHIFT]++;
|
||||
#endif
|
||||
@ -651,15 +717,18 @@ kmeminit(void *dummy)
|
||||
for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
|
||||
int size = kmemzones[indx].kz_size;
|
||||
char *name = kmemzones[indx].kz_name;
|
||||
int subzone;
|
||||
|
||||
kmemzones[indx].kz_zone = uma_zcreate(name, size,
|
||||
for (subzone = 0; subzone < numzones; subzone++) {
|
||||
kmemzones[indx].kz_zone[subzone] =
|
||||
uma_zcreate(name, size,
|
||||
#ifdef INVARIANTS
|
||||
mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
|
||||
mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
|
||||
#else
|
||||
NULL, NULL, NULL, NULL,
|
||||
NULL, NULL, NULL, NULL,
|
||||
#endif
|
||||
UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
|
||||
|
||||
UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
|
||||
}
|
||||
for (;i <= size; i+= KMEM_ZBASE)
|
||||
kmemsize[i >> KMEM_ZSHIFT] = indx;
|
||||
|
||||
@ -680,6 +749,7 @@ malloc_init(void *data)
|
||||
|
||||
mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO);
|
||||
mtp->ks_handle = mtip;
|
||||
mtip->mti_zone = mtp_get_subzone(mtp->ks_shortdesc);
|
||||
|
||||
mtx_lock(&malloc_mtx);
|
||||
mtp->ks_next = kmemstatistics;
|
||||
@ -902,7 +972,37 @@ DB_SHOW_COMMAND(malloc, db_show_malloc)
|
||||
(alloced - freed + 1023) / 1024, allocs);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if MALLOC_DEBUG_MAXZONES > 1
|
||||
DB_SHOW_COMMAND(multizone_matches, db_show_multizone_matches)
|
||||
{
|
||||
struct malloc_type_internal *mtip;
|
||||
struct malloc_type *mtp;
|
||||
u_int subzone;
|
||||
|
||||
if (!have_addr) {
|
||||
db_printf("Usage: show multizone_matches <malloc type/addr>\n");
|
||||
return;
|
||||
}
|
||||
mtp = (void *)addr;
|
||||
if (mtp->ks_magic != M_MAGIC) {
|
||||
db_printf("Magic %lx does not match expected %x\n",
|
||||
mtp->ks_magic, M_MAGIC);
|
||||
return;
|
||||
}
|
||||
|
||||
mtip = mtp->ks_handle;
|
||||
subzone = mtip->mti_zone;
|
||||
|
||||
for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
|
||||
mtip = mtp->ks_handle;
|
||||
if (mtip->mti_zone != subzone)
|
||||
continue;
|
||||
db_printf("%s\n", mtp->ks_shortdesc);
|
||||
}
|
||||
}
|
||||
#endif /* MALLOC_DEBUG_MAXZONES > 1 */
|
||||
#endif /* DDB */
|
||||
|
||||
#ifdef MALLOC_PROFILE
|
||||
|
||||
|
@ -76,6 +76,7 @@ options INVARIANTS # Enable calls of extra sanity checking
|
||||
options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS
|
||||
options WITNESS # Enable checks to detect deadlocks and cycles
|
||||
options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
|
||||
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
|
||||
# To make an SMP kernel, the next two lines are needed
|
||||
#options SMP # Symmetric MultiProcessor Kernel
|
||||
|
@ -74,6 +74,7 @@ options INVARIANTS #Enable calls of extra sanity checking
|
||||
options INVARIANT_SUPPORT #Extra sanity checks of internal structures, required by INVARIANTS
|
||||
options WITNESS #Enable checks to detect deadlocks and cycles
|
||||
options WITNESS_SKIPSPIN #Don't run witness on spinlocks for speed
|
||||
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
|
||||
# To make an SMP kernel, the next line is needed
|
||||
#options SMP # Symmetric MultiProcessor Kernel
|
||||
|
@ -73,6 +73,7 @@ options INVARIANTS # Enable calls of extra sanity checking
|
||||
options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS
|
||||
options WITNESS # Enable checks to detect deadlocks and cycles
|
||||
options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
|
||||
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
|
||||
# Make an SMP-capable kernel by default
|
||||
options SMP # Symmetric MultiProcessor Kernel
|
||||
|
@ -79,6 +79,7 @@ options DDB # Support DDB.
|
||||
#options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS
|
||||
#options WITNESS # Enable checks to detect deadlocks and cycles
|
||||
#options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
|
||||
#options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
|
||||
#options DEBUG_LOCKS
|
||||
#options DEBUG_VFS_LOCKS
|
||||
|
||||
|
@ -90,6 +90,7 @@ struct malloc_type_stats {
|
||||
struct malloc_type_internal {
|
||||
uint32_t mti_probes[DTMALLOC_PROBE_MAX];
|
||||
/* DTrace probe ID array. */
|
||||
u_char mti_zone;
|
||||
struct malloc_type_stats mti_stats[MAXCPU];
|
||||
};
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user