- Make the keg abstraction more complete. Permit a zone to have multiple
backend kegs so it may source compatible memory from multiple backends. This is useful for cases such as NUMA or different layouts for the same memory type. - Provide a new api for adding new backend kegs to secondary zones. - Provide a new flag for adjusting the layout of zones to stagger allocations better across cache lines. Sponsored by: Nokia
This commit is contained in:
Jeff Roberson
parent
30bf032c76
commit
e20a199fd5
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=187681
@ -329,7 +329,6 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
|
||||
int indx;
|
||||
caddr_t va;
|
||||
uma_zone_t zone;
|
||||
uma_keg_t keg;
|
||||
#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
|
||||
unsigned long osize = size;
|
||||
#endif
|
||||
@ -378,18 +377,16 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
|
||||
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
|
||||
indx = kmemsize[size >> KMEM_ZSHIFT];
|
||||
zone = kmemzones[indx].kz_zone;
|
||||
keg = zone->uz_keg;
|
||||
#ifdef MALLOC_PROFILE
|
||||
krequests[size >> KMEM_ZSHIFT]++;
|
||||
#endif
|
||||
va = uma_zalloc(zone, flags);
|
||||
if (va != NULL)
|
||||
size = keg->uk_size;
|
||||
size = zone->uz_size;
|
||||
malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);
|
||||
} else {
|
||||
size = roundup(size, PAGE_SIZE);
|
||||
zone = NULL;
|
||||
keg = NULL;
|
||||
va = uma_large_malloc(size, flags);
|
||||
malloc_type_allocated(mtp, va == NULL ? 0 : size);
|
||||
}
|
||||
|
||||
27
sys/vm/uma.h
27
sys/vm/uma.h
@ -204,6 +204,17 @@ uma_zone_t uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
|
||||
uma_zone_t uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
|
||||
uma_init zinit, uma_fini zfini, uma_zone_t master);
|
||||
|
||||
/*
|
||||
* Add a second master to a secondary zone. This provides multiple data
|
||||
* backends for objects with the same size. Both masters must have
|
||||
* compatible allocation flags. Presently, UMA_ZONE_MALLOC type zones are
|
||||
* the only supported.
|
||||
*
|
||||
* Returns:
|
||||
* Error on failure, 0 on success.
|
||||
*/
|
||||
int uma_zsecond_add(uma_zone_t zone, uma_zone_t master);
|
||||
|
||||
/*
|
||||
* Definitions for uma_zcreate flags
|
||||
*
|
||||
@ -230,6 +241,22 @@ uma_zone_t uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
|
||||
#define UMA_ZONE_SECONDARY 0x0200 /* Zone is a Secondary Zone */
|
||||
#define UMA_ZONE_REFCNT 0x0400 /* Allocate refcnts in slabs */
|
||||
#define UMA_ZONE_MAXBUCKET 0x0800 /* Use largest buckets */
|
||||
#define UMA_ZONE_CACHESPREAD 0x1000 /*
|
||||
* Spread memory start locations across
|
||||
* all possible cache lines. May
|
||||
* require many virtually contiguous
|
||||
* backend pages and can fail early.
|
||||
*/
|
||||
#define UMA_ZONE_VTOSLAB 0x2000 /* Zone uses vtoslab for lookup. */
|
||||
|
||||
/*
|
||||
* These flags are shared between the keg and zone. In zones wishing to add
|
||||
* new kegs these flags must be compatible. Some are determined based on
|
||||
* physical parameters of the request and may not be provided by the consumer.
|
||||
*/
|
||||
#define UMA_ZONE_INHERIT \
|
||||
(UMA_ZONE_OFFPAGE | UMA_ZONE_MALLOC | UMA_ZONE_HASH | \
|
||||
UMA_ZONE_REFCNT | UMA_ZONE_VTOSLAB)
|
||||
|
||||
/* Definitions for align */
|
||||
#define UMA_ALIGN_PTR (sizeof(void *) - 1) /* Alignment fit for ptr */
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -198,15 +198,15 @@ uma_dbg_getslab(uma_zone_t zone, void *item)
|
||||
uma_keg_t keg;
|
||||
u_int8_t *mem;
|
||||
|
||||
keg = zone->uz_keg;
|
||||
mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
|
||||
if (keg->uk_flags & UMA_ZONE_MALLOC) {
|
||||
if (zone->uz_flags & UMA_ZONE_VTOSLAB) {
|
||||
slab = vtoslab((vm_offset_t)mem);
|
||||
} else if (keg->uk_flags & UMA_ZONE_HASH) {
|
||||
slab = hash_sfind(&keg->uk_hash, mem);
|
||||
} else {
|
||||
mem += keg->uk_pgoff;
|
||||
slab = (uma_slab_t)mem;
|
||||
keg = LIST_FIRST(&zone->uz_kegs)->kl_keg;
|
||||
if (keg->uk_flags & UMA_ZONE_HASH)
|
||||
slab = hash_sfind(&keg->uk_hash, mem);
|
||||
else
|
||||
slab = (uma_slab_t)(mem + keg->uk_pgoff);
|
||||
}
|
||||
|
||||
return (slab);
|
||||
@ -224,13 +224,13 @@ uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
|
||||
uma_slabrefcnt_t slabref;
|
||||
int freei;
|
||||
|
||||
keg = zone->uz_keg;
|
||||
if (slab == NULL) {
|
||||
slab = uma_dbg_getslab(zone, item);
|
||||
if (slab == NULL)
|
||||
panic("uma: item %p did not belong to zone %s\n",
|
||||
item, zone->uz_name);
|
||||
}
|
||||
keg = slab->us_keg;
|
||||
|
||||
freei = ((unsigned long)item - (unsigned long)slab->us_data)
|
||||
/ keg->uk_rsize;
|
||||
@ -258,13 +258,13 @@ uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
|
||||
uma_slabrefcnt_t slabref;
|
||||
int freei;
|
||||
|
||||
keg = zone->uz_keg;
|
||||
if (slab == NULL) {
|
||||
slab = uma_dbg_getslab(zone, item);
|
||||
if (slab == NULL)
|
||||
panic("uma: Freed item %p did not belong to zone %s\n",
|
||||
item, zone->uz_name);
|
||||
}
|
||||
keg = slab->us_keg;
|
||||
|
||||
freei = ((unsigned long)item - (unsigned long)slab->us_data)
|
||||
/ keg->uk_rsize;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*-
|
||||
* Copyright (c) 2002, 2003, 2004, 2005 Jeffrey Roberson <jeff@FreeBSD.org>
|
||||
* Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
|
||||
* Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
|
||||
* All rights reserved.
|
||||
*
|
||||
@ -193,6 +193,7 @@ struct uma_keg {
|
||||
struct mtx uk_lock; /* Lock for the keg */
|
||||
struct uma_hash uk_hash;
|
||||
|
||||
char *uk_name; /* Name of creating zone. */
|
||||
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
|
||||
LIST_HEAD(,uma_slab) uk_part_slab; /* partially allocated slabs */
|
||||
LIST_HEAD(,uma_slab) uk_free_slab; /* empty slab list */
|
||||
@ -220,9 +221,7 @@ struct uma_keg {
|
||||
u_int16_t uk_ipers; /* Items per slab */
|
||||
u_int32_t uk_flags; /* Internal flags */
|
||||
};
|
||||
|
||||
/* Simpler reference to uma_keg for internal use. */
|
||||
typedef struct uma_keg * uma_keg_t;
|
||||
typedef struct uma_keg * uma_keg_t;
|
||||
|
||||
/* Page management structure */
|
||||
|
||||
@ -271,6 +270,8 @@ struct uma_slab_refcnt {
|
||||
|
||||
typedef struct uma_slab * uma_slab_t;
|
||||
typedef struct uma_slab_refcnt * uma_slabrefcnt_t;
|
||||
typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int);
|
||||
|
||||
|
||||
/*
|
||||
* These give us the size of one free item reference within our corresponding
|
||||
@ -282,6 +283,12 @@ typedef struct uma_slab_refcnt * uma_slabrefcnt_t;
|
||||
#define UMA_FRITMREF_SZ (sizeof(struct uma_slab_refcnt) - \
|
||||
sizeof(struct uma_slab_head))
|
||||
|
||||
struct uma_klink {
|
||||
LIST_ENTRY(uma_klink) kl_link;
|
||||
uma_keg_t kl_keg;
|
||||
};
|
||||
typedef struct uma_klink *uma_klink_t;
|
||||
|
||||
/*
|
||||
* Zone management structure
|
||||
*
|
||||
@ -291,12 +298,15 @@ typedef struct uma_slab_refcnt * uma_slabrefcnt_t;
|
||||
struct uma_zone {
|
||||
char *uz_name; /* Text name of the zone */
|
||||
struct mtx *uz_lock; /* Lock for the zone (keg's lock) */
|
||||
uma_keg_t uz_keg; /* Our underlying Keg */
|
||||
|
||||
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
|
||||
LIST_HEAD(,uma_bucket) uz_full_bucket; /* full buckets */
|
||||
LIST_HEAD(,uma_bucket) uz_free_bucket; /* Buckets for frees */
|
||||
|
||||
LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */
|
||||
struct uma_klink uz_klink; /* klink for first keg. */
|
||||
|
||||
uma_slaballoc uz_slab; /* Allocate a slab from the backend. */
|
||||
uma_ctor uz_ctor; /* Constructor for each allocation */
|
||||
uma_dtor uz_dtor; /* Destructor */
|
||||
uma_init uz_init; /* Initializer for each item */
|
||||
@ -305,6 +315,8 @@ struct uma_zone {
|
||||
u_int64_t uz_allocs; /* Total number of allocations */
|
||||
u_int64_t uz_frees; /* Total number of frees */
|
||||
u_int64_t uz_fails; /* Total number of alloc failures */
|
||||
u_int32_t uz_flags; /* Flags inherited from kegs */
|
||||
u_int32_t uz_size; /* Size inherited from kegs */
|
||||
uint16_t uz_fills; /* Outstanding bucket fills */
|
||||
uint16_t uz_count; /* Highest value ub_ptr can have */
|
||||
|
||||
@ -318,11 +330,17 @@ struct uma_zone {
|
||||
/*
|
||||
* These flags must not overlap with the UMA_ZONE flags specified in uma.h.
|
||||
*/
|
||||
#define UMA_ZFLAG_BUCKET 0x02000000 /* Bucket zone. */
|
||||
#define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */
|
||||
#define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */
|
||||
#define UMA_ZFLAG_PRIVALLOC 0x10000000 /* Use uz_allocf. */
|
||||
#define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */
|
||||
#define UMA_ZFLAG_FULL 0x40000000 /* Reached uz_maxpages */
|
||||
#define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */
|
||||
|
||||
#define UMA_ZFLAG_INHERIT (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | \
|
||||
UMA_ZFLAG_BUCKET)
|
||||
|
||||
#ifdef _KERNEL
|
||||
/* Internal prototypes */
|
||||
static __inline uma_slab_t hash_sfind(struct uma_hash *hash, u_int8_t *data);
|
||||
@ -331,17 +349,19 @@ void uma_large_free(uma_slab_t slab);
|
||||
|
||||
/* Lock Macros */
|
||||
|
||||
#define ZONE_LOCK_INIT(z, lc) \
|
||||
#define KEG_LOCK_INIT(k, lc) \
|
||||
do { \
|
||||
if ((lc)) \
|
||||
mtx_init((z)->uz_lock, (z)->uz_name, \
|
||||
(z)->uz_name, MTX_DEF | MTX_DUPOK); \
|
||||
mtx_init(&(k)->uk_lock, (k)->uk_name, \
|
||||
(k)->uk_name, MTX_DEF | MTX_DUPOK); \
|
||||
else \
|
||||
mtx_init((z)->uz_lock, (z)->uz_name, \
|
||||
mtx_init(&(k)->uk_lock, (k)->uk_name, \
|
||||
"UMA zone", MTX_DEF | MTX_DUPOK); \
|
||||
} while (0)
|
||||
|
||||
#define ZONE_LOCK_FINI(z) mtx_destroy((z)->uz_lock)
|
||||
#define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock)
|
||||
#define KEG_LOCK(k) mtx_lock(&(k)->uk_lock)
|
||||
#define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock)
|
||||
#define ZONE_LOCK(z) mtx_lock((z)->uz_lock)
|
||||
#define ZONE_UNLOCK(z) mtx_unlock((z)->uz_lock)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user