d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

o Move zone limit from keg level up to zone level. This means that now

Browse Source 
two zones sharing a keg may have different limits. Now this is going
  to work:

  zone = uma_zcreate();
  uma_zone_set_max(zone, limit);
  zone2 = uma_zsecond_create(zone);
  uma_zone_set_max(zone2, limit2);

  Kegs no longer have uk_maxpages field, but zones have uz_items. When
  set, it may be rounded up to minimum possible CPU bucket cache size.
  For small limits bucket cache can also be reconfigured to be smaller.
  Counter uz_items is updated whenever items transition from keg to a
  bucket cache or directly to a consumer. If zone has uz_maxitems set and
  it is reached, then we are going to sleep.

o Since new limits don't play well with multi-keg zones, remove them. The
  idea of multi-keg zones was introduced exactly 10 years ago, and never
  have had a practical usage. In discussion with Jeff we came to a wild
  agreement that if we ever want to reintroduce the idea of a smart allocator
  that would be able to choose between two (or more) totally different
  backing stores, that choice should be made one level higher than UMA,
  e.g. in malloc(9) or in mget(), or whatever and choice should be controlled
  by the caller.

o Sleeping code is improved to account number of sleepers and wake them one
  by one, to avoid thundering herd problem.

o Flag UMA_ZONE_NOBUCKETCACHE removed, instead uma_zone_set_maxcache()
  KPI added. Having no bucket cache basically means setting maxcache to 0.

o Now with many fields added and many removed (no multi-keg zones!) make
  sure that struct uma_zone is perfectly aligned.

Reviewed by:	markj, jeff
Tested by:	pho
Differential Revision:	https://reviews.freebsd.org/D17773
This commit is contained in:
glebius 2019-01-15 00:02:06 +00:00
parent e1660f9ead
commit f1a8621cf2
5 changed files with 264 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/libmemstat/memstat_uma.c
View File

@ -448,12 +448,7 @@ memstat_kvm_uma(struct memory_type_list *list, void *kvm_handle)
mtp->mt_memalloced = mtp->mt_numallocs * mtp->mt_size;
mtp->mt_memfreed = mtp->mt_numfrees * mtp->mt_size;
mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
if (kz.uk_ppera > 1)
mtp->mt_countlimit = kz.uk_maxpages /
kz.uk_ipers;
else
mtp->mt_countlimit = kz.uk_maxpages *
kz.uk_ipers;
mtp->mt_countlimit = uz.uz_max_items;
mtp->mt_byteslimit = mtp->mt_countlimit * mtp->mt_size;
mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
for (i = 0; i < ndomains; i++) {

27
sys/vm/uma.h
View File

@ -217,17 +217,6 @@ uma_zone_t uma_zcreate(const char *name, size_t size, uma_ctor ctor,
uma_zone_t uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
uma_init zinit, uma_fini zfini, uma_zone_t master);
/*
* Add a second master to a secondary zone. This provides multiple data
* backends for objects with the same size. Both masters must have
* compatible allocation flags. Presently, UMA_ZONE_MALLOC type zones are
* the only supported.
*
* Returns:
* Error on failure, 0 on success.
*/
int uma_zsecond_add(uma_zone_t zone, uma_zone_t master);
/*
* Create cache-only zones.
*
@ -285,10 +274,6 @@ uma_zone_t uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
* NUMA aware Zone. Implements a best
* effort first-touch policy.
*/
#define UMA_ZONE_NOBUCKETCACHE 0x20000 /*
* Don't cache full buckets. Limit
* UMA to per-cpu state.
*/
/*
* These flags are shared between the keg and zone. In zones wishing to add
@ -511,6 +496,18 @@ int uma_zone_reserve_kva(uma_zone_t zone, int nitems);
*/
int uma_zone_set_max(uma_zone_t zone, int nitems);
/*
* Sets a high limit on the number of items allowed in zone's bucket cache
*
* Arguments:
* zone The zone to limit
* nitems The requested upper limit on the number of items allowed
*
* Returns:
* int The effective value of nitems set
*/
int uma_zone_set_maxcache(uma_zone_t zone, int nitems);
/*
* Obtains the effective limit on the number of items in a zone
*

580
sys/vm/uma_core.c
View File

File diff suppressed because it is too large Load Diff

66
sys/vm/uma_int.h
View File

@ -231,7 +231,9 @@ typedef struct uma_domain * uma_domain_t;
*
*/
struct uma_keg {
struct mtx uk_lock; /* Lock for the keg */
struct mtx uk_lock; /* Lock for the keg must be first.
* See shared uz_keg/uz_lockptr
* member of struct uma_zone. */
struct uma_hash uk_hash;
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
@ -242,7 +244,6 @@ struct uma_keg {
uint32_t uk_reserve; /* Number of reserved items. */
uint32_t uk_size; /* Requested size of each item */
uint32_t uk_rsize; /* Real size of each item */
uint32_t uk_maxpages; /* Maximum number of pages to alloc */
uma_init uk_init; /* Keg's init routine */
uma_fini uk_fini; /* Keg's fini routine */
@ -304,12 +305,6 @@ struct uma_slab {
typedef struct uma_slab * uma_slab_t;
typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int, int);
struct uma_klink {
LIST_ENTRY(uma_klink) kl_link;
uma_keg_t kl_keg;
};
typedef struct uma_klink *uma_klink_t;
struct uma_zone_domain {
LIST_HEAD(,uma_bucket) uzd_buckets; /* full buckets */
long uzd_nitems; /* total item count */
@ -328,26 +323,30 @@ typedef struct uma_zone_domain * uma_zone_domain_t;
*/
struct uma_zone {
/* Offset 0, used in alloc/free fast/medium fast path and const. */
struct mtx *uz_lockptr;
const char *uz_name; /* Text name of the zone */
union {
uma_keg_t uz_keg; /* This zone's keg */
struct mtx *uz_lockptr; /* To keg or to self */
};
struct uma_zone_domain *uz_domain; /* per-domain buckets */
uint32_t uz_flags; /* Flags inherited from kegs */
uint32_t uz_size; /* Size inherited from kegs */
uma_ctor uz_ctor; /* Constructor for each allocation */
uma_dtor uz_dtor; /* Destructor */
uma_init uz_init; /* Initializer for each item */
uma_fini uz_fini; /* Finalizer for each item. */
uint64_t uz_items; /* Total items count */
uint64_t uz_max_items; /* Maximum number of items to alloc */
uint32_t uz_sleepers; /* Number of sleepers on memory */
uint16_t uz_count; /* Amount of items in full bucket */
uint16_t uz_count_max; /* Maximum amount of items there */
/* Offset 64, used in bucket replenish. */
uma_import uz_import; /* Import new memory to cache. */
uma_release uz_release; /* Release memory from cache. */
void *uz_arg; /* Import/release argument. */
uma_init uz_init; /* Initializer for each item */
uma_fini uz_fini; /* Finalizer for each item. */
uma_slaballoc uz_slab; /* Allocate a slab from the backend. */
uint16_t uz_count; /* Amount of items in full bucket */
uint16_t uz_count_min; /* Minimal amount of items there */
/* 32bit pad on 64bit. */
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */
uint64_t uz_bkt_count; /* Items in bucket cache */
uint64_t uz_bkt_max; /* Maximum bucket cache size */
/* Offset 128 Rare. */
/*
@ -356,19 +355,19 @@ struct uma_zone {
* members to reduce alignment overhead.
*/
struct mtx uz_lock; /* Lock for the zone */
struct uma_klink uz_klink; /* klink for first keg. */
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
const char *uz_name; /* Text name of the zone */
/* The next two fields are used to print a rate-limited warnings. */
const char *uz_warning; /* Warning to print on failure */
struct timeval uz_ratecheck; /* Warnings rate-limiting */
struct task uz_maxaction; /* Task to run when at limit */
uint16_t uz_count_min; /* Minimal amount of items in bucket */
/* 16 bytes of pad. */
/* Offset 256, atomic stats. */
volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */
volatile u_long uz_fails; /* Total number of alloc failures */
volatile u_long uz_frees; /* Total number of frees */
/* Offset 256, stats. */
uint64_t uz_allocs UMA_ALIGN; /* Total number of allocations */
uint64_t uz_sleeps; /* Total number of alloc sleeps */
uint64_t uz_frees; /* Total number of frees */
volatile u_long uz_fails; /* Total number of alloc failures */
/*
* This HAS to be the last item because we adjust the zone size
@ -382,25 +381,15 @@ struct uma_zone {
/*
* These flags must not overlap with the UMA_ZONE flags specified in uma.h.
*/
#define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */
#define UMA_ZFLAG_CACHE 0x04000000 /* uma_zcache_create()d it */
#define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */
#define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */
#define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */
#define UMA_ZFLAG_FULL 0x40000000 /* Reached uz_maxpages */
#define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */
#define UMA_ZFLAG_INHERIT \
(UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | UMA_ZFLAG_BUCKET)
static inline uma_keg_t
zone_first_keg(uma_zone_t zone)
{
uma_klink_t klink;
klink = LIST_FIRST(&zone->uz_kegs);
return (klink != NULL) ? klink->kl_keg : NULL;
}
#undef UMA_ALIGN
#ifdef _KERNEL
@ -425,6 +414,13 @@ void uma_large_free(uma_slab_t slab);
#define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock)
#define KEG_LOCK(k) mtx_lock(&(k)->uk_lock)
#define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock)
#define KEG_LOCK_ASSERT(k) mtx_assert(&(k)->uk_lock, MA_OWNED)
#define KEG_GET(zone, keg) do { \
(keg) = (zone)->uz_keg; \
KASSERT((void *)(keg) != (void *)&(zone)->uz_lock, \
("%s: Invalid zone %p type", __func__, (zone))); \
} while (0)
#define ZONE_LOCK_INIT(z, lc) \
do { \

3
sys/vm/vm_page.c
View File

@ -222,7 +222,8 @@ vm_page_init_cache_zones(void *dummy __unused)
vmd->vmd_pgcache = uma_zcache_create("vm pgcache",
sizeof(struct vm_page), NULL, NULL, NULL, NULL,
vm_page_import, vm_page_release, vmd,
UMA_ZONE_NOBUCKETCACHE | UMA_ZONE_MAXBUCKET | UMA_ZONE_VM);
UMA_ZONE_MAXBUCKET | UMA_ZONE_VM);
(void )uma_zone_set_maxcache(vmd->vmd_pgcache, 0);
}
}
SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL);