d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use per-domain keg locks. This provides both a lock and separate space

Browse Source 
accounting for each NUMA domain.  Independent keg domain locks are important
with cross-domain frees.  Hashed zones are non-numa and use a single keg
lock to protect the hash table.

Reviewed by:	markj, rlibby
Differential Revision:	https://reviews.freebsd.org/D22829
This commit is contained in:
Jeff Roberson 2020-01-04 03:30:08 +00:00
parent 727c691857
commit 8b987a7769
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=356349
3 changed files with 192 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lib/libmemstat/memstat_uma.c
View File

@ -311,10 +311,12 @@ memstat_kvm_uma(struct memory_type_list *list, void *kvm_handle)
LIST_HEAD(, uma_keg) uma_kegs;
struct memory_type *mtp;
struct uma_zone_domain uzd;
struct uma_domain ukd;
struct uma_bucket *ubp, ub;
struct uma_cache *ucp, *ucp_array;
struct uma_zone *uzp, uz;
struct uma_keg *kzp, kz;
uint64_t kegfree;
int hint_dontsearch, i, mp_maxid, ndomains, ret;
char name[MEMTYPE_MAXNAME];
cpuset_t all_cpus;
@ -454,18 +456,29 @@ memstat_kvm_uma(struct memory_type_list *list, void *kvm_handle)
for (i = 0; i < ndomains; i++) {
ret = kread(kvm, &uz.uz_domain[i], &uzd,
sizeof(uzd), 0);
if (ret != 0)
continue;
for (ubp =
TAILQ_FIRST(&uzd.uzd_buckets);
ubp != NULL;
ubp = TAILQ_NEXT(&ub, ub_link)) {
ret = kread(kvm, ubp, &ub,
sizeof(ub), 0);
if (ret != 0)
continue;
mtp->mt_zonefree += ub.ub_cnt;
}
}
if (!((kz.uk_flags & UMA_ZONE_SECONDARY) &&
LIST_FIRST(&kz.uk_zones) != uzp)) {
mtp->mt_kegfree = kz.uk_free;
kegfree = 0;
for (i = 0; i < ndomains; i++) {
ret = kread(kvm, &kzp->uk_domain[i],
&ukd, sizeof(ukd), 0);
if (ret != 0)
kegfree += ukd.ud_free;
}
mtp->mt_kegfree = kegfree;
mtp->mt_free += mtp->mt_kegfree;
}
mtp->mt_free += mtp->mt_zonefree;

284
sys/vm/uma_core.c
View File

@ -740,13 +740,20 @@ static void
zone_timeout(uma_zone_t zone, void *unused)
{
uma_keg_t keg;
u_int slabs;
u_int slabs, pages;
if ((zone->uz_flags & UMA_ZONE_HASH) == 0)
goto update_wss;
keg = zone->uz_keg;
KEG_LOCK(keg);
/*
* Hash zones are non-numa by definition so the first domain
* is the only one present.
*/
KEG_LOCK(keg, 0);
pages = keg->uk_domain[0].ud_pages;
/*
* Expand the keg hash table.
*
@ -754,9 +761,7 @@ zone_timeout(uma_zone_t zone, void *unused)
* What I'm trying to do here is completely reduce collisions. This
* may be a little aggressive. Should I allow for two collisions max?
*/
if (keg->uk_flags & UMA_ZONE_HASH &&
(slabs = keg->uk_pages / keg->uk_ppera) >
keg->uk_hash.uh_hashsize) {
if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) {
struct uma_hash newhash;
struct uma_hash oldhash;
int ret;
@ -767,9 +772,9 @@ zone_timeout(uma_zone_t zone, void *unused)
* I have to do everything in stages and check for
* races.
*/
KEG_UNLOCK(keg);
KEG_UNLOCK(keg, 0);
ret = hash_alloc(&newhash, 1 << fls(slabs));
KEG_LOCK(keg);
KEG_LOCK(keg, 0);
if (ret) {
if (hash_expand(&keg->uk_hash, &newhash)) {
oldhash = keg->uk_hash;
@ -777,12 +782,12 @@ zone_timeout(uma_zone_t zone, void *unused)
} else
oldhash = newhash;
KEG_UNLOCK(keg);
KEG_UNLOCK(keg, 0);
hash_free(&oldhash);
return;
goto update_wss;
}
}
KEG_UNLOCK(keg);
KEG_UNLOCK(keg, 0);
update_wss:
ZONE_LOCK(zone);
@ -1166,7 +1171,7 @@ keg_drain(uma_keg_t keg)
struct slabhead freeslabs = { 0 };
uma_domain_t dom;
uma_slab_t slab, tmp;
int i;
int i, n;
/*
* We don't want to take pages from statically allocated kegs at this
@ -1175,33 +1180,27 @@ keg_drain(uma_keg_t keg)
if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
return;
CTR3(KTR_UMA, "keg_drain %s(%p) free items: %u",
keg->uk_name, keg, keg->uk_free);
KEG_LOCK(keg);
if (keg->uk_free == 0)
goto finished;
for (i = 0; i < vm_ndomains; i++) {
CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u",
keg->uk_name, keg, i, dom->ud_free);
n = 0;
dom = &keg->uk_domain[i];
KEG_LOCK(keg, i);
LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) {
/* We have nowhere to free these to. */
if (slab->us_flags & UMA_SLAB_BOOT)
continue;
LIST_REMOVE(slab, us_link);
keg->uk_pages -= keg->uk_ppera;
keg->uk_free -= keg->uk_ipers;
if (keg->uk_flags & UMA_ZONE_HASH)
UMA_HASH_REMOVE(&keg->uk_hash, slab);
n++;
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&freeslabs, slab, us_link);
}
dom->ud_pages -= n * keg->uk_ppera;
dom->ud_free -= n * keg->uk_ipers;
KEG_UNLOCK(keg, i);
}
finished:
KEG_UNLOCK(keg);
while ((slab = LIST_FIRST(&freeslabs)) != NULL) {
LIST_REMOVE(slab, us_link);
keg_free_slab(keg, slab, keg->uk_ipers);
@ -1257,9 +1256,9 @@ zone_trim(uma_zone_t zone, void *unused)
}
/*
* Allocate a new slab for a keg. This does not insert the slab onto a list.
* The keg should be locked on entry and will be dropped and reacquired on
* return.
* Allocate a new slab for a keg and inserts it into the partial slab list.
* The keg should be unlocked on entry. If the allocation succeeds it will
* be locked on return.
*
* Arguments:
* flags Wait flags for the item initialization routine
@ -1273,6 +1272,7 @@ static uma_slab_t
keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
int aflags)
{
uma_domain_t dom;
uma_alloc allocf;
uma_slab_t slab;
unsigned long size;
@ -1282,10 +1282,8 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
KASSERT(domain >= 0 && domain < vm_ndomains,
("keg_alloc_slab: domain %d out of range", domain));
KEG_LOCK_ASSERT(keg);
allocf = keg->uk_allocf;
KEG_UNLOCK(keg);
allocf = keg->uk_allocf;
slab = NULL;
mem = NULL;
if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
@ -1319,6 +1317,10 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
}
uma_total_inc(size);
/* For HASH zones all pages go to the same uma_domain. */
if ((keg->uk_flags & UMA_ZONE_HASH) != 0)
domain = 0;
/* Point the slab into the allocated memory */
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
slab = (uma_slab_t )(mem + keg->uk_pgoff);
@ -1333,6 +1335,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
slab->us_freecount = keg->uk_ipers;
slab->us_flags = sflags;
slab->us_domain = domain;
BIT_FILL(keg->uk_ipers, &slab->us_free);
#ifdef INVARIANTS
BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg));
@ -1348,7 +1351,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
goto fail;
}
}
KEG_LOCK(keg);
KEG_LOCK(keg, domain);
CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)",
slab, keg->uk_name, keg);
@ -1356,13 +1359,19 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
if (keg->uk_flags & UMA_ZONE_HASH)
UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
keg->uk_pages += keg->uk_ppera;
keg->uk_free += keg->uk_ipers;
/*
* If we got a slab here it's safe to mark it partially used
* and return. We assume that the caller is going to remove
* at least one item.
*/
dom = &keg->uk_domain[domain];
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
dom->ud_pages += keg->uk_ppera;
dom->ud_free += keg->uk_ipers;
return (slab);
fail:
KEG_LOCK(keg);
return (NULL);
}
@ -1875,15 +1884,14 @@ keg_ctor(void *mem, int size, void *udata, int flags)
struct uma_kctor_args *arg = udata;
uma_keg_t keg = mem;
uma_zone_t zone;
int i;
bzero(keg, size);
keg->uk_size = arg->size;
keg->uk_init = arg->uminit;
keg->uk_fini = arg->fini;
keg->uk_align = arg->align;
keg->uk_free = 0;
keg->uk_reserve = 0;
keg->uk_pages = 0;
keg->uk_flags = arg->flags;
keg->uk_slabzone = NULL;
@ -1926,6 +1934,15 @@ keg_ctor(void *mem, int size, void *udata, int flags)
keg_small_init(keg);
}
/*
* Sets all kegs with memory that comes from the page array to a
* first-touch domain policy.
*/
#ifdef UMA_FIRSTTOUCH
if ((keg->uk_flags & UMA_ZONE_HASH) == 0)
keg->uk_flags |= UMA_ZONE_NUMA;
#endif
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
keg->uk_slabzone = slabzone;
@ -1954,9 +1971,10 @@ keg_ctor(void *mem, int size, void *udata, int flags)
keg->uk_freef = page_free;
/*
* Initialize keg's lock
* Initialize keg's locks.
*/
KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
for (i = 0; i < vm_ndomains; i++)
KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS));
/*
* If we're putting the slab header in the actual page we need to
@ -1983,10 +2001,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
if (keg->uk_flags & UMA_ZONE_HASH)
hash_alloc(&keg->uk_hash, 0);
CTR5(KTR_UMA, "keg_ctor %p zone %s(%p) out %d free %d\n",
keg, zone->uz_name, zone,
(keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
keg->uk_free);
CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)\n", keg, zone->uz_name, zone);
LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
@ -2009,6 +2024,7 @@ static void
zone_alloc_sysctl(uma_zone_t zone, void *unused)
{
uma_zone_domain_t zdom;
uma_domain_t dom;
uma_keg_t keg;
struct sysctl_oid *oid, *domainoid;
int domains, i, cnt;
@ -2056,6 +2072,10 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
/*
* keg if present.
*/
if ((zone->uz_flags & UMA_ZONE_HASH) == 0)
domains = vm_ndomains;
else
domains = 1;
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
"keg", CTLFLAG_RD, NULL, "");
keg = zone->uz_keg;
@ -2074,16 +2094,24 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"align", CTLFLAG_RD, &keg->uk_align, 0,
"item alignment mask");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"pages", CTLFLAG_RD, &keg->uk_pages, 0,
"Total pages currently allocated from VM");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"free", CTLFLAG_RD, &keg->uk_free, 0,
"items free in the slab layer");
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"efficiency", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE,
keg, 0, sysctl_handle_uma_slab_efficiency, "I",
"Slab utilization (100 - internal fragmentation %)");
domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(oid),
OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
for (i = 0; i < domains; i++) {
dom = &keg->uk_domain[i];
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD,
NULL, "");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"pages", CTLFLAG_RD, &dom->ud_pages, 0,
"Total pages currently allocated from VM");
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"free", CTLFLAG_RD, &dom->ud_free, 0,
"items free in the slab layer");
}
} else
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"name", CTLFLAG_RD, nokeg, "Keg name");
@ -2114,14 +2142,12 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
"Number of items in the bucket cache");
/*
* Per-domain information.
* Per-domain zone information.
*/
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
domains = vm_ndomains;
else
domains = 1;
domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
if ((zone->uz_flags & UMA_ZONE_NUMA) == 0)
domains = 1;
for (i = 0; i < domains; i++) {
zdom = &zone->uz_domain[i];
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
@ -2359,20 +2385,23 @@ static void
keg_dtor(void *arg, int size, void *udata)
{
uma_keg_t keg;
uint32_t free, pages;
int i;
keg = (uma_keg_t)arg;
KEG_LOCK(keg);
if (keg->uk_free != 0) {
printf("Freed UMA keg (%s) was not empty (%d items). "
" Lost %d pages of memory.\n",
keg->uk_name ? keg->uk_name : "",
keg->uk_free, keg->uk_pages);
free = pages = 0;
for (i = 0; i < vm_ndomains; i++) {
free += keg->uk_domain[i].ud_free;
pages += keg->uk_domain[i].ud_pages;
KEG_LOCK_FINI(keg, i);
}
KEG_UNLOCK(keg);
if (free != 0)
printf("Freed UMA keg (%s) was not empty (%u items). "
" Lost %u pages of memory.\n",
keg->uk_name ? keg->uk_name : "",
free, pages);
hash_free(&keg->uk_hash);
KEG_LOCK_FINI(keg);
}
/*
@ -2659,11 +2688,6 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"",
align, name));
/* Sets all zones to a first-touch domain policy. */
#ifdef UMA_FIRSTTOUCH
flags |= UMA_ZONE_NUMA;
#endif
/* This stuff is essential for the zone ctor */
memset(&args, 0, sizeof(args));
args.name = name;
@ -3135,7 +3159,7 @@ keg_first_slab(uma_keg_t keg, int domain, bool rr)
KASSERT(domain >= 0 && domain < vm_ndomains,
("keg_first_slab: domain %d out of range", domain));
KEG_LOCK_ASSERT(keg);
KEG_LOCK_ASSERT(keg, domain);
slab = NULL;
start = domain;
@ -3156,31 +3180,39 @@ keg_first_slab(uma_keg_t keg, int domain, bool rr)
return (NULL);
}
/*
* Fetch an existing slab from a free or partial list. Returns with the
* keg domain lock held if a slab was found or unlocked if not.
*/
static uma_slab_t
keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags)
{
uma_slab_t slab;
uint32_t reserve;
KEG_LOCK_ASSERT(keg);
/* HASH has a single free list. */
if ((keg->uk_flags & UMA_ZONE_HASH) != 0)
domain = 0;
KEG_LOCK(keg, domain);
reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve;
if (keg->uk_free <= reserve)
if (keg->uk_domain[domain].ud_free <= reserve ||
(slab = keg_first_slab(keg, domain, rr)) == NULL) {
KEG_UNLOCK(keg, domain);
return (NULL);
return (keg_first_slab(keg, domain, rr));
}
return (slab);
}
static uma_slab_t
keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
{
struct vm_domainset_iter di;
uma_domain_t dom;
uma_slab_t slab;
int aflags, domain;
bool rr;
restart:
KEG_LOCK_ASSERT(keg);
/*
* Use the keg's policy if upper layers haven't already specified a
* domain (as happens with first-touch zones).
@ -3211,23 +3243,13 @@ keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
break;
slab = keg_alloc_slab(keg, zone, domain, flags, aflags);
/*
* If we got a slab here it's safe to mark it partially used
* and return. We assume that the caller is going to remove
* at least one item.
*/
if (slab) {
dom = &keg->uk_domain[slab->us_domain];
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
if (slab != NULL)
return (slab);
}
if (!rr && (flags & M_WAITOK) == 0)
break;
if (rr && vm_domainset_iter_policy(&di, &domain) != 0) {
if ((flags & M_WAITOK) != 0) {
KEG_UNLOCK(keg);
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
KEG_LOCK(keg);
goto restart;
}
break;
@ -3239,9 +3261,9 @@ keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
* could have while we were unlocked. Check again before we
* fail.
*/
if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) {
if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL)
return (slab);
}
return (NULL);
}
@ -3252,18 +3274,18 @@ slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
void *item;
uint8_t freei;
KEG_LOCK_ASSERT(keg);
KEG_LOCK_ASSERT(keg, slab->us_domain);
dom = &keg->uk_domain[slab->us_domain];
freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1;
BIT_CLR(keg->uk_ipers, freei, &slab->us_free);
item = slab_item(slab, keg, freei);
slab->us_freecount--;
keg->uk_free--;
dom->ud_free--;
/* Move this slab to the full list */
if (slab->us_freecount == 0) {
LIST_REMOVE(slab, us_link);
dom = &keg->uk_domain[slab->us_domain];
LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link);
}
@ -3273,6 +3295,7 @@ slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
static int
zone_import(void *arg, void **bucket, int max, int domain, int flags)
{
uma_domain_t dom;
uma_zone_t zone;
uma_slab_t slab;
uma_keg_t keg;
@ -3284,7 +3307,6 @@ zone_import(void *arg, void **bucket, int max, int domain, int flags)
zone = arg;
slab = NULL;
keg = zone->uz_keg;
KEG_LOCK(keg);
/* Try to keep the buckets totally full */
for (i = 0; i < max; ) {
if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL)
@ -3292,9 +3314,10 @@ zone_import(void *arg, void **bucket, int max, int domain, int flags)
#ifdef NUMA
stripe = howmany(max, vm_ndomains);
#endif
dom = &keg->uk_domain[slab->us_domain];
while (slab->us_freecount && i < max) {
bucket[i++] = slab_alloc_item(keg, slab);
if (keg->uk_free <= keg->uk_reserve)
if (dom->ud_free <= keg->uk_reserve)
break;
#ifdef NUMA
/*
@ -3310,11 +3333,11 @@ zone_import(void *arg, void **bucket, int max, int domain, int flags)
break;
#endif
}
KEG_UNLOCK(keg, slab->us_domain);
/* Don't block if we allocated any successfully. */
flags &= ~M_WAITOK;
flags |= M_NOWAIT;
}
KEG_UNLOCK(keg);
return i;
}
@ -3866,11 +3889,10 @@ slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item)
uint8_t freei;
keg = zone->uz_keg;
KEG_LOCK_ASSERT(keg);
dom = &keg->uk_domain[slab->us_domain];
KEG_LOCK_ASSERT(keg, slab->us_domain);
/* Do we need to remove from any lists? */
dom = &keg->uk_domain[slab->us_domain];
if (slab->us_freecount+1 == keg->uk_ipers) {
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);
@ -3885,37 +3907,45 @@ slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item)
slab->us_freecount++;
/* Keg statistics. */
keg->uk_free++;
dom->ud_free++;
}
static void
zone_release(void *arg, void **bucket, int cnt)
{
struct mtx *lock;
uma_zone_t zone;
void *item;
uma_slab_t slab;
uma_keg_t keg;
uint8_t *mem;
void *item;
int i;
zone = arg;
keg = zone->uz_keg;
KEG_LOCK(keg);
lock = NULL;
if (__predict_false((zone->uz_flags & UMA_ZONE_HASH) != 0))
lock = KEG_LOCK(keg, 0);
for (i = 0; i < cnt; i++) {
item = bucket[i];
if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
if (zone->uz_flags & UMA_ZONE_HASH) {
slab = hash_sfind(&keg->uk_hash, mem);
} else {
mem += keg->uk_pgoff;
slab = (uma_slab_t)mem;
}
} else
if (__predict_true((zone->uz_flags & UMA_ZONE_VTOSLAB) != 0)) {
slab = vtoslab((vm_offset_t)item);
} else {
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
if ((zone->uz_flags & UMA_ZONE_HASH) != 0)
slab = hash_sfind(&keg->uk_hash, mem);
else
slab = (uma_slab_t)(mem + keg->uk_pgoff);
}
if (lock != KEG_LOCKPTR(keg, slab->us_domain)) {
if (lock != NULL)
mtx_unlock(lock);
lock = KEG_LOCK(keg, slab->us_domain);
}
slab_free_item(zone, slab, item);
}
KEG_UNLOCK(keg);
if (lock != NULL)
mtx_unlock(lock);
}
/*
@ -4203,7 +4233,6 @@ uma_prealloc(uma_zone_t zone, int items)
int aflags, domain, slabs;
KEG_GET(zone, keg);
KEG_LOCK(keg);
slabs = items / keg->uk_ipers;
if (slabs * keg->uk_ipers < items)
slabs++;
@ -4216,18 +4245,16 @@ uma_prealloc(uma_zone_t zone, int items)
aflags);
if (slab != NULL) {
dom = &keg->uk_domain[slab->us_domain];
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&dom->ud_free_slab, slab,
us_link);
KEG_UNLOCK(keg, slab->us_domain);
break;
}
if (vm_domainset_iter_policy(&di, &domain) != 0) {
KEG_UNLOCK(keg);
if (vm_domainset_iter_policy(&di, &domain) != 0)
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
KEG_LOCK(keg);
}
}
}
KEG_UNLOCK(keg);
}
/* See uma.h */
@ -4465,6 +4492,7 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
uma_keg_t kz;
uma_zone_t z;
uint64_t items;
uint32_t kfree, pages;
int count, error, i;
error = sysctl_wire_old_buffer(req, 0);
@ -4494,6 +4522,11 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
(void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
LIST_FOREACH(kz, &uma_kegs, uk_link) {
kfree = pages = 0;
for (i = 0; i < vm_ndomains; i++) {
kfree += kz->uk_domain[i].ud_free;
pages += kz->uk_domain[i].ud_pages;
}
LIST_FOREACH(z, &kz->uk_zones, uz_link) {
bzero(&uth, sizeof(uth));
ZONE_LOCK(z);
@ -4506,11 +4539,11 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
uth.uth_pages = (items / kz->uk_ipers) *
kz->uk_ppera;
} else
uth.uth_pages = kz->uk_pages;
uth.uth_pages = pages;
uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) *
kz->uk_ppera;
uth.uth_limit = z->uz_max_items;
uth.uth_keg_free = z->uz_keg->uk_free;
uth.uth_keg_free = kfree;
/*
* A zone is secondary is it is not the first entry
@ -4670,9 +4703,9 @@ uma_dbg_getslab(uma_zone_t zone, void *item)
keg = zone->uz_keg;
if ((keg->uk_flags & UMA_ZONE_HASH) == 0)
return ((uma_slab_t)(mem + keg->uk_pgoff));
KEG_LOCK(keg);
KEG_LOCK(keg, 0);
slab = hash_sfind(&keg->uk_hash, mem);
KEG_UNLOCK(keg);
KEG_UNLOCK(keg, 0);
return (slab);
}
@ -4791,11 +4824,12 @@ get_uma_stats(uma_keg_t kz, uma_zone_t z, uint64_t *allocs, uint64_t *used,
} else
uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps,
xdomain);
if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
(LIST_FIRST(&kz->uk_zones) != z)))
*cachefree += kz->uk_free;
for (i = 0; i < vm_ndomains; i++)
for (i = 0; i < vm_ndomains; i++) {
*cachefree += z->uz_domain[i].uzd_nitems;
if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
(LIST_FIRST(&kz->uk_zones) != z)))
*cachefree += kz->uk_domain[i].ud_free;
}
*used = *allocs - frees;
return (((int64_t)*used + *cachefree) * kz->uk_size);
}

36
sys/vm/uma_int.h
View File

@ -254,9 +254,12 @@ cache_uz_size(uma_cache_t cache)
* Per-domain slab lists. Embedded in the kegs.
*/
struct uma_domain {
struct mtx_padalign ud_lock; /* Lock for the domain lists. */
struct slabhead ud_part_slab; /* partially allocated slabs */
struct slabhead ud_free_slab; /* completely unallocated slabs */
struct slabhead ud_full_slab; /* fully allocated slabs */
uint32_t ud_pages; /* Total page count */
uint32_t ud_free; /* Count of items free in slabs */
} __aligned(CACHE_LINE_SIZE);
typedef struct uma_domain * uma_domain_t;
@ -268,14 +271,11 @@ typedef struct uma_domain * uma_domain_t;
*
*/
struct uma_keg {
struct mtx uk_lock; /* Lock for the keg. */
struct uma_hash uk_hash;
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
struct domainset_ref uk_dr; /* Domain selection policy. */
uint32_t uk_align; /* Alignment mask */
uint32_t uk_pages; /* Total page count */
uint32_t uk_free; /* Count of items free in slabs */
uint32_t uk_reserve; /* Number of reserved items. */
uint32_t uk_size; /* Requested size of each item */
uint32_t uk_rsize; /* Real size of each item */
@ -305,8 +305,8 @@ typedef struct uma_keg * uma_keg_t;
#ifdef _KERNEL
#define KEG_ASSERT_COLD(k) \
KASSERT((k)->uk_pages == 0, ("keg %s initialization after use.",\
(k)->uk_name))
KASSERT((k)->uk_domain[0].ud_pages == 0, \
("keg %s initialization after use.", (k)->uk_name))
/*
* Free bits per-slab.
@ -536,20 +536,22 @@ static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data);
/* Lock Macros */
#define KEG_LOCK_INIT(k, lc) \
do { \
if ((lc)) \
mtx_init(&(k)->uk_lock, (k)->uk_name, \
(k)->uk_name, MTX_DEF | MTX_DUPOK); \
else \
mtx_init(&(k)->uk_lock, (k)->uk_name, \
"UMA zone", MTX_DEF | MTX_DUPOK); \
#define KEG_LOCKPTR(k, d) (struct mtx *)&(k)->uk_domain[(d)].ud_lock
#define KEG_LOCK_INIT(k, d, lc) \
do { \
if ((lc)) \
mtx_init(KEG_LOCKPTR(k, d), (k)->uk_name, \
(k)->uk_name, MTX_DEF | MTX_DUPOK); \
else \
mtx_init(KEG_LOCKPTR(k, d), (k)->uk_name, \
"UMA zone", MTX_DEF | MTX_DUPOK); \
} while (0)
#define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock)
#define KEG_LOCK(k) mtx_lock(&(k)->uk_lock)
#define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock)
#define KEG_LOCK_ASSERT(k) mtx_assert(&(k)->uk_lock, MA_OWNED)
#define KEG_LOCK_FINI(k, d) mtx_destroy(KEG_LOCKPTR(k, d))
#define KEG_LOCK(k, d) \
({ mtx_lock(KEG_LOCKPTR(k, d)); KEG_LOCKPTR(k, d); })
#define KEG_UNLOCK(k, d) mtx_unlock(KEG_LOCKPTR(k, d))
#define KEG_LOCK_ASSERT(k, d) mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED)
#define KEG_GET(zone, keg) do { \
(keg) = (zone)->uz_keg; \