From cae33c14291339048a4b1365bb3fb45d82ef68b0 Mon Sep 17 00:00:00 2001
From: Jeff Roberson <jeff@FreeBSD.org>
Date: Fri, 19 Sep 2003 06:26:45 +0000
Subject: [PATCH]  - Initialize a pool of bucket zones so that we waste less
 space on zones that    don't cache as many items.  - Introduce the
 bucket_alloc(), bucket_free() functions to wrap bucket    allocation.  These
 functions select the appropriate bucket zone to    allocate from or free to. 
 - Rename ub_ptr to ub_cnt to reflect a change in its use.  ub_cnt now
 reflects    the count of free items in the bucket.  This gets rid of many
 unnatural    subtractions by 1 throughout the code.  - Add ub_entries which
 reflects the number of entries possibly held in a    bucket.

---
 sys/vm/uma_core.c | 214 ++++++++++++++++++++++++++++++----------------
 sys/vm/uma_int.h  |  13 +--
 2 files changed, 142 insertions(+), 85 deletions(-)

diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index cc3884fa25d4..225f15cc7932 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -100,11 +100,6 @@ static uma_zone_t slabzone;
  */
 static uma_zone_t hashzone;
 
-/*
- * Zone that buckets come from.
- */
-static uma_zone_t bucketzone;
-
 /*
  * Are we allowed to allocate buckets?
  */
@@ -150,6 +145,27 @@ struct uma_zctor_args {
 	u_int16_t flags;
 };
 
+struct uma_bucket_zone {
+	uma_zone_t	ubz_zone;
+	char		*ubz_name;
+	int		ubz_entries;
+};
+
+#define	BUCKET_MAX	128
+
+struct uma_bucket_zone bucket_zones[] = {
+	{ NULL, "16 Bucket", 16 },
+	{ NULL, "32 Bucket", 32 },
+	{ NULL, "64 Bucket", 64 },
+	{ NULL, "128 Bucket", 128 },
+	{ NULL, NULL, 0}
+};
+
+#define	BUCKET_SHIFT	4
+#define	BUCKET_ZONES	((BUCKET_MAX >> BUCKET_SHIFT) + 1)
+
+uint8_t bucket_size[BUCKET_ZONES];
+
 /* Prototypes.. */
 
 static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
@@ -174,6 +190,10 @@ static void uma_startup3(void);
 static void *uma_zalloc_internal(uma_zone_t, void *, int);
 static void uma_zfree_internal(uma_zone_t, void *, void *, int);
 static void bucket_enable(void);
+static void bucket_init(void);
+static uma_bucket_t bucket_alloc(int, int);
+static void bucket_free(uma_bucket_t);
+static void bucket_zone_drain(void);
 static int uma_zalloc_bucket(uma_zone_t zone, int flags);
 static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
 static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);
@@ -200,6 +220,76 @@ bucket_enable(void)
 		bucketdisable = 0;
 }
 
+static void
+bucket_init(void)
+{
+	struct uma_bucket_zone *ubz;
+	int i;
+	int j;
+
+	for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) {
+		int size;
+
+		ubz = &bucket_zones[j];
+		size = roundup(sizeof(struct uma_bucket), sizeof(void *));
+		size += sizeof(void *) * ubz->ubz_entries;
+		ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
+	    	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_INTERNAL);
+		for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT))
+			bucket_size[i >> BUCKET_SHIFT] = j;
+	}
+}
+
+static uma_bucket_t
+bucket_alloc(int entries, int bflags)
+{
+	struct uma_bucket_zone *ubz;
+	uma_bucket_t bucket;
+	int idx;
+
+	/*
+	 * This is to stop us from allocating per cpu buckets while we're
+	 * running out of UMA_BOOT_PAGES.  Otherwise, we would exhaust the
+	 * boot pages.  This also prevents us from allocating buckets in
+	 * low memory situations.
+	 */
+
+	if (bucketdisable)
+		return (NULL);
+	idx = howmany(entries, 1 << BUCKET_SHIFT);
+	ubz = &bucket_zones[bucket_size[idx]];
+	bucket = uma_zalloc_internal(ubz->ubz_zone, NULL, bflags);
+	if (bucket) {
+#ifdef INVARIANTS
+		bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
+#endif
+		bucket->ub_cnt = 0;
+		bucket->ub_entries = ubz->ubz_entries;
+	}
+
+	return (bucket);
+}
+
+static void
+bucket_free(uma_bucket_t bucket)
+{
+	struct uma_bucket_zone *ubz;
+	int idx;
+
+	idx = howmany(bucket->ub_entries, 1 << BUCKET_SHIFT);
+	ubz = &bucket_zones[bucket_size[idx]];
+	uma_zfree_internal(ubz->ubz_zone, bucket, NULL, 0);
+}
+
+static void
+bucket_zone_drain(void)
+{
+	struct uma_bucket_zone *ubz;
+
+	for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
+		zone_drain(ubz->ubz_zone);
+}
+
 
 /*
  * Routine called by timeout which is used to fire off some time interval
@@ -452,14 +542,14 @@ bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
 	if (zone->uz_flags & UMA_ZFLAG_MALLOC)
 		mzone = 1;
 
-	while (bucket->ub_ptr > -1)  {
-		item = bucket->ub_bucket[bucket->ub_ptr];
+	while (bucket->ub_cnt > 0)  {
+		bucket->ub_cnt--;
+		item = bucket->ub_bucket[bucket->ub_cnt];
 #ifdef INVARIANTS
-		bucket->ub_bucket[bucket->ub_ptr] = NULL;
+		bucket->ub_bucket[bucket->ub_cnt] = NULL;
 		KASSERT(item != NULL,
 		    ("bucket_drain: botched ptr, item is NULL"));
 #endif
-		bucket->ub_ptr--;
 		/* 
 		 * This is extremely inefficient.  The slab pointer was passed
 		 * to uma_zfree_arg, but we lost it because the buckets don't
@@ -514,11 +604,9 @@ cache_drain(uma_zone_t zone, int destroy)
 		bucket_drain(zone, cache->uc_freebucket);
 		if (destroy) {
 			if (cache->uc_allocbucket != NULL)
-				uma_zfree_internal(bucketzone,
-				    cache->uc_allocbucket, NULL, 0);
+				bucket_free(cache->uc_allocbucket);
 			if (cache->uc_freebucket != NULL)
-				uma_zfree_internal(bucketzone,
-				    cache->uc_freebucket, NULL, 0);
+				bucket_free(cache->uc_freebucket);
 			cache->uc_allocbucket = cache->uc_freebucket = NULL;
 		}
 	}
@@ -532,14 +620,14 @@ cache_drain(uma_zone_t zone, int destroy)
 		LIST_REMOVE(bucket, ub_link);
 		ZONE_UNLOCK(zone);
 		bucket_drain(zone, bucket);
-		uma_zfree_internal(bucketzone, bucket, NULL, 0);
+		bucket_free(bucket);
 		ZONE_LOCK(zone);
 	}
 
 	/* Now we do the free queue.. */
 	while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
 		LIST_REMOVE(bucket, ub_link);
-		uma_zfree_internal(bucketzone, bucket, NULL, 0);
+		bucket_free(bucket);
 	}
 
 	/* We unlock here, but they will all block until the zone is unlocked */
@@ -659,8 +747,7 @@ finished:
 
 }
 
-static __inline
-void
+static __inline void
 zone_drain(uma_zone_t zone)
 {
 	zone_drain_common(zone, 0);
@@ -1132,10 +1219,10 @@ zone_ctor(void *mem, int size, void *udata)
 	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
 		return;
 
-	if (zone->uz_ipers < UMA_BUCKET_SIZE)
-		zone->uz_count = zone->uz_ipers - 1;
+	if (zone->uz_ipers <= BUCKET_MAX)
+		zone->uz_count = zone->uz_ipers;
 	else
-		zone->uz_count = UMA_BUCKET_SIZE - 1;
+		zone->uz_count = BUCKET_MAX;
 }
 
 /* 
@@ -1266,9 +1353,7 @@ uma_startup(void *bootmem)
 	    NULL, NULL, NULL, NULL,
 	    UMA_ALIGN_PTR, UMA_ZONE_INTERNAL);
 
-	bucketzone = uma_zcreate("UMA Buckets", sizeof(struct uma_bucket),
-	    NULL, NULL, NULL, NULL,
-	    UMA_ALIGN_PTR, UMA_ZONE_INTERNAL);
+	bucket_init();
 
 #ifdef UMA_MD_SMALL_ALLOC
 	booted = 1;
@@ -1385,12 +1470,12 @@ zalloc_start:
 	bucket = cache->uc_allocbucket;
 
 	if (bucket) {
-		if (bucket->ub_ptr > -1) {
-			item = bucket->ub_bucket[bucket->ub_ptr];
+		if (bucket->ub_cnt > 0) {
+			bucket->ub_cnt--;
+			item = bucket->ub_bucket[bucket->ub_cnt];
 #ifdef INVARIANTS
-			bucket->ub_bucket[bucket->ub_ptr] = NULL;
+			bucket->ub_bucket[bucket->ub_cnt] = NULL;
 #endif
-			bucket->ub_ptr--;
 			KASSERT(item != NULL,
 			    ("uma_zalloc: Bucket pointer mangled."));
 			cache->uc_allocs++;
@@ -1410,7 +1495,7 @@ zalloc_start:
 			 * We have run out of items in our allocbucket.
 			 * See if we can switch with our free bucket.
 			 */
-			if (cache->uc_freebucket->ub_ptr > -1) {
+			if (cache->uc_freebucket->ub_cnt > 0) {
 				uma_bucket_t swap;
 
 #ifdef UMA_DEBUG_ALLOC
@@ -1431,7 +1516,7 @@ zalloc_start:
 
 	/* Our old one is now a free bucket */
 	if (cache->uc_allocbucket) {
-		KASSERT(cache->uc_allocbucket->ub_ptr == -1,
+		KASSERT(cache->uc_allocbucket->ub_cnt == 0,
 		    ("uma_zalloc_arg: Freeing a non free bucket."));
 		LIST_INSERT_HEAD(&zone->uz_free_bucket,
 		    cache->uc_allocbucket, ub_link);
@@ -1440,7 +1525,7 @@ zalloc_start:
 
 	/* Check the free list for a new alloc bucket */
 	if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) {
-		KASSERT(bucket->ub_ptr != -1,
+		KASSERT(bucket->ub_cnt != 0,
 		    ("uma_zalloc_arg: Returning an empty bucket."));
 
 		LIST_REMOVE(bucket, ub_link);
@@ -1452,7 +1537,7 @@ zalloc_start:
 	CPU_UNLOCK(cpu);
 
 	/* Bump up our uz_count so we get here less */
-	if (zone->uz_count < UMA_BUCKET_SIZE - 1)
+	if (zone->uz_count < BUCKET_MAX)
 		zone->uz_count++;
 
 	/*
@@ -1489,7 +1574,7 @@ uma_zone_slab(uma_zone_t zone, int flags)
 	 * things happen.  So instead we return a NULL bucket, and make
 	 * the code that allocates buckets smart enough to deal with it
 	 */ 
-	if (zone == bucketzone && zone->uz_recurse != 0)
+	if (zone->uz_flags & UMA_ZFLAG_INTERNAL && zone->uz_recurse != 0)
 		return (NULL);
 
 	slab = NULL;
@@ -1586,26 +1671,19 @@ uma_zalloc_bucket(uma_zone_t zone, int flags)
 	 */
 
 	if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
-		KASSERT(bucket->ub_ptr == -1,
+		KASSERT(bucket->ub_cnt == 0,
 		    ("uma_zalloc_bucket: Bucket on free list is not empty."));
 		LIST_REMOVE(bucket, ub_link);
 	} else {
 		int bflags;
 
-		bflags = flags;
+		bflags = (flags & ~M_ZERO);
 		if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
 			bflags |= M_NOVM;
 
 		ZONE_UNLOCK(zone);
-		bucket = uma_zalloc_internal(bucketzone,
-		    NULL, bflags);
+		bucket = bucket_alloc(zone->uz_count, bflags);
 		ZONE_LOCK(zone);
-		if (bucket != NULL) {
-#ifdef INVARIANTS
-			bzero(bucket, bucketzone->uz_size);
-#endif
-			bucket->ub_ptr = -1;
-		}
 	}
 
 	if (bucket == NULL)
@@ -1624,11 +1702,11 @@ uma_zalloc_bucket(uma_zone_t zone, int flags)
 	zone->uz_fills++;
 
 	/* Try to keep the buckets totally full */
-	while (bucket->ub_ptr < zone->uz_count &&
+	while (bucket->ub_cnt < bucket->ub_entries &&
 	    (slab = uma_zone_slab(zone, flags)) != NULL) {
 		while (slab->us_freecount &&
-		    bucket->ub_ptr < zone->uz_count) {
-			bucket->ub_bucket[++bucket->ub_ptr] =
+		    bucket->ub_cnt < bucket->ub_entries) {
+			bucket->ub_bucket[bucket->ub_cnt++] =
 			    uma_slab_alloc(zone, slab);
 		}
 		/* Don't block on the next fill */
@@ -1637,7 +1715,7 @@ uma_zalloc_bucket(uma_zone_t zone, int flags)
 
 	zone->uz_fills--;
 
-	if (bucket->ub_ptr != -1) {
+	if (bucket->ub_cnt != 0) {
 		LIST_INSERT_HEAD(&zone->uz_full_bucket,
 		    bucket, ub_link);
 		return (1);
@@ -1645,7 +1723,7 @@ uma_zalloc_bucket(uma_zone_t zone, int flags)
 #ifdef SMP
 done:
 #endif
-	uma_zfree_internal(bucketzone, bucket, NULL, 0);
+	bucket_free(bucket);
 
 	return (0);
 }
@@ -1670,15 +1748,6 @@ uma_zalloc_internal(uma_zone_t zone, void *udata, int flags)
 
 	item = NULL;
 
-	/*
-	 * This is to stop us from allocating per cpu buckets while we're
-	 * running out of UMA_BOOT_PAGES.  Otherwise, we would exhaust the
-	 * boot pages.
-	 */
-
-	if (bucketdisable && zone == bucketzone)
-		return (NULL);
-
 #ifdef UMA_DEBUG_ALLOC
 	printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
 #endif
@@ -1744,11 +1813,11 @@ zfree_start:
 		 * check to be slightly out of sync.
 		 */
 
-		if (bucket->ub_ptr < zone->uz_count) {
-			bucket->ub_ptr++;
-			KASSERT(bucket->ub_bucket[bucket->ub_ptr] == NULL,
+		if (bucket->ub_cnt < bucket->ub_entries) {
+			KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
 			    ("uma_zfree: Freeing to non free bucket index."));
-			bucket->ub_bucket[bucket->ub_ptr] = item;
+			bucket->ub_bucket[bucket->ub_cnt] = item;
+			bucket->ub_cnt++;
 #ifdef INVARIANTS
 			ZONE_LOCK(zone);
 			if (zone->uz_flags & UMA_ZFLAG_MALLOC)
@@ -1767,8 +1836,8 @@ zfree_start:
 			 * We have run out of space in our freebucket.
 			 * See if we can switch with our alloc bucket.
 			 */
-			if (cache->uc_allocbucket->ub_ptr < 
-			    cache->uc_freebucket->ub_ptr) {
+			if (cache->uc_allocbucket->ub_cnt < 
+			    cache->uc_freebucket->ub_cnt) {
 				uma_bucket_t swap;
 
 				swap = cache->uc_freebucket;
@@ -1798,8 +1867,8 @@ zfree_start:
 #ifdef UMA_DEBUG_ALLOC
 		printf("uma_zfree: Putting old bucket on the free list.\n");
 #endif
-		/* ub_ptr is pointing to the last free item */
-		KASSERT(bucket->ub_ptr != -1,
+		/* ub_cnt is pointing to the last free item */
+		KASSERT(bucket->ub_cnt != 0,
 		    ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
 		LIST_INSERT_HEAD(&zone->uz_full_bucket,
 		    bucket, ub_link);
@@ -1823,13 +1892,8 @@ zfree_start:
 
 	if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
 		bflags |= M_NOVM;
-#ifdef INVARIANTS
-	bflags |= M_ZERO;
-#endif
-	bucket = uma_zalloc_internal(bucketzone,
-	    NULL, bflags);
+	bucket = bucket_alloc(zone->uz_count, bflags);
 	if (bucket) {
-		bucket->ub_ptr = -1;
 		ZONE_LOCK(zone);
 		LIST_INSERT_HEAD(&zone->uz_free_bucket,
 		    bucket, ub_link);
@@ -2067,8 +2131,8 @@ uma_reclaim(void)
 	 * we visit again so that we can free pages that are empty once other
 	 * zones are drained.  We have to do the same for buckets.
 	 */
-	zone_drain(slabzone);
-	zone_drain(bucketzone);
+	zone_drain_common(slabzone, 0);
+	bucket_zone_drain();
 }
 
 void *
@@ -2182,14 +2246,14 @@ sysctl_vm_zone(SYSCTL_HANDLER_ARGS)
 					continue;
 				cache = &z->uz_cpu[cpu];
 				if (cache->uc_allocbucket != NULL)
-					cachefree += cache->uc_allocbucket->ub_ptr + 1;
+					cachefree += cache->uc_allocbucket->ub_cnt;
 				if (cache->uc_freebucket != NULL)
-					cachefree += cache->uc_freebucket->ub_ptr + 1;
+					cachefree += cache->uc_freebucket->ub_cnt;
 				CPU_UNLOCK(cpu);
 			}
 		}
 		LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) {
-			cachefree += bucket->ub_ptr + 1;
+			cachefree += bucket->ub_cnt;
 		}
 		totalfree = z->uz_free + cachefree;
 		len = snprintf(offset, linesize,
diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h
index 3554f0fa5b93..f7550a77dbed 100644
--- a/sys/vm/uma_int.h
+++ b/sys/vm/uma_int.h
@@ -173,18 +173,11 @@ struct uma_hash {
  * Structures for per cpu queues.
  */
 
-/*
- * This size was chosen so that the struct bucket size is roughly
- * 128 * sizeof(void *).  This is exactly true for x86, and for alpha
- * it will would be 32bits smaller if it didn't have alignment adjustments.
- */
-
-#define UMA_BUCKET_SIZE	125
-
 struct uma_bucket {
 	LIST_ENTRY(uma_bucket)	ub_link;	/* Link into the zone */
-	int16_t	ub_ptr;				/* Pointer to current item */
-	void	*ub_bucket[UMA_BUCKET_SIZE];	/* actual allocation storage */
+	int16_t	ub_cnt;				/* Count of free items. */
+	int16_t	ub_entries;			/* Max items. */
+	void	*ub_bucket[];			/* actual allocation storage */
 };
 
 typedef struct uma_bucket * uma_bucket_t;