d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sort uma_zone fields according to 64 byte cache line with adjacent line

Browse Source 
prefetch on 64bit architectures.  Prior to this, two lines were needed
for the fast path and each line may fetch an unused adjacent neighbor.
 - Move fields used by the fast path into a single line.
 - Move constants into the adjacent line which is mostly used for
   the spare bucket alloc 'medium path'.
 - Unpad the mtx which is only used by the fast path and place it in
   a line with rarely used data.  This aligns the cachelines better and
   eliminates 128 bytes of wasted space.

This gives a 45% improvement on a will-it-scale test on a 24 core machine.

Reviewed by:	mmacy
This commit is contained in:
Jeff Roberson 2018-06-23 08:10:09 +00:00
parent 0bcfb47363
commit 63b5557b2f
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=335579
1 changed files with 29 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
sys/vm/uma_int.h
View File

@ -222,9 +222,8 @@ typedef struct uma_domain * uma_domain_t;
*
*/
struct uma_keg {
struct mtx_padalign uk_lock; /* Lock for the keg */
struct mtx uk_lock; /* Lock for the keg */
struct uma_hash uk_hash;
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
uint32_t uk_cursor; /* Domain alloc cursor. */
@ -315,40 +314,48 @@ typedef struct uma_zone_domain * uma_zone_domain_t;
*
*/
struct uma_zone {
struct mtx_padalign uz_lock; /* Lock for the zone */
struct mtx_padalign *uz_lockptr;
const char *uz_name; /* Text name of the zone */
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
/* Offset 0, used in alloc/free fast/medium fast path and const. */
struct mtx *uz_lockptr;
const char *uz_name; /* Text name of the zone */
struct uma_zone_domain *uz_domain; /* per-domain buckets */
LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */
struct uma_klink uz_klink; /* klink for first keg. */
uma_slaballoc uz_slab; /* Allocate a slab from the backend. */
uint32_t uz_flags; /* Flags inherited from kegs */
uint32_t uz_size; /* Size inherited from kegs */
uma_ctor uz_ctor; /* Constructor for each allocation */
uma_dtor uz_dtor; /* Destructor */
uma_init uz_init; /* Initializer for each item */
uma_fini uz_fini; /* Finalizer for each item. */
/* Offset 64, used in bucket replenish. */
uma_import uz_import; /* Import new memory to cache. */
uma_release uz_release; /* Release memory from cache. */
void *uz_arg; /* Import/release argument. */
uma_slaballoc uz_slab; /* Allocate a slab from the backend. */
uint16_t uz_count; /* Amount of items in full bucket */
uint16_t uz_count_min; /* Minimal amount of items there */
/* 32bit pad on 64bit. */
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */
uint32_t uz_flags; /* Flags inherited from kegs */
uint32_t uz_size; /* Size inherited from kegs */
/* Offset 128 Rare. */
/*
* The lock is placed here to avoid adjacent line prefetcher
* in fast paths and to take up space near infrequently accessed
* members to reduce alignment overhead.
*/
struct mtx uz_lock; /* Lock for the zone */
struct uma_klink uz_klink; /* klink for first keg. */
/* The next two fields are used to print a rate-limited warnings. */
const char *uz_warning; /* Warning to print on failure */
struct timeval uz_ratecheck; /* Warnings rate-limiting */
struct task uz_maxaction; /* Task to run when at limit */
/* 16 bytes of pad. */
/* Offset 256, atomic stats. */
volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */
volatile u_long uz_fails; /* Total number of alloc failures */
volatile u_long uz_frees; /* Total number of frees */
uint64_t uz_sleeps; /* Total number of alloc sleeps */
uint16_t uz_count; /* Amount of items in full bucket */
uint16_t uz_count_min; /* Minimal amount of items there */
/* The next two fields are used to print a rate-limited warnings. */
const char *uz_warning; /* Warning to print on failure */
struct timeval uz_ratecheck; /* Warnings rate-limiting */
struct task uz_maxaction; /* Task to run when at limit */
/*
* This HAS to be the last item because we adjust the zone size