From 44a6c9cd90e20195eea935e0b0ad29ca944406ae Mon Sep 17 00:00:00 2001 From: kmacy Date: Wed, 22 Nov 2006 05:54:24 +0000 Subject: [PATCH] Integrate, but do not enable support for dynamically resizing TSBs --- sys/sun4v/include/pmap.h | 7 +++ sys/sun4v/include/tsb.h | 1 - sys/sun4v/sun4v/pmap.c | 95 ++++++++++++++++++++++++++++++++++++++++ sys/sun4v/sun4v/tsb.c | 12 +++++ 4 files changed, 114 insertions(+), 1 deletion(-) diff --git a/sys/sun4v/include/pmap.h b/sys/sun4v/include/pmap.h index ea40fdd12eff..3115fffedf72 100644 --- a/sys/sun4v/include/pmap.h +++ b/sys/sun4v/include/pmap.h @@ -46,7 +46,13 @@ #include #include +#define TSB_INIT_SHIFT 3 #define PMAP_CONTEXT_MAX 8192 +/* + * We don't want TSBs getting above 1MB - which is enough + * for a working set of 512MB - revisit in the future + */ +#define TSB_MAX_RESIZE (20 - TSB_INIT_SHIFT - PAGE_SHIFT) typedef struct pmap *pmap_t; typedef uint32_t pmap_cpumask_t; @@ -75,6 +81,7 @@ struct pmap { struct pmap_statistics pm_stats; uint32_t pm_tsb_miss_count; uint32_t pm_tsb_cap_miss_count; + vm_paddr_t pm_old_tsb_pa[TSB_MAX_RESIZE]; }; #define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx) diff --git a/sys/sun4v/include/tsb.h b/sys/sun4v/include/tsb.h index 6a247181d3b6..a2937073dd5d 100644 --- a/sys/sun4v/include/tsb.h +++ b/sys/sun4v/include/tsb.h @@ -51,7 +51,6 @@ extern hv_tsb_info_t kernel_td[MAX_TSB_INFO]; struct hv_tsb_info; -#define TSB_INIT_SHIFT 3 void tsb_init(struct hv_tsb_info *tsb, uint64_t *scratchval, uint64_t page_shift); diff --git a/sys/sun4v/sun4v/pmap.c b/sys/sun4v/sun4v/pmap.c index 9372fb8ba213..c91be020f061 100644 --- a/sys/sun4v/sun4v/pmap.c +++ b/sys/sun4v/sun4v/pmap.c @@ -195,10 +195,16 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m); static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); static void pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va); static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot); +static void pmap_tsb_reset(pmap_t pmap); +static void pmap_tsb_resize(pmap_t pmap); static void pmap_tte_hash_resize(pmap_t pmap); void pmap_set_ctx_panic(uint64_t error, vm_paddr_t tsb_ra, pmap_t pmap); +struct tsb_resize_info { + uint64_t tri_tsbscratch; + uint64_t tri_tsb_ra; +}; /* * Quick sort callout for comparing memory regions. @@ -409,6 +415,8 @@ pmap_activate(struct thread *td) pmap->pm_hashscratch = tte_hash_set_scratchpad_user(pmap->pm_hash, pmap->pm_context); pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb); + pmap->pm_tsb_miss_count = pmap->pm_tsb_cap_miss_count = 0; + PCPU_SET(curpmap, pmap); if (pmap->pm_context != 0) if ((err = hv_set_ctxnon0(1, pmap->pm_tsb_ra)) != H_EOK) @@ -1056,6 +1064,13 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, pmap->pm_tsb_cap_miss_count = 0; } #endif + /* + * 512 is an arbitrary number of tsb misses + */ + if (0 && pmap->pm_context != 0 && pmap->pm_tsb_miss_count > 512) + pmap_tsb_resize(pmap); + + vm_page_unlock_queues(); @@ -1624,6 +1639,7 @@ pmap_pinit0(pmap_t pmap) void pmap_pinit(pmap_t pmap) { + int i; pmap->pm_context = get_context(); pmap->pm_tsb_ra = vtophys(&pmap->pm_tsb); @@ -1632,7 +1648,11 @@ pmap_pinit(pmap_t pmap) pmap->pm_hash = tte_hash_create(pmap->pm_context, &pmap->pm_hashscratch); tsb_init(&pmap->pm_tsb, &pmap->pm_tsbscratch, TSB_INIT_SHIFT); vm_page_unlock_queues(); + pmap->pm_tsb_miss_count = pmap->pm_tsb_cap_miss_count = 0; pmap->pm_active = pmap->pm_tlbactive = 0; + for (i = 0; i < TSB_MAX_RESIZE; i++) + pmap->pm_old_tsb_pa[i] = 0; + TAILQ_INIT(&pmap->pm_pvlist); PMAP_LOCK_INIT(pmap); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1925,6 +1945,9 @@ pmap_remove_pages(pmap_t pmap) free_pv_entry(pv); } pmap->pm_hash = tte_hash_reset(pmap->pm_hash, &pmap->pm_hashscratch); + if (0) + pmap_tsb_reset(pmap); + vm_page_unlock_queues(); pmap_invalidate_all(pmap); @@ -1932,6 +1955,27 @@ pmap_remove_pages(pmap_t pmap) PMAP_UNLOCK(pmap); } +static void +pmap_tsb_reset(pmap_t pmap) +{ + int i; + + for (i = 1; i < TSB_MAX_RESIZE && pmap->pm_old_tsb_pa[i]; i++) { + pmap_free_contig_pages((void *)TLB_PHYS_TO_DIRECT(pmap->pm_old_tsb_pa[i]), + (1 << (TSB_INIT_SHIFT + i))); + pmap->pm_old_tsb_pa[i] = 0; + } + if (pmap->pm_old_tsb_pa[0] != 0) { + vm_paddr_t tsb_pa = pmap->pm_tsb.hvtsb_pa; + int size = tsb_size(&pmap->pm_tsb); + pmap->pm_tsb.hvtsb_ntte = (1 << (TSB_INIT_SHIFT + PAGE_SHIFT - TTE_SHIFT)); + pmap->pm_tsb.hvtsb_pa = pmap->pm_old_tsb_pa[0]; + pmap_free_contig_pages((void *)TLB_PHYS_TO_DIRECT(tsb_pa), size); + pmap->pm_tsbscratch = pmap->pm_tsb.hvtsb_pa | (uint64_t)TSB_INIT_SHIFT; + pmap->pm_old_tsb_pa[0] = 0; + } +} + void pmap_scrub_pages(vm_paddr_t pa, int64_t size) { @@ -1970,6 +2014,57 @@ pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va) } } +/* resize the tsb if the number of capacity misses is greater than 1/4 of + * the total + */ +static void +pmap_tsb_resize(pmap_t pmap) +{ + uint32_t miss_count; + uint32_t cap_miss_count; + struct tsb_resize_info info; + hv_tsb_info_t hvtsb; + uint64_t tsbscratch; + + KASSERT(pmap == PCPU_GET(curpmap), ("operating on non-current pmap")); + miss_count = pmap->pm_tsb_miss_count; + cap_miss_count = pmap->pm_tsb_cap_miss_count; + int npages_shift = tsb_page_shift(pmap); + + if (npages_shift < (TSB_INIT_SHIFT + TSB_MAX_RESIZE) && + cap_miss_count > (miss_count >> 1)) { + DPRINTF("resizing tsb for proc=%s pid=%d\n", + curthread->td_proc->p_comm, curthread->td_proc->p_pid); + pmap->pm_old_tsb_pa[npages_shift - TSB_INIT_SHIFT] = pmap->pm_tsb.hvtsb_pa; + + /* double TSB size */ + tsb_init(&hvtsb, &tsbscratch, npages_shift + 1); +#ifdef SMP + spinlock_enter(); + /* reset tsb */ + bcopy(&hvtsb, &pmap->pm_tsb, sizeof(hv_tsb_info_t)); + pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb); + + if (hv_set_ctxnon0(1, pmap->pm_tsb_ra) != H_EOK) + panic("failed to set TSB 0x%lx - context == %ld\n", + pmap->pm_tsb_ra, pmap->pm_context); + info.tri_tsbscratch = pmap->pm_tsbscratch; + info.tri_tsb_ra = pmap->pm_tsb_ra; + pmap_ipi(pmap, tl_tsbupdate, pmap->pm_context, vtophys(&info)); + pmap->pm_tlbactive = pmap->pm_active; + spinlock_exit(); +#else + bcopy(&hvtsb, &pmap->pm_tsb, sizeof(hvtsb)); + if (hv_set_ctxnon0(1, pmap->pm_tsb_ra) != H_EOK) + panic("failed to set TSB 0x%lx - context == %ld\n", + pmap->pm_tsb_ra, pmap->pm_context); + pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb); +#endif + } + pmap->pm_tsb_miss_count = 0; + pmap->pm_tsb_cap_miss_count = 0; +} + static void pmap_tte_hash_resize(pmap_t pmap) { diff --git a/sys/sun4v/sun4v/tsb.c b/sys/sun4v/sun4v/tsb.c index 1fc07adfe36b..15824bb2be76 100644 --- a/sys/sun4v/sun4v/tsb.c +++ b/sys/sun4v/sun4v/tsb.c @@ -273,3 +273,15 @@ tsb_set_scratchpad_user(hv_tsb_info_t *tsb) membar(Sync); return tsb_scratch; } + +int +tsb_size(hv_tsb_info_t *hvtsb) +{ + return (hvtsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT)); +} + +int +tsb_page_shift(pmap_t pmap) +{ + return (pmap->pm_tsbscratch & PAGE_MASK); +}