From e60b2fcbebd29dad02e2cbbd7be0cb92784586c4 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Wed, 3 Feb 2016 23:30:17 +0000 Subject: [PATCH] Redo r292484. Embed task(9) into zone, so that uz_maxaction is called in a context that can sleep, allowing consumers of the KPI to run their drain routines without any extra measures. Discussed with: jtl --- sys/amd64/amd64/uma_machdep.c | 1 + sys/kern/kern_malloc.c | 1 + sys/kern/kern_mbuf.c | 92 ++++------------------------------- sys/vm/memguard.c | 1 + sys/vm/uma.h | 2 +- sys/vm/uma_core.c | 9 ++-- sys/vm/uma_dbg.c | 1 + sys/vm/uma_int.h | 2 +- sys/vm/vm_page.c | 1 + 9 files changed, 22 insertions(+), 88 deletions(-) diff --git a/sys/amd64/amd64/uma_machdep.c b/sys/amd64/amd64/uma_machdep.c index db566ae89d47..fc343cabe52f 100644 --- a/sys/amd64/amd64/uma_machdep.c +++ b/sys/amd64/amd64/uma_machdep.c @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index e7c81d67e936..f7c67f2d7610 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c index 779c62af32bd..55964398b6a1 100644 --- a/sys/kern/kern_mbuf.c +++ b/sys/kern/kern_mbuf.c @@ -273,12 +273,6 @@ uma_zone_t zone_jumbo9; uma_zone_t zone_jumbo16; uma_zone_t zone_ext_refcnt; -/* - * Callout to assist us in freeing mbufs. - */ -static struct callout mb_reclaim_callout; -static struct mtx mb_reclaim_callout_mtx; - /* * Local prototypes. */ @@ -291,9 +285,8 @@ static void mb_dtor_pack(void *, int, void *); static int mb_zinit_pack(void *, int, int); static void mb_zfini_pack(void *, int); -static void mb_reclaim(void *); +static void mb_reclaim(uma_zone_t, int); static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int); -static void mb_maxaction(uma_zone_t); /* Ensure that MSIZE is a power of 2. */ CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); @@ -319,7 +312,7 @@ mbuf_init(void *dummy) if (nmbufs > 0) nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached"); - uma_zone_set_maxaction(zone_mbuf, mb_maxaction); + uma_zone_set_maxaction(zone_mbuf, mb_reclaim); zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, mb_ctor_clust, mb_dtor_clust, @@ -332,7 +325,7 @@ mbuf_init(void *dummy) if (nmbclusters > 0) nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached"); - uma_zone_set_maxaction(zone_clust, mb_maxaction); + uma_zone_set_maxaction(zone_clust, mb_reclaim); zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); @@ -349,7 +342,7 @@ mbuf_init(void *dummy) if (nmbjumbop > 0) nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached"); - uma_zone_set_maxaction(zone_jumbop, mb_maxaction); + uma_zone_set_maxaction(zone_jumbop, mb_reclaim); zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, mb_ctor_clust, mb_dtor_clust, @@ -363,7 +356,7 @@ mbuf_init(void *dummy) if (nmbjumbo9 > 0) nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached"); - uma_zone_set_maxaction(zone_jumbo9, mb_maxaction); + uma_zone_set_maxaction(zone_jumbo9, mb_reclaim); zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, mb_ctor_clust, mb_dtor_clust, @@ -377,20 +370,13 @@ mbuf_init(void *dummy) if (nmbjumbo16 > 0) nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached"); - uma_zone_set_maxaction(zone_jumbo16, mb_maxaction); + uma_zone_set_maxaction(zone_jumbo16, mb_reclaim); zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); - /* uma_prealloc() goes here... */ - - /* Initialize the mb_reclaim() callout. */ - mtx_init(&mb_reclaim_callout_mtx, "mb_reclaim_callout_mtx", NULL, - MTX_DEF); - callout_init(&mb_reclaim_callout, 1); - /* * Hook event handler for low-memory situation, used to * drain protocols and push data back to the caches (UMA @@ -677,81 +663,23 @@ m_pkthdr_init(struct mbuf *m, int how) } /* - * This is the protocol drain routine. + * This is the protocol drain routine. Called by UMA whenever any of the + * mbuf zones is closed to its limit. * * No locks should be held when this is called. The drain routines have to * presently acquire some locks which raises the possibility of lock order * reversal. */ static void -mb_reclaim(void *junk) +mb_reclaim(uma_zone_t zone __unused, int pending __unused) { struct domain *dp; struct protosw *pr; - WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, - "mb_reclaim()"); + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__); for (dp = domains; dp != NULL; dp = dp->dom_next) for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) if (pr->pr_drain != NULL) (*pr->pr_drain)(); } - -/* - * This is the function called by the mb_reclaim_callout, which is - * used when we hit the maximum for a zone. - * - * (See mb_maxaction() below.) - */ -static void -mb_reclaim_timer(void *junk __unused) -{ - - mtx_lock(&mb_reclaim_callout_mtx); - - /* - * Avoid running this function extra times by skipping this invocation - * if the callout has already been rescheduled. - */ - if (callout_pending(&mb_reclaim_callout) || - !callout_active(&mb_reclaim_callout)) { - mtx_unlock(&mb_reclaim_callout_mtx); - return; - } - mtx_unlock(&mb_reclaim_callout_mtx); - - mb_reclaim(NULL); - - mtx_lock(&mb_reclaim_callout_mtx); - callout_deactivate(&mb_reclaim_callout); - mtx_unlock(&mb_reclaim_callout_mtx); -} - -/* - * This function is called when we hit the maximum for a zone. - * - * At that point, we want to call the protocol drain routine to free up some - * mbufs. However, we will use the callout routines to schedule this to - * occur in another thread. (The thread calling this function holds the - * zone lock.) - */ -static void -mb_maxaction(uma_zone_t zone __unused) -{ - - /* - * If we can't immediately obtain the lock, either the callout - * is currently running, or another thread is scheduling the - * callout. - */ - if (!mtx_trylock(&mb_reclaim_callout_mtx)) - return; - - /* If not already scheduled/running, schedule the callout. */ - if (!callout_active(&mb_reclaim_callout)) { - callout_reset(&mb_reclaim_callout, 1, mb_reclaim_timer, NULL); - } - - mtx_unlock(&mb_reclaim_callout_mtx); -} diff --git a/sys/vm/memguard.c b/sys/vm/memguard.c index d4efc2b7123e..fb70a4186c73 100644 --- a/sys/vm/memguard.c +++ b/sys/vm/memguard.c @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include diff --git a/sys/vm/uma.h b/sys/vm/uma.h index d218e60886dc..21c61214315d 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -530,7 +530,7 @@ void uma_zone_set_warning(uma_zone_t zone, const char *warning); * Returns: * Nothing */ -typedef void (*uma_maxaction_t)(uma_zone_t); +typedef void (*uma_maxaction_t)(uma_zone_t, int); void uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t); /* diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 89ed213e82e6..0d45046e7e51 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -439,8 +440,9 @@ zone_log_warning(uma_zone_t zone) static inline void zone_maxaction(uma_zone_t zone) { - if (zone->uz_maxaction) - (*zone->uz_maxaction)(zone); + + if (zone->uz_maxaction.ta_func != NULL) + taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); } static void @@ -1590,7 +1592,6 @@ zone_ctor(void *mem, int size, void *udata, int flags) zone->uz_flags = 0; zone->uz_warning = NULL; timevalclear(&zone->uz_ratecheck); - zone->uz_maxaction = NULL; keg = arg->keg; ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS)); @@ -3027,7 +3028,7 @@ uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) { ZONE_LOCK(zone); - zone->uz_maxaction = maxaction; + TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); ZONE_UNLOCK(zone); } diff --git a/sys/vm/uma_dbg.c b/sys/vm/uma_dbg.c index dcef5c9480cb..7bf06d35b4e2 100644 --- a/sys/vm/uma_dbg.c +++ b/sys/vm/uma_dbg.c @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h index 5d7ecd331a7f..1b0d5d5dbf37 100644 --- a/sys/vm/uma_int.h +++ b/sys/vm/uma_int.h @@ -307,7 +307,7 @@ struct uma_zone { const char *uz_warning; /* Warning to print on failure */ struct timeval uz_ratecheck; /* Warnings rate-limiting */ - uma_maxaction_t uz_maxaction; /* Function to run when at limit */ + struct task uz_maxaction; /* Task to run when at limit */ /* * This HAS to be the last item because we adjust the zone size diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index d27eb2d4eaf0..b510696ed170 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -102,6 +102,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include