d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch from mutexes to atomics in GEOM_DEV I/O path.

Browse Source 
Mutexes in I/O path there were used twice per I/O to atomically access
several variables to close and/or destroy the device on last request
completion.  I found the way to fit all required info into one integer,
suitable for atomic operations.  It opened race window on device close,
but addition of timeout to the msleep() there should cover it.

Profiling shows removal of significant spinning time on those mutexes
and IOPS increase from ~600K to >800K to NVMe on 72-core systems.

MFC after:	1 month
Sponsored by:	iXsystems, Inc.
This commit is contained in:
Alexander Motin 2018-12-27 19:15:24 +00:00
parent 4d07b68988
commit 02a9923034
1 changed files with 25 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
sys/geom/geom_dev.c
View File

@ -64,7 +64,10 @@ struct g_dev_softc {
struct cdev *sc_dev;
struct cdev *sc_alias;
int sc_open;
int sc_active;
u_int sc_active;
#define SC_A_DESTROY (1 << 31)
#define SC_A_OPEN (1 << 30)
#define SC_A_ACTIVE (SC_A_OPEN - 1)
};
static d_open_t g_dev_open;
@ -420,9 +423,13 @@ g_dev_open(struct cdev *dev, int flags, int fmt, struct thread *td)
if (error == 0) {
sc = cp->private;
mtx_lock(&sc->sc_mtx);
if (sc->sc_open == 0 && sc->sc_active != 0)
if (sc->sc_open == 0 && (sc->sc_active & SC_A_ACTIVE) != 0)
wakeup(&sc->sc_active);
sc->sc_open += r + w + e;
if (sc->sc_open == 0)
atomic_clear_int(&sc->sc_active, SC_A_OPEN);
else
atomic_set_int(&sc->sc_active, SC_A_OPEN);
mtx_unlock(&sc->sc_mtx);
}
return (error);
@ -465,8 +472,12 @@ g_dev_close(struct cdev *dev, int flags, int fmt, struct thread *td)
sc = cp->private;
mtx_lock(&sc->sc_mtx);
sc->sc_open += r + w + e;
while (sc->sc_open == 0 && sc->sc_active != 0)
msleep(&sc->sc_active, &sc->sc_mtx, 0, "PRIBIO", 0);
if (sc->sc_open == 0)
atomic_clear_int(&sc->sc_active, SC_A_OPEN);
else
atomic_set_int(&sc->sc_active, SC_A_OPEN);
while (sc->sc_open == 0 && (sc->sc_active & SC_A_ACTIVE) != 0)
msleep(&sc->sc_active, &sc->sc_mtx, 0, "g_dev_close", hz / 10);
mtx_unlock(&sc->sc_mtx);
g_topology_lock();
error = g_access(cp, r, w, e);
@ -707,7 +718,7 @@ g_dev_done(struct bio *bp2)
struct g_consumer *cp;
struct g_dev_softc *sc;
struct bio *bp;
int destroy;
int active;
cp = bp2->bio_from;
sc = cp->private;
@ -727,17 +738,13 @@ g_dev_done(struct bio *bp2)
bp2, bp, bp2->bio_resid, (intmax_t)bp2->bio_completed);
}
g_destroy_bio(bp2);
destroy = 0;
mtx_lock(&sc->sc_mtx);
if ((--sc->sc_active) == 0) {
if (sc->sc_open == 0)
active = atomic_fetchadd_int(&sc->sc_active, -1) - 1;
if ((active & SC_A_ACTIVE) == 0) {
if ((active & SC_A_OPEN) == 0)
wakeup(&sc->sc_active);
if (sc->sc_dev == NULL)
destroy = 1;
if (active & SC_A_DESTROY)
g_post_event(g_dev_destroy, cp, M_NOWAIT, NULL);
}
mtx_unlock(&sc->sc_mtx);
if (destroy)
g_post_event(g_dev_destroy, cp, M_NOWAIT, NULL);
biodone(bp);
}
@ -769,10 +776,8 @@ g_dev_strategy(struct bio *bp)
return;
}
#endif
mtx_lock(&sc->sc_mtx);
KASSERT(sc->sc_open > 0, ("Closed device in g_dev_strategy"));
sc->sc_active++;
mtx_unlock(&sc->sc_mtx);
atomic_add_int(&sc->sc_active, 1);
for (;;) {
/*
@ -810,18 +815,16 @@ g_dev_callback(void *arg)
{
struct g_consumer *cp;
struct g_dev_softc *sc;
int destroy;
int active;
cp = arg;
sc = cp->private;
g_trace(G_T_TOPOLOGY, "g_dev_callback(%p(%s))", cp, cp->geom->name);
mtx_lock(&sc->sc_mtx);
sc->sc_dev = NULL;
sc->sc_alias = NULL;
destroy = (sc->sc_active == 0);
mtx_unlock(&sc->sc_mtx);
if (destroy)
active = atomic_fetchadd_int(&sc->sc_active, SC_A_DESTROY);
if ((active & SC_A_ACTIVE) == 0)
g_post_event(g_dev_destroy, cp, M_WAITOK, NULL);
}