diff --git a/sys/geom/vinum/geom_vinum_drive.c b/sys/geom/vinum/geom_vinum_drive.c index b2841a81e454..98b7c574cf6e 100644 --- a/sys/geom/vinum/geom_vinum_drive.c +++ b/sys/geom/vinum/geom_vinum_drive.c @@ -77,7 +77,8 @@ gv_config_new_drive(struct gv_drive *d) LIST_INSERT_HEAD(&d->freelist, fl, freelist); d->freelist_entries = 1; - TAILQ_INIT(&d->bqueue); + d->bqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO); + bioq_init(d->bqueue); mtx_init(&d->bqueue_mtx, "gv_drive", NULL, MTX_DEF); kthread_create(gv_drive_worker, d, NULL, 0, 0, "gv_d %s", d->name); d->flags |= GV_DRIVE_THREAD_ACTIVE; @@ -235,15 +236,12 @@ static void gv_drive_done(struct bio *bp) { struct gv_drive *d; - struct gv_bioq *bq; /* Put the BIO on the worker queue again. */ d = bp->bio_from->geom->softc; bp->bio_cflags |= GV_BIO_DONE; - bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO); - bq->bp = bp; mtx_lock(&d->bqueue_mtx); - TAILQ_INSERT_TAIL(&d->bqueue, bq, queue); + bioq_insert_tail(d->bqueue, bp); wakeup(d); mtx_unlock(&d->bqueue_mtx); } @@ -254,7 +252,6 @@ gv_drive_start(struct bio *bp) { struct gv_drive *d; struct gv_sd *s; - struct gv_bioq *bq; switch (bp->bio_cmd) { case BIO_READ: @@ -279,10 +276,8 @@ gv_drive_start(struct bio *bp) * Put the BIO on the worker queue, where the worker thread will pick * it up. */ - bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO); - bq->bp = bp; mtx_lock(&d->bqueue_mtx); - TAILQ_INSERT_TAIL(&d->bqueue, bq, queue); + bioq_disksort(d->bqueue, bp); wakeup(d); mtx_unlock(&d->bqueue_mtx); @@ -296,7 +291,6 @@ gv_drive_worker(void *arg) struct g_provider *pp; struct gv_drive *d; struct gv_sd *s; - struct gv_bioq *bq, *bq2; int error; d = arg; @@ -308,16 +302,13 @@ gv_drive_worker(void *arg) break; /* Take the first BIO from out queue. */ - bq = TAILQ_FIRST(&d->bqueue); - if (bq == NULL) { + bp = bioq_takefirst(d->bqueue); + if (bp == NULL) { msleep(d, &d->bqueue_mtx, PRIBIO, "-", hz/10); continue; } - TAILQ_REMOVE(&d->bqueue, bq, queue); mtx_unlock(&d->bqueue_mtx); - bp = bq->bp; - g_free(bq); pp = bp->bio_to; gp = pp->geom; @@ -371,11 +362,8 @@ gv_drive_worker(void *arg) mtx_lock(&d->bqueue_mtx); } - TAILQ_FOREACH_SAFE(bq, &d->bqueue, queue, bq2) { - TAILQ_REMOVE(&d->bqueue, bq, queue); + while ((bp = bioq_takefirst(d->bqueue)) != NULL) { mtx_unlock(&d->bqueue_mtx); - bp = bq->bp; - g_free(bq); if (bp->bio_cflags & GV_BIO_DONE) g_std_done(bp); else @@ -504,15 +492,19 @@ gv_drive_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) LIST_INSERT_HEAD(&d->freelist, fl, freelist); d->freelist_entries = 1; - TAILQ_INIT(&d->bqueue); - /* Save it into the main configuration. */ LIST_INSERT_HEAD(&sc->drives, d, drive); } /* - * Create a bio queue mutex and a worker thread, if necessary. + * Create bio queue, queue mutex and a worker thread, if + * necessary. */ + if (d->bqueue == NULL) { + d->bqueue = g_malloc(sizeof(struct bio_queue_head), + M_WAITOK | M_ZERO); + bioq_init(d->bqueue); + } if (mtx_initialized(&d->bqueue_mtx) == 0) mtx_init(&d->bqueue_mtx, "gv_drive", NULL, MTX_DEF); diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c index 6f68dd9bfd22..bdcce213f7fb 100644 --- a/sys/geom/vinum/geom_vinum_plex.c +++ b/sys/geom/vinum/geom_vinum_plex.c @@ -88,14 +88,11 @@ void gv_plex_done(struct bio *bp) { struct gv_plex *p; - struct gv_bioq *bq; p = bp->bio_from->geom->softc; bp->bio_cflags |= GV_BIO_DONE; - bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO); - bq->bp = bp; mtx_lock(&p->bqueue_mtx); - TAILQ_INSERT_TAIL(&p->bqueue, bq, queue); + bioq_insert_tail(p->bqueue, bp); wakeup(p); mtx_unlock(&p->bqueue_mtx); } @@ -236,7 +233,6 @@ static void gv_plex_start(struct bio *bp) { struct gv_plex *p; - struct gv_bioq *bq; switch(bp->bio_cmd) { case BIO_READ: @@ -260,10 +256,8 @@ gv_plex_start(struct bio *bp) return; } - bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO); - bq->bp = bp; mtx_lock(&p->bqueue_mtx); - TAILQ_INSERT_TAIL(&p->bqueue, bq, queue); + bioq_disksort(p->bqueue, bp); wakeup(p); mtx_unlock(&p->bqueue_mtx); } @@ -274,7 +268,6 @@ gv_plex_worker(void *arg) struct bio *bp; struct gv_plex *p; struct gv_sd *s; - struct gv_bioq *bq; p = arg; KASSERT(p != NULL, ("NULL p")); @@ -286,20 +279,15 @@ gv_plex_worker(void *arg) break; /* Take the first BIO from our queue. */ - bq = TAILQ_FIRST(&p->bqueue); - if (bq == NULL) { + bp = bioq_takefirst(p->bqueue); + if (bp == NULL) { msleep(p, &p->bqueue_mtx, PRIBIO, "-", hz/10); continue; } - TAILQ_REMOVE(&p->bqueue, bq, queue); mtx_unlock(&p->bqueue_mtx); - bp = bq->bp; - /* A completed request. */ if (bp->bio_cflags & GV_BIO_DONE) { - g_free(bq); - if (bp->bio_cflags & GV_BIO_SYNCREQ || bp->bio_cflags & GV_BIO_REBUILD) { s = bp->bio_to->private; @@ -327,19 +315,16 @@ gv_plex_worker(void *arg) if (gv_stripe_active(p, bp)) { /* Park the bio on the waiting queue. */ mtx_lock(&p->bqueue_mtx); - TAILQ_INSERT_TAIL(&p->wqueue, bq, queue); + bioq_disksort(p->wqueue, bp); mtx_unlock(&p->bqueue_mtx); } else { - g_free(bq); bp->bio_cflags &= ~GV_BIO_ONHOLD; g_io_request(bp, bp->bio_caller2); } /* A normal request to this plex. */ - } else { - g_free(bq); + } else gv_plex_normal_request(p, bp); - } mtx_lock(&p->bqueue_mtx); } @@ -380,7 +365,7 @@ gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) static int gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) { - struct bio *cbp, *pbp; + struct bio *pbp; int err, finished, i; err = 0; @@ -393,12 +378,12 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) finished = 0; } else if (wp->parity != NULL) { - cbp = wp->parity; + pbp = wp->parity; wp->parity = NULL; /* Check if the parity is correct. */ for (i = 0; i < wp->length; i++) { - if (bp->bio_data[i] != cbp->bio_data[i]) { + if (bp->bio_data[i] != pbp->bio_data[i]) { err = 1; break; } @@ -410,7 +395,7 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) /* ... but we rebuild it. */ if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) { - g_io_request(cbp, cbp->bio_caller2); + g_io_request(pbp, pbp->bio_caller2); finished = 0; } } @@ -421,7 +406,7 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) */ if (finished) { bp->bio_parent->bio_inbed++; - g_destroy_bio(cbp); + g_destroy_bio(pbp); } } @@ -459,7 +444,11 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp) TAILQ_REMOVE(&p->packets, wp, list); /* Bring the waiting bios back into the game. */ mtx_lock(&p->bqueue_mtx); - TAILQ_CONCAT(&p->bqueue, &p->wqueue, queue); + pbp = bioq_takefirst(p->wqueue); + while (pbp != NULL) { + bioq_disksort(p->bqueue, pbp); + pbp = bioq_takefirst(p->wqueue); + } mtx_unlock(&p->bqueue_mtx); } g_free(wp); @@ -499,7 +488,11 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp) TAILQ_REMOVE(&p->packets, wp, list); /* Bring the waiting bios back into the game. */ mtx_lock(&p->bqueue_mtx); - TAILQ_CONCAT(&p->bqueue, &p->wqueue, queue); + pbp = bioq_takefirst(p->wqueue); + while (pbp != NULL) { + bioq_disksort(p->bqueue, pbp); + pbp = bioq_takefirst(p->wqueue); + } mtx_unlock(&p->bqueue_mtx); g_free(wp); } @@ -662,10 +655,8 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp) gv_stripe_active(p, pbp)) { /* Park the bio on the waiting queue. */ pbp->bio_cflags |= GV_BIO_ONHOLD; - bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO); - bq->bp = pbp; mtx_lock(&p->bqueue_mtx); - TAILQ_INSERT_TAIL(&p->wqueue, bq, queue); + bioq_disksort(p->wqueue, pbp); mtx_unlock(&p->bqueue_mtx); } else g_io_request(pbp, pbp->bio_caller2); @@ -776,8 +767,19 @@ gv_plex_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) gv_update_vol_size(p->vol_sc, p->size); /* - * If necessary, create a bio queue mutex and a worker thread. + * If necessary, create bio queues, queue mutex and a worker + * thread. */ + if (p->bqueue == NULL) { + p->bqueue = g_malloc(sizeof(struct bio_queue_head), + M_WAITOK | M_ZERO); + bioq_init(p->bqueue); + } + if (p->wqueue == NULL) { + p->wqueue = g_malloc(sizeof(struct bio_queue_head), + M_WAITOK | M_ZERO); + bioq_init(p->wqueue); + } if (mtx_initialized(&p->bqueue_mtx) == 0) mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF); if (!(p->flags & GV_PLEX_THREAD_ACTIVE)) { @@ -798,8 +800,12 @@ gv_plex_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) p->geom = gp; TAILQ_INIT(&p->packets); - TAILQ_INIT(&p->bqueue); - TAILQ_INIT(&p->wqueue); + p->bqueue = g_malloc(sizeof(struct bio_queue_head), + M_WAITOK | M_ZERO); + bioq_init(p->bqueue); + p->wqueue = g_malloc(sizeof(struct bio_queue_head), + M_WAITOK | M_ZERO); + bioq_init(p->wqueue); mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF); kthread_create(gv_plex_worker, p, NULL, 0, 0, "gv_p %s", p->name); diff --git a/sys/geom/vinum/geom_vinum_subr.c b/sys/geom/vinum/geom_vinum_subr.c index ff91196e7823..ce79a1877824 100644 --- a/sys/geom/vinum/geom_vinum_subr.c +++ b/sys/geom/vinum/geom_vinum_subr.c @@ -792,6 +792,8 @@ gv_kill_drive_thread(struct gv_drive *d) d->flags &= ~GV_DRIVE_THREAD_ACTIVE; d->flags &= ~GV_DRIVE_THREAD_DIE; d->flags &= ~GV_DRIVE_THREAD_DEAD; + g_free(d->bqueue); + d->bqueue = NULL; mtx_destroy(&d->bqueue_mtx); } } @@ -807,6 +809,10 @@ gv_kill_plex_thread(struct gv_plex *p) p->flags &= ~GV_PLEX_THREAD_ACTIVE; p->flags &= ~GV_PLEX_THREAD_DIE; p->flags &= ~GV_PLEX_THREAD_DEAD; + g_free(p->bqueue); + g_free(p->wqueue); + p->bqueue = NULL; + p->wqueue = NULL; mtx_destroy(&p->bqueue_mtx); } } @@ -822,6 +828,8 @@ gv_kill_vol_thread(struct gv_volume *v) v->flags &= ~GV_VOL_THREAD_ACTIVE; v->flags &= ~GV_VOL_THREAD_DIE; v->flags &= ~GV_VOL_THREAD_DEAD; + g_free(v->bqueue); + v->bqueue = NULL; mtx_destroy(&v->bqueue_mtx); } } diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h index 3de2b44a3901..8f746f0460cb 100644 --- a/sys/geom/vinum/geom_vinum_var.h +++ b/sys/geom/vinum/geom_vinum_var.h @@ -198,7 +198,11 @@ struct gv_drive { LIST_HEAD(,gv_sd) subdisks; /* Subdisks on this drive. */ LIST_ENTRY(gv_drive) drive; /* Entry in the vinum config. */ - TAILQ_HEAD(,gv_bioq) bqueue; /* BIO queue of this drive. */ +#ifdef _KERNEL + struct bio_queue_head *bqueue; /* BIO queue of this drive. */ +#else + char *padding; +#endif struct mtx bqueue_mtx; /* Mtx. to protect the queue. */ struct g_geom *geom; /* The geom of this drive. */ @@ -277,8 +281,12 @@ struct gv_plex { off_t synced; /* Count of synced bytes. */ struct mtx bqueue_mtx; /* Lock for the BIO queue. */ - TAILQ_HEAD(,gv_bioq) bqueue; /* BIO queue. */ - TAILQ_HEAD(,gv_bioq) wqueue; /* Waiting BIO queue. */ +#ifdef _KERNEL + struct bio_queue_head *bqueue; /* BIO queue. */ + struct bio_queue_head *wqueue; /* Waiting BIO queue. */ +#else + char *bpad, *wpad; +#endif TAILQ_HEAD(,gv_raid5_packet) packets; /* RAID5 sub-requests. */ LIST_HEAD(,gv_sd) subdisks; /* List of attached subdisks. */ @@ -307,7 +315,11 @@ struct gv_volume { #define GV_VOL_THREAD_DEAD 0x04 /* The thread has died. */ struct mtx bqueue_mtx; /* Lock for the BIO queue. */ - TAILQ_HEAD(,gv_bioq) bqueue; /* BIO queue. */ +#ifdef _KERNEL + struct bio_queue_head *bqueue; /* BIO queue. */ +#else + char *padding; +#endif LIST_HEAD(,gv_plex) plexes; /* List of attached plexes. */ LIST_ENTRY(gv_volume) volume; /* Entry in vinum config. */ diff --git a/sys/geom/vinum/geom_vinum_volume.c b/sys/geom/vinum/geom_vinum_volume.c index 510d3f06ee92..afc1f2a33b56 100644 --- a/sys/geom/vinum/geom_vinum_volume.c +++ b/sys/geom/vinum/geom_vinum_volume.c @@ -79,14 +79,11 @@ static void gv_volume_done(struct bio *bp) { struct gv_volume *v; - struct gv_bioq *bq; v = bp->bio_from->geom->softc; bp->bio_cflags |= GV_BIO_DONE; - bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO); - bq->bp = bp; mtx_lock(&v->bqueue_mtx); - TAILQ_INSERT_TAIL(&v->bqueue, bq, queue); + bioq_insert_tail(v->bqueue, bp); wakeup(v); mtx_unlock(&v->bqueue_mtx); } @@ -95,7 +92,6 @@ static void gv_volume_start(struct bio *bp) { struct gv_volume *v; - struct gv_bioq *bq; switch(bp->bio_cmd) { case BIO_READ: @@ -114,10 +110,8 @@ gv_volume_start(struct bio *bp) return; } - bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO); - bq->bp = bp; mtx_lock(&v->bqueue_mtx); - TAILQ_INSERT_TAIL(&v->bqueue, bq, queue); + bioq_disksort(v->bqueue, bp); wakeup(v); mtx_unlock(&v->bqueue_mtx); } @@ -127,7 +121,6 @@ gv_vol_worker(void *arg) { struct bio *bp; struct gv_volume *v; - struct gv_bioq *bq; v = arg; KASSERT(v != NULL, ("NULL v")); @@ -138,17 +131,13 @@ gv_vol_worker(void *arg) break; /* Take the first BIO from our queue. */ - bq = TAILQ_FIRST(&v->bqueue); - if (bq == NULL) { + bp = bioq_takefirst(v->bqueue); + if (bp == NULL) { msleep(v, &v->bqueue_mtx, PRIBIO, "-", hz/10); continue; } - TAILQ_REMOVE(&v->bqueue, bq, queue); mtx_unlock(&v->bqueue_mtx); - bp = bq->bp; - g_free(bq); - if (bp->bio_cflags & GV_BIO_DONE) gv_vol_completed_request(v, bp); else @@ -169,7 +158,6 @@ gv_vol_completed_request(struct gv_volume *v, struct bio *bp) struct bio *pbp; struct g_geom *gp; struct g_consumer *cp, *cp2; - struct gv_bioq *bq; pbp = bp->bio_parent; @@ -196,10 +184,8 @@ gv_vol_completed_request(struct gv_volume *v, struct bio *bp) g_destroy_bio(bp); pbp->bio_children--; - bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO); - bq->bp = pbp; mtx_lock(&v->bqueue_mtx); - TAILQ_INSERT_TAIL(&v->bqueue, bq, queue); + bioq_disksort(v->bqueue, pbp); mtx_unlock(&v->bqueue_mtx); return; @@ -370,11 +356,15 @@ gv_volume_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) gp->access = gv_volume_access; gp->softc = v; first++; - TAILQ_INIT(&v->bqueue); } else gp = v->geom; - /* Create bio queue mutex and worker thread, if necessary. */ + /* Create bio queue, queue mutex, and worker thread, if necessary. */ + if (v->bqueue == NULL) { + v->bqueue = g_malloc(sizeof(struct bio_queue_head), + M_WAITOK | M_ZERO); + bioq_init(v->bqueue); + } if (mtx_initialized(&v->bqueue_mtx) == 0) mtx_init(&v->bqueue_mtx, "gv_plex", NULL, MTX_DEF);