From fa4a1febf719c322c3c7db0e6b45e827e6e116b9 Mon Sep 17 00:00:00 2001 From: Pawel Jakub Dawidek Date: Fri, 30 Jul 2004 23:13:45 +0000 Subject: [PATCH] Add GEOM_MIRROR class which provide RAID1 functionality and has many useful features. The gmirror(8) utility should be used for control of this class. There is no manual page yet, but I'm working on it with keramida@. Many useful tests provided by: simon (thank you!) Some ideas from: scottl, simon, phk --- sbin/geom/class/mirror/Makefile | 11 + sbin/geom/class/mirror/geom_mirror.c | 408 ++++ sys/geom/mirror/g_mirror.c | 2591 ++++++++++++++++++++++ sys/geom/mirror/g_mirror.h | 350 +++ sys/geom/mirror/g_mirror_ctl.c | 617 ++++++ sys/modules/geom/geom_mirror/Makefile | 9 + tools/regression/geom_mirror/Makefile | 8 + tools/regression/geom_mirror/runtests.sh | 10 + tools/regression/geom_mirror/test-1.sh | 32 + tools/regression/geom_mirror/test-2.sh | 57 + tools/regression/geom_mirror/test-3.sh | 68 + tools/regression/geom_mirror/test-4.sh | 68 + tools/regression/geom_mirror/test-5.sh | 68 + tools/regression/geom_mirror/test-6.sh | 50 + 14 files changed, 4347 insertions(+) create mode 100644 sbin/geom/class/mirror/Makefile create mode 100644 sbin/geom/class/mirror/geom_mirror.c create mode 100644 sys/geom/mirror/g_mirror.c create mode 100644 sys/geom/mirror/g_mirror.h create mode 100644 sys/geom/mirror/g_mirror_ctl.c create mode 100644 sys/modules/geom/geom_mirror/Makefile create mode 100644 tools/regression/geom_mirror/Makefile create mode 100644 tools/regression/geom_mirror/runtests.sh create mode 100644 tools/regression/geom_mirror/test-1.sh create mode 100644 tools/regression/geom_mirror/test-2.sh create mode 100644 tools/regression/geom_mirror/test-3.sh create mode 100644 tools/regression/geom_mirror/test-4.sh create mode 100644 tools/regression/geom_mirror/test-5.sh create mode 100644 tools/regression/geom_mirror/test-6.sh diff --git a/sbin/geom/class/mirror/Makefile b/sbin/geom/class/mirror/Makefile new file mode 100644 index 000000000000..fc6c36054252 --- /dev/null +++ b/sbin/geom/class/mirror/Makefile @@ -0,0 +1,11 @@ +# $FreeBSD$ + +.PATH: ${.CURDIR}/../../misc + +CLASS= mirror + +NOMAN= notyet +DPADD= ${LIBMD} +LDADD= -lmd + +.include diff --git a/sbin/geom/class/mirror/geom_mirror.c b/sbin/geom/class/mirror/geom_mirror.c new file mode 100644 index 000000000000..66174b7c2cf6 --- /dev/null +++ b/sbin/geom/class/mirror/geom_mirror.c @@ -0,0 +1,408 @@ +/*- + * Copyright (c) 2004 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +uint32_t lib_version = G_LIB_VERSION; +uint32_t version = G_MIRROR_VERSION; + +static char label_balance[] = "split", configure_balance[] = "none"; +static intmax_t label_slice = 4096, configure_slice = -1; + +static void mirror_main(struct gctl_req *req, unsigned f); +static void mirror_activate(struct gctl_req *req); +static void mirror_clear(struct gctl_req *req); +static void mirror_dump(struct gctl_req *req); +static void mirror_label(struct gctl_req *req); + +struct g_command class_commands[] = { + { "activate", G_FLAG_VERBOSE, mirror_main, G_NULL_OPTS }, + { "clear", G_FLAG_VERBOSE, mirror_main, G_NULL_OPTS }, + { "configure", G_FLAG_VERBOSE, NULL, + { + { 'a', "autosync", NULL, G_TYPE_NONE }, + { 'b', "balance", configure_balance, G_TYPE_STRING }, + { 'n', "noautosync", NULL, G_TYPE_NONE }, + { 's', "slice", &configure_slice, G_TYPE_NUMBER }, + G_OPT_SENTINEL + } + }, + { "deactivate", G_FLAG_VERBOSE, NULL, G_NULL_OPTS }, + { "dump", 0, mirror_main, G_NULL_OPTS }, + { "forget", G_FLAG_VERBOSE, NULL, G_NULL_OPTS }, + { "label", G_FLAG_VERBOSE, mirror_main, + { + { 'b', "balance", label_balance, G_TYPE_STRING }, + { 'n', "noautosync", NULL, G_TYPE_NONE }, + { 's', "slice", &label_slice, G_TYPE_NUMBER }, + G_OPT_SENTINEL + } + }, + { "insert", G_FLAG_VERBOSE, NULL, + { + { 'i', "inactive", NULL, G_TYPE_NONE }, + G_OPT_SENTINEL + } + }, + { "rebuild", G_FLAG_VERBOSE, NULL, G_NULL_OPTS }, + { "remove", G_FLAG_VERBOSE, NULL, G_NULL_OPTS }, + { "stop", G_FLAG_VERBOSE, NULL, + { + { 'f', "force", NULL, G_TYPE_NONE }, + G_OPT_SENTINEL + } + }, + G_CMD_SENTINEL +}; + +static int verbose = 0; + +void usage(const char *); +void +usage(const char *comm) +{ + fprintf(stderr, + "usage: %s label [-nv] [-b balance] [-s slice] name dev1 [dev2 [...]]\n" + " %s clear [-v] dev1 [dev2 [...]]\n" + " %s dump dev1 [dev2 [...]]\n" + " %s configure [-anv] [-b balance] [-s slice] name\n" + " %s rebuild [-v] name dev1 [dev2 [...]]\n" + " %s insert [-iv] name dev1 [dev2 [...]]\n" + " %s remove [-v] name dev1 [dev2 [...]]\n" + " %s activate [-v] name dev1 [dev2 [...]]\n" + " %s deactivate [-v] name dev1 [dev2 [...]]\n" + " %s forget dev1 [dev2 [...]]\n" + " %s stop [-fv] name\n", + comm, comm, comm, comm, comm, comm, comm, comm, comm, comm, comm); + exit(EXIT_FAILURE); +} + +static void +mirror_main(struct gctl_req *req, unsigned f) +{ + const char *name; + + if ((f & G_FLAG_VERBOSE) != 0) + verbose = 1; + + name = gctl_get_asciiparam(req, "verb"); + if (name == NULL) { + gctl_error(req, "No '%s' argument.", "verb"); + return; + } + if (strcmp(name, "label") == 0) + mirror_label(req); + else if (strcmp(name, "clear") == 0) + mirror_clear(req); + else if (strcmp(name, "dump") == 0) + mirror_dump(req); + else if (strcmp(name, "activate") == 0) + mirror_activate(req); + else + gctl_error(req, "Unknown command: %s.", name); +} + +static void +mirror_label(struct gctl_req *req) +{ + struct g_mirror_metadata md; + u_char sector[512]; + const char *str; + char param[16]; + int *nargs, *noautosync, bal, error, i; + unsigned sectorsize; + off_t mediasize; + intmax_t *valp; + + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 2) { + gctl_error(req, "Too few arguments."); + return; + } + + strlcpy(md.md_magic, G_MIRROR_MAGIC, sizeof(md.md_magic)); + md.md_version = G_MIRROR_VERSION; + str = gctl_get_asciiparam(req, "arg0"); + if (str == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + strlcpy(md.md_name, str, sizeof(md.md_name)); + md.md_mid = arc4random(); + md.md_all = *nargs - 1; + md.md_mflags = 0; + md.md_dflags = 0; + md.md_syncid = 1; + md.md_sync_offset = 0; + valp = gctl_get_paraml(req, "slice", sizeof(*valp)); + if (valp == NULL) { + gctl_error(req, "No '%s' argument.", "slice"); + return; + } + md.md_slice = *valp; + str = gctl_get_asciiparam(req, "balance"); + if (str == NULL) { + gctl_error(req, "No '%s' argument.", "balance"); + return; + } + bal = balance_id(str); + if (bal == -1) { + gctl_error(req, "Wrong balance algorithm."); + return; + } + md.md_balance = bal; + noautosync = gctl_get_paraml(req, "noautosync", sizeof(*noautosync)); + if (noautosync == NULL) { + gctl_error(req, "No '%s' argument.", "noautosync"); + return; + } + if (*noautosync) + md.md_mflags |= G_MIRROR_DEVICE_FLAG_NOAUTOSYNC; + + /* + * Calculate sectorsize by finding least common multiple from + * sectorsizes of every disk and find the smallest mediasize. + */ + mediasize = 0; + sectorsize = 0; + for (i = 1; i < *nargs; i++) { + unsigned ssize; + off_t msize; + + snprintf(param, sizeof(param), "arg%u", i); + str = gctl_get_asciiparam(req, param); + + msize = g_get_mediasize(str); + ssize = g_get_sectorsize(str); + if (msize == 0 || ssize == 0) { + gctl_error(req, "Can't get informations about %s: %s.", + str, strerror(errno)); + return; + } + msize -= ssize; + if (mediasize == 0 || (mediasize > 0 && msize < mediasize)) + mediasize = msize; + if (sectorsize == 0) + sectorsize = ssize; + else + sectorsize = g_lcm(sectorsize, ssize); + } + md.md_mediasize = mediasize; + md.md_sectorsize = sectorsize; + + /* + * Clear last sector first, to spoil all components if device exists. + */ + for (i = 1; i < *nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + str = gctl_get_asciiparam(req, param); + + error = g_metadata_clear(str, NULL); + if (error != 0) { + gctl_error(req, "Can't store metadata on %s: %s.", str, + strerror(error)); + return; + } + } + + /* + * Ok, store metadata (use disk number as priority). + */ + for (i = 1; i < *nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + str = gctl_get_asciiparam(req, param); + + md.md_did = arc4random(); + md.md_priority = i - 1; + mirror_metadata_encode(&md, sector); + error = g_metadata_store(str, sector, sizeof(sector)); + if (error != 0) { + fprintf(stderr, "Can't store metadata on %s: %s.\n", + str, strerror(error)); + gctl_error(req, "Not fully done."); + continue; + } + if (verbose) + printf("Metadata value stored on %s.\n", str); + } +} + +static void +mirror_clear(struct gctl_req *req) +{ + const char *name; + char param[16]; + int *nargs, error, i; + + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 1) { + gctl_error(req, "Too few arguments."); + return; + } + + for (i = 0; i < *nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + + error = g_metadata_clear(name, G_MIRROR_MAGIC); + if (error != 0) { + fprintf(stderr, "Can't clear metadata on %s: %s.\n", + name, strerror(error)); + gctl_error(req, "Not fully done."); + continue; + } + if (verbose) + printf("Metadata cleared on %s.\n", name); + } +} + +static void +mirror_dump(struct gctl_req *req) +{ + struct g_mirror_metadata md, tmpmd; + const char *name; + char param[16]; + int *nargs, error, i; + + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 1) { + gctl_error(req, "Too few arguments."); + return; + } + + for (i = 0; i < *nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + + error = g_metadata_read(name, (u_char *)&tmpmd, sizeof(tmpmd), + G_MIRROR_MAGIC); + if (error != 0) { + fprintf(stderr, "Can't read metadata from %s: %s.\n", + name, strerror(error)); + gctl_error(req, "Not fully done."); + continue; + } + if (mirror_metadata_decode((u_char *)&tmpmd, &md) != 0) { + fprintf(stderr, "MD5 hash mismatch for %s, skipping.\n", + name); + gctl_error(req, "Not fully done."); + continue; + } + printf("Metadata on %s:\n", name); + mirror_metadata_dump(&md); + printf("\n"); + } +} + +static void +mirror_activate(struct gctl_req *req) +{ + struct g_mirror_metadata md, tmpmd; + const char *name, *path; + int *nargs, error, i; + char param[16]; + + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 2) { + gctl_error(req, "Too few arguments."); + return; + } + name = gctl_get_asciiparam(req, "arg0"); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + + for (i = 1; i < *nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + path = gctl_get_asciiparam(req, param); + + error = g_metadata_read(path, (u_char *)&tmpmd, sizeof(tmpmd), + G_MIRROR_MAGIC); + if (error != 0) { + fprintf(stderr, "Cannot read metadata from %s: %s.\n", + path, strerror(error)); + gctl_error(req, "Not fully done."); + continue; + } + if (mirror_metadata_decode((u_char *)&tmpmd, &md) != 0) { + fprintf(stderr, + "MD5 hash mismatch for provider %s, skipping.\n", + path); + gctl_error(req, "Not fully done."); + continue; + } + if (strcmp(md.md_name, name) != 0) { + fprintf(stderr, + "Provider %s is not the mirror %s component.\n", + path, name); + gctl_error(req, "Not fully done."); + continue; + } + md.md_dflags &= ~G_MIRROR_DISK_FLAG_INACTIVE; + mirror_metadata_encode(&md, (u_char *)&tmpmd); + error = g_metadata_store(path, (u_char *)&tmpmd, sizeof(tmpmd)); + if (error != 0) { + fprintf(stderr, "Cannot write metadata from %s: %s.\n", + path, strerror(error)); + gctl_error(req, "Not fully done."); + continue; + } + if (verbose) + printf("Provider %s activated.\n", path); + } +} diff --git a/sys/geom/mirror/g_mirror.c b/sys/geom/mirror/g_mirror.c new file mode 100644 index 000000000000..132527b5ef77 --- /dev/null +++ b/sys/geom/mirror/g_mirror.c @@ -0,0 +1,2591 @@ +/*- + * Copyright (c) 2004 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static MALLOC_DEFINE(M_MIRROR, "mirror data", "GEOM_MIRROR Data"); + +SYSCTL_DECL(_kern_geom); +SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0, "GEOM_MIRROR stuff"); +u_int g_mirror_debug = 0; +SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RW, &g_mirror_debug, 0, + "Debug level"); +static u_int g_mirror_sync_block_size = 131072; +SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_block_size, CTLFLAG_RW, + &g_mirror_sync_block_size, 0, "Synchronization block size"); +static u_int g_mirror_timeout = 8; +SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RW, &g_mirror_timeout, + 0, "Time to wait on all mirror components"); +static u_int g_mirror_reqs_per_sync = 5; +SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, reqs_per_sync, CTLFLAG_RW, + &g_mirror_reqs_per_sync, 0, + "Number of regular I/O requests per synchronization request"); +static u_int g_mirror_syncs_per_sec = 100; +SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, syncs_per_sec, CTLFLAG_RW, + &g_mirror_syncs_per_sec, 0, + "Number of synchronizations requests per second"); + +#define MSLEEP(ident, mtx, priority, wmesg, timeout) do { \ + G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident)); \ + msleep((ident), (mtx), (priority), (wmesg), (timeout)); \ + G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident)); \ +} while (0) + + +static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp, + struct g_geom *gp); +static g_taste_t g_mirror_taste; + +struct g_class g_mirror_class = { + .name = G_MIRROR_CLASS_NAME, + .ctlreq = g_mirror_config, + .taste = g_mirror_taste, + .destroy_geom = g_mirror_destroy_geom +}; + + +static void g_mirror_destroy_provider(struct g_mirror_softc *sc); +static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state); +static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force); +static void g_mirror_dumpconf(struct sbuf *sb, const char *indent, + struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp); +static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type); + + +static const char * +g_mirror_disk_state2str(int state) +{ + + switch (state) { + case G_MIRROR_DISK_STATE_NONE: + return ("NONE"); + case G_MIRROR_DISK_STATE_NEW: + return ("NEW"); + case G_MIRROR_DISK_STATE_ACTIVE: + return ("ACTIVE"); + case G_MIRROR_DISK_STATE_STALE: + return ("STALE"); + case G_MIRROR_DISK_STATE_SYNCHRONIZING: + return ("SYNCHRONIZING"); + case G_MIRROR_DISK_STATE_DISCONNECTED: + return ("DISCONNECTED"); + case G_MIRROR_DISK_STATE_DESTROY: + return ("DESTROY"); + default: + return ("INVALID"); + } +} + +static const char * +g_mirror_device_state2str(int state) +{ + + switch (state) { + case G_MIRROR_DEVICE_STATE_STARTING: + return ("STARTING"); + case G_MIRROR_DEVICE_STATE_RUNNING: + return ("RUNNING"); + default: + return ("INVALID"); + } +} + +static const char * +g_mirror_get_diskname(struct g_mirror_disk *disk) +{ + + if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL) + return ("[unknown]"); + return (disk->d_name); +} + +/* + * --- Events handling functions --- + * Events in geom_mirror are used to maintain disks and device status + * from one thread to simplify locking. + */ +static void +g_mirror_event_free(struct g_mirror_event *ep) +{ + + free(ep, M_MIRROR); +} + +int +g_mirror_event_send(void *arg, int state, int flags) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + struct g_mirror_event *ep; + int error; + + ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK); + G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep); + if ((flags & G_MIRROR_EVENT_DEVICE) != 0) { + disk = NULL; + sc = arg; + } else { + disk = arg; + sc = disk->d_softc; + } + ep->e_disk = disk; + ep->e_state = state; + ep->e_flags = flags; + ep->e_error = 0; + mtx_lock(&sc->sc_events_mtx); + TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next); + mtx_unlock(&sc->sc_events_mtx); + G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc); + mtx_lock(&sc->sc_queue_mtx); + wakeup(sc); + mtx_unlock(&sc->sc_queue_mtx); + if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0) + return (0); + g_topology_assert(); + G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep); + g_topology_unlock(); + while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) { + mtx_lock(&sc->sc_events_mtx); + MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event", + hz * 5); + } + /* Don't even try to use 'sc' here, because it could be already dead. */ + g_topology_lock(); + error = ep->e_error; + g_mirror_event_free(ep); + return (error); +} + +static struct g_mirror_event * +g_mirror_event_get(struct g_mirror_softc *sc) +{ + struct g_mirror_event *ep; + + mtx_lock(&sc->sc_events_mtx); + ep = TAILQ_FIRST(&sc->sc_events); + if (ep != NULL) + TAILQ_REMOVE(&sc->sc_events, ep, e_next); + mtx_unlock(&sc->sc_events_mtx); + return (ep); +} + +static void +g_mirror_event_cancel(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + struct g_mirror_event *ep, *tmpep; + + g_topology_assert(); + + sc = disk->d_softc; + mtx_lock(&sc->sc_events_mtx); + TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) { + if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) + continue; + if (ep->e_disk != disk) + continue; + TAILQ_REMOVE(&sc->sc_events, ep, e_next); + if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) + g_mirror_event_free(ep); + else { + ep->e_error = ECANCELED; + wakeup(ep); + } + } + mtx_unlock(&sc->sc_events_mtx); +} + +/* + * Return the number of disks in given state. + * If state is equal to -1, count all connected disks. + */ +u_int +g_mirror_ndisks(struct g_mirror_softc *sc, int state) +{ + struct g_mirror_disk *disk; + u_int n = 0; + + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (state == -1 || disk->d_state == state) + n++; + } + return (n); +} + +/* + * Find a disk in mirror by its disk ID. + */ +static struct g_mirror_disk * +g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id) +{ + struct g_mirror_disk *disk; + + g_topology_assert(); + + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_id == id) + return (disk); + } + return (NULL); +} + +static u_int +g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp) +{ + struct bio *bp; + u_int nreqs = 0; + + mtx_lock(&sc->sc_queue_mtx); + TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) { + if (bp->bio_from == cp) + nreqs++; + } + mtx_unlock(&sc->sc_queue_mtx); + return (nreqs); +} + +static void +g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp) +{ + + g_topology_assert(); + + cp->private = NULL; + if (cp->nstart != cp->nend) { + G_MIRROR_DEBUG(2, + "I/O requests for %s exist, can't destroy it now.", + cp->provider->name); + return; + } + if (g_mirror_nrequests(sc, cp) > 0) { + G_MIRROR_DEBUG(2, + "I/O requests for %s in queue, can't destroy it now.", + cp->provider->name); + return; + } + G_MIRROR_DEBUG(2, "Consumer %s destroyed.", cp->provider->name); + g_detach(cp); + g_destroy_consumer(cp); +} + +static int +g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp) +{ + int error; + + g_topology_assert(); + KASSERT(disk->d_consumer == NULL, + ("Disk already connected (device %s).", disk->d_softc->sc_name)); + + disk->d_consumer = g_new_consumer(disk->d_softc->sc_geom); + disk->d_consumer->private = disk; + error = g_attach(disk->d_consumer, pp); + if (error != 0) + return (error); + G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk)); + return (0); +} + +static void +g_mirror_disconnect_disk(struct g_mirror_disk *disk) +{ + struct g_consumer *cp; + + g_topology_assert(); + + cp = disk->d_consumer; + if (cp == NULL) + return; + if (cp->provider != NULL) { + G_MIRROR_DEBUG(2, "Disk %s disconnected.", + g_mirror_get_diskname(disk)); + if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0) { + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", + cp->provider->name, -cp->acr, -cp->acw, -cp->ace, + 0); + g_access(cp, -cp->acr, -cp->acw, -cp->ace); + } + g_mirror_kill_consumer(disk->d_softc, cp); + } else { + g_destroy_consumer(cp); + } +} + +/* + * Initialize disk. This means allocate memory, create consumer, attach it + * to the provider and open access (r1w1e1) to it. + */ +static struct g_mirror_disk * +g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp, + struct g_mirror_metadata *md, int *errorp) +{ + struct g_mirror_disk *disk; + int error; + + disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO); + if (disk == NULL) { + error = ENOMEM; + goto fail; + } + disk->d_softc = sc; + error = g_mirror_connect_disk(disk, pp); + if (error != 0) + goto fail; + disk->d_id = md->md_did; + disk->d_state = G_MIRROR_DISK_STATE_NONE; + disk->d_priority = md->md_priority; + disk->d_delay.sec = 0; + disk->d_delay.frac = 0; + binuptime(&disk->d_last_used); + disk->d_flags = md->md_dflags; + disk->d_sync.ds_consumer = NULL; + disk->d_sync.ds_offset = md->md_sync_offset; + disk->d_sync.ds_offset_done = md->md_sync_offset; + disk->d_sync.ds_syncid = md->md_syncid; + if (errorp != NULL) + *errorp = 0; + return (disk); +fail: + if (errorp != NULL) + *errorp = error; + if (disk != NULL) { + g_mirror_disconnect_disk(disk); + free(disk, M_MIRROR); + } + return (NULL); +} + +/* + * Free the disk. + */ +static void +g_mirror_free_disk(struct g_mirror_disk *disk) +{ + + g_topology_assert(); + + g_mirror_disconnect_disk(disk); + free(disk, M_MIRROR); +} + +static void +g_mirror_destroy_disk(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + + g_topology_assert(); + + LIST_REMOVE(disk, d_next); + g_mirror_event_cancel(disk); + sc = disk->d_softc; + if (sc->sc_hint == disk) + sc->sc_hint = NULL; + switch (disk->d_state) { + case G_MIRROR_DISK_STATE_SYNCHRONIZING: + g_mirror_sync_stop(disk, 1); + /* FALLTHROUGH */ + case G_MIRROR_DISK_STATE_NEW: + case G_MIRROR_DISK_STATE_STALE: + case G_MIRROR_DISK_STATE_ACTIVE: + g_mirror_free_disk(disk); + break; + default: + KASSERT(0 == 1, ("Wrong disk state (%s, %s).", + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + } +} + +static void +g_mirror_destroy_device(struct g_mirror_softc *sc) +{ + struct g_mirror_disk *disk; + struct g_mirror_event *ep; + struct g_geom *gp; + + g_topology_assert(); + + gp = sc->sc_geom; + if (sc->sc_provider != NULL) + g_mirror_destroy_provider(sc); + for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL; + disk = LIST_FIRST(&sc->sc_disks)) { + g_mirror_destroy_disk(disk); + } + while ((ep = g_mirror_event_get(sc)) != NULL) { + if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) + g_mirror_event_free(ep); + else { + ep->e_error = ECANCELED; + ep->e_flags |= G_MIRROR_EVENT_DONE; + G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep); + mtx_lock(&sc->sc_events_mtx); + wakeup(ep); + mtx_unlock(&sc->sc_events_mtx); + } + } + callout_drain(&sc->sc_callout); + gp->softc = NULL; + uma_zdestroy(sc->sc_sync.ds_zone); + g_wither_geom(sc->sc_sync.ds_geom, ENXIO); + mtx_destroy(&sc->sc_queue_mtx); + mtx_destroy(&sc->sc_events_mtx); + G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name); + g_wither_geom(gp, ENXIO); +} + +static void +g_mirror_orphan(struct g_consumer *cp) +{ + struct g_mirror_disk *disk; + + g_topology_assert(); + + disk = cp->private; + if (disk == NULL) + return; + disk->d_softc->sc_bump_syncid = G_MIRROR_BUMP_ON_FIRST_WRITE; + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); +} + +/* + * Function should return the next active disk on the list. + * It is possible that it will be the same disk as given. + * If there are no active disks on list, NULL is returned. + */ +static __inline struct g_mirror_disk * +g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk) +{ + struct g_mirror_disk *dp; + + for (dp = LIST_NEXT(disk, d_next); dp != disk; + dp = LIST_NEXT(dp, d_next)) { + if (dp == NULL) + dp = LIST_FIRST(&sc->sc_disks); + if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE) + break; + } + if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE) + return (NULL); + return (dp); +} + +static struct g_mirror_disk * +g_mirror_get_disk(struct g_mirror_softc *sc) +{ + struct g_mirror_disk *disk; + + if (sc->sc_hint == NULL) { + sc->sc_hint = LIST_FIRST(&sc->sc_disks); + if (sc->sc_hint == NULL) + return (NULL); + } + disk = sc->sc_hint; + if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) { + disk = g_mirror_find_next(sc, disk); + if (disk == NULL) + return (NULL); + } + sc->sc_hint = g_mirror_find_next(sc, disk); + return (disk); +} + +static int +g_mirror_clear_metadata(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + struct g_consumer *cp; + off_t offset, length; + u_char *sector; + int close = 0, error = 0; + + g_topology_assert(); + + sc = disk->d_softc; + cp = disk->d_consumer; + KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name)); + KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name)); + length = cp->provider->sectorsize; + offset = cp->provider->mediasize - length; + sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO); + /* + * Open consumer if it wasn't opened and remember to close it. + */ + if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) { + error = g_access(cp, 0, 1, 1); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", + cp->provider->name, 0, 1, 1, error); + if (error == 0) + close = 1; +#ifdef INVARIANTS + } else { + KASSERT(cp->acw > 0 && cp->ace > 0, + ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, + cp->acr, cp->acw, cp->ace)); +#endif + } + if (error == 0) { + g_topology_unlock(); + error = g_write_data(cp, offset, sector, length); + g_topology_lock(); + } + free(sector, M_MIRROR); + if (close) { + g_access(cp, 0, -1, -1); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", + cp->provider->name, 0, -1, -1, 0); + } + if (error != 0) { + G_MIRROR_DEBUG(0, "Cannot clear metadata on disk %s.", + g_mirror_get_diskname(disk)); + disk->d_softc->sc_bump_syncid = G_MIRROR_BUMP_IMMEDIATELY; + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + return (error); + } + G_MIRROR_DEBUG(2, "Metadata on %s cleared.", + g_mirror_get_diskname(disk)); + return (0); +} + +void +g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk, + struct g_mirror_metadata *md) +{ + + strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic)); + md->md_version = G_MIRROR_VERSION; + strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name)); + md->md_mid = sc->sc_id; + md->md_all = sc->sc_ndisks; + md->md_slice = sc->sc_slice; + md->md_balance = sc->sc_balance; + md->md_mediasize = sc->sc_mediasize; + md->md_sectorsize = sc->sc_sectorsize; + md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK); + if (disk == NULL) { + md->md_did = arc4random(); + md->md_priority = 0; + md->md_syncid = 0; + md->md_dflags = 0; + md->md_sync_offset = 0; + } else { + md->md_did = disk->d_id; + md->md_priority = disk->d_priority; + md->md_syncid = disk->d_sync.ds_syncid; + md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK); + if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) + md->md_sync_offset = disk->d_sync.ds_offset_done; + else + md->md_sync_offset = 0; + } +} + +void +g_mirror_update_metadata(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + struct g_mirror_metadata md; + struct g_consumer *cp; + off_t offset, length; + u_char *sector; + int close = 0, error = 0; + + g_topology_assert(); + + sc = disk->d_softc; + cp = disk->d_consumer; + KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name)); + KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name)); + length = cp->provider->sectorsize; + offset = cp->provider->mediasize - length; + sector = malloc((size_t)length, M_MIRROR, M_WAITOK); + /* + * Open consumer if it wasn't opened and remember to close it. + */ + if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) { + error = g_access(cp, 0, 1, 1); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", + cp->provider->name, 0, 1, 1, error); + if (error == 0) + close = 1; +#ifdef INVARIANTS + } else { + KASSERT(cp->acw > 0 && cp->ace > 0, + ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, + cp->acr, cp->acw, cp->ace)); +#endif + } + if (error == 0) { + g_mirror_fill_metadata(sc, disk, &md); + mirror_metadata_encode(&md, sector); + g_topology_unlock(); + error = g_write_data(cp, offset, sector, length); + g_topology_lock(); + } + free(sector, M_MIRROR); + if (close) { + g_access(cp, 0, -1, -1); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", + cp->provider->name, 0, -1, -1, 0); + } + if (error != 0) { + G_MIRROR_DEBUG(0, + "Cannot update metadata on disk %s (error=%d).", + g_mirror_get_diskname(disk), error); + disk->d_softc->sc_bump_syncid = G_MIRROR_BUMP_IMMEDIATELY; + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + return; + } + G_MIRROR_DEBUG(2, "Metadata on %s updated.", + g_mirror_get_diskname(disk)); +} + +static void +g_mirror_bump_syncid(struct g_mirror_softc *sc) +{ + struct g_mirror_disk *disk; + + g_topology_assert(); + KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0, + ("%s called with no active disks (device=%s).", __func__, + sc->sc_name)); + + sc->sc_syncid++; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE || + disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) { + disk->d_sync.ds_syncid = sc->sc_syncid; + g_mirror_update_metadata(disk); + } + } +} + +static __inline int +bintime_cmp(struct bintime *bt1, struct bintime *bt2) +{ + + if (bt1->sec < bt2->sec) + return (-1); + else if (bt1->sec > bt2->sec) + return (1); + if (bt1->frac < bt2->frac) + return (-1); + else if (bt1->frac > bt2->frac) + return (1); + return (0); +} + +static void +g_mirror_update_delay(struct g_mirror_disk *disk, struct bio *bp) +{ + + if (disk->d_softc->sc_balance != G_MIRROR_BALANCE_LOAD) + return; + binuptime(&disk->d_delay); + bintime_sub(&disk->d_delay, &bp->bio_t0); +} + +static void +g_mirror_done(struct bio *bp) +{ + struct g_mirror_softc *sc; + + sc = bp->bio_from->geom->softc; + bp->bio_flags = BIO_FLAG1; + mtx_lock(&sc->sc_queue_mtx); + bioq_disksort(&sc->sc_queue, bp); + wakeup(sc); + mtx_unlock(&sc->sc_queue_mtx); +} + +static void +g_mirror_regular_request(struct bio *bp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + struct bio *pbp; + + g_topology_assert_not(); + + pbp = bp->bio_parent; + sc = pbp->bio_to->geom->softc; + disk = bp->bio_from->private; + if (disk == NULL) { + g_topology_lock(); + g_mirror_kill_consumer(sc, bp->bio_from); + g_topology_unlock(); + } else { + g_mirror_update_delay(disk, bp); + } + + pbp->bio_inbed++; + KASSERT(pbp->bio_inbed <= pbp->bio_children, + ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed, + pbp->bio_children)); + if (bp->bio_error == 0 && pbp->bio_error == 0) { + G_MIRROR_LOGREQ(3, bp, "Request delivered."); + g_destroy_bio(bp); + if (pbp->bio_children == pbp->bio_inbed) { + G_MIRROR_LOGREQ(3, pbp, "Request delivered."); + pbp->bio_completed = pbp->bio_length; + g_io_deliver(pbp, pbp->bio_error); + } + return; + } else if (bp->bio_error != 0) { + if (pbp->bio_error == 0) + pbp->bio_error = bp->bio_error; + G_MIRROR_LOGREQ(0, bp, "Request failed (error=%d).", + bp->bio_error); + if (disk != NULL) { + sc->sc_bump_syncid = G_MIRROR_BUMP_IMMEDIATELY; + g_mirror_event_send(disk, + G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + } + switch (pbp->bio_cmd) { + case BIO_DELETE: + case BIO_WRITE: + pbp->bio_inbed--; + pbp->bio_children--; + break; + } + } + g_destroy_bio(bp); + + switch (pbp->bio_cmd) { + case BIO_READ: + if (pbp->bio_children == pbp->bio_inbed) { + pbp->bio_error = 0; + mtx_lock(&sc->sc_queue_mtx); + bioq_disksort(&sc->sc_queue, pbp); + G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc); + wakeup(sc); + mtx_unlock(&sc->sc_queue_mtx); + } + break; + case BIO_DELETE: + case BIO_WRITE: + if (pbp->bio_children == 0) { + /* + * All requests failed. + */ + } else if (pbp->bio_inbed < pbp->bio_children) { + /* Do nothing. */ + break; + } else if (pbp->bio_children == pbp->bio_inbed) { + /* Some requests succeeded. */ + pbp->bio_error = 0; + pbp->bio_completed = pbp->bio_length; + } + g_io_deliver(pbp, pbp->bio_error); + break; + default: + KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd)); + break; + } +} + +static void +g_mirror_sync_done(struct bio *bp) +{ + struct g_mirror_softc *sc; + + G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered."); + sc = bp->bio_from->geom->softc; + bp->bio_flags = BIO_FLAG2; + mtx_lock(&sc->sc_queue_mtx); + bioq_disksort(&sc->sc_queue, bp); + wakeup(sc); + mtx_unlock(&sc->sc_queue_mtx); +} + +static void +g_mirror_start(struct bio *bp) +{ + struct g_mirror_softc *sc; + + sc = bp->bio_to->geom->softc; + /* + * If sc == NULL or there are no valid disks, provider's error + * should be set and g_mirror_start() should not be called at all. + */ + KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING, + ("Provider's error should be set (error=%d)(mirror=%s).", + bp->bio_to->error, bp->bio_to->name)); + G_MIRROR_LOGREQ(3, bp, "Request received."); + + switch (bp->bio_cmd) { + case BIO_READ: + case BIO_WRITE: + case BIO_DELETE: + break; + case BIO_GETATTR: + default: + g_io_deliver(bp, EOPNOTSUPP); + return; + } + mtx_lock(&sc->sc_queue_mtx); + bioq_disksort(&sc->sc_queue, bp); + G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc); + wakeup(sc); + mtx_unlock(&sc->sc_queue_mtx); +} + +/* + * Send one synchronization request. + */ +static void +g_mirror_sync_one(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + struct bio *bp; + + sc = disk->d_softc; + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING, + ("Disk %s is not marked for synchronization.", + g_mirror_get_diskname(disk))); + + bp = g_new_bio(); + if (bp == NULL) + return; + bp->bio_parent = NULL; + bp->bio_cmd = BIO_READ; + bp->bio_offset = disk->d_sync.ds_offset; + bp->bio_length = MIN(sc->sc_sync.ds_block, + sc->sc_mediasize - bp->bio_offset); + bp->bio_flags = 0; + bp->bio_done = g_mirror_sync_done; + bp->bio_data = uma_zalloc(sc->sc_sync.ds_zone, M_NOWAIT | M_ZERO); + if (bp->bio_data == NULL) { + g_destroy_bio(bp); + return; + } + disk->d_sync.ds_offset += bp->bio_length; + bp->bio_to = sc->sc_provider; + G_MIRROR_LOGREQ(3, bp, "Sending synchronization request."); + g_io_request(bp, disk->d_sync.ds_consumer); +} + +static void +g_mirror_sync_request(struct bio *bp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + + sc = bp->bio_from->geom->softc; + disk = bp->bio_from->private; + if (disk == NULL) { + g_topology_lock(); + g_mirror_kill_consumer(sc, bp->bio_from); + g_topology_unlock(); + uma_zfree(sc->sc_sync.ds_zone, bp->bio_data); + g_destroy_bio(bp); + return; + } + + /* + * Synchronization request. + */ + switch (bp->bio_cmd) { + case BIO_READ: + { + struct g_consumer *cp; + + if (bp->bio_error != 0) { + G_MIRROR_LOGREQ(0, bp, + "Synchronization request failed (error=%d).", + bp->bio_error); + uma_zfree(sc->sc_sync.ds_zone, bp->bio_data); + g_destroy_bio(bp); + return; + } + bp->bio_cmd = BIO_WRITE; + bp->bio_flags = 0; + G_MIRROR_LOGREQ(3, bp, "Synchronization request finished."); + cp = disk->d_consumer; + KASSERT(cp->acr == 0 && cp->acw == 1 && cp->ace == 1, + ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, + cp->acr, cp->acw, cp->ace)); + g_io_request(bp, cp); + return; + } + case BIO_WRITE: + uma_zfree(sc->sc_sync.ds_zone, bp->bio_data); + if (bp->bio_error != 0) { + G_MIRROR_LOGREQ(0, bp, + "Synchronization request failed (error=%d).", + bp->bio_error); + g_destroy_bio(bp); + disk->d_softc->sc_bump_syncid = G_MIRROR_BUMP_IMMEDIATELY; + g_mirror_event_send(disk, + G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + return; + } + G_MIRROR_LOGREQ(3, bp, "Synchronization request finished."); + g_destroy_bio(bp); + disk->d_sync.ds_offset_done = bp->bio_offset + bp->bio_length; + if (bp->bio_offset + bp->bio_length == + sc->sc_provider->mediasize) { + /* + * Disk up-to-date, activate it. + */ + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE, + G_MIRROR_EVENT_DONTWAIT); + return; + } else if ((disk->d_sync.ds_offset_done % + (sc->sc_sync.ds_block * 100)) == 0) { + /* + * Update offset_done on every 100 blocks. + * XXX: This should be configurable. + */ + g_topology_lock(); + g_mirror_update_metadata(disk); + g_topology_unlock(); + } + return; + default: + KASSERT(1 == 0, ("Invalid command here: %u (device=%s)", + bp->bio_cmd, sc->sc_name)); + break; + } +} + +static void +g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp) +{ + struct g_mirror_disk *disk; + struct g_consumer *cp; + struct bio *cbp; + + disk = g_mirror_get_disk(sc); + if (disk == NULL) { + if (bp->bio_error == 0) + bp->bio_error = ENXIO; + g_io_deliver(bp, bp->bio_error); + return; + } + cbp = g_clone_bio(bp); + if (cbp == NULL) { + if (bp->bio_error == 0) + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + return; + } + /* + * Fill in the component buf structure. + */ + cp = disk->d_consumer; + cbp->bio_done = g_mirror_done; + cbp->bio_to = cp->provider; + G_MIRROR_LOGREQ(3, cbp, "Sending request."); + KASSERT(cp->acr > 0 && cp->ace > 0, + ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr, + cp->acw, cp->ace)); + g_io_request(cbp, cp); +} + +static void +g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp) +{ + struct g_mirror_disk *disk, *dp; + struct g_consumer *cp; + struct bio *cbp; + struct bintime curtime; + + binuptime(&curtime); + /* + * Find a disk which the smallest load. + */ + disk = NULL; + LIST_FOREACH(dp, &sc->sc_disks, d_next) { + if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE) + continue; + /* If disk wasn't used for more than 2 sec, use it. */ + if (curtime.sec - dp->d_last_used.sec >= 2) { + disk = dp; + break; + } + if (disk == NULL || + bintime_cmp(&dp->d_delay, &disk->d_delay) < 0) { + disk = dp; + } + } + cbp = g_clone_bio(bp); + if (cbp == NULL) { + if (bp->bio_error == 0) + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + return; + } + /* + * Fill in the component buf structure. + */ + cp = disk->d_consumer; + cbp->bio_done = g_mirror_done; + cbp->bio_to = cp->provider; + binuptime(&disk->d_last_used); + G_MIRROR_LOGREQ(3, cbp, "Sending request."); + KASSERT(cp->acr > 0 && cp->ace > 0, + ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr, + cp->acw, cp->ace)); + g_io_request(cbp, cp); +} + +static void +g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp) +{ + struct bio_queue_head queue; + struct g_mirror_disk *disk; + struct g_consumer *cp; + struct bio *cbp; + off_t left, mod, offset, slice; + u_char *data; + u_int ndisks; + + if (bp->bio_length <= sc->sc_slice) { + g_mirror_request_round_robin(sc, bp); + return; + } + ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE); + slice = bp->bio_length / ndisks; + mod = slice % sc->sc_provider->sectorsize; + if (mod != 0) + slice += sc->sc_provider->sectorsize - mod; + /* + * Allocate all bios before sending any request, so we can + * return ENOMEM in nice and clean way. + */ + left = bp->bio_length; + offset = bp->bio_offset; + data = bp->bio_data; + bioq_init(&queue); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) + continue; + cbp = g_clone_bio(bp); + if (cbp == NULL) { + for (cbp = bioq_first(&queue); cbp != NULL; + cbp = bioq_first(&queue)) { + bioq_remove(&queue, cbp); + g_destroy_bio(cbp); + } + if (bp->bio_error == 0) + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + return; + } + bioq_insert_tail(&queue, cbp); + cbp->bio_done = g_mirror_done; + cbp->bio_caller1 = disk; + cbp->bio_to = disk->d_consumer->provider; + cbp->bio_offset = offset; + cbp->bio_data = data; + cbp->bio_length = MIN(left, slice); + left -= cbp->bio_length; + if (left == 0) + break; + offset += cbp->bio_length; + data += cbp->bio_length; + } + for (cbp = bioq_first(&queue); cbp != NULL; cbp = bioq_first(&queue)) { + bioq_remove(&queue, cbp); + G_MIRROR_LOGREQ(3, cbp, "Sending request."); + disk = cbp->bio_caller1; + cbp->bio_caller1 = NULL; + cp = disk->d_consumer; + KASSERT(cp->acr > 0 && cp->ace > 0, + ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, + cp->acr, cp->acw, cp->ace)); + g_io_request(cbp, disk->d_consumer); + } +} + +static void +g_mirror_register_request(struct bio *bp) +{ + struct g_mirror_softc *sc; + + sc = bp->bio_to->geom->softc; + switch (bp->bio_cmd) { + case BIO_READ: + switch (sc->sc_balance) { + case G_MIRROR_BALANCE_ROUND_ROBIN: + g_mirror_request_round_robin(sc, bp); + break; + case G_MIRROR_BALANCE_LOAD: + g_mirror_request_load(sc, bp); + break; + case G_MIRROR_BALANCE_SPLIT: + g_mirror_request_split(sc, bp); + break; + } + return; + case BIO_WRITE: + case BIO_DELETE: + { + struct g_mirror_disk *disk; + struct bio_queue_head queue; + struct g_consumer *cp; + struct bio *cbp; + + /* + * Allocate all bios before sending any request, so we can + * return ENOMEM in nice and clean way. + */ + bioq_init(&queue); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + switch (disk->d_state) { + case G_MIRROR_DISK_STATE_ACTIVE: + break; + case G_MIRROR_DISK_STATE_SYNCHRONIZING: + if (bp->bio_offset >= disk->d_sync.ds_offset) + continue; + break; + default: + continue; + } + cbp = g_clone_bio(bp); + if (cbp == NULL) { + for (cbp = bioq_first(&queue); cbp != NULL; + cbp = bioq_first(&queue)) { + bioq_remove(&queue, cbp); + g_destroy_bio(cbp); + } + if (bp->bio_error == 0) + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + return; + } + bioq_insert_tail(&queue, cbp); + } + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + switch (disk->d_state) { + case G_MIRROR_DISK_STATE_ACTIVE: + break; + case G_MIRROR_DISK_STATE_SYNCHRONIZING: + if (bp->bio_offset >= disk->d_sync.ds_offset) + continue; + break; + default: + continue; + } + cbp = bioq_first(&queue); + KASSERT(cbp != NULL, ("NULL cbp! (device %s).", + sc->sc_name)); + bioq_remove(&queue, cbp); + cp = disk->d_consumer; + cbp->bio_done = g_mirror_done; + cbp->bio_to = cp->provider; + G_MIRROR_LOGREQ(3, cbp, "Sending request."); + KASSERT(cp->acw > 0 && cp->ace > 0, + ("Consumer %s not opened (r%dw%de%d).", + cp->provider->name, cp->acr, cp->acw, cp->ace)); + g_io_request(cbp, cp); + } + /* + * Bump syncid on first write. + */ + if (sc->sc_bump_syncid == G_MIRROR_BUMP_ON_FIRST_WRITE) { + sc->sc_bump_syncid = 0; + g_topology_lock(); + g_mirror_bump_syncid(sc); + g_topology_unlock(); + } + return; + } + default: + KASSERT(1 == 0, ("Invalid command here: %u (device=%s)", + bp->bio_cmd, sc->sc_name)); + break; + } +} + +/* + * Worker thread. + */ +static void +g_mirror_worker(void *arg) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + struct g_mirror_event *ep; + struct bio *bp; + u_int nreqs; + + sc = arg; + curthread->td_base_pri = PRIBIO; + + nreqs = 0; + for (;;) { + G_MIRROR_DEBUG(5, "%s: Let's see...", __func__); + /* + * First take a look at events. + * This is important to handle events before any I/O requests. + */ + ep = g_mirror_event_get(sc); + if (ep != NULL) { + g_topology_lock(); + if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) { + /* Update only device status. */ + G_MIRROR_DEBUG(3, + "Running event for device %s.", + sc->sc_name); + ep->e_error = 0; + g_mirror_update_device(sc, 1); + } else { + /* Update disk status. */ + G_MIRROR_DEBUG(3, "Running event for disk %s.", + g_mirror_get_diskname(ep->e_disk)); + ep->e_error = g_mirror_update_disk(ep->e_disk, + ep->e_state); + if (ep->e_error == 0) + g_mirror_update_device(sc, 0); + } + g_topology_unlock(); + if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) { + KASSERT(ep->e_error == 0, + ("Error cannot be handled.")); + g_mirror_event_free(ep); + } else { + ep->e_flags |= G_MIRROR_EVENT_DONE; + G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, + ep); + mtx_lock(&sc->sc_events_mtx); + wakeup(ep); + mtx_unlock(&sc->sc_events_mtx); + } + if ((sc->sc_flags & + G_MIRROR_DEVICE_FLAG_DESTROY) != 0) { +end: + if ((sc->sc_flags & + G_MIRROR_DEVICE_FLAG_WAIT) != 0) { + G_MIRROR_DEBUG(4, "%s: Waking up %p.", + __func__, &sc->sc_worker); + wakeup(&sc->sc_worker); + sc->sc_worker = NULL; + } else { + g_topology_lock(); + g_mirror_destroy_device(sc); + g_topology_unlock(); + free(sc, M_MIRROR); + } + kthread_exit(0); + } + G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__); + continue; + } + /* + * Now I/O requests. + */ + /* Get first request from the queue. */ + mtx_lock(&sc->sc_queue_mtx); + bp = bioq_first(&sc->sc_queue); + if (bp == NULL) { + if ((sc->sc_flags & + G_MIRROR_DEVICE_FLAG_DESTROY) != 0) { + mtx_unlock(&sc->sc_queue_mtx); + goto end; + } + } + if (sc->sc_sync.ds_ndisks > 0 && + (bp == NULL || nreqs > g_mirror_reqs_per_sync)) { + mtx_unlock(&sc->sc_queue_mtx); + /* + * It is time for synchronization... + */ + nreqs = 0; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state != + G_MIRROR_DISK_STATE_SYNCHRONIZING) { + continue; + } + if (disk->d_sync.ds_offset >= + sc->sc_provider->mediasize) { + continue; + } + if (disk->d_sync.ds_offset > + disk->d_sync.ds_offset_done) { + continue; + } + g_mirror_sync_one(disk); + } + G_MIRROR_DEBUG(5, "%s: I'm here 2.", __func__); + goto sleep; + } + if (bp == NULL) { + MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1", 0); + G_MIRROR_DEBUG(5, "%s: I'm here 3.", __func__); + continue; + } + nreqs++; + bioq_remove(&sc->sc_queue, bp); + mtx_unlock(&sc->sc_queue_mtx); + + if ((bp->bio_flags & BIO_FLAG1) != 0) { + g_mirror_regular_request(bp); + } else if ((bp->bio_flags & BIO_FLAG2) != 0) { + u_int timeout, sps; + + g_mirror_sync_request(bp); +sleep: + sps = atomic_load_acq_int(&g_mirror_syncs_per_sec); + if (sps == 0) { + G_MIRROR_DEBUG(5, "%s: I'm here 5.", __func__); + continue; + } + mtx_lock(&sc->sc_queue_mtx); + if (bioq_first(&sc->sc_queue) != NULL) { + mtx_unlock(&sc->sc_queue_mtx); + G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__); + continue; + } + timeout = hz / sps; + if (timeout == 0) + timeout = 1; + MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w2", + timeout); + } else { + g_mirror_register_request(bp); + } + G_MIRROR_DEBUG(5, "%s: I'm here 6.", __func__); + } +} + +/* + * Open disk's consumer if needed. + */ +static void +g_mirror_update_access(struct g_mirror_disk *disk) +{ + struct g_provider *pp; + struct g_consumer *cp; + int acr, acw, ace, cpw, error; + + g_topology_assert(); + + cp = disk->d_consumer; + pp = disk->d_softc->sc_provider; + if (pp == NULL) { + acr = -cp->acr; + acw = -cp->acw; + ace = -cp->ace; + } else { + acr = pp->acr - cp->acr; + acw = pp->acw - cp->acw; + ace = pp->ace - cp->ace; + /* Grab an extra "exclusive" bit. */ + if (pp->acr > 0 || pp->acw > 0 || pp->ace > 0) + ace++; + } + if (acr == 0 && acw == 0 && ace == 0) + return; + cpw = cp->acw; + error = g_access(cp, acr, acw, ace); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", cp->provider->name, acr, + acw, ace, error); + if (error != 0) { + disk->d_softc->sc_bump_syncid = G_MIRROR_BUMP_ON_FIRST_WRITE; + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + return; + } + if (cpw == 0 && cp->acw > 0) { + G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.", + g_mirror_get_diskname(disk), disk->d_softc->sc_name); + disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY; + } else if (cpw > 0 && cp->acw == 0) { + G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.", + g_mirror_get_diskname(disk), disk->d_softc->sc_name); + disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY; + } +} + +static void +g_mirror_sync_start(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + struct g_consumer *cp; + int error; + + g_topology_assert(); + + sc = disk->d_softc; + KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING, + ("Device not in RUNNING state (%s, %u).", sc->sc_name, + sc->sc_state)); + cp = disk->d_consumer; + KASSERT(cp->acr == 0 && cp->acw == 0 && cp->ace == 0, + ("Consumer %s already opened.", cp->provider->name)); + + G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name, + g_mirror_get_diskname(disk)); + error = g_access(cp, 0, 1, 1); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", cp->provider->name, 0, 1, + 1, error); + if (error != 0) { + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + return; + } + disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY; + KASSERT(disk->d_sync.ds_consumer == NULL, + ("Sync consumer already exists (device=%s, disk=%s).", + sc->sc_name, g_mirror_get_diskname(disk))); + disk->d_sync.ds_consumer = g_new_consumer(sc->sc_sync.ds_geom); + disk->d_sync.ds_consumer->private = disk; + error = g_attach(disk->d_sync.ds_consumer, disk->d_softc->sc_provider); + KASSERT(error == 0, ("Cannot attach to %s (error=%d).", + disk->d_softc->sc_name, error)); + error = g_access(disk->d_sync.ds_consumer, 1, 0, 0); + KASSERT(error == 0, ("Cannot open %s (error=%d).", + disk->d_softc->sc_name, error)); + sc->sc_sync.ds_ndisks++; +} + +/* + * Stop synchronization process. + * type: 0 - synchronization finished + * 1 - synchronization stopped + */ +static void +g_mirror_sync_stop(struct g_mirror_disk *disk, int type) +{ + struct g_consumer *cp; + + g_topology_assert(); + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING, + ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + if (disk->d_sync.ds_consumer == NULL) + return; + + if (type == 0) { + G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.", + disk->d_softc->sc_name, g_mirror_get_diskname(disk)); + } else /* if (type == 1) */ { + G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.", + disk->d_softc->sc_name, g_mirror_get_diskname(disk)); + } + cp = disk->d_sync.ds_consumer; + g_access(cp, -1, 0, 0); + g_mirror_kill_consumer(disk->d_softc, cp); + disk->d_sync.ds_consumer = NULL; + disk->d_softc->sc_sync.ds_ndisks--; + cp = disk->d_consumer; + KASSERT(cp->acr == 0 && cp->acw == 1 && cp->ace == 1, + ("Consumer %s not opened.", cp->provider->name)); + g_access(cp, 0, -1, -1); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", cp->provider->name, 0, -1, + -1, 0); + disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY; +} + +static void +g_mirror_launch_provider(struct g_mirror_softc *sc) +{ + struct g_mirror_disk *disk; + struct g_provider *pp; + + g_topology_assert(); + + pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name); + pp->mediasize = sc->sc_mediasize; + pp->sectorsize = sc->sc_sectorsize; + sc->sc_provider = pp; + g_error_provider(pp, 0); + G_MIRROR_DEBUG(0, "Device %s: provider %s launched.", sc->sc_name, + pp->name); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) + g_mirror_sync_start(disk); + } +} + +static void +g_mirror_destroy_provider(struct g_mirror_softc *sc) +{ + struct g_mirror_disk *disk; + struct bio *bp; + + g_topology_assert(); + KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).", + sc->sc_name)); + + g_error_provider(sc->sc_provider, ENXIO); + mtx_lock(&sc->sc_queue_mtx); + while ((bp = bioq_first(&sc->sc_queue)) != NULL) { + bioq_remove(&sc->sc_queue, bp); + g_io_deliver(bp, ENXIO); + } + mtx_unlock(&sc->sc_queue_mtx); + G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name, + sc->sc_provider->name); + sc->sc_provider->flags |= G_PF_WITHER; + g_orphan_provider(sc->sc_provider, ENXIO); + sc->sc_provider = NULL; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) + g_mirror_sync_stop(disk, 1); + } +} + +static void +g_mirror_go(void *arg) +{ + struct g_mirror_softc *sc; + + sc = arg; + G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name); + g_mirror_event_send(sc, 0, + G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE); +} + +static u_int +g_mirror_determine_state(struct g_mirror_disk *disk) +{ + struct g_mirror_softc *sc; + u_int state; + + sc = disk->d_softc; + if (sc->sc_syncid == disk->d_sync.ds_syncid) { + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) { + /* Disk does not need synchronization. */ + state = G_MIRROR_DISK_STATE_ACTIVE; + } else { + if ((sc->sc_flags & + G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 || + (disk->d_flags & + G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) { + /* + * We can start synchronization from + * the stored offset. + */ + state = G_MIRROR_DISK_STATE_SYNCHRONIZING; + } else { + state = G_MIRROR_DISK_STATE_STALE; + } + } + } else if (disk->d_sync.ds_syncid < sc->sc_syncid) { + /* + * Reset all synchronization data for this disk, + * because if it even was synchronized, it was + * synchronized to disks with different syncid. + */ + disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING; + disk->d_sync.ds_offset = 0; + disk->d_sync.ds_offset_done = 0; + disk->d_sync.ds_syncid = sc->sc_syncid; + if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 || + (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) { + state = G_MIRROR_DISK_STATE_SYNCHRONIZING; + } else { + state = G_MIRROR_DISK_STATE_STALE; + } + } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ { + /* + * Not good, NOT GOOD! + * It means that mirror was started on stale disks + * and more fresh disk just arrive. + * If there were writes, mirror is fucked up, sorry. + * I think the best choice here is don't touch + * this disk and inform the user laudly. + */ + G_MIRROR_DEBUG(0, "Device %s was started before the freshest " + "disk (%s) arrives!! It will not be connected to the " + "running device.", sc->sc_name, + g_mirror_get_diskname(disk)); + g_mirror_destroy_disk(disk); + state = G_MIRROR_DISK_STATE_NONE; + /* Return immediately, because disk was destroyed. */ + return (state); + } + G_MIRROR_DEBUG(3, "State for %s disk: %s.", + g_mirror_get_diskname(disk), g_mirror_disk_state2str(state)); + return (state); +} + +/* + * Update device state. + */ +static void +g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force) +{ + struct g_mirror_disk *disk; + u_int state; + + g_topology_assert(); + + switch (sc->sc_state) { + case G_MIRROR_DEVICE_STATE_STARTING: + { + struct g_mirror_disk *pdisk; + u_int dirty, ndisks, syncid; + + KASSERT(sc->sc_provider == NULL, + ("Non-NULL provider in STARTING state (%s).", sc->sc_name)); + /* + * Are we ready? We are, if all disks are connected or + * if we have any disks and 'force' is true. + */ + if ((force && g_mirror_ndisks(sc, -1) > 0) || + sc->sc_ndisks == g_mirror_ndisks(sc, -1)) { + ; + } else if (g_mirror_ndisks(sc, -1) == 0) { + /* + * Disks went down in starting phase, so destroy + * device. + */ + callout_drain(&sc->sc_callout); + sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY; + return; + } else { + return; + } + + /* + * Activate all disks with the biggest syncid. + */ + if (force) { + /* + * If called with 'force' true, we're called from + * timeout * procedure, so don't bother canceling + * timeout. + */ + ndisks = 0; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) { + ndisks++; + } + } + if (ndisks == 0) { + int timeout; + + /* No valid disks still, wait some more. */ + timeout = + atomic_load_acq_int(&g_mirror_timeout); + callout_reset(&sc->sc_callout, timeout * hz, + g_mirror_go, sc); + return; + } + } else { + /* Cancel timeout. */ + callout_drain(&sc->sc_callout); + } + + /* + * Find disk with the biggest syncid. + */ + syncid = 0; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_sync.ds_syncid > syncid) + syncid = disk->d_sync.ds_syncid; + } + + /* + * Here we need to look for dirty disks and if all disks + * with the biggest syncid are dirty, we have to choose + * one with the biggest priority and rebuild the rest. + */ + /* + * Find the number of dirty disks with the biggest syncid. + * Find the number of disks with the biggest syncid. + * While here, find a disk with the biggest priority. + */ + dirty = ndisks = 0; + pdisk = NULL; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_sync.ds_syncid != syncid) + continue; + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) { + continue; + } + ndisks++; + if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) { + dirty++; + if (pdisk == NULL || + pdisk->d_priority < disk->d_priority) { + pdisk = disk; + } + } + } + if (dirty == 0) { + /* No dirty disks at all, great. */ + } else if (dirty == ndisks) { + /* + * Force synchronization for all dirty disks except one + * with the biggest priority. + */ + KASSERT(pdisk != NULL, ("pdisk == NULL")); + G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a " + "master disk for synchronization.", + g_mirror_get_diskname(pdisk), sc->sc_name); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_sync.ds_syncid != syncid) + continue; + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) { + continue; + } + KASSERT((disk->d_flags & + G_MIRROR_DISK_FLAG_DIRTY) != 0, + ("Disk %s isn't marked as dirty.", + g_mirror_get_diskname(disk))); + /* Skip the disk with the biggest priority. */ + if (disk == pdisk) + continue; + disk->d_sync.ds_syncid = 0; + } + } else if (dirty < ndisks) { + /* + * Force synchronization for all dirty disks. + * We have some non-dirty disks. + */ + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_sync.ds_syncid != syncid) + continue; + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) { + continue; + } + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_DIRTY) == 0) { + continue; + } + disk->d_sync.ds_syncid = 0; + } + } + + /* Reset hint. */ + sc->sc_hint = NULL; + sc->sc_syncid = syncid; + if (force) { + /* Remember to bump syncid on first write. */ + sc->sc_bump_syncid = G_MIRROR_BUMP_ON_FIRST_WRITE; + } + state = G_MIRROR_DEVICE_STATE_RUNNING; + G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.", + sc->sc_name, g_mirror_device_state2str(sc->sc_state), + g_mirror_device_state2str(state)); + sc->sc_state = state; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + state = g_mirror_determine_state(disk); + g_mirror_event_send(disk, state, + G_MIRROR_EVENT_DONTWAIT); + if (state == G_MIRROR_DISK_STATE_STALE) { + sc->sc_bump_syncid = + G_MIRROR_BUMP_ON_FIRST_WRITE; + } + } + break; + } + case G_MIRROR_DEVICE_STATE_RUNNING: + /* + * Bump syncid here, if we need to do it immediately. + */ + if (sc->sc_bump_syncid == G_MIRROR_BUMP_IMMEDIATELY) { + sc->sc_bump_syncid = 0; + g_mirror_bump_syncid(sc); + } + if (g_mirror_ndisks(sc, -1) == 0) { + /* + * No disks at all, we need to destroy device. + */ + sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY; + } else if (g_mirror_ndisks(sc, + G_MIRROR_DISK_STATE_ACTIVE) == 0 && + g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) { + /* + * No active disks, destroy provider. + */ + if (sc->sc_provider != NULL) + g_mirror_destroy_provider(sc); + } else if (g_mirror_ndisks(sc, + G_MIRROR_DISK_STATE_ACTIVE) > 0) { + /* + * We have active disks, launch provider if it doesn't + * exist. + */ + if (sc->sc_provider == NULL) + g_mirror_launch_provider(sc); + } + break; + default: + KASSERT(1 == 0, ("Wrong device state (%s, %s).", + sc->sc_name, g_mirror_device_state2str(sc->sc_state))); + break; + } +} + +/* + * Update disk state and device state if needed. + */ +#define DISK_STATE_CHANGED() G_MIRROR_DEBUG(1, \ + "Disk %s state changed from %s to %s (device %s).", \ + g_mirror_get_diskname(disk), \ + g_mirror_disk_state2str(disk->d_state), \ + g_mirror_disk_state2str(state), sc->sc_name) +static int +g_mirror_update_disk(struct g_mirror_disk *disk, u_int state) +{ + struct g_mirror_softc *sc; + + g_topology_assert(); + + sc = disk->d_softc; +again: + G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.", + g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state), + g_mirror_disk_state2str(state)); + switch (state) { + case G_MIRROR_DISK_STATE_NEW: + /* + * Possible scenarios: + * 1. New disk arrive. + */ + /* Previous state should be NONE. */ + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE, + ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + DISK_STATE_CHANGED(); + + disk->d_state = state; + LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next); + G_MIRROR_DEBUG(0, "Device %s: provider %s detected.", + sc->sc_name, g_mirror_get_diskname(disk)); + if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) + break; + KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING, + ("Wrong device state (%s, %s, %s, %s).", sc->sc_name, + g_mirror_device_state2str(sc->sc_state), + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + state = g_mirror_determine_state(disk); + if (state != G_MIRROR_DISK_STATE_NONE) + goto again; + break; + case G_MIRROR_DISK_STATE_ACTIVE: + /* + * Possible scenarios: + * 1. New disk does not need synchronization. + * 2. Synchronization process finished successfully. + */ + KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING, + ("Wrong device state (%s, %s, %s, %s).", sc->sc_name, + g_mirror_device_state2str(sc->sc_state), + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + /* Previous state should be NEW or SYNCHRONIZING. */ + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW || + disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING, + ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + DISK_STATE_CHANGED(); + + if (disk->d_state == G_MIRROR_DISK_STATE_NEW) + disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY; + else if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) { + disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING; + disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC; + g_mirror_sync_stop(disk, 0); + } + disk->d_state = state; + disk->d_sync.ds_offset = 0; + disk->d_sync.ds_offset_done = 0; + g_mirror_update_access(disk); + g_mirror_update_metadata(disk); + G_MIRROR_DEBUG(0, "Device %s: provider %s activated.", + sc->sc_name, g_mirror_get_diskname(disk)); + break; + case G_MIRROR_DISK_STATE_STALE: + /* + * Possible scenarios: + * 1. Stale disk was connected. + */ + /* Previous state should be NEW. */ + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW, + ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING, + ("Wrong device state (%s, %s, %s, %s).", sc->sc_name, + g_mirror_device_state2str(sc->sc_state), + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + /* + * STALE state is only possible if device is marked + * NOAUTOSYNC. + */ + KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0, + ("Wrong device state (%s, %s, %s, %s).", sc->sc_name, + g_mirror_device_state2str(sc->sc_state), + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + DISK_STATE_CHANGED(); + + disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY; + disk->d_state = state; + g_mirror_update_metadata(disk); + G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.", + sc->sc_name, g_mirror_get_diskname(disk)); + break; + case G_MIRROR_DISK_STATE_SYNCHRONIZING: + /* + * Possible scenarios: + * 1. Disk which needs synchronization was connected. + */ + /* Previous state should be NEW. */ + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW, + ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING, + ("Wrong device state (%s, %s, %s, %s).", sc->sc_name, + g_mirror_device_state2str(sc->sc_state), + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + DISK_STATE_CHANGED(); + + if (disk->d_state == G_MIRROR_DISK_STATE_NEW) + disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY; + disk->d_state = state; + if (sc->sc_provider != NULL) { + g_mirror_sync_start(disk); + g_mirror_update_metadata(disk); + } + break; + case G_MIRROR_DISK_STATE_DISCONNECTED: + /* + * Possible scenarios: + * 1. Device wasn't running yet, but disk disappear. + * 2. Disk was active and disapppear. + * 3. Disk disappear during synchronization process. + */ + if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) { + /* + * Previous state should be ACTIVE, STALE or + * SYNCHRONIZING. + */ + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE || + disk->d_state == G_MIRROR_DISK_STATE_STALE || + disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING, + ("Wrong disk state (%s, %s).", + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) { + /* Previous state should be NEW. */ + KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW, + ("Wrong disk state (%s, %s).", + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); + /* + * Reset bumping syncid if disk disappeared in STARTING + * state. + */ + if (sc->sc_bump_syncid == G_MIRROR_BUMP_ON_FIRST_WRITE) + sc->sc_bump_syncid = 0; +#ifdef INVARIANTS + } else { + KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).", + sc->sc_name, + g_mirror_device_state2str(sc->sc_state), + g_mirror_get_diskname(disk), + g_mirror_disk_state2str(disk->d_state))); +#endif + } + DISK_STATE_CHANGED(); + G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.", + sc->sc_name, g_mirror_get_diskname(disk)); + + g_mirror_destroy_disk(disk); + break; + case G_MIRROR_DISK_STATE_DESTROY: + { + int error; + + error = g_mirror_clear_metadata(disk); + if (error != 0) + return (error); + DISK_STATE_CHANGED(); + G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", + sc->sc_name, g_mirror_get_diskname(disk)); + + g_mirror_destroy_disk(disk); + sc->sc_ndisks--; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + g_mirror_update_metadata(disk); + } + break; + } + default: + KASSERT(1 == 0, ("Unknown state (%u).", state)); + break; + } + return (0); +} +#undef DISK_STATE_CHANGED + +static int +g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md) +{ + struct g_provider *pp; + u_char *buf; + int error; + + g_topology_assert(); + + error = g_access(cp, 1, 0, 0); + if (error != 0) + return (error); + pp = cp->provider; + g_topology_unlock(); + /* Metadata are stored on last sector. */ + buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize, + &error); + g_topology_lock(); + if (buf == NULL) { + g_access(cp, -1, 0, 0); + return (error); + } + if (error != 0) { + g_access(cp, -1, 0, 0); + g_free(buf); + return (error); + } + error = g_access(cp, -1, 0, 0); + KASSERT(error == 0, ("Cannot decrease access count for %s.", pp->name)); + + /* Decode metadata. */ + error = mirror_metadata_decode(buf, md); + g_free(buf); + if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0) + return (EINVAL); + if (error != 0) { + G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.", + cp->provider->name); + return (error); + } + + return (0); +} + +static int +g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp, + struct g_mirror_metadata *md) +{ + + if (g_mirror_id2disk(sc, md->md_did) != NULL) { + G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.", + pp->name, md->md_did); + return (EEXIST); + } + if (md->md_all != sc->sc_ndisks) { + G_MIRROR_DEBUG(1, + "Invalid '%s' field on disk %s (device %s), skipping.", + "md_all", pp->name, sc->sc_name); + return (EINVAL); + } + if (md->md_slice != sc->sc_slice) { + G_MIRROR_DEBUG(1, + "Invalid '%s' field on disk %s (device %s), skipping.", + "md_slice", pp->name, sc->sc_name); + return (EINVAL); + } + if (md->md_balance != sc->sc_balance) { + G_MIRROR_DEBUG(1, + "Invalid '%s' field on disk %s (device %s), skipping.", + "md_balance", pp->name, sc->sc_name); + return (EINVAL); + } + if (md->md_mediasize != sc->sc_mediasize) { + G_MIRROR_DEBUG(1, + "Invalid '%s' field on disk %s (device %s), skipping.", + "md_mediasize", pp->name, sc->sc_name); + return (EINVAL); + } + if (sc->sc_mediasize > pp->mediasize) { + G_MIRROR_DEBUG(1, + "Invalid size of disk %s (device %s), skipping.", pp->name, + sc->sc_name); + return (EINVAL); + } + if (md->md_sectorsize != sc->sc_sectorsize) { + G_MIRROR_DEBUG(1, + "Invalid '%s' field on disk %s (device %s), skipping.", + "md_sectorsize", pp->name, sc->sc_name); + return (EINVAL); + } + if ((sc->sc_sectorsize % pp->sectorsize) != 0) { + G_MIRROR_DEBUG(1, + "Invalid sector size of disk %s (device %s), skipping.", + pp->name, sc->sc_name); + return (EINVAL); + } + if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) { + G_MIRROR_DEBUG(1, + "Invalid device flags on disk %s (device %s), skipping.", + pp->name, sc->sc_name); + return (EINVAL); + } + if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) { + G_MIRROR_DEBUG(1, + "Invalid disk flags on disk %s (device %s), skipping.", + pp->name, sc->sc_name); + return (EINVAL); + } + return (0); +} + +static int +g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp, + struct g_mirror_metadata *md) +{ + struct g_mirror_disk *disk; + int error; + + g_topology_assert(); + G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name); + + error = g_mirror_check_metadata(sc, pp, md); + if (error != 0) + return (error); + disk = g_mirror_init_disk(sc, pp, md, &error); + if (disk == NULL) + return (error); + error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW, + G_MIRROR_EVENT_WAIT); + return (error); +} + +static int +g_mirror_access(struct g_provider *pp, int acr, int acw, int ace) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + int dcr, dcw, dce, err, error; + + g_topology_assert(); + G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr, + acw, ace); + + dcr = pp->acr + acr; + dcw = pp->acw + acw; + dce = pp->ace + ace; + + /* On first open, grab an extra "exclusive" bit */ + if (pp->acr == 0 && pp->acw == 0 && pp->ace == 0) + ace++; + /* ... and let go of it on last close */ + if (dcr == 0 && dcw == 0 && dce == 0) + ace--; + + sc = pp->geom->softc; + if (sc == NULL || LIST_EMPTY(&sc->sc_disks)) { + if (acr <= 0 && acw <= 0 && ace <= 0) + return (0); + else + return (ENXIO); + } + error = ENXIO; + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) + continue; + err = g_access(disk->d_consumer, acr, acw, ace); + G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", + g_mirror_get_diskname(disk), acr, acw, ace, err); + if (err == 0) { + /* + * Mark disk as dirty on open and unmark on close. + */ + if (pp->acw == 0 && dcw > 0) { + G_MIRROR_DEBUG(1, + "Disk %s (device %s) marked as dirty.", + g_mirror_get_diskname(disk), sc->sc_name); + disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY; + g_mirror_update_metadata(disk); + } else if (pp->acw > 0 && dcw == 0) { + G_MIRROR_DEBUG(1, + "Disk %s (device %s) marked as clean.", + g_mirror_get_diskname(disk), sc->sc_name); + disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY; + g_mirror_update_metadata(disk); + } + error = 0; + } else { + sc->sc_bump_syncid = G_MIRROR_BUMP_ON_FIRST_WRITE; + g_mirror_event_send(disk, + G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + } + } + return (error); +} + +static struct g_geom * +g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md) +{ + struct g_mirror_softc *sc; + struct g_geom *gp; + int error, timeout; + + g_topology_assert(); + G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name, + md->md_mid); + + /* One disk is minimum. */ + if (md->md_all < 1) + return (NULL); + /* + * Action geom. + */ + gp = g_new_geomf(mp, "%s", md->md_name); + sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO); + gp->start = g_mirror_start; + gp->spoiled = g_mirror_orphan; + gp->orphan = g_mirror_orphan; + gp->access = g_mirror_access; + gp->dumpconf = g_mirror_dumpconf; + + sc->sc_id = md->md_mid; + sc->sc_slice = md->md_slice; + sc->sc_balance = md->md_balance; + sc->sc_mediasize = md->md_mediasize; + sc->sc_sectorsize = md->md_sectorsize; + sc->sc_ndisks = md->md_all; + sc->sc_flags = md->md_mflags; + sc->sc_bump_syncid = 0; + bioq_init(&sc->sc_queue); + mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF); + LIST_INIT(&sc->sc_disks); + TAILQ_INIT(&sc->sc_events); + mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF); + callout_init(&sc->sc_callout, CALLOUT_MPSAFE); + sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING; + gp->softc = sc; + sc->sc_geom = gp; + sc->sc_provider = NULL; + /* + * Synchronization geom. + */ + gp = g_new_geomf(mp, "%s.sync", md->md_name); + gp->softc = sc; + gp->spoiled = g_mirror_orphan; + gp->orphan = g_mirror_orphan; + sc->sc_sync.ds_geom = gp; + sc->sc_sync.ds_block = atomic_load_acq_int(&g_mirror_sync_block_size); + sc->sc_sync.ds_ndisks = 0; + sc->sc_sync.ds_zone = uma_zcreate("gmirror:sync", sc->sc_sync.ds_block, + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + error = kthread_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0, + "g_mirror %s", md->md_name); + if (error != 0) { + G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.", + sc->sc_name); + uma_zdestroy(sc->sc_sync.ds_zone); + g_destroy_geom(sc->sc_sync.ds_geom); + mtx_destroy(&sc->sc_events_mtx); + mtx_destroy(&sc->sc_queue_mtx); + g_destroy_geom(sc->sc_geom); + free(sc, M_MIRROR); + return (NULL); + } + + G_MIRROR_DEBUG(0, "Device %s created (id=%u).", sc->sc_name, sc->sc_id); + + /* + * Run timeout. + */ + timeout = atomic_load_acq_int(&g_mirror_timeout); + callout_reset(&sc->sc_callout, timeout * hz, g_mirror_go, sc); + return (sc->sc_geom); +} + +int +g_mirror_destroy(struct g_mirror_softc *sc, boolean_t force) +{ + struct g_provider *pp; + + g_topology_assert(); + + if (sc == NULL) + return (ENXIO); + pp = sc->sc_provider; + if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) { + if (force) { + G_MIRROR_DEBUG(0, "Device %s is still open, so it " + "can't be definitely removed.", pp->name); + } else { + G_MIRROR_DEBUG(1, + "Device %s is still open (r%dw%de%d).", pp->name, + pp->acr, pp->acw, pp->ace); + return (EBUSY); + } + } + + sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY; + sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT; + G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc); + mtx_lock(&sc->sc_queue_mtx); + wakeup(sc); + mtx_unlock(&sc->sc_queue_mtx); + G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker); + while (sc->sc_worker != NULL) + tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5); + G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker); + g_mirror_destroy_device(sc); + free(sc, M_MIRROR); + return (0); +} + +static void +g_mirror_taste_orphan(struct g_consumer *cp) +{ + + KASSERT(1 == 0, ("%s called while tasting %s.", __func__, + cp->provider->name)); +} + +static struct g_geom * +g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) +{ + struct g_mirror_metadata md; + struct g_mirror_softc *sc; + struct g_consumer *cp; + struct g_geom *gp; + int error; + + g_topology_assert(); + g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name); + G_MIRROR_DEBUG(2, "Tasting %s.", pp->name); + + gp = g_new_geomf(mp, "mirror:taste"); + /* + * This orphan function should be never called. + */ + gp->orphan = g_mirror_taste_orphan; + cp = g_new_consumer(gp); + g_attach(cp, pp); + error = g_mirror_read_metadata(cp, &md); + g_detach(cp); + g_destroy_consumer(cp); + g_destroy_geom(gp); + if (error != 0) + return (NULL); + gp = NULL; + + if (md.md_version > G_MIRROR_VERSION) { + printf("geom_mirror.ko module is too old to handle %s.\n", + pp->name); + return (NULL); + } + if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) { + G_MIRROR_DEBUG(0, + "Device %s: provider %s marked as inactive, skipping.", + md.md_name, pp->name); + return (NULL); + } + if (g_mirror_debug >= 2) + mirror_metadata_dump(&md); + + /* + * Let's check if device already exists. + */ + LIST_FOREACH(gp, &mp->geom, geom) { + sc = gp->softc; + if (sc == NULL) + continue; + if (sc->sc_sync.ds_geom == gp) + continue; + if (strcmp(md.md_name, sc->sc_name) != 0) + continue; + if (md.md_mid != sc->sc_id) { + G_MIRROR_DEBUG(0, "Device %s already configured.", + sc->sc_name); + return (NULL); + } + break; + } + if (gp == NULL) { + gp = g_mirror_create(mp, &md); + if (gp == NULL) { + G_MIRROR_DEBUG(0, "Cannot create device %s.mirror", + md.md_name); + return (NULL); + } + sc = gp->softc; + } + G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name); + error = g_mirror_add_disk(sc, pp, &md); + if (error != 0) { + G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).", + pp->name, gp->name, error); + if (LIST_EMPTY(&sc->sc_disks)) + g_mirror_destroy(sc, 1); + return (NULL); + } + return (gp); +} + +static int +g_mirror_destroy_geom(struct gctl_req *req __unused, + struct g_class *mp __unused, struct g_geom *gp) +{ + + return (g_mirror_destroy(gp->softc, 0)); +} + +static void +g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, + struct g_consumer *cp, struct g_provider *pp) +{ + struct g_mirror_softc *sc; + + g_topology_assert(); + + sc = gp->softc; + if (sc == NULL) + return; + /* Skip synchronization geom. */ + if (gp == sc->sc_sync.ds_geom) + return; + if (pp != NULL) { + /* Nothing here. */ + } else if (cp != NULL) { + struct g_mirror_disk *disk; + + disk = cp->private; + if (disk == NULL) + return; + sbuf_printf(sb, "%s%u\n", indent, (u_int)disk->d_id); + if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) { + sbuf_printf(sb, "%s", indent); + if (disk->d_sync.ds_offset_done == 0) + sbuf_printf(sb, "0%%"); + else { + sbuf_printf(sb, "%u%%", + (u_int)((disk->d_sync.ds_offset_done * 100) / + sc->sc_provider->mediasize)); + } + sbuf_printf(sb, "\n"); + } + sbuf_printf(sb, "%s%u\n", indent, + disk->d_sync.ds_syncid); + sbuf_printf(sb, "%s", indent); + if (disk->d_flags == 0) + sbuf_printf(sb, "NONE"); + else { + int first = 1; + + if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) { + if (!first) + sbuf_printf(sb, ", "); + else + first = 0; + sbuf_printf(sb, "DIRTY"); + } + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_INACTIVE) != 0) { + if (!first) + sbuf_printf(sb, ", "); + else + first = 0; + sbuf_printf(sb, "INACTIVE"); + } + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) { + if (!first) + sbuf_printf(sb, ", "); + else + first = 0; + sbuf_printf(sb, "SYNCHRONIZING"); + } + if ((disk->d_flags & + G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) { + if (!first) + sbuf_printf(sb, ", "); + else + first = 0; + sbuf_printf(sb, "FORCE_SYNC"); + } + } + sbuf_printf(sb, "\n"); + sbuf_printf(sb, "%s%u\n", indent, + disk->d_sync.ds_syncid); + sbuf_printf(sb, "%s%s\n", indent, + g_mirror_disk_state2str(disk->d_state)); + } else { + sbuf_printf(sb, "%s%u\n", indent, (u_int)sc->sc_id); + sbuf_printf(sb, "%s%u\n", indent, sc->sc_syncid); + sbuf_printf(sb, "%s", indent); + if (sc->sc_flags == 0) + sbuf_printf(sb, "NONE"); + else { + int first = 1; + + if ((sc->sc_flags & + G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0) { + if (!first) + sbuf_printf(sb, ", "); + else + first = 0; + sbuf_printf(sb, "NOAUTOSYNC"); + } + } + sbuf_printf(sb, "\n"); + sbuf_printf(sb, "%s%u\n", indent, + (u_int)sc->sc_slice); + sbuf_printf(sb, "%s%s\n", indent, + balance_name(sc->sc_balance)); + sbuf_printf(sb, "%s%u\n", indent, + sc->sc_ndisks); + } +} + +DECLARE_GEOM_CLASS(g_mirror_class, g_mirror); diff --git a/sys/geom/mirror/g_mirror.h b/sys/geom/mirror/g_mirror.h new file mode 100644 index 000000000000..0386bab19e91 --- /dev/null +++ b/sys/geom/mirror/g_mirror.h @@ -0,0 +1,350 @@ +/*- + * Copyright (c) 2004 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _G_MIRROR_H_ +#define _G_MIRROR_H_ + +#include +#include + +#define G_MIRROR_CLASS_NAME "MIRROR" + +#define G_MIRROR_MAGIC "GEOM::MIRROR" +#define G_MIRROR_VERSION 0 + +#define G_MIRROR_BALANCE_NONE 0 +#define G_MIRROR_BALANCE_ROUND_ROBIN 1 +#define G_MIRROR_BALANCE_LOAD 2 +#define G_MIRROR_BALANCE_SPLIT 3 +#define G_MIRROR_BALANCE_MIN G_MIRROR_BALANCE_NONE +#define G_MIRROR_BALANCE_MAX G_MIRROR_BALANCE_SPLIT + +#define G_MIRROR_DISK_FLAG_DIRTY 0x0000000000000001ULL +#define G_MIRROR_DISK_FLAG_SYNCHRONIZING 0x0000000000000002ULL +#define G_MIRROR_DISK_FLAG_FORCE_SYNC 0x0000000000000004ULL +#define G_MIRROR_DISK_FLAG_INACTIVE 0x0000000000000008ULL +#define G_MIRROR_DISK_FLAG_MASK (G_MIRROR_DISK_FLAG_DIRTY | \ + G_MIRROR_DISK_FLAG_SYNCHRONIZING | \ + G_MIRROR_DISK_FLAG_FORCE_SYNC | \ + G_MIRROR_DISK_FLAG_INACTIVE) + +#define G_MIRROR_DEVICE_FLAG_NOAUTOSYNC 0x0000000000000001ULL +#define G_MIRROR_DEVICE_FLAG_MASK (G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) + +#ifdef _KERNEL +extern u_int g_mirror_debug; + +#define G_MIRROR_DEBUG(lvl, ...) do { \ + if (g_mirror_debug >= (lvl)) { \ + printf("GEOM_MIRROR"); \ + if (g_mirror_debug > 0) \ + printf("[%u]", lvl); \ + printf(": "); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ +} while (0) +#define G_MIRROR_LOGREQ(lvl, bp, ...) do { \ + if (g_mirror_debug >= (lvl)) { \ + printf("GEOM_MIRROR"); \ + if (g_mirror_debug > 0) \ + printf("[%u]", lvl); \ + printf(": "); \ + printf(__VA_ARGS__); \ + printf(" "); \ + g_print_bio(bp); \ + printf("\n"); \ + } \ +} while (0) + +/* + * Informations needed for synchronization. + */ +struct g_mirror_disk_sync { + struct g_consumer *ds_consumer; /* Consumer connected to our mirror. */ + off_t ds_offset; /* Offset of next request to send. */ + off_t ds_offset_done; /* Offset of already synchronized + region. */ + u_int ds_syncid; /* Disk's synchronization ID. */ +}; + +/* + * Informations needed for synchronization. + */ +struct g_mirror_device_sync { + struct g_geom *ds_geom; /* Synchronization geom. */ + size_t ds_block; /* Synchronization request size. */ + u_int ds_ndisks; /* Number of disks in SYNCHRONIZING + state. */ + uma_zone_t ds_zone; /* UMA zone for synchronization + blocks. */ +}; + +#define G_MIRROR_DISK_STATE_NONE 0 +#define G_MIRROR_DISK_STATE_NEW 1 +#define G_MIRROR_DISK_STATE_ACTIVE 2 +#define G_MIRROR_DISK_STATE_STALE 3 +#define G_MIRROR_DISK_STATE_SYNCHRONIZING 4 +#define G_MIRROR_DISK_STATE_DISCONNECTED 5 +#define G_MIRROR_DISK_STATE_DESTROY 6 +struct g_mirror_disk { + uint32_t d_id; /* Disk ID. */ + struct g_consumer *d_consumer; /* Consumer. */ + struct g_mirror_softc *d_softc; /* Back-pointer to softc. */ + struct proc *d_worker; + int d_state; /* Disk state. */ + u_int d_priority; /* Disk priority. */ + struct bintime d_delay; /* Disk delay. */ + struct bintime d_last_used; /* When disk was last used. */ + uint64_t d_flags; /* Additional flags. */ + struct g_mirror_disk_sync d_sync;/* Sync information. */ + LIST_ENTRY(g_mirror_disk) d_next; +}; +#define d_name d_consumer->provider->name + +#define G_MIRROR_EVENT_DONTWAIT 0x1 +#define G_MIRROR_EVENT_WAIT 0x2 +#define G_MIRROR_EVENT_DEVICE 0x4 +#define G_MIRROR_EVENT_DONE 0x8 +struct g_mirror_event { + struct g_mirror_disk *e_disk; + int e_state; + int e_flags; + int e_error; + TAILQ_ENTRY(g_mirror_event) e_next; +}; + +#define G_MIRROR_DEVICE_FLAG_DESTROY 0x0100000000000000ULL +#define G_MIRROR_DEVICE_FLAG_WAIT 0x0200000000000000ULL + +#define G_MIRROR_DEVICE_STATE_STARTING 0 +#define G_MIRROR_DEVICE_STATE_RUNNING 1 + +#define G_MIRROR_BUMP_ON_FIRST_WRITE 1 +#define G_MIRROR_BUMP_IMMEDIATELY 2 +struct g_mirror_softc { + u_int sc_state; /* Device state. */ + uint32_t sc_slice; /* Slice size. */ + uint8_t sc_balance; /* Balance algorithm. */ + uint64_t sc_mediasize; /* Device size. */ + uint32_t sc_sectorsize; /* Sector size. */ + uint64_t sc_flags; /* Additional flags. */ + + struct g_geom *sc_geom; + struct g_provider *sc_provider; + + uint32_t sc_id; /* Mirror unique ID. */ + + struct bio_queue_head sc_queue; + struct mtx sc_queue_mtx; + struct proc *sc_worker; + + LIST_HEAD(, g_mirror_disk) sc_disks; + u_int sc_ndisks; /* Number of disks. */ + struct g_mirror_disk *sc_hint; + + u_int sc_syncid; /* Synchronization ID. */ + int sc_bump_syncid; + struct g_mirror_device_sync sc_sync; + + TAILQ_HEAD(, g_mirror_event) sc_events; + struct mtx sc_events_mtx; + + struct callout sc_callout; +}; +#define sc_name sc_geom->name + +u_int g_mirror_ndisks(struct g_mirror_softc *sc, int state); +int g_mirror_destroy(struct g_mirror_softc *sc, boolean_t force); +int g_mirror_event_send(void *arg, int state, int flags); +struct g_mirror_metadata; +void g_mirror_fill_metadata(struct g_mirror_softc *sc, + struct g_mirror_disk *disk, struct g_mirror_metadata *md); +void g_mirror_update_metadata(struct g_mirror_disk *disk); + +g_ctl_req_t g_mirror_config; +#endif /* _KERNEL */ + +struct g_mirror_metadata { + char md_magic[16]; /* Magic value. */ + uint32_t md_version; /* Version number. */ + char md_name[16]; /* Mirror name. */ + uint32_t md_mid; /* Mirror unique ID. */ + uint32_t md_did; /* Disk unique ID. */ + uint8_t md_all; /* Number of disks in mirror. */ + uint32_t md_syncid; /* Synchronization ID. */ + uint8_t md_priority; /* Disk priority. */ + uint32_t md_slice; /* Slice size. */ + uint8_t md_balance; /* Balance type. */ + uint64_t md_mediasize; /* Size of the smallest + disk in mirror. */ + uint32_t md_sectorsize; /* Sector size. */ + uint64_t md_sync_offset; /* Synchronized offset. */ + uint64_t md_mflags; /* Additional mirror flags. */ + uint64_t md_dflags; /* Additional disk flags. */ + u_char md_hash[16]; /* MD5 hash. */ +}; +static __inline void +mirror_metadata_encode(struct g_mirror_metadata *md, u_char *data) +{ + MD5_CTX ctx; + + bcopy(md->md_magic, data, 16); + le32enc(data + 16, md->md_version); + bcopy(md->md_name, data + 20, 16); + le32enc(data + 36, md->md_mid); + le32enc(data + 40, md->md_did); + *(data + 44) = md->md_all; + le32enc(data + 45, md->md_syncid); + *(data + 49) = md->md_priority; + le32enc(data + 50, md->md_slice); + *(data + 54) = md->md_balance; + le64enc(data + 55, md->md_mediasize); + le32enc(data + 63, md->md_sectorsize); + le64enc(data + 67, md->md_sync_offset); + le64enc(data + 75, md->md_mflags); + le64enc(data + 83, md->md_dflags); + MD5Init(&ctx); + MD5Update(&ctx, data, 91); + MD5Final(md->md_hash, &ctx); + bcopy(md->md_hash, data + 91, 16); +} +static __inline int +mirror_metadata_decode(const u_char *data, struct g_mirror_metadata *md) +{ + MD5_CTX ctx; + + bcopy(data, md->md_magic, 16); + md->md_version = le32dec(data + 16); + bcopy(data + 20, md->md_name, 16); + md->md_mid = le32dec(data + 36); + md->md_did = le32dec(data + 40); + md->md_all = *(data + 44); + md->md_syncid = le32dec(data + 45); + md->md_priority = *(data + 49); + md->md_slice = le32dec(data + 50); + md->md_balance = *(data + 54); + md->md_mediasize = le64dec(data + 55); + md->md_sectorsize = le32dec(data + 63); + md->md_sync_offset = le64dec(data + 67); + md->md_mflags = le64dec(data + 75); + md->md_dflags = le64dec(data + 83); + bcopy(data + 91, md->md_hash, 16); + MD5Init(&ctx); + MD5Update(&ctx, data, 91); + MD5Final(md->md_hash, &ctx); + if (bcmp(md->md_hash, data + 91, 16) != 0) + return (EINVAL); + return (0); +} + +static __inline const char * +balance_name(u_int balance) +{ + static const char *algorithms[] = { + [G_MIRROR_BALANCE_NONE] = "none", + [G_MIRROR_BALANCE_ROUND_ROBIN] = "round-robin", + [G_MIRROR_BALANCE_LOAD] = "load", + [G_MIRROR_BALANCE_SPLIT] = "split", + [G_MIRROR_BALANCE_MAX + 1] = "unknown" + }; + + if (balance > G_MIRROR_BALANCE_MAX) + balance = G_MIRROR_BALANCE_MAX + 1; + + return (algorithms[balance]); +} + +static __inline int +balance_id(const char *name) +{ + static const char *algorithms[] = { + [G_MIRROR_BALANCE_NONE] = "none", + [G_MIRROR_BALANCE_ROUND_ROBIN] = "round-robin", + [G_MIRROR_BALANCE_LOAD] = "load", + [G_MIRROR_BALANCE_SPLIT] = "split" + }; + int n; + + for (n = G_MIRROR_BALANCE_MIN; n <= G_MIRROR_BALANCE_MAX; n++) { + if (strcmp(name, algorithms[n]) == 0) + return (n); + } + return (-1); +} + +static __inline void +mirror_metadata_dump(const struct g_mirror_metadata *md) +{ + static const char hex[] = "0123456789abcdef"; + char hash[16 * 2 + 1]; + u_int i; + + printf(" magic: %s\n", md->md_magic); + printf(" version: %u\n", (u_int)md->md_version); + printf(" name: %s\n", md->md_name); + printf(" mid: %u\n", (u_int)md->md_mid); + printf(" did: %u\n", (u_int)md->md_did); + printf(" all: %u\n", (u_int)md->md_all); + printf(" syncid: %u\n", (u_int)md->md_syncid); + printf(" priority: %u\n", (u_int)md->md_priority); + printf(" slice: %u\n", (u_int)md->md_slice); + printf(" balance: %s\n", balance_name((u_int)md->md_balance)); + printf(" mediasize: %jd\n", (intmax_t)md->md_mediasize); + printf("sectorsize: %u\n", (u_int)md->md_sectorsize); + printf("syncoffset: %jd\n", (intmax_t)md->md_sync_offset); + printf(" mflags:"); + if (md->md_mflags == 0) + printf(" NONE"); + else { + if ((md->md_mflags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0) + printf(" NOAUTOSYNC"); + } + printf("\n"); + printf(" dflags:"); + if (md->md_mflags == 0) + printf(" NONE"); + else { + if ((md->md_dflags & G_MIRROR_DISK_FLAG_DIRTY) != 0) + printf(" DIRTY"); + if ((md->md_dflags & G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) + printf(" SYNCHRONIZING"); + if ((md->md_dflags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) + printf(" FORCE_SYNC"); + if ((md->md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) + printf(" INACTIVE"); + } + printf("\n"); + bzero(hash, sizeof(hash)); + for (i = 0; i < 16; i++) { + hash[i * 2] = hex[md->md_hash[i] >> 4]; + hash[i * 2 + 1] = hex[md->md_hash[i] & 0x0f]; + } + printf(" MD5 hash: %s\n", hash); +} +#endif /* !_G_MIRROR_H_ */ diff --git a/sys/geom/mirror/g_mirror_ctl.c b/sys/geom/mirror/g_mirror_ctl.c new file mode 100644 index 000000000000..62d6b44fcdff --- /dev/null +++ b/sys/geom/mirror/g_mirror_ctl.c @@ -0,0 +1,617 @@ +/*- + * Copyright (c) 2004 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static struct g_mirror_softc * +g_mirror_find_device(struct g_class *mp, const char *name) +{ + struct g_mirror_softc *sc; + struct g_geom *gp; + + g_topology_assert(); + LIST_FOREACH(gp, &mp->geom, geom) { + sc = gp->softc; + if (sc == NULL) + continue; + if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) + continue; + if (strcmp(gp->name, name) == 0 || + strcmp(sc->sc_name, name) == 0) { + return (sc); + } + } + return (NULL); +} + +static struct g_mirror_disk * +g_mirror_find_disk(struct g_mirror_softc *sc, const char *name) +{ + struct g_mirror_disk *disk; + + g_topology_assert(); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_consumer == NULL) + continue; + if (disk->d_consumer->provider == NULL) + continue; + if (strcmp(disk->d_consumer->provider->name, name) == 0) + return (disk); + } + return (NULL); +} + +static void +g_mirror_ctl_configure(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + const char *name, *balancep; + intmax_t *slicep; + uint32_t slice; + uint8_t balance; + int *nargs, *autosync, *noautosync, do_sync = 0; + + g_topology_assert(); + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (*nargs != 1) { + gctl_error(req, "Invalid number of arguments."); + return; + } + name = gctl_get_asciiparam(req, "arg0"); + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + if (g_mirror_ndisks(sc, -1) < sc->sc_ndisks) { + gctl_error(req, "Not all disks connected."); + return; + } + balancep = gctl_get_asciiparam(req, "balance"); + if (strcmp(balancep, "none") == 0) + balance = sc->sc_balance; + else { + if (balance_id(balancep) == -1) { + gctl_error(req, "Invalid balance algorithm."); + return; + } + balance = balance_id(balancep); + } + slicep = gctl_get_paraml(req, "slice", sizeof(*slicep)); + if (*slicep == -1) + slice = sc->sc_slice; + else + slice = *slicep; + autosync = gctl_get_paraml(req, "autosync", sizeof(*autosync)); + if (autosync == NULL) { + gctl_error(req, "No '%s' argument.", "autosync"); + return; + } + noautosync = gctl_get_paraml(req, "noautosync", sizeof(*noautosync)); + if (noautosync == NULL) { + gctl_error(req, "No '%s' argument.", "noautosync"); + return; + } + if (sc->sc_balance == balance && sc->sc_slice == slice && !*autosync && + !*noautosync) { + gctl_error(req, "Nothing has changed."); + return; + } + if (*autosync && *noautosync) { + gctl_error(req, "'%s' and '%s' specified.", "autosync", + "noautosync"); + return; + } + sc->sc_balance = balance; + sc->sc_slice = slice; + if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0) { + if (*autosync) { + sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_NOAUTOSYNC; + do_sync = 1; + } + } else { + if (*noautosync) + sc->sc_flags |= G_MIRROR_DEVICE_FLAG_NOAUTOSYNC; + } + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (do_sync) { + if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) + disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC; + } + g_mirror_update_metadata(disk); + if (do_sync) { + if (disk->d_state == G_MIRROR_DISK_STATE_STALE) { + g_mirror_event_send(disk, + G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_DONTWAIT); + } + } + } +} + +static void +g_mirror_ctl_rebuild(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + const char *name; + char param[16]; + int *nargs; + u_int i; + + g_topology_assert(); + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 2) { + gctl_error(req, "Too few arguments."); + return; + } + name = gctl_get_asciiparam(req, "arg0"); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + + for (i = 1; i < (u_int)*nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", i); + return; + } + disk = g_mirror_find_disk(sc, name); + if (disk == NULL) { + gctl_error(req, "No such provider: %s.", name); + return; + } + if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1 && + disk->d_state == G_MIRROR_DISK_STATE_ACTIVE) { + /* + * This is the last active disk. There will be nothing + * to rebuild it from, so deny this request. + */ + gctl_error(req, + "Provider %s is the last active provider in %s.", + name, sc->sc_geom->name); + return; + } + /* + * Do rebuild by resetting syncid and disconnecting disk. + * It'll be retasted, connected to the mirror and + * synchronized. + */ + disk->d_sync.ds_syncid = 0; + if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0) + disk->d_flags |= G_MIRROR_DISK_FLAG_FORCE_SYNC; + g_mirror_update_metadata(disk); + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_WAIT); + } +} + +static void +g_mirror_ctl_insert(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + struct g_mirror_metadata md; + struct g_provider *pp; + struct g_consumer *cp; + const char *name; + char param[16]; + u_char *sector; + u_int i, n; + int error, *nargs, *inactive; + struct { + struct g_provider *provider; + struct g_consumer *consumer; + } *disks; + + g_topology_assert(); + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 2) { + gctl_error(req, "Too few arguments."); + return; + } + inactive = gctl_get_paraml(req, "inactive", sizeof(*inactive)); + if (inactive == NULL) { + gctl_error(req, "No '%s' argument.", "inactive"); + return; + } + name = gctl_get_asciiparam(req, "arg0"); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + if (g_mirror_ndisks(sc, -1) < sc->sc_ndisks) { + gctl_error(req, "Not all disks connected."); + return; + } + + disks = g_malloc(sizeof(*disks) * (*nargs), M_WAITOK | M_ZERO); + for (i = 1, n = 0; i < (u_int)*nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", i); + continue; + } + if (strncmp(name, "/dev/", strlen("/dev/")) == 0) + name += strlen("/dev/"); + pp = g_provider_by_name(name); + if (pp == NULL) { + gctl_error(req, "Unknown provider %s.", name); + continue; + } + if (sc->sc_provider->mediasize > pp->mediasize) { + gctl_error(req, "Provider %s too small.", name); + continue; + } + if ((sc->sc_provider->sectorsize % pp->sectorsize) != 0) { + gctl_error(req, "Invalid sectorsize of provider %s.", + name); + continue; + } + cp = g_new_consumer(sc->sc_geom); + if (g_attach(cp, pp) != 0) { + g_destroy_consumer(cp); + gctl_error(req, "Cannot attach to provider %s.", name); + continue; + } + if (g_access(cp, 0, 1, 1) != 0) { + g_detach(cp); + g_destroy_consumer(cp); + gctl_error(req, "Cannot access provider %s.", name); + continue; + } + disks[n].provider = pp; + disks[n].consumer = cp; + n++; + } + if (n == 0) { + g_free(disks); + return; + } + sc->sc_ndisks += n; +again: + for (i = 0; i < n; i++) { + if (disks[i].consumer == NULL) + continue; + g_mirror_fill_metadata(sc, NULL, &md); + if (*inactive) + md.md_dflags |= G_MIRROR_DISK_FLAG_INACTIVE; + pp = disks[i].provider; + sector = g_malloc(pp->sectorsize, M_WAITOK); + mirror_metadata_encode(&md, sector); + error = g_write_data(disks[i].consumer, + pp->mediasize - pp->sectorsize, sector, pp->sectorsize); + g_free(sector); + if (error != 0) { + gctl_error(req, "Cannot store metadata on %s.", + pp->name); + g_access(disks[i].consumer, 0, -1, -1); + g_detach(disks[i].consumer); + g_destroy_consumer(disks[i].consumer); + disks[i].consumer = NULL; + disks[i].provider = NULL; + sc->sc_ndisks--; + goto again; + } + } + if (i == 0) { + /* All writes failed. */ + g_free(disks); + return; + } + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + g_mirror_update_metadata(disk); + } + /* + * Release provider and wait for retaste. + */ + for (i = 0; i < n; i++) { + if (disks[i].consumer == NULL) + continue; + g_access(disks[i].consumer, 0, -1, -1); + g_detach(disks[i].consumer); + g_destroy_consumer(disks[i].consumer); + } + g_free(disks); +} + +static void +g_mirror_ctl_remove(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + const char *name; + char param[16]; + int *nargs; + u_int i; + + g_topology_assert(); + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 2) { + gctl_error(req, "Too few arguments."); + return; + } + name = gctl_get_asciiparam(req, "arg0"); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + if (g_mirror_ndisks(sc, -1) < sc->sc_ndisks) { + gctl_error(req, "Not all disks connected."); + return; + } + + for (i = 1; i < (u_int)*nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", i); + return; + } + disk = g_mirror_find_disk(sc, name); + if (disk == NULL) { + gctl_error(req, "No such provider: %s.", name); + return; + } + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DESTROY, + G_MIRROR_EVENT_WAIT); + } +} + +static void +g_mirror_ctl_deactivate(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + const char *name; + char param[16]; + int *nargs; + u_int i; + + g_topology_assert(); + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 2) { + gctl_error(req, "Too few arguments."); + return; + } + name = gctl_get_asciiparam(req, "arg0"); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + + for (i = 1; i < (u_int)*nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", i); + return; + } + disk = g_mirror_find_disk(sc, name); + if (disk == NULL) { + gctl_error(req, "No such provider: %s.", name); + return; + } + /* + * Do rebuild by resetting syncid and disconnecting disk. + * It'll be retasted, connected to the mirror and + * synchronized. + */ + disk->d_flags |= G_MIRROR_DISK_FLAG_INACTIVE; + disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC; + g_mirror_update_metadata(disk); + sc->sc_bump_syncid = G_MIRROR_BUMP_ON_FIRST_WRITE; + g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED, + G_MIRROR_EVENT_WAIT); + } +} + +static void +g_mirror_ctl_forget(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + struct g_mirror_disk *disk; + const char *name; + char param[16]; + int *nargs; + u_int i; + + g_topology_assert(); + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 1) { + gctl_error(req, "Missing device(s)."); + return; + } + + for (i = 0; i < (u_int)*nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", i); + return; + } + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + if (g_mirror_ndisks(sc, -1) == sc->sc_ndisks) { + G_MIRROR_DEBUG(1, + "All disks connected in %s, skipping.", + sc->sc_name); + continue; + } + sc->sc_ndisks = g_mirror_ndisks(sc, -1); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + g_mirror_update_metadata(disk); + } + } +} + +static void +g_mirror_ctl_stop(struct gctl_req *req, struct g_class *mp) +{ + struct g_mirror_softc *sc; + int *force, *nargs, error; + const char *name; + char param[16]; + u_int i; + + g_topology_assert(); + + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs < 1) { + gctl_error(req, "Missing device(s)."); + return; + } + force = gctl_get_paraml(req, "force", sizeof(*force)); + if (force == NULL) { + gctl_error(req, "No '%s' argument.", "force"); + return; + } + + for (i = 0; i < (u_int)*nargs; i++) { + snprintf(param, sizeof(param), "arg%u", i); + name = gctl_get_asciiparam(req, param); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", i); + return; + } + sc = g_mirror_find_device(mp, name); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", name); + return; + } + error = g_mirror_destroy(sc, *force); + if (error != 0) { + gctl_error(req, "Cannot destroy device %s (error=%d).", + sc->sc_geom->name, error); + return; + } + } +} + +void +g_mirror_config(struct gctl_req *req, struct g_class *mp, const char *verb) +{ + uint32_t *version; + + g_topology_assert(); + + version = gctl_get_paraml(req, "version", sizeof(*version)); + if (version == NULL) { + gctl_error(req, "No '%s' argument.", "version"); + return; + } + if (*version != G_MIRROR_VERSION) { + gctl_error(req, "Userland and kernel parts are out of sync."); + return; + } + + if (strcmp(verb, "configure") == 0) + g_mirror_ctl_configure(req, mp); + else if (strcmp(verb, "rebuild") == 0) + g_mirror_ctl_rebuild(req, mp); + else if (strcmp(verb, "insert") == 0) + g_mirror_ctl_insert(req, mp); + else if (strcmp(verb, "remove") == 0) + g_mirror_ctl_remove(req, mp); + else if (strcmp(verb, "deactivate") == 0) + g_mirror_ctl_deactivate(req, mp); + else if (strcmp(verb, "forget") == 0) + g_mirror_ctl_forget(req, mp); + else if (strcmp(verb, "stop") == 0) + g_mirror_ctl_stop(req, mp); + else + gctl_error(req, "Unknown verb."); +} diff --git a/sys/modules/geom/geom_mirror/Makefile b/sys/modules/geom/geom_mirror/Makefile new file mode 100644 index 000000000000..3e653ce540c4 --- /dev/null +++ b/sys/modules/geom/geom_mirror/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +.PATH: ${.CURDIR}/../../../geom/mirror + +KMOD= geom_mirror +SRCS= g_mirror.c +SRCS+= g_mirror_ctl.c + +.include diff --git a/tools/regression/geom_mirror/Makefile b/tools/regression/geom_mirror/Makefile new file mode 100644 index 000000000000..88687b191b6b --- /dev/null +++ b/tools/regression/geom_mirror/Makefile @@ -0,0 +1,8 @@ +# +# $FreeBSD$ +# +# Regression tests for geom_mirror. +# + +test: + @sh runtests.sh diff --git a/tools/regression/geom_mirror/runtests.sh b/tools/regression/geom_mirror/runtests.sh new file mode 100644 index 000000000000..7e30b95f3069 --- /dev/null +++ b/tools/regression/geom_mirror/runtests.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# $FreeBSD$ + +dir=`dirname $0` + +gmirror load >/dev/null 2>&1 +for ts in `dirname $0`/test-*.sh; do + sh $ts +done +gmirror unload >/dev/null 2>&1 diff --git a/tools/regression/geom_mirror/test-1.sh b/tools/regression/geom_mirror/test-1.sh new file mode 100644 index 000000000000..e521564211cd --- /dev/null +++ b/tools/regression/geom_mirror/test-1.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# $FreeBSD$ + +name="test" +base=`basename $0` +us0=45 +us1=`expr $us0 + 1` +us2=`expr $us0 + 2` + +mdconfig -a -t malloc -s 1M -u $us0 || exit 1 +mdconfig -a -t malloc -s 2M -u $us1 || exit 1 +mdconfig -a -t malloc -s 3M -u $us2 || exit 1 +sleep 1 + +gmirror label $name /dev/md${us0} /dev/md${us1} /dev/md${us2} || exit 1 + +# Size of created device should be 1MB - 512b. + +size=`diskinfo /dev/mirror/${name} | awk '{print $3}'` + +if [ $size -eq 1048064 ]; then + echo "PASS" +else + echo "FAIL" +fi + +gmirror remove $name md${us0} +gmirror remove $name md${us1} +gmirror remove $name md${us2} +mdconfig -d -u $us0 +mdconfig -d -u $us1 +mdconfig -d -u $us2 diff --git a/tools/regression/geom_mirror/test-2.sh b/tools/regression/geom_mirror/test-2.sh new file mode 100644 index 000000000000..89bfdd4df85c --- /dev/null +++ b/tools/regression/geom_mirror/test-2.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# $FreeBSD$ + +name="test" +base=`basename $0` +balance="round-robin" +us0=45 +us1=`expr $us0 + 1` +us2=`expr $us0 + 2` +ddbs=2048 +nblocks1=1024 +nblocks2=`expr $nblocks1 / \( $ddbs / 512 \)` +src=`mktemp /tmp/$base.XXXXXX` || exit 1 +dst=`mktemp /tmp/$base.XXXXXX` || exit 1 + +dd if=/dev/random of=${src} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us0 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us1 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us2 || exit 1 +sleep 1 + +gmirror label -b $balance $name /dev/md${us0} /dev/md${us1} /dev/md${us2} || exit 1 + +dd if=${src} of=/dev/mirror/${name} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi +dd if=/dev/md${us0} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi +dd if=/dev/md${us1} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +dd if=/dev/md${us2} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us0} md${us1} md${us2} +mdconfig -d -u $us0 +mdconfig -d -u $us1 +mdconfig -d -u $us2 +rm -f ${src} ${dst} diff --git a/tools/regression/geom_mirror/test-3.sh b/tools/regression/geom_mirror/test-3.sh new file mode 100644 index 000000000000..67a41516ea59 --- /dev/null +++ b/tools/regression/geom_mirror/test-3.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# $FreeBSD$ + +name="test" +base=`basename $0` +balance="round-robin" +us0=45 +us1=`expr $us0 + 1` +us2=`expr $us0 + 2` +ddbs=2048 +nblocks1=1024 +nblocks2=`expr $nblocks1 / \( $ddbs / 512 \)` +src=`mktemp /tmp/$base.XXXXXX` || exit 1 +dst=`mktemp /tmp/$base.XXXXXX` || exit 1 + +dd if=/dev/random of=${src} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us0 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us1 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us2 || exit 1 +sleep 1 + +gmirror label -b $balance $name /dev/md${us0} /dev/md${us1} /dev/md${us2} || exit 1 + +dd if=${src} of=/dev/mirror/${name} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us0} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us1} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us2} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +# mirror/${name} should be removed. +if [ -c /dev/${name} ]; then + echo "FAIL" +else + echo "PASS" +fi + +mdconfig -d -u $us0 +mdconfig -d -u $us1 +mdconfig -d -u $us2 +rm -f ${src} ${dst} diff --git a/tools/regression/geom_mirror/test-4.sh b/tools/regression/geom_mirror/test-4.sh new file mode 100644 index 000000000000..0b764cad67bc --- /dev/null +++ b/tools/regression/geom_mirror/test-4.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# $FreeBSD$ + +name="test" +base=`basename $0` +balance="load" +us0=45 +us1=`expr $us0 + 1` +us2=`expr $us0 + 2` +ddbs=2048 +nblocks1=1024 +nblocks2=`expr $nblocks1 / \( $ddbs / 512 \)` +src=`mktemp /tmp/$base.XXXXXX` || exit 1 +dst=`mktemp /tmp/$base.XXXXXX` || exit 1 + +dd if=/dev/random of=${src} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us0 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us1 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us2 || exit 1 +sleep 1 + +gmirror label -b $balance $name /dev/md${us0} /dev/md${us1} /dev/md${us2} || exit 1 + +dd if=${src} of=/dev/mirror/${name} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us0} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us1} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us2} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +# mirror/${name} should be removed. +if [ -c /dev/${name} ]; then + echo "FAIL" +else + echo "PASS" +fi + +mdconfig -d -u $us0 +mdconfig -d -u $us1 +mdconfig -d -u $us2 +rm -f ${src} ${dst} diff --git a/tools/regression/geom_mirror/test-5.sh b/tools/regression/geom_mirror/test-5.sh new file mode 100644 index 000000000000..d1958d587cb3 --- /dev/null +++ b/tools/regression/geom_mirror/test-5.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# $FreeBSD$ + +name="test" +base=`basename $0` +balance="split" +us0=45 +us1=`expr $us0 + 1` +us2=`expr $us0 + 2` +ddbs=8192 +nblocks1=1024 +nblocks2=`expr $nblocks1 / \( $ddbs / 512 \)` +src=`mktemp /tmp/$base.XXXXXX` || exit 1 +dst=`mktemp /tmp/$base.XXXXXX` || exit 1 + +dd if=/dev/random of=${src} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us0 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us1 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us2 || exit 1 +sleep 1 + +gmirror label -b $balance -s `expr $ddbs / 2` $name /dev/md${us0} /dev/md${us1} /dev/md${us2} || exit 1 + +dd if=${src} of=/dev/mirror/${name} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us0} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us1} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us2} +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +# mirror/${name} should be removed. +if [ -c /dev/${name} ]; then + echo "FAIL" +else + echo "PASS" +fi + +mdconfig -d -u $us0 +mdconfig -d -u $us1 +mdconfig -d -u $us2 +rm -f ${src} ${dst} diff --git a/tools/regression/geom_mirror/test-6.sh b/tools/regression/geom_mirror/test-6.sh new file mode 100644 index 000000000000..db36726b2929 --- /dev/null +++ b/tools/regression/geom_mirror/test-6.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# $FreeBSD$ + +name="test" +base=`basename $0` +balance="split" +us0=45 +us1=`expr $us0 + 1` +us2=`expr $us0 + 2` +ddbs=8192 +nblocks1=1024 +nblocks2=`expr $nblocks1 / \( $ddbs / 512 \)` +src=`mktemp /tmp/$base.XXXXXX` || exit 1 +dst=`mktemp /tmp/$base.XXXXXX` || exit 1 + +dd if=/dev/random of=${src} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us0 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us1 || exit 1 +mdconfig -a -t malloc -s `expr $nblocks1 + 1` -u $us2 || exit 1 +sleep 1 + +gmirror label -b $balance -s `expr $ddbs / 2` $name /dev/md${us0} /dev/md${us1} || exit 1 + +dd if=${src} of=/dev/mirror/${name} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +dd if=/dev/zero of=/dev/md${us2} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 + +dd if=/dev/mirror/${name} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +# Connect disk to the mirror. +gmirror insert ${name} md${us2} +# Wait for synchronization. +sleep 1 +dd if=/dev/md${us2} of=${dst} bs=$ddbs count=$nblocks2 >/dev/null 2>&1 +if [ `md5 -q ${src}` != `md5 -q ${dst}` ]; then + echo "FAIL" +else + echo "PASS" +fi + +gmirror remove $name md${us0} md${us1} md${us2} +mdconfig -d -u $us0 +mdconfig -d -u $us1 +mdconfig -d -u $us2 +rm -f ${src} ${dst}