From 0c629e2884b5fd625155722bad18754e9f95451c Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Sat, 21 Jan 2017 19:38:26 +0000 Subject: [PATCH] Add initial support for CTL module unloading. It is only a first step and not perfect, but better then nothing. The main blocker is CAM target frontend, that can not be unloaded, since CAM does not have mechanism to unregister periph driver now. MFC after: 2 weeks --- sys/cam/ctl/ctl.c | 142 ++++++++++++++++++----------- sys/cam/ctl/ctl_backend.c | 71 +++++---------- sys/cam/ctl/ctl_backend.h | 12 ++- sys/cam/ctl/ctl_backend_block.c | 50 +++++++--- sys/cam/ctl/ctl_backend_ramdisk.c | 17 ++-- sys/cam/ctl/ctl_frontend.c | 35 +++---- sys/cam/ctl/ctl_frontend.h | 11 ++- sys/cam/ctl/ctl_frontend_cam_sim.c | 43 +++++---- sys/cam/ctl/ctl_frontend_ioctl.c | 18 ++-- sys/cam/ctl/ctl_frontend_iscsi.c | 23 ++++- sys/cam/ctl/ctl_private.h | 3 + sys/cam/ctl/ctl_tpc_local.c | 21 +++-- sys/cam/ctl/scsi_ctl.c | 14 +-- 13 files changed, 264 insertions(+), 196 deletions(-) diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c index a90d21831d1a..b80a7d878746 100644 --- a/sys/cam/ctl/ctl.c +++ b/sys/cam/ctl/ctl.c @@ -424,7 +424,7 @@ static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event, static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest); static void ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest); static int ctl_init(void); -void ctl_shutdown(void); +static int ctl_shutdown(void); static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td); static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td); static void ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio); @@ -520,6 +520,8 @@ static const struct ctl_cmd_entry * ctl_validate_command(struct ctl_scsiio *ctsio); static int ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry); +static int ctl_ha_init(void); +static int ctl_ha_shutdown(void); static uint64_t ctl_get_prkey(struct ctl_lun *lun, uint32_t residx); static void ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx); @@ -561,6 +563,49 @@ MODULE_VERSION(ctl, 1); static struct ctl_frontend ha_frontend = { .name = "ha", + .init = ctl_ha_init, + .shutdown = ctl_ha_shutdown, +}; + +static int +ctl_ha_init(void) +{ + struct ctl_softc *softc = control_softc; + + if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC, + &softc->othersc_pool) != 0) + return (ENOMEM); + if (ctl_ha_msg_init(softc) != CTL_HA_STATUS_SUCCESS) { + ctl_pool_free(softc->othersc_pool); + return (EIO); + } + if (ctl_ha_msg_register(CTL_HA_CHAN_CTL, ctl_isc_event_handler) + != CTL_HA_STATUS_SUCCESS) { + ctl_ha_msg_destroy(softc); + ctl_pool_free(softc->othersc_pool); + return (EIO); + } + return (0); +}; + +static int +ctl_ha_shutdown(void) +{ + struct ctl_softc *softc = control_softc; + struct ctl_port *port; + + ctl_ha_msg_shutdown(softc); + if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL) != CTL_HA_STATUS_SUCCESS) + return (EIO); + if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS) + return (EIO); + ctl_pool_free(softc->othersc_pool); + while ((port = STAILQ_FIRST(&ha_frontend.port_list)) != NULL) { + ctl_port_deregister(port); + free(port->port_name, M_CTL); + free(port, M_CTL); + } + return (0); }; static void @@ -1782,7 +1827,6 @@ ctl_init(void) { struct make_dev_args args; struct ctl_softc *softc; - void *other_pool; int i, error; softc = control_softc = malloc(sizeof(*control_softc), M_DEVBUF, @@ -1855,15 +1899,6 @@ ctl_init(void) STAILQ_INIT(&softc->be_list); ctl_tpc_init(softc); - if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC, - &other_pool) != 0) - { - printf("ctl: can't allocate %d entry other SC pool, " - "exiting\n", CTL_POOL_ENTRIES_OTHER_SC); - return (ENOMEM); - } - softc->othersc_pool = other_pool; - if (worker_threads <= 0) worker_threads = max(1, mp_ncpus / 4); if (worker_threads > CTL_MAX_THREADS) @@ -1883,22 +1918,19 @@ ctl_init(void) &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i); if (error != 0) { printf("error creating CTL work thread!\n"); - ctl_pool_free(other_pool); return (error); } } error = kproc_kthread_add(ctl_lun_thread, softc, - &softc->ctl_proc, NULL, 0, 0, "ctl", "lun"); + &softc->ctl_proc, &softc->lun_thread, 0, 0, "ctl", "lun"); if (error != 0) { printf("error creating CTL lun thread!\n"); - ctl_pool_free(other_pool); return (error); } error = kproc_kthread_add(ctl_thresh_thread, softc, - &softc->ctl_proc, NULL, 0, 0, "ctl", "thresh"); + &softc->ctl_proc, &softc->thresh_thread, 0, 0, "ctl", "thresh"); if (error != 0) { printf("error creating CTL threshold thread!\n"); - ctl_pool_free(other_pool); return (error); } @@ -1907,58 +1939,54 @@ ctl_init(void) softc, 0, ctl_ha_role_sysctl, "I", "HA role for this head"); if (softc->is_single == 0) { - ctl_frontend_register(&ha_frontend); - if (ctl_ha_msg_init(softc) != CTL_HA_STATUS_SUCCESS) { - printf("ctl_init: ctl_ha_msg_init failed.\n"); + if (ctl_frontend_register(&ha_frontend) != 0) softc->is_single = 1; - } else - if (ctl_ha_msg_register(CTL_HA_CHAN_CTL, ctl_isc_event_handler) - != CTL_HA_STATUS_SUCCESS) { - printf("ctl_init: ctl_ha_msg_register failed.\n"); - softc->is_single = 1; - } } return (0); } -void +static int ctl_shutdown(void) { struct ctl_softc *softc = control_softc; - struct ctl_lun *lun, *next_lun; + int i; - if (softc->is_single == 0) { - ctl_ha_msg_shutdown(softc); - if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL) - != CTL_HA_STATUS_SUCCESS) - printf("%s: ctl_ha_msg_deregister failed.\n", __func__); - if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS) - printf("%s: ctl_ha_msg_destroy failed.\n", __func__); + if (softc->is_single == 0) ctl_frontend_deregister(&ha_frontend); + + destroy_dev(softc->dev); + + /* Shutdown CTL threads. */ + softc->shutdown = 1; + for (i = 0; i < worker_threads; i++) { + struct ctl_thread *thr = &softc->threads[i]; + while (thr->thread != NULL) { + wakeup(thr); + if (thr->thread != NULL) + pause("CTL thr shutdown", 1); + } + mtx_destroy(&thr->queue_lock); + } + while (softc->lun_thread != NULL) { + wakeup(&softc->pending_lun_queue); + if (softc->lun_thread != NULL) + pause("CTL thr shutdown", 1); + } + while (softc->thresh_thread != NULL) { + wakeup(softc->thresh_thread); + if (softc->thresh_thread != NULL) + pause("CTL thr shutdown", 1); } - - mtx_lock(&softc->ctl_lock); - - STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) - ctl_free_lun(lun); - - mtx_unlock(&softc->ctl_lock); - -#if 0 - ctl_shutdown_thread(softc->work_thread); - mtx_destroy(&softc->queue_lock); -#endif ctl_tpc_shutdown(softc); uma_zdestroy(softc->io_zone); mtx_destroy(&softc->ctl_lock); - destroy_dev(softc->dev); - sysctl_ctx_free(&softc->sysctl_ctx); free(softc, M_DEVBUF); control_softc = NULL; + return (0); } static int @@ -1969,7 +1997,7 @@ ctl_module_event_handler(module_t mod, int what, void *arg) case MOD_LOAD: return (ctl_init()); case MOD_UNLOAD: - return (EBUSY); + return (ctl_shutdown()); default: return (EOPNOTSUPP); } @@ -13268,7 +13296,7 @@ ctl_work_thread(void *arg) CTL_DEBUG_PRINT(("ctl_work_thread starting\n")); - for (;;) { + while (!softc->shutdown) { /* * We handle the queues in this order: * - ISC @@ -13318,6 +13346,8 @@ ctl_work_thread(void *arg) /* Sleep until we have something to do. */ mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0); } + thr->thread = NULL; + kthread_exit(); } static void @@ -13328,7 +13358,7 @@ ctl_lun_thread(void *arg) CTL_DEBUG_PRINT(("ctl_lun_thread starting\n")); - for (;;) { + while (!softc->shutdown) { mtx_lock(&softc->ctl_lock); be_lun = STAILQ_FIRST(&softc->pending_lun_queue); if (be_lun != NULL) { @@ -13342,6 +13372,8 @@ ctl_lun_thread(void *arg) mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock, PDROP | PRIBIO, "-", 0); } + softc->lun_thread = NULL; + kthread_exit(); } static void @@ -13357,7 +13389,7 @@ ctl_thresh_thread(void *arg) CTL_DEBUG_PRINT(("ctl_thresh_thread starting\n")); - for (;;) { + while (!softc->shutdown) { mtx_lock(&softc->ctl_lock); STAILQ_FOREACH(lun, &softc->lun_list, links) { if ((lun->flags & CTL_LUN_DISABLED) || @@ -13442,9 +13474,11 @@ ctl_thresh_thread(void *arg) mtx_lock(&softc->ctl_lock); } } - mtx_unlock(&softc->ctl_lock); - pause("-", CTL_LBP_PERIOD * hz); + mtx_sleep(&softc->thresh_thread, &softc->ctl_lock, + PDROP | PRIBIO, "-", CTL_LBP_PERIOD * hz); } + softc->thresh_thread = NULL; + kthread_exit(); } static void diff --git a/sys/cam/ctl/ctl_backend.c b/sys/cam/ctl/ctl_backend.c index 86f7d3c35a70..bac7e857b6fc 100644 --- a/sys/cam/ctl/ctl_backend.c +++ b/sys/cam/ctl/ctl_backend.c @@ -67,11 +67,10 @@ ctl_backend_register(struct ctl_backend_driver *be) { struct ctl_softc *softc = control_softc; struct ctl_backend_driver *be_tmp; + int error; + /* Sanity check, make sure this isn't a duplicate registration. */ mtx_lock(&softc->ctl_lock); - /* - * Sanity check, make sure this isn't a duplicate registration. - */ STAILQ_FOREACH(be_tmp, &softc->be_list, links) { if (strcmp(be_tmp->name, be->name) == 0) { mtx_unlock(&softc->ctl_lock); @@ -79,39 +78,24 @@ ctl_backend_register(struct ctl_backend_driver *be) } } mtx_unlock(&softc->ctl_lock); - - /* - * Call the backend's initialization routine. - */ - be->init(); - - mtx_lock(&softc->ctl_lock); - - STAILQ_INSERT_TAIL(&softc->be_list, be, links); - - softc->num_backends++; - - /* - * Don't want to increment the usage count for internal consumers, - * we won't be able to unload otherwise. - */ - /* XXX KDM find a substitute for this? */ -#if 0 - if ((be->flags & CTL_BE_FLAG_INTERNAL) == 0) - MOD_INC_USE_COUNT; -#endif - #ifdef CS_BE_CONFIG_MOVE_DONE_IS_NOT_USED be->config_move_done = ctl_config_move_done; #endif - /* XXX KDM fix this! */ be->num_luns = 0; -#if 0 - atomic_set(&be->num_luns, 0); -#endif + /* Call the backend's initialization routine. */ + if (be->init != NULL) { + if ((error = be->init()) != 0) { + printf("%s backend init error: %d\n", + be->name, error); + return (error); + } + } + + mtx_lock(&softc->ctl_lock); + STAILQ_INSERT_TAIL(&softc->be_list, be, links); + softc->num_backends++; mtx_unlock(&softc->ctl_lock); - return (0); } @@ -119,30 +103,21 @@ int ctl_backend_deregister(struct ctl_backend_driver *be) { struct ctl_softc *softc = control_softc; + int error; - mtx_lock(&softc->ctl_lock); - -#if 0 - if (atomic_read(&be->num_luns) != 0) { -#endif - /* XXX KDM fix this! */ - if (be->num_luns != 0) { - mtx_unlock(&softc->ctl_lock); - return (-1); + /* Call the backend's shutdown routine. */ + if (be->shutdown != NULL) { + if ((error = be->shutdown()) != 0) { + printf("%s backend shutdown error: %d\n", + be->name, error); + return (error); + } } + mtx_lock(&softc->ctl_lock); STAILQ_REMOVE(&softc->be_list, be, ctl_backend_driver, links); - softc->num_backends--; - - /* XXX KDM find a substitute for this? */ -#if 0 - if ((be->flags & CTL_BE_FLAG_INTERNAL) == 0) - MOD_DEC_USE_COUNT; -#endif - mtx_unlock(&softc->ctl_lock); - return (0); } diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h index d8ada8377738..4202efc137d3 100644 --- a/sys/cam/ctl/ctl_backend.h +++ b/sys/cam/ctl/ctl_backend.h @@ -55,12 +55,13 @@ typedef enum { { \ switch (type) { \ case MOD_LOAD: \ - ctl_backend_register( \ - (struct ctl_backend_driver *)data); \ + return (ctl_backend_register( \ + (struct ctl_backend_driver *)data)); \ break; \ case MOD_UNLOAD: \ - printf(#name " module unload - not possible for this module type\n"); \ - return EINVAL; \ + return (ctl_backend_deregister( \ + (struct ctl_backend_driver *)data)); \ + break; \ default: \ return EOPNOTSUPP; \ } \ @@ -179,10 +180,10 @@ struct ctl_be_lun { typedef enum { CTL_BE_FLAG_NONE = 0x00, /* no flags */ CTL_BE_FLAG_HAS_CONFIG = 0x01, /* can do config reads, writes */ - CTL_BE_FLAG_INTERNAL = 0x02 /* don't inc mod refcount */ } ctl_backend_flags; typedef int (*be_init_t)(void); +typedef int (*be_shutdown_t)(void); typedef int (*be_func_t)(union ctl_io *io); typedef void (*be_vfunc_t)(union ctl_io *io); typedef int (*be_ioctl_t)(struct cdev *dev, u_long cmd, caddr_t addr, int flag, @@ -194,6 +195,7 @@ struct ctl_backend_driver { char name[CTL_BE_NAME_LEN]; /* passed to CTL */ ctl_backend_flags flags; /* passed to CTL */ be_init_t init; /* passed to CTL */ + be_shutdown_t shutdown; /* passed to CTL */ be_func_t data_submit; /* passed to CTL */ be_func_t data_move_done; /* passed to CTL */ be_func_t config_read; /* passed to CTL */ diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c index 2bc8d697cd18..4a6141ff143f 100644 --- a/sys/cam/ctl/ctl_backend_block.c +++ b/sys/cam/ctl/ctl_backend_block.c @@ -183,6 +183,7 @@ struct ctl_be_block_lun { */ struct ctl_be_block_softc { struct mtx lock; + uma_zone_t beio_zone; int num_luns; STAILQ_HEAD(, ctl_be_block_lun) lun_list; }; @@ -273,13 +274,15 @@ static int ctl_be_block_config_write(union ctl_io *io); static int ctl_be_block_config_read(union ctl_io *io); static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb); static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname); -int ctl_be_block_init(void); +static int ctl_be_block_init(void); +static int ctl_be_block_shutdown(void); static struct ctl_backend_driver ctl_be_block_driver = { .name = "block", .flags = CTL_BE_FLAG_HAS_CONFIG, .init = ctl_be_block_init, + .shutdown = ctl_be_block_shutdown, .data_submit = ctl_be_block_submit, .data_move_done = ctl_be_block_move_done, .config_read = ctl_be_block_config_read, @@ -292,14 +295,12 @@ static struct ctl_backend_driver ctl_be_block_driver = MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend"); CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver); -static uma_zone_t beio_zone; - static struct ctl_be_block_io * ctl_alloc_beio(struct ctl_be_block_softc *softc) { struct ctl_be_block_io *beio; - beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO); + beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO); beio->softc = softc; return (beio); } @@ -332,7 +333,7 @@ ctl_free_beio(struct ctl_be_block_io *beio) duplicate_free, beio->num_segs); } - uma_zfree(beio_zone, beio); + uma_zfree(beio->softc->beio_zone, beio); } static void @@ -2859,19 +2860,40 @@ ctl_be_block_lun_attr(void *be_lun, const char *attrname) return (lun->getattr(lun, attrname)); } -int +static int ctl_be_block_init(void) { - struct ctl_be_block_softc *softc; - int retval; - - softc = &backend_block_softc; - retval = 0; + struct ctl_be_block_softc *softc = &backend_block_softc; mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); - beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), + softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); STAILQ_INIT(&softc->lun_list); - - return (retval); + return (0); +} + + +static int +ctl_be_block_shutdown(void) +{ + struct ctl_be_block_softc *softc = &backend_block_softc; + struct ctl_be_block_lun *lun, *next_lun; + + mtx_lock(&softc->lock); + STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) { + /* + * Drop our lock here. Since ctl_invalidate_lun() can call + * back into us, this could potentially lead to a recursive + * lock of the same mutex, which would cause a hang. + */ + mtx_unlock(&softc->lock); + ctl_disable_lun(&lun->cbe_lun); + ctl_invalidate_lun(&lun->cbe_lun); + mtx_lock(&softc->lock); + } + mtx_unlock(&softc->lock); + + uma_zdestroy(softc->beio_zone); + mtx_destroy(&softc->lock); + return (0); } diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c index 71ed07ad909a..5b85d9c700fb 100644 --- a/sys/cam/ctl/ctl_backend_ramdisk.c +++ b/sys/cam/ctl/ctl_backend_ramdisk.c @@ -108,8 +108,8 @@ struct ctl_be_ramdisk_softc { static struct ctl_be_ramdisk_softc rd_softc; extern struct ctl_softc *control_softc; -int ctl_backend_ramdisk_init(void); -void ctl_backend_ramdisk_shutdown(void); +static int ctl_backend_ramdisk_init(void); +static int ctl_backend_ramdisk_shutdown(void); static int ctl_backend_ramdisk_move_done(union ctl_io *io); static int ctl_backend_ramdisk_submit(union ctl_io *io); static void ctl_backend_ramdisk_continue(union ctl_io *io); @@ -133,6 +133,7 @@ static struct ctl_backend_driver ctl_be_ramdisk_driver = .name = "ramdisk", .flags = CTL_BE_FLAG_HAS_CONFIG, .init = ctl_backend_ramdisk_init, + .shutdown = ctl_backend_ramdisk_shutdown, .data_submit = ctl_backend_ramdisk_submit, .data_move_done = ctl_backend_ramdisk_move_done, .config_read = ctl_backend_ramdisk_config_read, @@ -170,7 +171,7 @@ ctl_backend_ramdisk_init(void) return (0); } -void +static int ctl_backend_ramdisk_shutdown(void) { struct ctl_be_ramdisk_softc *softc = &rd_softc; @@ -192,20 +193,16 @@ ctl_backend_ramdisk_shutdown(void) mtx_lock(&softc->lock); } mtx_unlock(&softc->lock); - + #ifdef CTL_RAMDISK_PAGES for (i = 0; i < softc->num_pages; i++) free(softc->ramdisk_pages[i], M_RAMDISK); - free(softc->ramdisk_pages, M_RAMDISK); #else free(softc->ramdisk_buffer, M_RAMDISK); #endif - - if (ctl_backend_deregister(&ctl_be_ramdisk_driver) != 0) { - printf("ctl_backend_ramdisk_shutdown: " - "ctl_backend_deregister() failed!\n"); - } + mtx_destroy(&softc->lock); + return (0); } static int diff --git a/sys/cam/ctl/ctl_frontend.c b/sys/cam/ctl/ctl_frontend.c index 03ee9da7566f..765a31d96a22 100644 --- a/sys/cam/ctl/ctl_frontend.c +++ b/sys/cam/ctl/ctl_frontend.c @@ -70,12 +70,11 @@ ctl_frontend_register(struct ctl_frontend *fe) { struct ctl_softc *softc = control_softc; struct ctl_frontend *fe_tmp; + int error; KASSERT(softc != NULL, ("CTL is not initialized")); - /* - * Sanity check, make sure this isn't a duplicate registration. - */ + /* Sanity check, make sure this isn't a duplicate registration. */ mtx_lock(&softc->ctl_lock); STAILQ_FOREACH(fe_tmp, &softc->fe_list, links) { if (strcmp(fe_tmp->name, fe->name) == 0) { @@ -86,11 +85,14 @@ ctl_frontend_register(struct ctl_frontend *fe) mtx_unlock(&softc->ctl_lock); STAILQ_INIT(&fe->port_list); - /* - * Call the frontend's initialization routine. - */ - if (fe->init != NULL) - fe->init(); + /* Call the frontend's initialization routine. */ + if (fe->init != NULL) { + if ((error = fe->init()) != 0) { + printf("%s frontend init error: %d\n", + fe->name, error); + return (error); + } + } mtx_lock(&softc->ctl_lock); softc->num_frontends++; @@ -103,20 +105,21 @@ int ctl_frontend_deregister(struct ctl_frontend *fe) { struct ctl_softc *softc = control_softc; + int error; - if (!STAILQ_EMPTY(&fe->port_list)) - return (-1); + /* Call the frontend's shutdown routine.*/ + if (fe->shutdown != NULL) { + if ((error = fe->shutdown()) != 0) { + printf("%s frontend shutdown error: %d\n", + fe->name, error); + return (error); + } + } mtx_lock(&softc->ctl_lock); STAILQ_REMOVE(&softc->fe_list, fe, ctl_frontend, links); softc->num_frontends--; mtx_unlock(&softc->ctl_lock); - - /* - * Call the frontend's shutdown routine. - */ - if (fe->shutdown != NULL) - fe->shutdown(); return (0); } diff --git a/sys/cam/ctl/ctl_frontend.h b/sys/cam/ctl/ctl_frontend.h index 37e6824eaa8d..38eb863a08f4 100644 --- a/sys/cam/ctl/ctl_frontend.h +++ b/sys/cam/ctl/ctl_frontend.h @@ -49,7 +49,7 @@ typedef enum { } ctl_port_status; typedef int (*fe_init_t)(void); -typedef void (*fe_shutdown_t)(void); +typedef int (*fe_shutdown_t)(void); typedef void (*port_func_t)(void *onoff_arg); typedef int (*port_info_func_t)(void *onoff_arg, struct sbuf *sb); typedef int (*lun_func_t)(void *arg, int lun_id); @@ -61,12 +61,13 @@ typedef int (*fe_ioctl_t)(struct cdev *dev, u_long cmd, caddr_t addr, int flag, { \ switch (type) { \ case MOD_LOAD: \ - ctl_frontend_register( \ - (struct ctl_frontend *)data); \ + return (ctl_frontend_register( \ + (struct ctl_frontend *)data)); \ break; \ case MOD_UNLOAD: \ - printf(#name " module unload - not possible for this module type\n"); \ - return EINVAL; \ + return (ctl_frontend_deregister( \ + (struct ctl_frontend *)data)); \ + break; \ default: \ return EOPNOTSUPP; \ } \ diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c index 406adde9d3f8..00a960e9e28c 100644 --- a/sys/cam/ctl/ctl_frontend_cam_sim.c +++ b/sys/cam/ctl/ctl_frontend_cam_sim.c @@ -94,15 +94,14 @@ struct cfcs_softc { CAM_SNS_BUF_PHYS | CAM_CDB_PHYS | CAM_SENSE_PTR | \ CAM_SENSE_PHYS) -int cfcs_init(void); +static int cfcs_init(void); +static int cfcs_shutdown(void); static void cfcs_poll(struct cam_sim *sim); static void cfcs_online(void *arg); static void cfcs_offline(void *arg); static void cfcs_datamove(union ctl_io *io); static void cfcs_done(union ctl_io *io); void cfcs_action(struct cam_sim *sim, union ccb *ccb); -static void cfcs_async(void *callback_arg, uint32_t code, - struct cam_path *path, void *arg); struct cfcs_softc cfcs_softc; /* @@ -121,14 +120,14 @@ static struct ctl_frontend cfcs_frontend = { .name = "camsim", .init = cfcs_init, + .shutdown = cfcs_shutdown, }; CTL_FRONTEND_DECLARE(ctlcfcs, cfcs_frontend); -int +static int cfcs_init(void) { struct cfcs_softc *softc; - struct ccb_setasync csa; struct ctl_port *port; int retval; @@ -214,13 +213,6 @@ cfcs_init(void) goto bailout; } - xpt_setup_ccb(&csa.ccb_h, softc->path, CAM_PRIORITY_NONE); - csa.ccb_h.func_code = XPT_SASYNC_CB; - csa.event_enable = AC_LOST_DEVICE; - csa.callback = cfcs_async; - csa.callback_arg = softc->sim; - xpt_action((union ccb *)&csa); - mtx_unlock(&softc->lock); return (retval); @@ -236,6 +228,27 @@ bailout: return (retval); } +static int +cfcs_shutdown(void) +{ + struct cfcs_softc *softc = &cfcs_softc; + struct ctl_port *port = &softc->port; + int error; + + ctl_port_offline(port); + + mtx_lock(&softc->lock); + xpt_free_path(softc->path); + xpt_bus_deregister(cam_sim_path(softc->sim)); + cam_sim_free(softc->sim, /*free_devq*/ TRUE); + mtx_unlock(&softc->lock); + mtx_destroy(&softc->lock); + + if ((error = ctl_port_deregister(port)) != 0) + printf("%s: cam_sim port deregistration failed\n", __func__); + return (error); +} + static void cfcs_poll(struct cam_sim *sim) { @@ -801,9 +814,3 @@ cfcs_action(struct cam_sim *sim, union ccb *ccb) break; } } - -static void -cfcs_async(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) -{ - -} diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c index 3f5c23c0c55e..4063c97392df 100644 --- a/sys/cam/ctl/ctl_frontend_ioctl.c +++ b/sys/cam/ctl/ctl_frontend_ioctl.c @@ -76,7 +76,7 @@ struct cfi_softc { static struct cfi_softc cfi_softc; static int cfi_init(void); -static void cfi_shutdown(void); +static int cfi_shutdown(void); static void cfi_datamove(union ctl_io *io); static void cfi_done(union ctl_io *io); @@ -93,6 +93,7 @@ cfi_init(void) { struct cfi_softc *isoftc = &cfi_softc; struct ctl_port *port; + int error = 0; memset(isoftc, 0, sizeof(*isoftc)); @@ -108,24 +109,25 @@ cfi_init(void) port->targ_port = -1; port->max_initiators = 1; - if (ctl_port_register(port) != 0) { + if ((error = ctl_port_register(port)) != 0) { printf("%s: ioctl port registration failed\n", __func__); - return (0); + return (error); } ctl_port_online(port); return (0); } -void +static int cfi_shutdown(void) { struct cfi_softc *isoftc = &cfi_softc; - struct ctl_port *port; + struct ctl_port *port = &isoftc->port; + int error = 0; - port = &isoftc->port; ctl_port_offline(port); - if (ctl_port_deregister(&isoftc->port) != 0) - printf("%s: ctl_frontend_deregister() failed\n", __func__); + if ((error = ctl_port_deregister(port)) != 0) + printf("%s: ioctl port deregistration failed\n", __func__); + return (error); } /* diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c index c3ffbf29915b..55b56ae18cc6 100644 --- a/sys/cam/ctl/ctl_frontend_iscsi.c +++ b/sys/cam/ctl/ctl_frontend_iscsi.c @@ -144,7 +144,8 @@ SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN, #define PDU_TOTAL_TRANSFER_LEN(X) (X)->ip_prv1 #define PDU_R2TSN(X) (X)->ip_prv2 -int cfiscsi_init(void); +static int cfiscsi_init(void); +static int cfiscsi_shutdown(void); static void cfiscsi_online(void *arg); static void cfiscsi_offline(void *arg); static int cfiscsi_info(void *arg, struct sbuf *sb); @@ -182,6 +183,7 @@ static struct ctl_frontend cfiscsi_frontend = .name = "iscsi", .init = cfiscsi_init, .ioctl = cfiscsi_ioctl, + .shutdown = cfiscsi_shutdown, }; CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend); MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1); @@ -1321,7 +1323,7 @@ cfiscsi_session_delete(struct cfiscsi_session *cs) free(cs, M_CFISCSI); } -int +static int cfiscsi_init(void) { struct cfiscsi_softc *softc; @@ -1344,6 +1346,23 @@ cfiscsi_init(void) return (0); } +static int +cfiscsi_shutdown(void) +{ + struct cfiscsi_softc *softc = &cfiscsi_softc; + + if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets)) + return (EBUSY); + + uma_zdestroy(cfiscsi_data_wait_zone); +#ifdef ICL_KERNEL_PROXY + cv_destroy(&softc->accept_cv); +#endif + cv_destroy(&softc->sessions_cv); + mtx_destroy(&softc->lock); + return (0); +} + #ifdef ICL_KERNEL_PROXY static void cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id) diff --git a/sys/cam/ctl/ctl_private.h b/sys/cam/ctl/ctl_private.h index 3f46769806b1..40f0e6127401 100644 --- a/sys/cam/ctl/ctl_private.h +++ b/sys/cam/ctl/ctl_private.h @@ -470,7 +470,10 @@ struct ctl_softc { STAILQ_HEAD(, ctl_backend_driver) be_list; struct uma_zone *io_zone; uint32_t cur_pool_id; + int shutdown; struct ctl_thread threads[CTL_MAX_THREADS]; + struct thread *lun_thread; + struct thread *thresh_thread; TAILQ_HEAD(tpc_tokens, tpc_token) tpc_tokens; struct callout tpc_timeout; struct mtx tpc_lock; diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c index 47b7ed6c9e34..e5e77b437a26 100644 --- a/sys/cam/ctl/ctl_tpc_local.c +++ b/sys/cam/ctl/ctl_tpc_local.c @@ -65,7 +65,7 @@ struct tpcl_softc { static struct tpcl_softc tpcl_softc; static int tpcl_init(void); -static void tpcl_shutdown(void); +static int tpcl_shutdown(void); static void tpcl_datamove(union ctl_io *io); static void tpcl_done(union ctl_io *io); @@ -84,7 +84,7 @@ tpcl_init(void) struct tpcl_softc *tsoftc = &tpcl_softc; struct ctl_port *port; struct scsi_transportid_spi *tid; - int len; + int error, len; memset(tsoftc, 0, sizeof(*tsoftc)); @@ -100,9 +100,9 @@ tpcl_init(void) port->targ_port = -1; port->max_initiators = 1; - if (ctl_port_register(port) != 0) { - printf("%s: ctl_port_register() failed with error\n", __func__); - return (0); + if ((error = ctl_port_register(port)) != 0) { + printf("%s: tpc port registration failed\n", __func__); + return (error); } len = sizeof(struct scsi_transportid_spi); @@ -118,16 +118,17 @@ tpcl_init(void) return (0); } -void +static int tpcl_shutdown(void) { struct tpcl_softc *tsoftc = &tpcl_softc; - struct ctl_port *port; + struct ctl_port *port = &tsoftc->port; + int error; - port = &tsoftc->port; ctl_port_offline(port); - if (ctl_port_deregister(&tsoftc->port) != 0) - printf("%s: ctl_frontend_deregister() failed\n", __func__); + if ((error = ctl_port_deregister(port)) != 0) + printf("%s: tpc port deregistration failed\n", __func__); + return (error); } static void diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c index a32a67200c9b..1d3e0485933b 100644 --- a/sys/cam/ctl/scsi_ctl.c +++ b/sys/cam/ctl/scsi_ctl.c @@ -188,8 +188,8 @@ MALLOC_DEFINE(M_CTLFE, "CAM CTL FE", "CAM CTL FE interface"); #define PRIV_CCB(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[0]) #define PRIV_INFO(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[1]) -int ctlfeinitialize(void); -void ctlfeshutdown(void); +static int ctlfeinitialize(void); +static int ctlfeshutdown(void); static periph_init_t ctlfeperiphinit; static void ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg); @@ -227,13 +227,15 @@ static struct ctl_frontend ctlfe_frontend = }; CTL_FRONTEND_DECLARE(ctlfe, ctlfe_frontend); -void +static int ctlfeshutdown(void) { - return; + + /* CAM does not support periph driver unregister now. */ + return (EBUSY); } -int +static int ctlfeinitialize(void) { @@ -243,7 +245,7 @@ ctlfeinitialize(void) return (0); } -void +static void ctlfeperiphinit(void) { cam_status status;