diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c index cd0111416b9a..7772910e2d5e 100644 --- a/sys/cam/ctl/ctl.c +++ b/sys/cam/ctl/ctl.c @@ -357,7 +357,6 @@ static struct ctl_logical_block_provisioning_page lbp_page_changeable = {{ static int rcv_sync_msg; static int persis_offset; static uint8_t ctl_pause_rtr; -static int ctl_is_single = 1; SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer"); static int worker_threads = -1; @@ -969,13 +968,43 @@ ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest) dest->io_hdr.status = src->hdr.status; } +static int +ctl_ha_state_sysctl(SYSCTL_HANDLER_ARGS) +{ + struct ctl_softc *softc = (struct ctl_softc *)arg1; + struct ctl_lun *lun; + int error, value, i; + + if (softc->flags & CTL_FLAG_ACTIVE_SHELF) + value = 0; + else + value = 1; + + error = sysctl_handle_int(oidp, &value, 0, req); + if ((error != 0) || (req->newptr == NULL)) + return (error); + + mtx_lock(&softc->ctl_lock); + if (value == 0) + softc->flags |= CTL_FLAG_ACTIVE_SHELF; + else + softc->flags &= ~CTL_FLAG_ACTIVE_SHELF; + STAILQ_FOREACH(lun, &softc->lun_list, links) { + mtx_lock(&lun->lun_lock); + for (i = 0; i < CTL_MAX_INITIATORS; i++) + lun->pending_ua[i] |= CTL_UA_ASYM_ACC_CHANGE; + mtx_unlock(&lun->lun_lock); + } + mtx_unlock(&softc->ctl_lock); + return (0); +} + static int ctl_init(void) { struct ctl_softc *softc; struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool; struct ctl_port *port; - uint8_t sc_id =0; int i, error, retval; //int isc_retval; @@ -1033,16 +1062,17 @@ ctl_init(void) * In Copan's HA scheme, the "master" and "slave" roles are * figured out through the slot the controller is in. Although it * is an active/active system, someone has to be in charge. - */ -#ifdef NEEDTOPORT - scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id); -#endif - - if (sc_id == 0) { - softc->flags |= CTL_FLAG_MASTER_SHELF; - persis_offset = 0; + */ + SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0, + "HA head ID (0 - no HA)"); + if (softc->ha_id == 0) { + softc->flags |= CTL_FLAG_ACTIVE_SHELF; + softc->is_single = 1; + softc->port_offset = 0; } else - persis_offset = CTL_MAX_INITIATORS; + softc->port_offset = (softc->ha_id - 1) * CTL_MAX_PORTS; + persis_offset = softc->port_offset * CTL_MAX_INIT_PER_PORT; /* * XXX KDM need to figure out where we want to get our target ID @@ -1155,12 +1185,15 @@ ctl_init(void) port->max_targets = 15; port->max_target_id = 15; - if (ctl_port_register(&softc->ioctl_info.port, - (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) { + if (ctl_port_register(&softc->ioctl_info.port) != 0) { printf("ctl: ioctl front end registration failed, will " "continue anyway\n"); } + SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "ha_state", CTLTYPE_INT | CTLFLAG_RWTUN, + softc, 0, ctl_ha_state_sysctl, "I", "HA state for this head"); + #ifdef CTL_IO_DELAY if (sizeof(struct callout) > CTL_TIMER_BYTES) { printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n", @@ -1262,10 +1295,10 @@ ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td) int ctl_port_enable(ctl_port_type port_type) { - struct ctl_softc *softc; + struct ctl_softc *softc = control_softc; struct ctl_port *port; - if (ctl_is_single == 0) { + if (softc->is_single == 0) { union ctl_ha_msg msg_info; int isc_retval; @@ -1290,8 +1323,6 @@ ctl_port_enable(ctl_port_type port_type) #endif } - softc = control_softc; - STAILQ_FOREACH(port, &softc->port_list, links) { if (port_type & port->port_type) { @@ -7440,8 +7471,8 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio) { struct scsi_maintenance_in *cdb; int retval; - int alloc_len, ext, total_len = 0, g, p, pc, pg; - int num_target_port_groups, num_target_ports, single; + int alloc_len, ext, total_len = 0, g, p, pc, pg, gs, os; + int num_target_port_groups, num_target_ports; struct ctl_lun *lun; struct ctl_softc *softc; struct ctl_port *port; @@ -7475,8 +7506,7 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio) return(retval); } - single = ctl_is_single; - if (single) + if (softc->is_single) num_target_port_groups = 1; else num_target_port_groups = NUM_TARGET_PORT_GROUPS; @@ -7532,18 +7562,26 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio) tpg_desc = &rtg_ptr->groups[0]; } - pg = ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS; mtx_lock(&softc->ctl_lock); + pg = softc->port_offset / CTL_MAX_PORTS; + if (softc->flags & CTL_FLAG_ACTIVE_SHELF) { + if (softc->ha_mode == CTL_HA_MODE_ACT_STBY) { + gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + os = TPG_ASYMMETRIC_ACCESS_STANDBY; + } else if (lun->flags & CTL_LUN_PRIMARY_SC) { + gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + os = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED; + } else { + gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED; + os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + } + } else { + gs = TPG_ASYMMETRIC_ACCESS_STANDBY; + os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + } for (g = 0; g < num_target_port_groups; g++) { - if (g == pg) - tpg_desc->pref_state = TPG_PRIMARY | - TPG_ASYMMETRIC_ACCESS_OPTIMIZED; - else - tpg_desc->pref_state = - TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED; - tpg_desc->support = TPG_AO_SUP; - if (!single) - tpg_desc->support |= TPG_AN_SUP; + tpg_desc->pref_state = (g == pg) ? gs : os; + tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP; scsi_ulto2b(g + 1, tpg_desc->target_port_group); tpg_desc->status = TPG_IMPLICIT; pc = 0; @@ -10192,12 +10230,11 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len) struct ctl_lun *lun; struct ctl_port *port; int data_len, num_target_ports, iid_len, id_len, g, pg, p; - int num_target_port_groups, single; + int num_target_port_groups; lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr; - single = ctl_is_single; - if (single) + if (softc->is_single) num_target_port_groups = 1; else num_target_port_groups = NUM_TARGET_PORT_GROUPS; @@ -10257,10 +10294,7 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len) pd = &sp->design[0]; mtx_lock(&softc->ctl_lock); - if (softc->flags & CTL_FLAG_MASTER_SHELF) - pg = 0; - else - pg = 1; + pg = softc->port_offset / CTL_MAX_PORTS; for (g = 0; g < num_target_port_groups; g++) { STAILQ_FOREACH(port, &softc->port_list, links) { if ((port->status & CTL_PORT_STATUS_ONLINE) == 0) @@ -11342,15 +11376,12 @@ ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun, * If this shelf is a secondary shelf controller, we have to reject * any media access commands. */ -#if 0 - /* No longer needed for HA */ - if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0) - && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) { + if ((ctl_softc->flags & CTL_FLAG_ACTIVE_SHELF) == 0 && + (entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0) { ctl_set_lun_standby(ctsio); retval = 1; goto bailout; } -#endif if (entry->pattern & CTL_LUN_PAT_WRITE) { if (lun->flags & CTL_LUN_READONLY) { @@ -14405,7 +14436,7 @@ ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state) // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap) if (c->state == CTL_HA_STATE_UNKNOWN ) { - ctl_is_single = 0; + control_softc->is_single = 0; if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler) != CTL_HA_STATUS_SUCCESS) { printf("ctl_isc_start: ctl_ha_msg_create failed.\n"); @@ -14415,14 +14446,14 @@ ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state) && CTL_HA_STATE_IS_SINGLE(state)){ // HA->SINGLE transition ctl_failover(); - ctl_is_single = 1; + control_softc->is_single = 1; } else { printf("ctl_isc_start:Invalid state transition %X->%X\n", c->state, state); ret = CTL_HA_COMP_STATUS_ERROR; } if (CTL_HA_STATE_IS_SINGLE(state)) - ctl_is_single = 1; + control_softc->is_single = 1; c->state = state; c->status = ret; diff --git a/sys/cam/ctl/ctl_cmd_table.c b/sys/cam/ctl/ctl_cmd_table.c index fedc11001e96..5ae50575e674 100644 --- a/sys/cam/ctl/ctl_cmd_table.c +++ b/sys/cam/ctl/ctl_cmd_table.c @@ -800,12 +800,18 @@ const struct ctl_cmd_entry ctl_cmd_table[256] = /* 3B WRITE BUFFER */ {ctl_write_buffer, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH | + CTL_CMD_FLAG_OK_ON_STOPPED | + CTL_CMD_FLAG_OK_ON_INOPERABLE | + CTL_CMD_FLAG_OK_ON_OFFLINE | CTL_FLAG_DATA_OUT, CTL_LUN_PAT_NONE, 10, {0x1f, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07}}, /* 3C READ BUFFER */ {ctl_read_buffer, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH | + CTL_CMD_FLAG_OK_ON_STOPPED | + CTL_CMD_FLAG_OK_ON_INOPERABLE | + CTL_CMD_FLAG_OK_ON_OFFLINE | CTL_FLAG_DATA_IN | CTL_CMD_FLAG_ALLOW_ON_PR_WRESV, CTL_LUN_PAT_NONE, diff --git a/sys/cam/ctl/ctl_frontend.c b/sys/cam/ctl/ctl_frontend.c index 8a992cc0ba74..70470933bd52 100644 --- a/sys/cam/ctl/ctl_frontend.c +++ b/sys/cam/ctl/ctl_frontend.c @@ -136,7 +136,7 @@ ctl_frontend_find(char *frontend_name) } int -ctl_port_register(struct ctl_port *port, int master_shelf) +ctl_port_register(struct ctl_port *port) { struct ctl_io_pool *pool; int port_num; @@ -193,7 +193,7 @@ ctl_port_register(struct ctl_port *port, int master_shelf) STAILQ_INIT(&port->options); mtx_lock(&control_softc->ctl_lock); - port->targ_port = port_num + (master_shelf != 0 ? 0 : CTL_MAX_PORTS); + port->targ_port = port_num + control_softc->port_offset; STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links); STAILQ_INSERT_TAIL(&control_softc->port_list, port, links); control_softc->ctl_ports[port_num] = port; diff --git a/sys/cam/ctl/ctl_frontend.h b/sys/cam/ctl/ctl_frontend.h index 825ff50a0bca..06ae5a14330c 100644 --- a/sys/cam/ctl/ctl_frontend.h +++ b/sys/cam/ctl/ctl_frontend.h @@ -278,7 +278,7 @@ struct ctl_frontend * ctl_frontend_find(char *frontend_name); * This may block until resources are allocated. Called at FETD module load * time. Returns 0 for success, non-zero for failure. */ -int ctl_port_register(struct ctl_port *port, int master_SC); +int ctl_port_register(struct ctl_port *port); /* * Called at FETD module unload time. diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c index 7cdd5b7972c2..93c12f48e417 100644 --- a/sys/cam/ctl/ctl_frontend_cam_sim.c +++ b/sys/cam/ctl/ctl_frontend_cam_sim.c @@ -164,7 +164,7 @@ cfcs_init(void) port->max_targets = 1; port->max_target_id = 15; - retval = ctl_port_register(port, /*master_SC*/ 1); + retval = ctl_port_register(port); if (retval != 0) { printf("%s: ctl_port_register() failed with error %d!\n", __func__, retval); diff --git a/sys/cam/ctl/ctl_frontend_internal.c b/sys/cam/ctl/ctl_frontend_internal.c index ac972f306420..1d74737f3ef0 100644 --- a/sys/cam/ctl/ctl_frontend_internal.c +++ b/sys/cam/ctl/ctl_frontend_internal.c @@ -244,8 +244,6 @@ cfi_init(void) memset(softc, 0, sizeof(*softc)); mtx_init(&softc->lock, "CTL frontend mutex", NULL, MTX_DEF); - softc->flags |= CTL_FLAG_MASTER_SHELF; - STAILQ_INIT(&softc->lun_list); STAILQ_INIT(&softc->metatask_list); sprintf(softc->fe_name, "kernel"); @@ -264,7 +262,7 @@ cfi_init(void) port->max_targets = 15; port->max_target_id = 15; - if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) + if (ctl_port_register(port) != 0) { printf("%s: internal frontend registration failed\n", __func__); return (0); diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c index 408b86a21fc7..b434a6b932f2 100644 --- a/sys/cam/ctl/ctl_frontend_iscsi.c +++ b/sys/cam/ctl/ctl_frontend_iscsi.c @@ -2046,7 +2046,7 @@ cfiscsi_ioctl_port_create(struct ctl_req *req) desc->length = idlen; strlcpy(desc->identifier, target, idlen); - retval = ctl_port_register(port, /*master_SC*/ 1); + retval = ctl_port_register(port); if (retval != 0) { ctl_free_opts(&port->options); cfiscsi_target_release(ct); diff --git a/sys/cam/ctl/ctl_ha.h b/sys/cam/ctl/ctl_ha.h index 6293c7ce1701..0c004b3c498b 100644 --- a/sys/cam/ctl/ctl_ha.h +++ b/sys/cam/ctl/ctl_ha.h @@ -38,6 +38,8 @@ /* * CTL High Availability Modes: * + * CTL_HA_MODE_ACT_STBY: One side is in Active state and processing commands, + * the other side is in Standby state, returning errors. * CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write * mirroring and read re-direction are assumed to * happen in the back end. @@ -46,6 +48,7 @@ */ typedef enum { + CTL_HA_MODE_ACT_STBY, CTL_HA_MODE_SER_ONLY, CTL_HA_MODE_XFER } ctl_ha_mode; diff --git a/sys/cam/ctl/ctl_private.h b/sys/cam/ctl/ctl_private.h index 7b66c7f8a974..e3ebe7a8258a 100644 --- a/sys/cam/ctl/ctl_private.h +++ b/sys/cam/ctl/ctl_private.h @@ -442,7 +442,7 @@ struct ctl_lun { typedef enum { CTL_FLAG_REAL_SYNC = 0x02, - CTL_FLAG_MASTER_SHELF = 0x04 + CTL_FLAG_ACTIVE_SHELF = 0x04 } ctl_gen_flags; #define CTL_MAX_THREADS 16 @@ -467,6 +467,10 @@ struct ctl_softc { int num_luns; ctl_gen_flags flags; ctl_ha_mode ha_mode; + int ha_id; + int ha_state; + int is_single; + int port_offset; int inquiry_pq_no_lun; struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_tree; diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c index 8fb797865ad2..97a5f984e267 100644 --- a/sys/cam/ctl/ctl_tpc_local.c +++ b/sys/cam/ctl/ctl_tpc_local.c @@ -87,7 +87,6 @@ CTL_FRONTEND_DECLARE(ctltpc, tpcl_frontend); static int tpcl_init(void) { - struct ctl_softc *softc = control_softc; struct tpcl_softc *tsoftc = &tpcl_softc; struct ctl_port *port; struct scsi_transportid_spi *tid; @@ -112,7 +111,7 @@ tpcl_init(void) port->max_target_id = 0; port->max_initiators = 1; - if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) + if (ctl_port_register(port) != 0) { printf("%s: tpc frontend registration failed\n", __func__); return (0); diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c index ff80f3e6f02b..cbdf36ccab18 100644 --- a/sys/cam/ctl/scsi_ctl.c +++ b/sys/cam/ctl/scsi_ctl.c @@ -425,7 +425,7 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) printf("%s: calling ctl_port_register() for %s%d\n", __func__, cpi->dev_name, cpi->unit_number); #endif - retval = ctl_port_register(port, /*master_SC*/ 1); + retval = ctl_port_register(port); if (retval != 0) { printf("%s: ctl_port_register() failed with " "error %d!\n", __func__, retval);