d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Partially reconstruct Active/Standby clusting.

Browse Source 
In this mode one head is in Active state, supporting all commands, while
another is in Standby state, supporting only minimal LUN discovery subset.

It is still incomplete since Standby state requires reservation support,
which is impossible to do right without having interlink between heads.
But it allows to run some basic experiments.
This commit is contained in:
Alexander Motin 2014-11-21 06:27:37 +00:00
parent 836856e3e6
commit 23b30f5600
11 changed files with 99 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
sys/cam/ctl/ctl.c
View File

@ -357,7 +357,6 @@ static struct ctl_logical_block_provisioning_page lbp_page_changeable = {{
static int rcv_sync_msg;
static int persis_offset;
static uint8_t ctl_pause_rtr;
static int ctl_is_single = 1;
SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
static int worker_threads = -1;
@ -969,13 +968,43 @@ ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
dest->io_hdr.status = src->hdr.status;
}
static int
ctl_ha_state_sysctl(SYSCTL_HANDLER_ARGS)
{
struct ctl_softc *softc = (struct ctl_softc *)arg1;
struct ctl_lun *lun;
int error, value, i;
if (softc->flags & CTL_FLAG_ACTIVE_SHELF)
value = 0;
else
value = 1;
error = sysctl_handle_int(oidp, &value, 0, req);
if ((error != 0) || (req->newptr == NULL))
return (error);
mtx_lock(&softc->ctl_lock);
if (value == 0)
softc->flags |= CTL_FLAG_ACTIVE_SHELF;
else
softc->flags &= ~CTL_FLAG_ACTIVE_SHELF;
STAILQ_FOREACH(lun, &softc->lun_list, links) {
mtx_lock(&lun->lun_lock);
for (i = 0; i < CTL_MAX_INITIATORS; i++)
lun->pending_ua[i] |= CTL_UA_ASYM_ACC_CHANGE;
mtx_unlock(&lun->lun_lock);
}
mtx_unlock(&softc->ctl_lock);
return (0);
}
static int
ctl_init(void)
{
struct ctl_softc *softc;
struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
struct ctl_port *port;
uint8_t sc_id =0;
int i, error, retval;
//int isc_retval;
@ -1033,16 +1062,17 @@ ctl_init(void)
* In Copan's HA scheme, the "master" and "slave" roles are
* figured out through the slot the controller is in. Although it
* is an active/active system, someone has to be in charge.
*/
#ifdef NEEDTOPORT
scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
#endif
if (sc_id == 0) {
softc->flags |= CTL_FLAG_MASTER_SHELF;
persis_offset = 0;
*/
SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0,
"HA head ID (0 - no HA)");
if (softc->ha_id == 0) {
softc->flags |= CTL_FLAG_ACTIVE_SHELF;
softc->is_single = 1;
softc->port_offset = 0;
} else
persis_offset = CTL_MAX_INITIATORS;
softc->port_offset = (softc->ha_id - 1) * CTL_MAX_PORTS;
persis_offset = softc->port_offset * CTL_MAX_INIT_PER_PORT;
/*
* XXX KDM need to figure out where we want to get our target ID
@ -1155,12 +1185,15 @@ ctl_init(void)
port->max_targets = 15;
port->max_target_id = 15;
if (ctl_port_register(&softc->ioctl_info.port,
(softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
if (ctl_port_register(&softc->ioctl_info.port) != 0) {
printf("ctl: ioctl front end registration failed, will "
"continue anyway\n");
}
SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
OID_AUTO, "ha_state", CTLTYPE_INT | CTLFLAG_RWTUN,
softc, 0, ctl_ha_state_sysctl, "I", "HA state for this head");
#ifdef CTL_IO_DELAY
if (sizeof(struct callout) > CTL_TIMER_BYTES) {
printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
@ -1262,10 +1295,10 @@ ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
int
ctl_port_enable(ctl_port_type port_type)
{
struct ctl_softc *softc;
struct ctl_softc *softc = control_softc;
struct ctl_port *port;
if (ctl_is_single == 0) {
if (softc->is_single == 0) {
union ctl_ha_msg msg_info;
int isc_retval;
@ -1290,8 +1323,6 @@ ctl_port_enable(ctl_port_type port_type)
#endif
}
softc = control_softc;
STAILQ_FOREACH(port, &softc->port_list, links) {
if (port_type & port->port_type)
{
@ -7440,8 +7471,8 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
{
struct scsi_maintenance_in *cdb;
int retval;
int alloc_len, ext, total_len = 0, g, p, pc, pg;
int num_target_port_groups, num_target_ports, single;
int alloc_len, ext, total_len = 0, g, p, pc, pg, gs, os;
int num_target_port_groups, num_target_ports;
struct ctl_lun *lun;
struct ctl_softc *softc;
struct ctl_port *port;
@ -7475,8 +7506,7 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
return(retval);
}
single = ctl_is_single;
if (single)
if (softc->is_single)
num_target_port_groups = 1;
else
num_target_port_groups = NUM_TARGET_PORT_GROUPS;
@ -7532,18 +7562,26 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
tpg_desc = &rtg_ptr->groups[0];
}
pg = ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS;
mtx_lock(&softc->ctl_lock);
pg = softc->port_offset / CTL_MAX_PORTS;
if (softc->flags & CTL_FLAG_ACTIVE_SHELF) {
if (softc->ha_mode == CTL_HA_MODE_ACT_STBY) {
gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
os = TPG_ASYMMETRIC_ACCESS_STANDBY;
} else if (lun->flags & CTL_LUN_PRIMARY_SC) {
gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
os = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
} else {
gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
}
} else {
gs = TPG_ASYMMETRIC_ACCESS_STANDBY;
os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
}
for (g = 0; g < num_target_port_groups; g++) {
if (g == pg)
tpg_desc->pref_state = TPG_PRIMARY |
TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
else
tpg_desc->pref_state =
TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
tpg_desc->support = TPG_AO_SUP;
if (!single)
tpg_desc->support |= TPG_AN_SUP;
tpg_desc->pref_state = (g == pg) ? gs : os;
tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP;
scsi_ulto2b(g + 1, tpg_desc->target_port_group);
tpg_desc->status = TPG_IMPLICIT;
pc = 0;
@ -10192,12 +10230,11 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
struct ctl_lun *lun;
struct ctl_port *port;
int data_len, num_target_ports, iid_len, id_len, g, pg, p;
int num_target_port_groups, single;
int num_target_port_groups;
lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
single = ctl_is_single;
if (single)
if (softc->is_single)
num_target_port_groups = 1;
else
num_target_port_groups = NUM_TARGET_PORT_GROUPS;
@ -10257,10 +10294,7 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
pd = &sp->design[0];
mtx_lock(&softc->ctl_lock);
if (softc->flags & CTL_FLAG_MASTER_SHELF)
pg = 0;
else
pg = 1;
pg = softc->port_offset / CTL_MAX_PORTS;
for (g = 0; g < num_target_port_groups; g++) {
STAILQ_FOREACH(port, &softc->port_list, links) {
if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
@ -11342,15 +11376,12 @@ ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
* If this shelf is a secondary shelf controller, we have to reject
* any media access commands.
*/
#if 0
/* No longer needed for HA */
if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
&& ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
if ((ctl_softc->flags & CTL_FLAG_ACTIVE_SHELF) == 0 &&
(entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0) {
ctl_set_lun_standby(ctsio);
retval = 1;
goto bailout;
}
#endif
if (entry->pattern & CTL_LUN_PAT_WRITE) {
if (lun->flags & CTL_LUN_READONLY) {
@ -14405,7 +14436,7 @@ ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
// UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
if (c->state == CTL_HA_STATE_UNKNOWN ) {
ctl_is_single = 0;
control_softc->is_single = 0;
if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
!= CTL_HA_STATUS_SUCCESS) {
printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
@ -14415,14 +14446,14 @@ ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
&& CTL_HA_STATE_IS_SINGLE(state)){
// HA->SINGLE transition
ctl_failover();
ctl_is_single = 1;
control_softc->is_single = 1;
} else {
printf("ctl_isc_start:Invalid state transition %X->%X\n",
c->state, state);
ret = CTL_HA_COMP_STATUS_ERROR;
}
if (CTL_HA_STATE_IS_SINGLE(state))
ctl_is_single = 1;
control_softc->is_single = 1;
c->state = state;
c->status = ret;

6
sys/cam/ctl/ctl_cmd_table.c
View File

@ -800,12 +800,18 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
/* 3B WRITE BUFFER */
{ctl_write_buffer, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH |
CTL_CMD_FLAG_OK_ON_STOPPED |
CTL_CMD_FLAG_OK_ON_INOPERABLE |
CTL_CMD_FLAG_OK_ON_OFFLINE |
CTL_FLAG_DATA_OUT,
CTL_LUN_PAT_NONE,
10, {0x1f, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07}},
/* 3C READ BUFFER */
{ctl_read_buffer, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
CTL_CMD_FLAG_OK_ON_STOPPED |
CTL_CMD_FLAG_OK_ON_INOPERABLE |
CTL_CMD_FLAG_OK_ON_OFFLINE |
CTL_FLAG_DATA_IN |
CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
CTL_LUN_PAT_NONE,

4
sys/cam/ctl/ctl_frontend.c
View File

@ -136,7 +136,7 @@ ctl_frontend_find(char *frontend_name)
}
int
ctl_port_register(struct ctl_port *port, int master_shelf)
ctl_port_register(struct ctl_port *port)
{
struct ctl_io_pool *pool;
int port_num;
@ -193,7 +193,7 @@ ctl_port_register(struct ctl_port *port, int master_shelf)
STAILQ_INIT(&port->options);
mtx_lock(&control_softc->ctl_lock);
port->targ_port = port_num + (master_shelf != 0 ? 0 : CTL_MAX_PORTS);
port->targ_port = port_num + control_softc->port_offset;
STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links);
STAILQ_INSERT_TAIL(&control_softc->port_list, port, links);
control_softc->ctl_ports[port_num] = port;

2
sys/cam/ctl/ctl_frontend.h
View File

@ -278,7 +278,7 @@ struct ctl_frontend * ctl_frontend_find(char *frontend_name);
* This may block until resources are allocated. Called at FETD module load
* time. Returns 0 for success, non-zero for failure.
*/
int ctl_port_register(struct ctl_port *port, int master_SC);
int ctl_port_register(struct ctl_port *port);
/*
* Called at FETD module unload time.

2
sys/cam/ctl/ctl_frontend_cam_sim.c
View File

@ -164,7 +164,7 @@ cfcs_init(void)
port->max_targets = 1;
port->max_target_id = 15;
retval = ctl_port_register(port, /*master_SC*/ 1);
retval = ctl_port_register(port);
if (retval != 0) {
printf("%s: ctl_port_register() failed with error %d!\n",
__func__, retval);

4
sys/cam/ctl/ctl_frontend_internal.c
View File

@ -244,8 +244,6 @@ cfi_init(void)
memset(softc, 0, sizeof(*softc));
mtx_init(&softc->lock, "CTL frontend mutex", NULL, MTX_DEF);
softc->flags |= CTL_FLAG_MASTER_SHELF;
STAILQ_INIT(&softc->lun_list);
STAILQ_INIT(&softc->metatask_list);
sprintf(softc->fe_name, "kernel");
@ -264,7 +262,7 @@ cfi_init(void)
port->max_targets = 15;
port->max_target_id = 15;
if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0)
if (ctl_port_register(port) != 0)
{
printf("%s: internal frontend registration failed\n", __func__);
return (0);

2
sys/cam/ctl/ctl_frontend_iscsi.c
View File

@ -2046,7 +2046,7 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
desc->length = idlen;
strlcpy(desc->identifier, target, idlen);
retval = ctl_port_register(port, /*master_SC*/ 1);
retval = ctl_port_register(port);
if (retval != 0) {
ctl_free_opts(&port->options);
cfiscsi_target_release(ct);

3
sys/cam/ctl/ctl_ha.h
View File

@ -38,6 +38,8 @@
/*
* CTL High Availability Modes:
*
* CTL_HA_MODE_ACT_STBY: One side is in Active state and processing commands,
* the other side is in Standby state, returning errors.
* CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write
* mirroring and read re-direction are assumed to
* happen in the back end.
@ -46,6 +48,7 @@
*/
typedef enum {
CTL_HA_MODE_ACT_STBY,
CTL_HA_MODE_SER_ONLY,
CTL_HA_MODE_XFER
} ctl_ha_mode;

6
sys/cam/ctl/ctl_private.h
View File

@ -442,7 +442,7 @@ struct ctl_lun {
typedef enum {
CTL_FLAG_REAL_SYNC = 0x02,
CTL_FLAG_MASTER_SHELF = 0x04
CTL_FLAG_ACTIVE_SHELF = 0x04
} ctl_gen_flags;
#define CTL_MAX_THREADS 16
@ -467,6 +467,10 @@ struct ctl_softc {
int num_luns;
ctl_gen_flags flags;
ctl_ha_mode ha_mode;
int ha_id;
int ha_state;
int is_single;
int port_offset;
int inquiry_pq_no_lun;
struct sysctl_ctx_list sysctl_ctx;
struct sysctl_oid *sysctl_tree;

3
sys/cam/ctl/ctl_tpc_local.c
View File

@ -87,7 +87,6 @@ CTL_FRONTEND_DECLARE(ctltpc, tpcl_frontend);
static int
tpcl_init(void)
{
struct ctl_softc *softc = control_softc;
struct tpcl_softc *tsoftc = &tpcl_softc;
struct ctl_port *port;
struct scsi_transportid_spi *tid;
@ -112,7 +111,7 @@ tpcl_init(void)
port->max_target_id = 0;
port->max_initiators = 1;
if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0)
if (ctl_port_register(port) != 0)
{
printf("%s: tpc frontend registration failed\n", __func__);
return (0);

2
sys/cam/ctl/scsi_ctl.c
View File

@ -425,7 +425,7 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
printf("%s: calling ctl_port_register() for %s%d\n",
__func__, cpi->dev_name, cpi->unit_number);
#endif
retval = ctl_port_register(port, /*master_SC*/ 1);
retval = ctl_port_register(port);
if (retval != 0) {
printf("%s: ctl_port_register() failed with "
"error %d!\n", __func__, retval);