d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modify target port groups logic in CTL.

Browse Source 
- Introduce "ha_shared" port option, which being set to "on" moves the
port into separate port group, shared between HA nodes.  This allows to
better handle cases when iSCSI portals are bound to CARP address that can
dynamically move between nodes.  Some initiators (at least VMware) don't
detect that after iSCSI reconnect they've attached to different SCSI port
from different port group, that totally breakes ALUA status parsing.
In theory, I believe, it should be enough to have different iSCSI portal
group tags on different nodes to make initiators detect this condition,
but it seems like VMware ignores those values, and even full LUN retaste
forced by UA does not help.
 - Make CTL report up to three port groups: 1 -- non-HA mode or ports
with "ha_shared" option set, 2 -- HA node 1, 3 -- HA node 2.
 - Report Transitioning state for all port groups when HA interlink is
connected, but neither of nodes is primary for the LUN.

MFC after:	2 weeks
This commit is contained in:
Alexander Motin 2015-11-11 13:18:38 +00:00
parent cd3903c620
commit 6bd364b523
4 changed files with 82 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
sys/cam/ctl/ctl.c
View File

@ -1819,13 +1819,13 @@ ctl_init(void)
SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0,
"HA head ID (0 - no HA)");
if (softc->ha_id == 0 || softc->ha_id > NUM_TARGET_PORT_GROUPS) {
if (softc->ha_id == 0 || softc->ha_id > NUM_HA_SHELVES) {
softc->flags |= CTL_FLAG_ACTIVE_SHELF;
softc->is_single = 1;
softc->port_cnt = CTL_MAX_PORTS;
softc->port_min = 0;
} else {
softc->port_cnt = CTL_MAX_PORTS / NUM_TARGET_PORT_GROUPS;
softc->port_cnt = CTL_MAX_PORTS / NUM_HA_SHELVES;
softc->port_min = (softc->ha_id - 1) * softc->port_cnt;
}
softc->port_max = softc->port_min + softc->port_cnt;
@ -7137,8 +7137,8 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
{
struct scsi_maintenance_in *cdb;
int retval;
int alloc_len, ext, total_len = 0, g, pc, pg, gs, os;
int num_target_port_groups, num_target_ports;
int alloc_len, ext, total_len = 0, g, pc, pg, ts, os;
int num_ha_groups, num_target_ports, shared_group;
struct ctl_lun *lun;
struct ctl_softc *softc;
struct ctl_port *port;
@ -7172,11 +7172,8 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
return(retval);
}
if (softc->is_single)
num_target_port_groups = 1;
else
num_target_port_groups = NUM_TARGET_PORT_GROUPS;
num_target_ports = 0;
shared_group = (softc->is_single != 0);
mtx_lock(&softc->ctl_lock);
STAILQ_FOREACH(port, &softc->port_list, links) {
if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
@ -7184,15 +7181,18 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
continue;
num_target_ports++;
if (port->status & CTL_PORT_STATUS_HA_SHARED)
shared_group = 1;
}
mtx_unlock(&softc->ctl_lock);
num_ha_groups = (softc->is_single) ? 0 : NUM_HA_SHELVES;
if (ext)
total_len = sizeof(struct scsi_target_group_data_extended);
else
total_len = sizeof(struct scsi_target_group_data);
total_len += sizeof(struct scsi_target_port_group_descriptor) *
num_target_port_groups +
(shared_group + num_ha_groups) +
sizeof(struct scsi_target_port_descriptor) * num_target_ports;
alloc_len = scsi_4btoul(cdb->length);
@ -7229,24 +7229,62 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
mtx_lock(&softc->ctl_lock);
pg = softc->port_min / softc->port_cnt;
if (softc->ha_link == CTL_HA_LINK_OFFLINE)
gs = TPG_ASYMMETRIC_ACCESS_UNAVAILABLE;
else if (softc->ha_link == CTL_HA_LINK_UNKNOWN)
gs = TPG_ASYMMETRIC_ACCESS_TRANSITIONING;
else if (softc->ha_mode == CTL_HA_MODE_ACT_STBY)
gs = TPG_ASYMMETRIC_ACCESS_STANDBY;
else
gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
if (lun->flags & CTL_LUN_PRIMARY_SC) {
os = gs;
gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
} else
os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
for (g = 0; g < num_target_port_groups; g++) {
tpg_desc->pref_state = (g == pg) ? gs : os;
if (lun->flags & (CTL_LUN_PRIMARY_SC | CTL_LUN_PEER_SC_PRIMARY)) {
/* Some shelf is known to be primary. */
if (softc->ha_link == CTL_HA_LINK_OFFLINE)
os = TPG_ASYMMETRIC_ACCESS_UNAVAILABLE;
else if (softc->ha_link == CTL_HA_LINK_UNKNOWN)
os = TPG_ASYMMETRIC_ACCESS_TRANSITIONING;
else if (softc->ha_mode == CTL_HA_MODE_ACT_STBY)
os = TPG_ASYMMETRIC_ACCESS_STANDBY;
else
os = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
if (lun->flags & CTL_LUN_PRIMARY_SC) {
ts = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
} else {
ts = os;
os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
}
} else {
/* No known primary shelf. */
if (softc->ha_link == CTL_HA_LINK_OFFLINE) {
ts = TPG_ASYMMETRIC_ACCESS_UNAVAILABLE;
os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
} else if (softc->ha_link == CTL_HA_LINK_UNKNOWN) {
ts = TPG_ASYMMETRIC_ACCESS_TRANSITIONING;
os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
} else {
ts = os = TPG_ASYMMETRIC_ACCESS_TRANSITIONING;
}
}
if (shared_group) {
tpg_desc->pref_state = ts;
tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP |
TPG_U_SUP | TPG_T_SUP;
scsi_ulto2b(g + 1, tpg_desc->target_port_group);
scsi_ulto2b(1, tpg_desc->target_port_group);
tpg_desc->status = TPG_IMPLICIT;
pc = 0;
STAILQ_FOREACH(port, &softc->port_list, links) {
if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
continue;
if (!softc->is_single &&
(port->status & CTL_PORT_STATUS_HA_SHARED) == 0)
continue;
if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
continue;
scsi_ulto2b(port->targ_port, tpg_desc->descriptors[pc].
relative_target_port_identifier);
pc++;
}
tpg_desc->target_port_count = pc;
tpg_desc = (struct scsi_target_port_group_descriptor *)
&tpg_desc->descriptors[pc];
}
for (g = 0; g < num_ha_groups; g++) {
tpg_desc->pref_state = (g == pg) ? ts : os;
tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP |
TPG_U_SUP | TPG_T_SUP;
scsi_ulto2b(2 + g, tpg_desc->target_port_group);
tpg_desc->status = TPG_IMPLICIT;
pc = 0;
STAILQ_FOREACH(port, &softc->port_list, links) {
@ -7255,6 +7293,8 @@ ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
continue;
if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
continue;
if (port->status & CTL_PORT_STATUS_HA_SHARED)
continue;
if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
continue;
scsi_ulto2b(port->targ_port, tpg_desc->descriptors[pc].
@ -9626,7 +9666,7 @@ ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
struct ctl_softc *softc;
struct ctl_lun *lun;
struct ctl_port *port;
int data_len;
int data_len, g;
uint8_t proto;
softc = control_softc;
@ -9720,8 +9760,11 @@ ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
SVPD_ID_TYPE_TPORTGRP;
desc->length = 4;
scsi_ulto2b(ctsio->io_hdr.nexus.targ_port / softc->port_cnt + 1,
&desc->identifier[2]);
if (softc->is_single || port->status & CTL_PORT_STATUS_HA_SHARED)
g = 1;
else
g = 2 + ctsio->io_hdr.nexus.targ_port / softc->port_cnt;
scsi_ulto2b(g, &desc->identifier[2]);
desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
sizeof(struct scsi_vpd_id_trgt_port_grp_id));

8
sys/cam/ctl/ctl_frontend.c
View File

@ -310,6 +310,7 @@ ctl_port_online(struct ctl_port *port)
{
struct ctl_softc *softc = port->ctl_softc;
struct ctl_lun *lun;
const char *value;
uint32_t l;
if (port->lun_enable != NULL) {
@ -328,6 +329,13 @@ ctl_port_online(struct ctl_port *port)
if (port->port_online != NULL)
port->port_online(port->onoff_arg);
mtx_lock(&softc->ctl_lock);
if (softc->is_single == 0) {
value = ctl_get_opt(&port->options, "ha_shared");
if (value != NULL && strcmp(value, "on") == 0)
port->status |= CTL_PORT_STATUS_HA_SHARED;
else
port->status &= ~CTL_PORT_STATUS_HA_SHARED;
}
port->status |= CTL_PORT_STATUS_ONLINE;
STAILQ_FOREACH(lun, &softc->lun_list, links) {
if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)

3
sys/cam/ctl/ctl_frontend.h
View File

@ -41,7 +41,8 @@
typedef enum {
CTL_PORT_STATUS_NONE = 0x00,
CTL_PORT_STATUS_ONLINE = 0x01
CTL_PORT_STATUS_ONLINE = 0x01,
CTL_PORT_STATUS_HA_SHARED = 0x02
} ctl_port_status;
typedef int (*fe_init_t)(void);

5
sys/cam/ctl/ctl_private.h
View File

@ -356,10 +356,7 @@ struct ctl_devid {
uint8_t data[];
};
/*
* For report target port groups.
*/
#define NUM_TARGET_PORT_GROUPS 2
#define NUM_HA_SHELVES 2
#define CTL_WRITE_BUFFER_SIZE 262144