d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

iscsi: start all sessions for a target node on the same lcore

Browse Source 
This ensures against races, when an existing session to a target node
stalls, causing the initiator to create a new session.  These new
session's connection may get migrated to a different core than the
core of the stalled session.

In practice, this does not happen, but is a common occurrence when
debugging the iSCSI target using gdb.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I1864c2ca0c330dc4faeeb1312adac7a02c8281dc
This commit is contained in:
Jim Harris 2016-09-21 14:11:39 -07:00
parent 2029c9faf9
commit f167fac386
2 changed files with 50 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
lib/iscsi/conn.c
View File

@ -92,7 +92,7 @@ void spdk_iscsi_conn_idle_do_work(void *arg);
static void spdk_iscsi_conn_full_feature_migrate(struct spdk_event *event);
static struct spdk_event *spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn,
int lcore);
int *lcore);
static void spdk_iscsi_conn_stop_poller(struct spdk_iscsi_conn *conn, spdk_event_fn fn_after_stop,
int lcore);
@ -630,9 +630,38 @@ spdk_iscsi_conn_check_shutdown(struct rte_timer *timer, void *arg)
}
static struct spdk_event *
spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn, int lcore)
spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn, int *_lcore)
{
struct spdk_event *event;
struct spdk_iscsi_tgt_node *target;
int lcore;
lcore = spdk_iscsi_conn_allocate_reactor(conn->portal->cpumask);
if (conn->sess->session_type == SESSION_TYPE_NORMAL) {
target = conn->sess->target;
pthread_mutex_lock(&target->mutex);
target->num_active_conns++;
if (target->num_active_conns == 1) {
/**
* This is the only active connection for this target node.
* Save the lcore in the target node so it can be used for
* any other connections to this target node.
*/
target->lcore = lcore;
} else {
/**
* There are other active connections for this target node.
* Ignore the lcore specified by the allocator and use the
* the target node's lcore to ensure this connection runs on
* the same lcore as other connections for this target node.
*/
lcore = target->lcore;
}
pthread_mutex_unlock(&target->mutex);
}
if (_lcore != NULL) {
*_lcore = lcore;
}
event = spdk_event_allocate(lcore, spdk_iscsi_conn_full_feature_migrate, conn, NULL, NULL);
@ -647,7 +676,14 @@ static void
spdk_iscsi_conn_stop_poller(struct spdk_iscsi_conn *conn, spdk_event_fn fn_after_stop, int lcore)
{
struct spdk_event *event;
struct spdk_iscsi_tgt_node *target;
if (conn->sess->session_type == SESSION_TYPE_NORMAL) {
target = conn->sess->target;
pthread_mutex_lock(&target->mutex);
target->num_active_conns--;
pthread_mutex_unlock(&target->mutex);
}
rte_atomic32_dec(&g_num_connections[spdk_app_get_current_core()]);
spdk_net_framework_clear_socket_association(conn->sock);
event = spdk_event_allocate(lcore, fn_after_stop, conn, NULL, NULL);
@ -664,7 +700,7 @@ void spdk_shutdown_iscsi_conns(void)
*/
STAILQ_FOREACH_SAFE(conn, &g_idle_conn_list_head, link, tmp) {
STAILQ_REMOVE(&g_idle_conn_list_head, conn, spdk_iscsi_conn, link);
spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, rte_get_master_lcore()));
spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, NULL));
conn->is_idle = 0;
del_idle_conn(conn);
}
@ -1255,8 +1291,7 @@ spdk_iscsi_conn_login_do_work(void *arg)
* did, migrate it to a dedicated reactor for the target node.
*/
if (conn->login_phase == ISCSI_FULL_FEATURE_PHASE) {
lcore = spdk_iscsi_conn_allocate_reactor(conn->portal->cpumask);
event = spdk_iscsi_conn_get_migrate_event(conn, lcore);
event = spdk_iscsi_conn_get_migrate_event(conn, &lcore);
rte_atomic32_dec(&g_num_connections[spdk_app_get_current_core()]);
rte_atomic32_inc(&g_num_connections[lcore]);
spdk_net_framework_clear_socket_association(conn->sock);
@ -1321,7 +1356,7 @@ void spdk_iscsi_conn_idle_do_work(void *arg)
}
if (tconn->pending_activate_event) {
uint32_t lcore;
int lcore;
spdk_trace_record(TRACE_ISCSI_CONN_ACTIVE, tconn->id, 0, 0, 0);
@ -1332,10 +1367,9 @@ void spdk_iscsi_conn_idle_do_work(void *arg)
tconn->is_idle = 0;
del_idle_conn(tconn);
/* migrate work item to new core */
lcore = spdk_iscsi_conn_allocate_reactor(tconn->portal->cpumask);
rte_atomic32_inc(&g_num_connections[lcore]);
spdk_net_framework_clear_socket_association(tconn->sock);
spdk_event_call(spdk_iscsi_conn_get_migrate_event(tconn, lcore));
spdk_event_call(spdk_iscsi_conn_get_migrate_event(tconn, &lcore));
rte_atomic32_inc(&g_num_connections[lcore]);
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "add conn id = %d, cid = %d poller = %p to lcore = %d active\n",
tconn->id, tconn->cid, &tconn->poller, lcore);
}
@ -1355,7 +1389,7 @@ __add_idle_conn(spdk_event_t e)
* process.
*/
if (conn->state == ISCSI_CONN_STATE_EXITING) {
spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, rte_get_master_lcore()));
spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, NULL));
return;
}

6
lib/iscsi/tgt_node.h
View File

@ -67,6 +67,12 @@ struct spdk_iscsi_tgt_node {
int queue_depth;
struct spdk_scsi_dev *dev;
/**
* Counts number of active iSCSI connections associated with this
* target node.
*/
uint32_t num_active_conns;
int lcore;
int maxmap;
struct spdk_iscsi_tgt_node_map map[MAX_TARGET_MAP];