From ba3a2d31c810f11860330921fc4863274988172a Mon Sep 17 00:00:00 2001 From: Edward Tomasz Napierala Date: Wed, 16 Apr 2014 10:06:37 +0000 Subject: [PATCH] Make it possible for the iSCSI target side to operate in both normal and ICL_KERNEL_PROXY mode, and fix some bit rot so the latter actually works again. Sponsored by: The FreeBSD Foundation --- sys/cam/ctl/ctl_frontend_iscsi.c | 76 ++++++++++------ sys/cam/ctl/ctl_ioctl.h | 17 ++-- usr.sbin/ctld/ctld.c | 151 +++++++++++++++---------------- usr.sbin/ctld/kernel.c | 11 ++- usr.sbin/ctld/pdu.c | 32 +++++-- 5 files changed, 163 insertions(+), 124 deletions(-) diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c index 57517c3f08c2..411aafc748c7 100644 --- a/sys/cam/ctl/ctl_frontend_iscsi.c +++ b/sys/cam/ctl/ctl_frontend_iscsi.c @@ -1419,9 +1419,7 @@ cfiscsi_ioctl_handoff(struct ctl_iscsi *ci) struct cfiscsi_session *cs; struct cfiscsi_target *ct; struct ctl_iscsi_handoff_params *cihp; -#ifndef ICL_KERNEL_PROXY int error; -#endif cihp = (struct ctl_iscsi_handoff_params *)&(ci->data); softc = &cfiscsi_softc; @@ -1446,28 +1444,40 @@ cfiscsi_ioctl_handoff(struct ctl_iscsi *ci) } #ifdef ICL_KERNEL_PROXY - mtx_lock(&cfiscsi_softc.lock); - TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) { - if (cs->cs_id == cihp->socket) - break; - } - if (cs == NULL) { - mtx_unlock(&cfiscsi_softc.lock); - snprintf(ci->error_str, sizeof(ci->error_str), "connection not found"); - ci->status = CTL_ISCSI_ERROR; - cfiscsi_target_release(ct); - return; - } - mtx_unlock(&cfiscsi_softc.lock); -#else - cs = cfiscsi_session_new(softc); - if (cs == NULL) { - ci->status = CTL_ISCSI_ERROR; + if (cihp->socket > 0 && cihp->connection_id > 0) { snprintf(ci->error_str, sizeof(ci->error_str), - "%s: cfiscsi_session_new failed", __func__); + "both socket and connection_id set"); + ci->status = CTL_ISCSI_ERROR; cfiscsi_target_release(ct); return; } + if (cihp->socket == 0) { + mtx_lock(&cfiscsi_softc.lock); + TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) { + if (cs->cs_id == cihp->socket) + break; + } + if (cs == NULL) { + mtx_unlock(&cfiscsi_softc.lock); + snprintf(ci->error_str, sizeof(ci->error_str), + "connection not found"); + ci->status = CTL_ISCSI_ERROR; + cfiscsi_target_release(ct); + return; + } + mtx_unlock(&cfiscsi_softc.lock); + } else { +#endif + cs = cfiscsi_session_new(softc); + if (cs == NULL) { + ci->status = CTL_ISCSI_ERROR; + snprintf(ci->error_str, sizeof(ci->error_str), + "%s: cfiscsi_session_new failed", __func__); + cfiscsi_target_release(ct); + return; + } +#ifdef ICL_KERNEL_PROXY + } #endif cs->cs_target = ct; @@ -1495,16 +1505,18 @@ cfiscsi_ioctl_handoff(struct ctl_iscsi *ci) cihp->initiator_alias, sizeof(cs->cs_initiator_alias)); #ifdef ICL_KERNEL_PROXY - cs->cs_login_phase = false; -#else - error = icl_conn_handoff(cs->cs_conn, cihp->socket); - if (error != 0) { - cfiscsi_session_delete(cs); - ci->status = CTL_ISCSI_ERROR; - snprintf(ci->error_str, sizeof(ci->error_str), - "%s: icl_conn_handoff failed with error %d", - __func__, error); - return; + if (cihp->socket > 0) { +#endif + error = icl_conn_handoff(cs->cs_conn, cihp->socket); + if (error != 0) { + cfiscsi_session_delete(cs); + ci->status = CTL_ISCSI_ERROR; + snprintf(ci->error_str, sizeof(ci->error_str), + "%s: icl_conn_handoff failed with error %d", + __func__, error); + return; + } +#ifdef ICL_KERNEL_PROXY } #endif @@ -1514,6 +1526,8 @@ cfiscsi_ioctl_handoff(struct ctl_iscsi *ci) cfiscsi_session_register_initiator(cs); #ifdef ICL_KERNEL_PROXY + cs->cs_login_phase = false; + /* * First PDU of the Full Feature phase has likely already arrived. * We have to pick it up and execute properly. @@ -1831,7 +1845,9 @@ cfiscsi_ioctl_send(struct ctl_iscsi *ci) icl_pdu_append_data(ip, data, datalen, M_WAITOK); free(data, M_CFISCSI); } + CFISCSI_SESSION_LOCK(cs); icl_pdu_queue(ip); + CFISCSI_SESSION_UNLOCK(cs); ci->status = CTL_ISCSI_OK; } diff --git a/sys/cam/ctl/ctl_ioctl.h b/sys/cam/ctl/ctl_ioctl.h index f869edf1ed04..d9aab930addb 100644 --- a/sys/cam/ctl/ctl_ioctl.h +++ b/sys/cam/ctl/ctl_ioctl.h @@ -640,15 +640,7 @@ struct ctl_iscsi_handoff_params { char initiator_addr[CTL_ISCSI_ADDR_LEN]; char initiator_alias[CTL_ISCSI_ALIAS_LEN]; char target_name[CTL_ISCSI_NAME_LEN]; -#ifdef ICL_KERNEL_PROXY - int connection_id; - /* - * XXX - */ int socket; -#else - int socket; -#endif int portal_group_tag; /* @@ -662,7 +654,12 @@ struct ctl_iscsi_handoff_params { uint32_t max_burst_length; uint32_t first_burst_length; uint32_t immediate_data; +#ifdef ICL_KERNEL_PROXY + int connection_id; + int spare[3]; +#else int spare[4]; +#endif }; struct ctl_iscsi_list_params { @@ -715,7 +712,7 @@ struct ctl_iscsi_send_params { void *spare2; size_t data_segment_len; void *data_segment; - int spare[4]; + int spare3[4]; }; struct ctl_iscsi_receive_params { @@ -725,7 +722,7 @@ struct ctl_iscsi_receive_params { void *spare2; size_t data_segment_len; void *data_segment; - int spare[4]; + int spare3[4]; }; struct ctl_iscsi_close_params { diff --git a/usr.sbin/ctld/ctld.c b/usr.sbin/ctld/ctld.c index 1090a52d02fd..354d0edf1604 100644 --- a/usr.sbin/ctld/ctld.c +++ b/usr.sbin/ctld/ctld.c @@ -48,6 +48,8 @@ #include "ctld.h" +bool proxy_mode = false; + static volatile bool sighup_received = false; static volatile bool sigterm_received = false; static volatile bool sigalrm_received = false; @@ -553,14 +555,6 @@ portal_group_add_listen(struct portal_group *pg, const char *value, bool iser) const char *port; int error, colons = 0; -#ifndef ICL_KERNEL_PROXY - if (iser) { - log_warnx("ctld(8) compiled without ICL_KERNEL_PROXY " - "does not support iSER protocol"); - return (-1); - } -#endif - portal = portal_new(pg); portal->p_listen = checked_strdup(value); portal->p_iser = iser; @@ -1180,9 +1174,7 @@ conf_apply(struct conf *oldconf, struct conf *newconf) struct portal *oldp, *newp; pid_t otherpid; int changed, cumulated_error = 0, error; -#ifndef ICL_KERNEL_PROXY int one = 1; -#endif if (oldconf->conf_debug != newconf->conf_debug) { log_debugx("changing debug level to %d", newconf->conf_debug); @@ -1415,10 +1407,14 @@ conf_apply(struct conf *oldconf, struct conf *newconf) } #ifdef ICL_KERNEL_PROXY - log_debugx("listening on %s, portal-group \"%s\" using ICL proxy", - newp->p_listen, newpg->pg_name); - kernel_listen(newp->p_ai, newp->p_iser); -#else + if (proxy_mode) { + log_debugx("listening on %s, portal-group \"%s\" using ICL proxy", + newp->p_listen, newpg->pg_name); + kernel_listen(newp->p_ai, newp->p_iser); + continue; + } +#endif + assert(proxy_mode == false); assert(newp->p_iser == false); log_debugx("listening on %s, portal-group \"%s\"", @@ -1461,7 +1457,6 @@ conf_apply(struct conf *oldconf, struct conf *newconf) cumulated_error++; continue; } -#endif /* !ICL_KERNEL_PROXY */ } } @@ -1579,11 +1574,9 @@ static void handle_connection(struct portal *portal, int fd, bool dont_fork) { struct connection *conn; -#ifndef ICL_KERNEL_PROXY struct sockaddr_storage ss; socklen_t sslen = sizeof(ss); int error; -#endif pid_t pid; char host[NI_MAXHOST + 1]; struct conf *conf; @@ -1619,20 +1612,25 @@ handle_connection(struct portal *portal, int fd, bool dont_fork) /* * XXX */ - log_set_peer_addr("XXX"); -#else - error = getpeername(fd, (struct sockaddr *)&ss, &sslen); - if (error != 0) - log_err(1, "getpeername"); - error = getnameinfo((struct sockaddr *)&ss, sslen, - host, sizeof(host), NULL, 0, NI_NUMERICHOST); - if (error != 0) - log_errx(1, "getaddrinfo: %s", gai_strerror(error)); + if (proxy_mode) { + log_set_peer_addr("XXX"); + } else { +#endif + assert(proxy_mode == false); + error = getpeername(fd, (struct sockaddr *)&ss, &sslen); + if (error != 0) + log_err(1, "getpeername"); + error = getnameinfo((struct sockaddr *)&ss, sslen, + host, sizeof(host), NULL, 0, NI_NUMERICHOST); + if (error != 0) + log_errx(1, "getaddrinfo: %s", gai_strerror(error)); - log_debugx("accepted connection from %s; portal group \"%s\"", - host, portal->p_portal_group->pg_name); - log_set_peer_addr(host); - setproctitle("%s", host); + log_debugx("accepted connection from %s; portal group \"%s\"", + host, portal->p_portal_group->pg_name); + log_set_peer_addr(host); + setproctitle("%s", host); +#ifdef ICL_KERNEL_PROXY + } #endif conn = connection_new(portal, fd, host); @@ -1650,7 +1648,6 @@ handle_connection(struct portal *portal, int fd, bool dont_fork) exit(0); } -#ifndef ICL_KERNEL_PROXY static int fd_add(int fd, fd_set *fdset, int nfds) { @@ -1666,7 +1663,6 @@ fd_add(int fd, fd_set *fdset, int nfds) nfds = fd; return (nfds); } -#endif static void main_loop(struct conf *conf, bool dont_fork) @@ -1675,10 +1671,9 @@ main_loop(struct conf *conf, bool dont_fork) struct portal *portal; #ifdef ICL_KERNEL_PROXY int connection_id; -#else +#endif fd_set fdset; int error, nfds, client_fd; -#endif pidfile_write(conf->conf_pidfh); @@ -1687,42 +1682,48 @@ main_loop(struct conf *conf, bool dont_fork) return; #ifdef ICL_KERNEL_PROXY - connection_id = kernel_accept(); - if (connection_id == 0) - continue; + if (proxy_mode) { + connection_id = kernel_accept(); + if (connection_id == 0) + continue; - /* - * XXX: This is obviously temporary. - */ - pg = TAILQ_FIRST(&conf->conf_portal_groups); - portal = TAILQ_FIRST(&pg->pg_portals); + /* + * XXX: This is obviously temporary. + */ + pg = TAILQ_FIRST(&conf->conf_portal_groups); + portal = TAILQ_FIRST(&pg->pg_portals); - handle_connection(portal, connection_id, dont_fork); -#else - FD_ZERO(&fdset); - nfds = 0; - TAILQ_FOREACH(pg, &conf->conf_portal_groups, pg_next) { - TAILQ_FOREACH(portal, &pg->pg_portals, p_next) - nfds = fd_add(portal->p_socket, &fdset, nfds); - } - error = select(nfds + 1, &fdset, NULL, NULL, NULL); - if (error <= 0) { - if (errno == EINTR) - return; - log_err(1, "select"); - } - TAILQ_FOREACH(pg, &conf->conf_portal_groups, pg_next) { - TAILQ_FOREACH(portal, &pg->pg_portals, p_next) { - if (!FD_ISSET(portal->p_socket, &fdset)) - continue; - client_fd = accept(portal->p_socket, NULL, 0); - if (client_fd < 0) - log_err(1, "accept"); - handle_connection(portal, client_fd, dont_fork); - break; + handle_connection(portal, connection_id, dont_fork); + } else { +#endif + assert(proxy_mode == false); + + FD_ZERO(&fdset); + nfds = 0; + TAILQ_FOREACH(pg, &conf->conf_portal_groups, pg_next) { + TAILQ_FOREACH(portal, &pg->pg_portals, p_next) + nfds = fd_add(portal->p_socket, &fdset, nfds); } + error = select(nfds + 1, &fdset, NULL, NULL, NULL); + if (error <= 0) { + if (errno == EINTR) + return; + log_err(1, "select"); + } + TAILQ_FOREACH(pg, &conf->conf_portal_groups, pg_next) { + TAILQ_FOREACH(portal, &pg->pg_portals, p_next) { + if (!FD_ISSET(portal->p_socket, &fdset)) + continue; + client_fd = accept(portal->p_socket, NULL, 0); + if (client_fd < 0) + log_err(1, "accept"); + handle_connection(portal, client_fd, dont_fork); + break; + } + } +#ifdef ICL_KERNEL_PROXY } -#endif /* !ICL_KERNEL_PROXY */ +#endif } } @@ -1788,7 +1789,7 @@ main(int argc, char **argv) int debug = 0, ch, error; bool dont_daemonize = false; - while ((ch = getopt(argc, argv, "df:")) != -1) { + while ((ch = getopt(argc, argv, "df:R")) != -1) { switch (ch) { case 'd': dont_daemonize = true; @@ -1797,6 +1798,13 @@ main(int argc, char **argv) case 'f': config_path = optarg; break; + case 'R': +#ifndef ICL_KERNEL_PROXY + log_errx(1, "ctld(8) compiled without ICL_KERNEL_PROXY " + "does not support iSER protocol"); +#endif + proxy_mode = true; + break; case '?': default: usage(); @@ -1818,12 +1826,10 @@ main(int argc, char **argv) newconf->conf_debug = debug; } -#ifdef ICL_KERNEL_PROXY log_debugx("enabling CTL iSCSI port"); error = kernel_port_on(); if (error != 0) log_errx(1, "failed to enable CTL iSCSI port, exiting"); -#endif error = conf_apply(oldconf, newconf); if (error != 0) @@ -1833,13 +1839,6 @@ main(int argc, char **argv) register_signals(); -#ifndef ICL_KERNEL_PROXY - log_debugx("enabling CTL iSCSI port"); - error = kernel_port_on(); - if (error != 0) - log_errx(1, "failed to enable CTL iSCSI port, exiting"); -#endif - if (dont_daemonize == false) { log_debugx("daemonizing"); if (daemon(0, 0) == -1) { diff --git a/usr.sbin/ctld/kernel.c b/usr.sbin/ctld/kernel.c index 012dc7cd2636..3c4a547097cc 100644 --- a/usr.sbin/ctld/kernel.c +++ b/usr.sbin/ctld/kernel.c @@ -65,11 +65,13 @@ #include #include +#include "ctld.h" + #ifdef ICL_KERNEL_PROXY #include #endif -#include "ctld.h" +extern bool proxy_mode; static int ctl_fd = 0; @@ -599,7 +601,14 @@ kernel_handoff(struct connection *conn) } strlcpy(req.data.handoff.target_name, conn->conn_target->t_name, sizeof(req.data.handoff.target_name)); +#ifdef ICL_KERNEL_PROXY + if (proxy_mode) + req.data.handoff.connection_id = conn->conn_socket; + else + req.data.handoff.socket = conn->conn_socket; +#else req.data.handoff.socket = conn->conn_socket; +#endif req.data.handoff.portal_group_tag = conn->conn_portal->p_portal_group->pg_tag; if (conn->conn_header_digest == CONN_DIGEST_CRC32C) diff --git a/usr.sbin/ctld/pdu.c b/usr.sbin/ctld/pdu.c index eb8921e33356..084423c7178c 100644 --- a/usr.sbin/ctld/pdu.c +++ b/usr.sbin/ctld/pdu.c @@ -44,6 +44,8 @@ #include #endif +extern bool proxy_mode; + static int pdu_ahs_length(const struct pdu *pdu) { @@ -101,11 +103,13 @@ pdu_new_response(struct pdu *request) #ifdef ICL_KERNEL_PROXY -void -pdu_receive(struct pdu *pdu) +static void +pdu_receive_proxy(struct pdu *pdu) { size_t len; + assert(proxy_mode); + kernel_receive(pdu); len = pdu_ahs_length(pdu); @@ -117,15 +121,17 @@ pdu_receive(struct pdu *pdu) pdu->pdu_data_len = len; } -void -pdu_send(struct pdu *pdu) +static void +pdu_send_proxy(struct pdu *pdu) { + assert(proxy_mode); + pdu_set_data_segment_length(pdu, pdu->pdu_data_len); kernel_send(pdu); } -#else /* !ICL_KERNEL_PROXY */ +#endif /* ICL_KERNEL_PROXY */ static size_t pdu_padding(const struct pdu *pdu) @@ -161,6 +167,13 @@ pdu_receive(struct pdu *pdu) size_t len, padding; char dummy[4]; +#ifdef ICL_KERNEL_PROXY + if (proxy_mode) + return (pdu_receive_proxy(pdu)); +#endif + + assert(proxy_mode == false); + pdu_read(pdu->pdu_connection->conn_socket, (char *)pdu->pdu_bhs, sizeof(*pdu->pdu_bhs)); @@ -202,6 +215,13 @@ pdu_send(struct pdu *pdu) struct iovec iov[3]; int iovcnt; +#ifdef ICL_KERNEL_PROXY + if (proxy_mode) + return (pdu_send_proxy(pdu)); +#endif + + assert(proxy_mode == false); + pdu_set_data_segment_length(pdu, pdu->pdu_data_len); iov[0].iov_base = pdu->pdu_bhs; iov[0].iov_len = sizeof(*pdu->pdu_bhs); @@ -234,8 +254,6 @@ pdu_send(struct pdu *pdu) log_errx(1, "short write"); } -#endif /* !ICL_KERNEL_PROXY */ - void pdu_delete(struct pdu *pdu) {