d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor CTL datamove KPI.

Browse Source 
- Make frontends call unified CTL core method ctl_datamove_done()
to report move completion.  It allows to reduce code duplication
in differerent backends by accounting DMA time in common code.
 - Add to ctl_datamove_done() and be_move_done() callback samethr
argument, reporting whether the callback is called in the same
context as ctl_datamove().  It allows for some cases like iSCSI
write with immediate data or camsim frontend write save one context
switch, since we know that the context is sleepable.
 - Remove data_move_done() methods from struct ctl_backend_driver,
unused since forever.

MFC after:	 1 month
This commit is contained in:
Alexander Motin 2021-02-21 16:45:14 -05:00
parent 1158508a80
commit 2c7dc6bae9
12 changed files with 94 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
sys/cam/ctl/ctl.c
View File

@ -535,9 +535,9 @@ static void ctl_done_timer_wakeup(void *arg);
static void ctl_send_datamove_done(union ctl_io *io, int have_lock); static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq); static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
static int ctl_datamove_remote_dm_write_cb(union ctl_io *io); static int ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr);
static void ctl_datamove_remote_write(union ctl_io *io); static void ctl_datamove_remote_write(union ctl_io *io);
static int ctl_datamove_remote_dm_read_cb(union ctl_io *io); static int ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr);
static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq); static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
static int ctl_datamove_remote_sgl_setup(union ctl_io *io); static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command, static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
@ -736,7 +736,7 @@ ctl_ha_datamove(union ctl_io *io)
sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries, sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries,
M_WAITOK) > CTL_HA_STATUS_SUCCESS) { M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
io->io_hdr.port_status = 31341; io->io_hdr.port_status = 31341;
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
return; return;
} }
msg.dt.sent_sg_entries = sg_entries_sent; msg.dt.sent_sg_entries = sg_entries_sent;
@ -753,7 +753,7 @@ ctl_ha_datamove(union ctl_io *io)
if (lun) if (lun)
mtx_unlock(&lun->lun_lock); mtx_unlock(&lun->lun_lock);
io->io_hdr.port_status = 31342; io->io_hdr.port_status = 31342;
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
return; return;
} }
io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE; io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
@ -5028,7 +5028,7 @@ ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
* make it down to say RAIDCore's configuration code. * make it down to say RAIDCore's configuration code.
*/ */
int int
ctl_config_move_done(union ctl_io *io) ctl_config_move_done(union ctl_io *io, bool samethr)
{ {
int retval; int retval;
@ -5036,18 +5036,6 @@ ctl_config_move_done(union ctl_io *io)
KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type)); ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
if ((io->io_hdr.port_status != 0) &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
/*retry_count*/ io->io_hdr.port_status);
} else if (io->scsiio.kern_data_resid != 0 &&
(io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_invalid_field_ciu(&io->scsiio);
}
if (ctl_debug & CTL_DEBUG_CDB_DATA) if (ctl_debug & CTL_DEBUG_CDB_DATA)
ctl_data_print(io); ctl_data_print(io);
if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) || if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
@ -12303,7 +12291,7 @@ ctl_handle_isc(union ctl_io *io)
ctl_datamove_remote(io); ctl_datamove_remote(io);
break; break;
case CTL_MSG_DATAMOVE_DONE: /* Only used in XFER mode */ case CTL_MSG_DATAMOVE_DONE: /* Only used in XFER mode */
io->scsiio.be_move_done(io); ctl_datamove_done(io, false);
break; break;
case CTL_MSG_FAILOVER: case CTL_MSG_FAILOVER:
ctl_failover_lun(io); ctl_failover_lun(io);
@ -12462,6 +12450,45 @@ ctl_datamove_timer_wakeup(void *arg)
} }
#endif /* CTL_IO_DELAY */ #endif /* CTL_IO_DELAY */
static void
ctl_datamove_done_process(union ctl_io *io)
{
#ifdef CTL_TIME_IO
struct bintime cur_bt;
#endif
KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
#ifdef CTL_TIME_IO
getbinuptime(&cur_bt);
bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
bintime_add(&io->io_hdr.dma_bt, &cur_bt);
#endif
io->io_hdr.num_dmas++;
if ((io->io_hdr.port_status != 0) &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
/*retry_count*/ io->io_hdr.port_status);
} else if (io->scsiio.kern_data_resid != 0 &&
(io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_invalid_field_ciu(&io->scsiio);
} else if (ctl_debug & CTL_DEBUG_CDB_DATA)
ctl_data_print(io);
}
void
ctl_datamove_done(union ctl_io *io, bool samethr)
{
ctl_datamove_done_process(io);
io->scsiio.be_move_done(io, samethr);
}
void void
ctl_datamove(union ctl_io *io) ctl_datamove(union ctl_io *io)
{ {
@ -12475,39 +12502,7 @@ ctl_datamove(union ctl_io *io)
io->scsiio.kern_data_resid = io->scsiio.kern_data_len; io->scsiio.kern_data_resid = io->scsiio.kern_data_len;
#ifdef CTL_TIME_IO #ifdef CTL_TIME_IO
if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) { getbinuptime(&io->io_hdr.dma_start_bt);
char str[256];
char path_str[64];
struct sbuf sb;
ctl_scsi_path_string(io, path_str, sizeof(path_str));
sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
sbuf_cat(&sb, path_str);
switch (io->io_hdr.io_type) {
case CTL_IO_SCSI:
ctl_scsi_command_string(&io->scsiio, NULL, &sb);
sbuf_printf(&sb, "\n");
sbuf_cat(&sb, path_str);
sbuf_printf(&sb, "Tag: 0x%04x/%d, Prio: %d\n",
io->scsiio.tag_num, io->scsiio.tag_type,
io->scsiio.priority);
break;
case CTL_IO_TASK:
sbuf_printf(&sb, "Task Action: %d Tag: 0x%04x/%d\n",
io->taskio.task_action,
io->taskio.tag_num, io->taskio.tag_type);
break;
default:
panic("%s: Invalid CTL I/O type %d\n",
__func__, io->io_hdr.io_type);
}
sbuf_cat(&sb, path_str);
sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
(intmax_t)time_uptime - io->io_hdr.start_time);
sbuf_finish(&sb);
printf("%s", sbuf_data(&sb));
}
#endif /* CTL_TIME_IO */ #endif /* CTL_TIME_IO */
#ifdef CTL_IO_DELAY #ifdef CTL_IO_DELAY
@ -12542,18 +12537,15 @@ ctl_datamove(union ctl_io *io)
io->io_hdr.nexus.targ_port, io->io_hdr.nexus.targ_port,
io->io_hdr.nexus.targ_lun); io->io_hdr.nexus.targ_lun);
io->io_hdr.port_status = 31337; io->io_hdr.port_status = 31337;
/* ctl_datamove_done_process(io);
* Note that the backend, in this case, will get the io->scsiio.be_move_done(io, true);
* callback in its context. In other cases it may get
* called in the frontend's interrupt thread context.
*/
io->scsiio.be_move_done(io);
return; return;
} }
/* Don't confuse frontend with zero length data move. */ /* Don't confuse frontend with zero length data move. */
if (io->scsiio.kern_data_len == 0) { if (io->scsiio.kern_data_len == 0) {
io->scsiio.be_move_done(io); ctl_datamove_done_process(io);
io->scsiio.be_move_done(io, true);
return; return;
} }
@ -12640,7 +12632,7 @@ ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
* need to push it over to the remote controller's memory. * need to push it over to the remote controller's memory.
*/ */
static int static int
ctl_datamove_remote_dm_write_cb(union ctl_io *io) ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr)
{ {
int retval; int retval;
@ -12679,7 +12671,7 @@ ctl_datamove_remote_write(union ctl_io *io)
} }
static int static int
ctl_datamove_remote_dm_read_cb(union ctl_io *io) ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr)
{ {
uint32_t i; uint32_t i;

3
sys/cam/ctl/ctl.h
View File

@ -170,7 +170,8 @@ int ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
int ctl_ie_log_sense_handler(struct ctl_scsiio *ctsio, int ctl_ie_log_sense_handler(struct ctl_scsiio *ctsio,
struct ctl_page_index *page_index, struct ctl_page_index *page_index,
int pc); int pc);
int ctl_config_move_done(union ctl_io *io); int ctl_config_move_done(union ctl_io *io, bool samethr);
void ctl_datamove_done(union ctl_io *io, bool samethr);
void ctl_datamove(union ctl_io *io); void ctl_datamove(union ctl_io *io);
void ctl_serseq_done(union ctl_io *io); void ctl_serseq_done(union ctl_io *io);
void ctl_done(union ctl_io *io); void ctl_done(union ctl_io *io);

1
sys/cam/ctl/ctl_backend.h
View File

@ -186,7 +186,6 @@ struct ctl_backend_driver {
be_init_t init; /* passed to CTL */ be_init_t init; /* passed to CTL */
be_shutdown_t shutdown; /* passed to CTL */ be_shutdown_t shutdown; /* passed to CTL */
be_func_t data_submit; /* passed to CTL */ be_func_t data_submit; /* passed to CTL */
be_func_t data_move_done; /* passed to CTL */
be_func_t config_read; /* passed to CTL */ be_func_t config_read; /* passed to CTL */
be_func_t config_write; /* passed to CTL */ be_func_t config_write; /* passed to CTL */
be_ioctl_t ioctl; /* passed to CTL */ be_ioctl_t ioctl; /* passed to CTL */

56
sys/cam/ctl/ctl_backend_block.c
View File

@ -235,7 +235,7 @@ SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
static void ctl_free_beio(struct ctl_be_block_io *beio); static void ctl_free_beio(struct ctl_be_block_io *beio);
static void ctl_complete_beio(struct ctl_be_block_io *beio); static void ctl_complete_beio(struct ctl_be_block_io *beio);
static int ctl_be_block_move_done(union ctl_io *io); static int ctl_be_block_move_done(union ctl_io *io, bool samethr);
static void ctl_be_block_biodone(struct bio *bio); static void ctl_be_block_biodone(struct bio *bio);
static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
struct ctl_be_block_io *beio); struct ctl_be_block_io *beio);
@ -291,7 +291,6 @@ static struct ctl_backend_driver ctl_be_block_driver =
.init = ctl_be_block_init, .init = ctl_be_block_init,
.shutdown = ctl_be_block_shutdown, .shutdown = ctl_be_block_shutdown,
.data_submit = ctl_be_block_submit, .data_submit = ctl_be_block_submit,
.data_move_done = ctl_be_block_move_done,
.config_read = ctl_be_block_config_read, .config_read = ctl_be_block_config_read,
.config_write = ctl_be_block_config_write, .config_write = ctl_be_block_config_write,
.ioctl = ctl_be_block_ioctl, .ioctl = ctl_be_block_ioctl,
@ -432,46 +431,23 @@ ctl_be_block_compare(union ctl_io *io)
} }
static int static int
ctl_be_block_move_done(union ctl_io *io) ctl_be_block_move_done(union ctl_io *io, bool samethr)
{ {
struct ctl_be_block_io *beio; struct ctl_be_block_io *beio;
struct ctl_be_block_lun *be_lun; struct ctl_be_block_lun *be_lun;
struct ctl_lba_len_flags *lbalen; struct ctl_lba_len_flags *lbalen;
#ifdef CTL_TIME_IO
struct bintime cur_bt;
#endif
beio = (struct ctl_be_block_io *)PRIV(io)->ptr; beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
be_lun = beio->lun; be_lun = beio->lun;
DPRINTF("entered\n"); DPRINTF("entered\n");
#ifdef CTL_TIME_IO
getbinuptime(&cur_bt);
bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
bintime_add(&io->io_hdr.dma_bt, &cur_bt);
#endif
io->io_hdr.num_dmas++;
io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
/* /*
* We set status at this point for read commands, and write * We set status at this point for read and compare commands.
* commands with errors.
*/ */
if (io->io_hdr.flags & CTL_FLAG_ABORT) { if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
; (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
} else if ((io->io_hdr.port_status != 0) &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
/*retry_count*/ io->io_hdr.port_status);
} else if (io->scsiio.kern_data_resid != 0 &&
(io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_invalid_field_ciu(&io->scsiio);
} else if ((io->io_hdr.port_status == 0) &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
lbalen = ARGS(beio->io); lbalen = ARGS(beio->io);
if (lbalen->flags & CTL_LLF_READ) { if (lbalen->flags & CTL_LLF_READ) {
ctl_set_success(&io->scsiio); ctl_set_success(&io->scsiio);
@ -492,18 +468,22 @@ ctl_be_block_move_done(union ctl_io *io)
} }
/* /*
* At this point, we have a write and the DMA completed * At this point, we have a write and the DMA completed successfully.
* successfully. We now have to queue it to the task queue to * If we were called synchronously in the original thread then just
* dispatch, otherwise we now have to queue it to the task queue to
* execute the backend I/O. That is because we do blocking * execute the backend I/O. That is because we do blocking
* memory allocations, and in the file backing case, blocking I/O. * memory allocations, and in the file backing case, blocking I/O.
* This move done routine is generally called in the SIM's * This move done routine is generally called in the SIM's
* interrupt context, and therefore we cannot block. * interrupt context, and therefore we cannot block.
*/ */
if (samethr) {
be_lun->dispatch(be_lun, beio);
} else {
mtx_lock(&be_lun->queue_lock); mtx_lock(&be_lun->queue_lock);
STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
mtx_unlock(&be_lun->queue_lock); mtx_unlock(&be_lun->queue_lock);
taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
}
return (0); return (0);
} }
@ -598,9 +578,6 @@ ctl_be_block_biodone(struct bio *bio)
ctl_set_success(&io->scsiio); ctl_set_success(&io->scsiio);
ctl_serseq_done(io); ctl_serseq_done(io);
} }
#ifdef CTL_TIME_IO
getbinuptime(&io->io_hdr.dma_start_bt);
#endif
ctl_datamove(io); ctl_datamove(io);
} }
} }
@ -811,9 +788,6 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
ctl_set_success(&io->scsiio); ctl_set_success(&io->scsiio);
ctl_serseq_done(io); ctl_serseq_done(io);
} }
#ifdef CTL_TIME_IO
getbinuptime(&io->io_hdr.dma_start_bt);
#endif
ctl_datamove(io); ctl_datamove(io);
} }
} }
@ -980,9 +954,6 @@ ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
ctl_set_success(&io->scsiio); ctl_set_success(&io->scsiio);
ctl_serseq_done(io); ctl_serseq_done(io);
} }
#ifdef CTL_TIME_IO
getbinuptime(&io->io_hdr.dma_start_bt);
#endif
ctl_datamove(io); ctl_datamove(io);
} }
} }
@ -1672,9 +1643,6 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
be_lun->dispatch(be_lun, beio); be_lun->dispatch(be_lun, beio);
} else { } else {
SDT_PROBE0(cbb, , write, alloc_done); SDT_PROBE0(cbb, , write, alloc_done);
#ifdef CTL_TIME_IO
getbinuptime(&io->io_hdr.dma_start_bt);
#endif
ctl_datamove(io); ctl_datamove(io);
} }
} }

36
sys/cam/ctl/ctl_backend_ramdisk.c
View File

@ -139,7 +139,7 @@ extern struct ctl_softc *control_softc;
static int ctl_backend_ramdisk_init(void); static int ctl_backend_ramdisk_init(void);
static int ctl_backend_ramdisk_shutdown(void); static int ctl_backend_ramdisk_shutdown(void);
static int ctl_backend_ramdisk_move_done(union ctl_io *io); static int ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr);
static void ctl_backend_ramdisk_compare(union ctl_io *io); static void ctl_backend_ramdisk_compare(union ctl_io *io);
static void ctl_backend_ramdisk_rw(union ctl_io *io); static void ctl_backend_ramdisk_rw(union ctl_io *io);
static int ctl_backend_ramdisk_submit(union ctl_io *io); static int ctl_backend_ramdisk_submit(union ctl_io *io);
@ -164,7 +164,6 @@ static struct ctl_backend_driver ctl_be_ramdisk_driver =
.init = ctl_backend_ramdisk_init, .init = ctl_backend_ramdisk_init,
.shutdown = ctl_backend_ramdisk_shutdown, .shutdown = ctl_backend_ramdisk_shutdown,
.data_submit = ctl_backend_ramdisk_submit, .data_submit = ctl_backend_ramdisk_submit,
.data_move_done = ctl_backend_ramdisk_move_done,
.config_read = ctl_backend_ramdisk_config_read, .config_read = ctl_backend_ramdisk_config_read,
.config_write = ctl_backend_ramdisk_config_write, .config_write = ctl_backend_ramdisk_config_write,
.ioctl = ctl_backend_ramdisk_ioctl, .ioctl = ctl_backend_ramdisk_ioctl,
@ -402,38 +401,17 @@ ctl_backend_ramdisk_cmp(union ctl_io *io)
} }
static int static int
ctl_backend_ramdisk_move_done(union ctl_io *io) ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr)
{ {
struct ctl_be_ramdisk_lun *be_lun = struct ctl_be_ramdisk_lun *be_lun =
(struct ctl_be_ramdisk_lun *)CTL_BACKEND_LUN(io); (struct ctl_be_ramdisk_lun *)CTL_BACKEND_LUN(io);
#ifdef CTL_TIME_IO
struct bintime cur_bt;
#endif
CTL_DEBUG_PRINT(("ctl_backend_ramdisk_move_done\n")); CTL_DEBUG_PRINT(("ctl_backend_ramdisk_move_done\n"));
#ifdef CTL_TIME_IO
getbinuptime(&cur_bt);
bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
bintime_add(&io->io_hdr.dma_bt, &cur_bt);
#endif
io->io_hdr.num_dmas++;
if (io->scsiio.kern_sg_entries > 0) if (io->scsiio.kern_sg_entries > 0)
free(io->scsiio.kern_data_ptr, M_RAMDISK); free(io->scsiio.kern_data_ptr, M_RAMDISK);
io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
if (io->io_hdr.flags & CTL_FLAG_ABORT) { if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
; (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
} else if (io->io_hdr.port_status != 0 &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
/*retry_count*/ io->io_hdr.port_status);
} else if (io->scsiio.kern_data_resid != 0 &&
(io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
(io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
ctl_set_invalid_field_ciu(&io->scsiio);
} else if ((io->io_hdr.port_status == 0) &&
((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
if (ARGS(io)->flags & CTL_LLF_COMPARE) { if (ARGS(io)->flags & CTL_LLF_COMPARE) {
/* We have data block ready for comparison. */ /* We have data block ready for comparison. */
if (ctl_backend_ramdisk_cmp(io)) if (ctl_backend_ramdisk_cmp(io))
@ -471,9 +449,6 @@ ctl_backend_ramdisk_compare(union ctl_io *io)
io->scsiio.kern_sg_entries = 0; io->scsiio.kern_sg_entries = 0;
io->io_hdr.flags |= CTL_FLAG_ALLOCATED; io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
PRIV(io)->len += lbas; PRIV(io)->len += lbas;
#ifdef CTL_TIME_IO
getbinuptime(&io->io_hdr.dma_start_bt);
#endif
ctl_datamove(io); ctl_datamove(io);
} }
@ -534,9 +509,6 @@ nospc:
ctl_set_success(&io->scsiio); ctl_set_success(&io->scsiio);
ctl_serseq_done(io); ctl_serseq_done(io);
} }
#ifdef CTL_TIME_IO
getbinuptime(&io->io_hdr.dma_start_bt);
#endif
ctl_datamove(io); ctl_datamove(io);
} }

2
sys/cam/ctl/ctl_frontend_cam_sim.c
View File

@ -415,7 +415,7 @@ cfcs_datamove(union ctl_io *io)
xpt_done(ccb); xpt_done(ccb);
} }
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
} }
static void static void

2
sys/cam/ctl/ctl_frontend_ioctl.c
View File

@ -566,7 +566,7 @@ cfi_submit_wait(union ctl_io *io)
* will immediately call back and wake us up, * will immediately call back and wake us up,
* probably using our own context. * probably using our own context.
*/ */
io->scsiio.be_move_done(io); ctl_datamove_done(io, false);
break; break;
case CTL_IOCTL_DONE: case CTL_IOCTL_DONE:
mtx_unlock(&params.ioctl_mtx); mtx_unlock(&params.ioctl_mtx);

23
sys/cam/ctl/ctl_frontend_iscsi.c
View File

@ -933,7 +933,7 @@ cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
cfiscsi_data_wait_free(cs, cdw); cfiscsi_data_wait_free(cs, cdw);
io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG; io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
if (done) if (done)
io->scsiio.be_move_done(io); ctl_datamove_done(io, false);
else else
cfiscsi_datamove_out(io); cfiscsi_datamove_out(io);
} }
@ -1146,7 +1146,7 @@ cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
*/ */
cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG; cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42; cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io); ctl_datamove_done(cdw->cdw_ctl_io, false);
cfiscsi_data_wait_free(cs, cdw); cfiscsi_data_wait_free(cs, cdw);
CFISCSI_SESSION_LOCK(cs); CFISCSI_SESSION_LOCK(cs);
} }
@ -2487,7 +2487,7 @@ cfiscsi_datamove_in(union ctl_io *io)
CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, " CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
"already sent the expected len", buffer_offset); "already sent the expected len", buffer_offset);
#endif #endif
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
return; return;
} }
@ -2508,7 +2508,7 @@ cfiscsi_datamove_in(union ctl_io *io)
CFISCSI_SESSION_WARN(cs, "failed to " CFISCSI_SESSION_WARN(cs, "failed to "
"allocate memory; dropping connection"); "allocate memory; dropping connection");
ctl_set_busy(&io->scsiio); ctl_set_busy(&io->scsiio);
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs); cfiscsi_session_terminate(cs);
return; return;
} }
@ -2566,7 +2566,7 @@ cfiscsi_datamove_in(union ctl_io *io)
"allocate memory; dropping connection"); "allocate memory; dropping connection");
icl_pdu_free(response); icl_pdu_free(response);
ctl_set_busy(&io->scsiio); ctl_set_busy(&io->scsiio);
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs); cfiscsi_session_terminate(cs);
return; return;
} }
@ -2656,7 +2656,7 @@ cfiscsi_datamove_in(union ctl_io *io)
cfiscsi_pdu_queue_cb(response, cb); cfiscsi_pdu_queue_cb(response, cb);
} }
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
} }
static void static void
@ -2685,9 +2685,10 @@ cfiscsi_datamove_out(union ctl_io *io)
*/ */
expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length); expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
if (io->scsiio.kern_rel_offset >= expected_len) { if (io->scsiio.kern_rel_offset >= expected_len) {
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
return; return;
} }
datamove_len = MIN(io->scsiio.kern_data_len, datamove_len = MIN(io->scsiio.kern_data_len,
expected_len - io->scsiio.kern_rel_offset); expected_len - io->scsiio.kern_rel_offset);
@ -2703,7 +2704,7 @@ cfiscsi_datamove_out(union ctl_io *io)
CFISCSI_SESSION_WARN(cs, "failed to " CFISCSI_SESSION_WARN(cs, "failed to "
"allocate memory; dropping connection"); "allocate memory; dropping connection");
ctl_set_busy(&io->scsiio); ctl_set_busy(&io->scsiio);
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs); cfiscsi_session_terminate(cs);
return; return;
} }
@ -2750,7 +2751,7 @@ cfiscsi_datamove_out(union ctl_io *io)
done = cfiscsi_handle_data_segment(request, cdw); done = cfiscsi_handle_data_segment(request, cdw);
if (done) { if (done) {
cfiscsi_data_wait_free(cs, cdw); cfiscsi_data_wait_free(cs, cdw);
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
return; return;
} }
} }
@ -2773,7 +2774,7 @@ cfiscsi_datamove_out(union ctl_io *io)
CFISCSI_SESSION_WARN(cs, "failed to " CFISCSI_SESSION_WARN(cs, "failed to "
"allocate memory; dropping connection"); "allocate memory; dropping connection");
ctl_set_busy(&io->scsiio); ctl_set_busy(&io->scsiio);
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
cfiscsi_session_terminate(cs); cfiscsi_session_terminate(cs);
return; return;
} }
@ -2954,7 +2955,7 @@ cfiscsi_task_management_done(union ctl_io *io)
cdw, cdw_next); cdw, cdw_next);
io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG; io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43; cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io); ctl_datamove_done(cdw->cdw_ctl_io, false);
cfiscsi_data_wait_free(cs, cdw); cfiscsi_data_wait_free(cs, cdw);
} }
CFISCSI_SESSION_UNLOCK(cs); CFISCSI_SESSION_UNLOCK(cs);

2
sys/cam/ctl/ctl_io.h
View File

@ -329,7 +329,7 @@ struct ctl_scsiio {
ctl_tag_type tag_type; /* simple, ordered, head of queue,etc.*/ ctl_tag_type tag_type; /* simple, ordered, head of queue,etc.*/
uint8_t cdb_len; /* CDB length */ uint8_t cdb_len; /* CDB length */
uint8_t cdb[CTL_MAX_CDBLEN]; /* CDB */ uint8_t cdb[CTL_MAX_CDBLEN]; /* CDB */
int (*be_move_done)(union ctl_io *io); /* called by fe */ int (*be_move_done)(union ctl_io *io, bool samethr); /* called by fe */
int (*io_cont)(union ctl_io *io); /* to continue processing */ int (*io_cont)(union ctl_io *io); /* to continue processing */
ctl_ref kern_data_ref; /* Method to reference/release data */ ctl_ref kern_data_ref; /* Method to reference/release data */
void *kern_data_arg; /* Opaque argument for kern_data_ref() */ void *kern_data_arg; /* Opaque argument for kern_data_ref() */

2
sys/cam/ctl/ctl_tpc_local.c
View File

@ -254,7 +254,7 @@ tpcl_datamove(union ctl_io *io)
__func__, ctsio->ext_data_len, ctsio->kern_data_len)); __func__, ctsio->ext_data_len, ctsio->kern_data_len));
bailout: bailout:
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
} }
static void static void

3
sys/cam/ctl/scsi_ctl.c
View File

@ -1394,8 +1394,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb)
xpt_release_ccb(done_ccb); xpt_release_ccb(done_ccb);
mtx_unlock(mtx); mtx_unlock(mtx);
/* Call the backend move done callback */ ctl_datamove_done(io, false);
io->scsiio.be_move_done(io);
} }
return; return;
} }

6
sys/dev/usb/storage/cfumass.c
View File

@ -475,7 +475,7 @@ cfumass_terminate(struct cfumass_softc *sc)
if (sc->sc_ctl_io != NULL) { if (sc->sc_ctl_io != NULL) {
CFUMASS_DEBUG(sc, "terminating CTL transfer"); CFUMASS_DEBUG(sc, "terminating CTL transfer");
ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio); ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio);
sc->sc_ctl_io->scsiio.be_move_done(sc->sc_ctl_io); ctl_datamove_done(sc->sc_ctl_io, false);
sc->sc_ctl_io = NULL; sc->sc_ctl_io = NULL;
} }
@ -730,7 +730,7 @@ cfumass_t_data_callback(struct usb_xfer *xfer, usb_error_t usb_error)
sc->sc_current_residue == 0 || sc->sc_current_residue == 0 ||
io->scsiio.kern_data_resid == 0) { io->scsiio.kern_data_resid == 0) {
sc->sc_ctl_io = NULL; sc->sc_ctl_io = NULL;
io->scsiio.be_move_done(io); ctl_datamove_done(io, false);
break; break;
} }
/* FALLTHROUGH */ /* FALLTHROUGH */
@ -887,7 +887,7 @@ cfumass_datamove(union ctl_io *io)
fail: fail:
ctl_set_data_phase_error(&io->scsiio); ctl_set_data_phase_error(&io->scsiio);
io->scsiio.be_move_done(io); ctl_datamove_done(io, true);
sc->sc_ctl_io = NULL; sc->sc_ctl_io = NULL;
} }