d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rewrite CTL statistics in more simple and scalable way.

Browse Source 
Instead of collecting statistics for each combination of ports and logical
units, that consumed ~45KB per LU with present number of ports, collect
separate statistics for every port and every logical unit separately, that
consume only 176 bytes per each single LU/port.  This reduces struct
ctl_lun size down to just 6KB.

Also new IOCTL API/ABI does not hardcode number of LUs/ports, and should
allow handling of very large quantities.

MFC after:	2 weeks (probably keeping old API enabled for some time)
This commit is contained in:
Alexander Motin 2017-01-09 18:18:15 +00:00
parent b8bf08b1fa
commit bb8f9017b3
8 changed files with 454 additions and 365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
sys/cam/ctl/ctl.c
View File

@ -1,7 +1,7 @@
/*- /*-
* Copyright (c) 2003-2009 Silicon Graphics International Corp. * Copyright (c) 2003-2009 Silicon Graphics International Corp.
* Copyright (c) 2012 The FreeBSD Foundation * Copyright (c) 2012 The FreeBSD Foundation
* Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org> * Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Portions of this software were developed by Edward Tomasz Napierala * Portions of this software were developed by Edward Tomasz Napierala
@ -2567,6 +2567,7 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
struct thread *td) struct thread *td)
{ {
struct ctl_softc *softc = dev->si_drv1; struct ctl_softc *softc = dev->si_drv1;
struct ctl_port *port;
struct ctl_lun *lun; struct ctl_lun *lun;
int retval; int retval;
@ -2778,6 +2779,7 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
#endif /* CTL_IO_DELAY */ #endif /* CTL_IO_DELAY */
break; break;
} }
#ifdef CTL_LEGACY_STATS
case CTL_GETSTATS: { case CTL_GETSTATS: {
struct ctl_stats *stats = (struct ctl_stats *)addr; struct ctl_stats *stats = (struct ctl_stats *)addr;
int i; int i;
@ -2790,26 +2792,26 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
stats->status = CTL_SS_OK; stats->status = CTL_SS_OK;
stats->fill_len = 0; stats->fill_len = 0;
STAILQ_FOREACH(lun, &softc->lun_list, links) { STAILQ_FOREACH(lun, &softc->lun_list, links) {
if (stats->fill_len + sizeof(lun->stats) > if (stats->fill_len + sizeof(lun->legacy_stats) >
stats->alloc_len) { stats->alloc_len) {
stats->status = CTL_SS_NEED_MORE_SPACE; stats->status = CTL_SS_NEED_MORE_SPACE;
break; break;
} }
retval = copyout(&lun->stats, &stats->lun_stats[i++], retval = copyout(&lun->legacy_stats, &stats->lun_stats[i++],
sizeof(lun->stats)); sizeof(lun->legacy_stats));
if (retval != 0) if (retval != 0)
break; break;
stats->fill_len += sizeof(lun->stats); stats->fill_len += sizeof(lun->legacy_stats);
} }
stats->num_luns = softc->num_luns; stats->num_luns = softc->num_luns;
#ifdef CTL_TIME_IO
stats->flags = CTL_STATS_FLAG_TIME_VALID;
#else
stats->flags = CTL_STATS_FLAG_NONE; stats->flags = CTL_STATS_FLAG_NONE;
#ifdef CTL_TIME_IO
stats->flags |= CTL_STATS_FLAG_TIME_VALID;
#endif #endif
getnanouptime(&stats->timestamp); getnanouptime(&stats->timestamp);
break; break;
} }
#endif /* CTL_LEGACY_STATS */
case CTL_ERROR_INJECT: { case CTL_ERROR_INJECT: {
struct ctl_error_desc *err_desc, *new_err_desc; struct ctl_error_desc *err_desc, *new_err_desc;
@ -3397,6 +3399,72 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
ctl_isc_announce_port(port); ctl_isc_announce_port(port);
break; break;
} }
case CTL_GET_LUN_STATS: {
struct ctl_get_io_stats *stats = (struct ctl_get_io_stats *)addr;
int i;
/*
* XXX KDM no locking here. If the LUN list changes,
* things can blow up.
*/
i = 0;
stats->status = CTL_SS_OK;
stats->fill_len = 0;
STAILQ_FOREACH(lun, &softc->lun_list, links) {
if (lun->lun < stats->first_item)
continue;
if (stats->fill_len + sizeof(lun->stats) >
stats->alloc_len) {
stats->status = CTL_SS_NEED_MORE_SPACE;
break;
}
retval = copyout(&lun->stats, &stats->stats[i++],
sizeof(lun->stats));
if (retval != 0)
break;
stats->fill_len += sizeof(lun->stats);
}
stats->num_items = softc->num_luns;
stats->flags = CTL_STATS_FLAG_NONE;
#ifdef CTL_TIME_IO
stats->flags |= CTL_STATS_FLAG_TIME_VALID;
#endif
getnanouptime(&stats->timestamp);
break;
}
case CTL_GET_PORT_STATS: {
struct ctl_get_io_stats *stats = (struct ctl_get_io_stats *)addr;
int i;
/*
* XXX KDM no locking here. If the LUN list changes,
* things can blow up.
*/
i = 0;
stats->status = CTL_SS_OK;
stats->fill_len = 0;
STAILQ_FOREACH(port, &softc->port_list, links) {
if (port->targ_port < stats->first_item)
continue;
if (stats->fill_len + sizeof(port->stats) >
stats->alloc_len) {
stats->status = CTL_SS_NEED_MORE_SPACE;
break;
}
retval = copyout(&port->stats, &stats->stats[i++],
sizeof(port->stats));
if (retval != 0)
break;
stats->fill_len += sizeof(port->stats);
}
stats->num_items = softc->num_ports;
stats->flags = CTL_STATS_FLAG_NONE;
#ifdef CTL_TIME_IO
stats->flags |= CTL_STATS_FLAG_TIME_VALID;
#endif
getnanouptime(&stats->timestamp);
break;
}
default: { default: {
/* XXX KDM should we fix this? */ /* XXX KDM should we fix this? */
#if 0 #if 0
@ -4391,7 +4459,7 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
struct scsi_vpd_id_descriptor *desc; struct scsi_vpd_id_descriptor *desc;
struct scsi_vpd_id_t10 *t10id; struct scsi_vpd_id_t10 *t10id;
const char *eui, *naa, *scsiname, *uuid, *vendor, *value; const char *eui, *naa, *scsiname, *uuid, *vendor, *value;
int lun_number, i, lun_malloced; int lun_number, lun_malloced;
int devidlen, idlen1, idlen2 = 0, len; int devidlen, idlen1, idlen2 = 0, len;
if (be_lun == NULL) if (be_lun == NULL)
@ -4613,13 +4681,16 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
ctl_softc->num_luns++; ctl_softc->num_luns++;
/* Setup statistics gathering */ /* Setup statistics gathering */
lun->stats.device_type = be_lun->lun_type; #ifdef CTL_LEGACY_STATS
lun->stats.lun_number = lun_number; lun->legacy_stats.device_type = be_lun->lun_type;
lun->stats.blocksize = be_lun->blocksize; lun->legacy_stats.lun_number = lun_number;
lun->legacy_stats.blocksize = be_lun->blocksize;
if (be_lun->blocksize == 0) if (be_lun->blocksize == 0)
lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE; lun->legacy_stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
for (i = 0;i < CTL_MAX_PORTS;i++) for (len = 0; len < CTL_MAX_PORTS; len++)
lun->stats.ports[i].targ_port = i; lun->legacy_stats.ports[len].targ_port = len;
#endif /* CTL_LEGACY_STATS */
lun->stats.item = lun_number;
mtx_unlock(&ctl_softc->ctl_lock); mtx_unlock(&ctl_softc->ctl_lock);
@ -6685,9 +6756,7 @@ ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
{ {
struct ctl_lun *lun = CTL_LUN(ctsio); struct ctl_lun *lun = CTL_LUN(ctsio);
struct stat_page *data; struct stat_page *data;
uint64_t rn, wn, rb, wb; struct bintime *t;
struct bintime rt, wt;
int i;
data = (struct stat_page *)page_index->page_data; data = (struct stat_page *)page_index->page_data;
@ -6695,28 +6764,21 @@ ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
data->sap.hdr.param_control = SLP_LBIN; data->sap.hdr.param_control = SLP_LBIN;
data->sap.hdr.param_len = sizeof(struct scsi_log_stat_and_perf) - data->sap.hdr.param_len = sizeof(struct scsi_log_stat_and_perf) -
sizeof(struct scsi_log_param_header); sizeof(struct scsi_log_param_header);
rn = wn = rb = wb = 0; scsi_u64to8b(lun->stats.operations[CTL_STATS_READ],
bintime_clear(&rt); data->sap.read_num);
bintime_clear(&wt); scsi_u64to8b(lun->stats.operations[CTL_STATS_WRITE],
for (i = 0; i < CTL_MAX_PORTS; i++) { data->sap.write_num);
rn += lun->stats.ports[i].operations[CTL_STATS_READ]; if (lun->be_lun->blocksize > 0) {
wn += lun->stats.ports[i].operations[CTL_STATS_WRITE]; scsi_u64to8b(lun->stats.bytes[CTL_STATS_WRITE] /
rb += lun->stats.ports[i].bytes[CTL_STATS_READ]; lun->be_lun->blocksize, data->sap.recvieved_lba);
wb += lun->stats.ports[i].bytes[CTL_STATS_WRITE]; scsi_u64to8b(lun->stats.bytes[CTL_STATS_READ] /
bintime_add(&rt, &lun->stats.ports[i].time[CTL_STATS_READ]); lun->be_lun->blocksize, data->sap.transmitted_lba);
bintime_add(&wt, &lun->stats.ports[i].time[CTL_STATS_WRITE]);
} }
scsi_u64to8b(rn, data->sap.read_num); t = &lun->stats.time[CTL_STATS_READ];
scsi_u64to8b(wn, data->sap.write_num); scsi_u64to8b((uint64_t)t->sec * 1000 + t->frac / (UINT64_MAX / 1000),
if (lun->stats.blocksize > 0) {
scsi_u64to8b(wb / lun->stats.blocksize,
data->sap.recvieved_lba);
scsi_u64to8b(rb / lun->stats.blocksize,
data->sap.transmitted_lba);
}
scsi_u64to8b((uint64_t)rt.sec * 1000 + rt.frac / (UINT64_MAX / 1000),
data->sap.read_int); data->sap.read_int);
scsi_u64to8b((uint64_t)wt.sec * 1000 + wt.frac / (UINT64_MAX / 1000), t = &lun->stats.time[CTL_STATS_WRITE];
scsi_u64to8b((uint64_t)t->sec * 1000 + t->frac / (UINT64_MAX / 1000),
data->sap.write_int); data->sap.write_int);
scsi_u64to8b(0, data->sap.weighted_num); scsi_u64to8b(0, data->sap.weighted_num);
scsi_u64to8b(0, data->sap.weighted_int); scsi_u64to8b(0, data->sap.weighted_int);
@ -13053,13 +13115,13 @@ static void
ctl_process_done(union ctl_io *io) ctl_process_done(union ctl_io *io)
{ {
struct ctl_softc *softc = CTL_SOFTC(io); struct ctl_softc *softc = CTL_SOFTC(io);
struct ctl_port *port = CTL_PORT(io);
struct ctl_lun *lun = CTL_LUN(io); struct ctl_lun *lun = CTL_LUN(io);
void (*fe_done)(union ctl_io *io); void (*fe_done)(union ctl_io *io);
union ctl_ha_msg msg; union ctl_ha_msg msg;
uint32_t targ_port = io->io_hdr.nexus.targ_port;
CTL_DEBUG_PRINT(("ctl_process_done\n")); CTL_DEBUG_PRINT(("ctl_process_done\n"));
fe_done = softc->ctl_ports[targ_port]->fe_done; fe_done = port->fe_done;
#ifdef CTL_TIME_IO #ifdef CTL_TIME_IO
if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) { if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
@ -13162,11 +13224,13 @@ ctl_process_done(union ctl_io *io)
*/ */
if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS && if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
io->io_hdr.io_type == CTL_IO_SCSI) { io->io_hdr.io_type == CTL_IO_SCSI) {
#ifdef CTL_TIME_IO
struct bintime cur_bt;
#endif
int type; int type;
#ifdef CTL_TIME_IO
struct bintime bt;
getbinuptime(&bt);
bintime_sub(&bt, &io->io_hdr.start_bt);
#endif
if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
CTL_FLAG_DATA_IN) CTL_FLAG_DATA_IN)
type = CTL_STATS_READ; type = CTL_STATS_READ;
@ -13176,18 +13240,38 @@ ctl_process_done(union ctl_io *io)
else else
type = CTL_STATS_NO_IO; type = CTL_STATS_NO_IO;
lun->stats.ports[targ_port].bytes[type] += #ifdef CTL_LEGACY_STATS
uint32_t targ_port = port->targ_port;
lun->legacy_stats.ports[targ_port].bytes[type] +=
io->scsiio.kern_total_len; io->scsiio.kern_total_len;
lun->stats.ports[targ_port].operations[type]++; lun->legacy_stats.ports[targ_port].operations[type] ++;
#ifdef CTL_TIME_IO lun->legacy_stats.ports[targ_port].num_dmas[type] +=
bintime_add(&lun->stats.ports[targ_port].dma_time[type],
&io->io_hdr.dma_bt);
getbinuptime(&cur_bt);
bintime_sub(&cur_bt, &io->io_hdr.start_bt);
bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
#endif
lun->stats.ports[targ_port].num_dmas[type] +=
io->io_hdr.num_dmas; io->io_hdr.num_dmas;
#ifdef CTL_TIME_IO
bintime_add(&lun->legacy_stats.ports[targ_port].dma_time[type],
&io->io_hdr.dma_bt);
bintime_add(&lun->legacy_stats.ports[targ_port].time[type],
&bt);
#endif
#endif /* CTL_LEGACY_STATS */
lun->stats.bytes[type] += io->scsiio.kern_total_len;
lun->stats.operations[type] ++;
lun->stats.dmas[type] += io->io_hdr.num_dmas;
#ifdef CTL_TIME_IO
bintime_add(&lun->stats.dma_time[type], &io->io_hdr.dma_bt);
bintime_add(&lun->stats.time[type], &bt);
#endif
mtx_lock(&port->port_lock);
port->stats.bytes[type] += io->scsiio.kern_total_len;
port->stats.operations[type] ++;
port->stats.dmas[type] += io->io_hdr.num_dmas;
#ifdef CTL_TIME_IO
bintime_add(&port->stats.dma_time[type], &io->io_hdr.dma_bt);
bintime_add(&port->stats.time[type], &bt);
#endif
mtx_unlock(&port->port_lock);
} }
/* /*

51
sys/cam/ctl/ctl_backend.h
View File

@ -1,6 +1,6 @@
/*- /*-
* Copyright (c) 2003 Silicon Graphics International Corp. * Copyright (c) 2003 Silicon Graphics International Corp.
* Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org> * Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -40,54 +40,7 @@
#ifndef _CTL_BACKEND_H_ #ifndef _CTL_BACKEND_H_
#define _CTL_BACKEND_H_ #define _CTL_BACKEND_H_
/* #include <cam/ctl/ctl_ioctl.h>
* XXX KDM move this to another header file?
*/
#define CTL_BE_NAME_LEN 32
/*
* The ID_REQ flag is used to say that the caller has requested a
* particular LUN ID in the req_lun_id field. If we cannot allocate that
* LUN ID, the ctl_add_lun() call will fail.
*
* The STOPPED flag tells us that the LUN should default to the powered
* off state. It will return 0x04,0x02 until it is powered up. ("Logical
* unit not ready, initializing command required.")
*
* The NO_MEDIA flag tells us that the LUN has no media inserted.
*
* The PRIMARY flag tells us that this LUN is registered as a Primary LUN
* which is accessible via the Master shelf controller in an HA. This flag
* being set indicates a Primary LUN. This flag being reset represents a
* Secondary LUN controlled by the Secondary controller in an HA
* configuration. Flag is applicable at this time to T_DIRECT types.
*
* The SERIAL_NUM flag tells us that the serial_num field is filled in and
* valid for use in SCSI INQUIRY VPD page 0x80.
*
* The DEVID flag tells us that the device_id field is filled in and
* valid for use in SCSI INQUIRY VPD page 0x83.
*
* The DEV_TYPE flag tells us that the device_type field is filled in.
*
* The EJECTED flag tells us that the removable LUN has tray open.
*
* The UNMAP flag tells us that this LUN supports UNMAP.
*
* The OFFLINE flag tells us that this LUN can not access backing store.
*/
typedef enum {
CTL_LUN_FLAG_ID_REQ = 0x01,
CTL_LUN_FLAG_STOPPED = 0x02,
CTL_LUN_FLAG_NO_MEDIA = 0x04,
CTL_LUN_FLAG_PRIMARY = 0x08,
CTL_LUN_FLAG_SERIAL_NUM = 0x10,
CTL_LUN_FLAG_DEVID = 0x20,
CTL_LUN_FLAG_DEV_TYPE = 0x40,
CTL_LUN_FLAG_UNMAP = 0x80,
CTL_LUN_FLAG_EJECTED = 0x100,
CTL_LUN_FLAG_READONLY = 0x200
} ctl_backend_lun_flags;
typedef enum { typedef enum {
CTL_LUN_SERSEQ_OFF, CTL_LUN_SERSEQ_OFF,

24
sys/cam/ctl/ctl_frontend.c
View File

@ -1,5 +1,6 @@
/*- /*-
* Copyright (c) 2003 Silicon Graphics International Corp. * Copyright (c) 2003 Silicon Graphics International Corp.
* Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -192,13 +193,14 @@ ctl_port_register(struct ctl_port *port)
mtx_unlock(&softc->ctl_lock); mtx_unlock(&softc->ctl_lock);
return (retval); return (retval);
} }
port->targ_port = port_num;
port->ctl_pool_ref = pool; port->ctl_pool_ref = pool;
if (port->options.stqh_first == NULL) if (port->options.stqh_first == NULL)
STAILQ_INIT(&port->options); STAILQ_INIT(&port->options);
port->stats.item = port_num;
mtx_init(&port->port_lock, "CTL port", NULL, MTX_DEF);
mtx_lock(&softc->ctl_lock); mtx_lock(&softc->ctl_lock);
port->targ_port = port_num;
STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links); STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links);
for (tport = NULL, nport = STAILQ_FIRST(&softc->port_list); for (tport = NULL, nport = STAILQ_FIRST(&softc->port_list);
nport != NULL && nport->targ_port < port_num; nport != NULL && nport->targ_port < port_num;
@ -218,17 +220,11 @@ int
ctl_port_deregister(struct ctl_port *port) ctl_port_deregister(struct ctl_port *port)
{ {
struct ctl_softc *softc = port->ctl_softc; struct ctl_softc *softc = port->ctl_softc;
struct ctl_io_pool *pool; struct ctl_io_pool *pool = (struct ctl_io_pool *)port->ctl_pool_ref;
int retval, i; int i;
retval = 0; if (port->targ_port == -1)
return (1);
pool = (struct ctl_io_pool *)port->ctl_pool_ref;
if (port->targ_port == -1) {
retval = 1;
goto bailout;
}
mtx_lock(&softc->ctl_lock); mtx_lock(&softc->ctl_lock);
STAILQ_REMOVE(&softc->port_list, port, ctl_port, links); STAILQ_REMOVE(&softc->port_list, port, ctl_port, links);
@ -251,9 +247,9 @@ ctl_port_deregister(struct ctl_port *port)
for (i = 0; i < port->max_initiators; i++) for (i = 0; i < port->max_initiators; i++)
free(port->wwpn_iid[i].name, M_CTL); free(port->wwpn_iid[i].name, M_CTL);
free(port->wwpn_iid, M_CTL); free(port->wwpn_iid, M_CTL);
mtx_destroy(&port->port_lock);
bailout: return (0);
return (retval);
} }
void void

5
sys/cam/ctl/ctl_frontend.h
View File

@ -1,5 +1,6 @@
/*- /*-
* Copyright (c) 2003 Silicon Graphics International Corp. * Copyright (c) 2003 Silicon Graphics International Corp.
* Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -39,6 +40,8 @@
#ifndef _CTL_FRONTEND_H_ #ifndef _CTL_FRONTEND_H_
#define _CTL_FRONTEND_H_ #define _CTL_FRONTEND_H_
#include <cam/ctl/ctl_ioctl.h>
typedef enum { typedef enum {
CTL_PORT_STATUS_NONE = 0x00, CTL_PORT_STATUS_NONE = 0x00,
CTL_PORT_STATUS_ONLINE = 0x01, CTL_PORT_STATUS_ONLINE = 0x01,
@ -243,6 +246,8 @@ struct ctl_port {
struct ctl_devid *port_devid; /* passed to CTL */ struct ctl_devid *port_devid; /* passed to CTL */
struct ctl_devid *target_devid; /* passed to CTL */ struct ctl_devid *target_devid; /* passed to CTL */
struct ctl_devid *init_devid; /* passed to CTL */ struct ctl_devid *init_devid; /* passed to CTL */
struct ctl_io_stats stats; /* used by CTL */
struct mtx port_lock; /* used by CTL */
STAILQ_ENTRY(ctl_port) fe_links; /* used by CTL */ STAILQ_ENTRY(ctl_port) fe_links; /* used by CTL */
STAILQ_ENTRY(ctl_port) links; /* used by CTL */ STAILQ_ENTRY(ctl_port) links; /* used by CTL */
}; };

97
sys/cam/ctl/ctl_ioctl.h
View File

@ -1,6 +1,7 @@
/*- /*-
* Copyright (c) 2003 Silicon Graphics International Corp. * Copyright (c) 2003 Silicon Graphics International Corp.
* Copyright (c) 2011 Spectra Logic Corporation * Copyright (c) 2011 Spectra Logic Corporation
* Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -80,6 +81,9 @@
/* Hopefully this won't conflict with new misc devices that pop up */ /* Hopefully this won't conflict with new misc devices that pop up */
#define CTL_MINOR 225 #define CTL_MINOR 225
/* Legacy statistics accumulated for every port for every LU. */
//#define CTL_LEGACY_STATS 1
typedef enum { typedef enum {
CTL_DELAY_TYPE_NONE, CTL_DELAY_TYPE_NONE,
CTL_DELAY_TYPE_CONT, CTL_DELAY_TYPE_CONT,
@ -116,6 +120,18 @@ typedef enum {
} ctl_stat_types; } ctl_stat_types;
#define CTL_STATS_NUM_TYPES 3 #define CTL_STATS_NUM_TYPES 3
typedef enum {
CTL_SS_OK,
CTL_SS_NEED_MORE_SPACE,
CTL_SS_ERROR
} ctl_stats_status;
typedef enum {
CTL_STATS_FLAG_NONE = 0x00,
CTL_STATS_FLAG_TIME_VALID = 0x01
} ctl_stats_flags;
#ifdef CTL_LEGACY_STATS
typedef enum { typedef enum {
CTL_LUN_STATS_NO_BLOCKSIZE = 0x01 CTL_LUN_STATS_NO_BLOCKSIZE = 0x01
} ctl_lun_stats_flags; } ctl_lun_stats_flags;
@ -137,17 +153,6 @@ struct ctl_lun_io_stats {
struct ctl_lun_io_port_stats ports[CTL_MAX_PORTS]; struct ctl_lun_io_port_stats ports[CTL_MAX_PORTS];
}; };
typedef enum {
CTL_SS_OK,
CTL_SS_NEED_MORE_SPACE,
CTL_SS_ERROR
} ctl_stats_status;
typedef enum {
CTL_STATS_FLAG_NONE = 0x00,
CTL_STATS_FLAG_TIME_VALID = 0x01
} ctl_stats_flags;
struct ctl_stats { struct ctl_stats {
int alloc_len; /* passed to kernel */ int alloc_len; /* passed to kernel */
struct ctl_lun_io_stats *lun_stats; /* passed to/from kernel */ struct ctl_lun_io_stats *lun_stats; /* passed to/from kernel */
@ -157,6 +162,27 @@ struct ctl_stats {
ctl_stats_flags flags; /* passed to userland */ ctl_stats_flags flags; /* passed to userland */
struct timespec timestamp; /* passed to userland */ struct timespec timestamp; /* passed to userland */
}; };
#endif /* CTL_LEGACY_STATS */
struct ctl_io_stats {
uint32_t item;
uint64_t bytes[CTL_STATS_NUM_TYPES];
uint64_t operations[CTL_STATS_NUM_TYPES];
uint64_t dmas[CTL_STATS_NUM_TYPES];
struct bintime time[CTL_STATS_NUM_TYPES];
struct bintime dma_time[CTL_STATS_NUM_TYPES];
};
struct ctl_get_io_stats {
struct ctl_io_stats *stats; /* passed to/from kernel */
size_t alloc_len; /* passed to kernel */
size_t fill_len; /* passed to userland */
int first_item; /* passed to kernel */
int num_items; /* passed to userland */
ctl_stats_status status; /* passed to userland */
ctl_stats_flags flags; /* passed to userland */
struct timespec timestamp; /* passed to userland */
};
/* /*
* The types of errors that can be injected: * The types of errors that can be injected:
@ -342,12 +368,54 @@ typedef enum {
CTL_LUNREQ_MODIFY, CTL_LUNREQ_MODIFY,
} ctl_lunreq_type; } ctl_lunreq_type;
/*
* The ID_REQ flag is used to say that the caller has requested a
* particular LUN ID in the req_lun_id field. If we cannot allocate that
* LUN ID, the ctl_add_lun() call will fail.
*
* The STOPPED flag tells us that the LUN should default to the powered
* off state. It will return 0x04,0x02 until it is powered up. ("Logical
* unit not ready, initializing command required.")
*
* The NO_MEDIA flag tells us that the LUN has no media inserted.
*
* The PRIMARY flag tells us that this LUN is registered as a Primary LUN
* which is accessible via the Master shelf controller in an HA. This flag
* being set indicates a Primary LUN. This flag being reset represents a
* Secondary LUN controlled by the Secondary controller in an HA
* configuration. Flag is applicable at this time to T_DIRECT types.
*
* The SERIAL_NUM flag tells us that the serial_num field is filled in and
* valid for use in SCSI INQUIRY VPD page 0x80.
*
* The DEVID flag tells us that the device_id field is filled in and
* valid for use in SCSI INQUIRY VPD page 0x83.
*
* The DEV_TYPE flag tells us that the device_type field is filled in.
*
* The EJECTED flag tells us that the removable LUN has tray open.
*
* The UNMAP flag tells us that this LUN supports UNMAP.
*
* The OFFLINE flag tells us that this LUN can not access backing store.
*/
typedef enum {
CTL_LUN_FLAG_ID_REQ = 0x01,
CTL_LUN_FLAG_STOPPED = 0x02,
CTL_LUN_FLAG_NO_MEDIA = 0x04,
CTL_LUN_FLAG_PRIMARY = 0x08,
CTL_LUN_FLAG_SERIAL_NUM = 0x10,
CTL_LUN_FLAG_DEVID = 0x20,
CTL_LUN_FLAG_DEV_TYPE = 0x40,
CTL_LUN_FLAG_UNMAP = 0x80,
CTL_LUN_FLAG_EJECTED = 0x100,
CTL_LUN_FLAG_READONLY = 0x200
} ctl_backend_lun_flags;
/* /*
* LUN creation parameters: * LUN creation parameters:
* *
* flags: Various LUN flags, see ctl_backend.h for a * flags: Various LUN flags, see above.
* description of the flag values and meanings.
* *
* device_type: The SCSI device type. e.g. 0 for Direct Access, * device_type: The SCSI device type. e.g. 0 for Direct Access,
* 3 for Processor, etc. Only certain backends may * 3 for Processor, etc. Only certain backends may
@ -465,6 +533,7 @@ union ctl_lunreq_data {
* kern_be_args: For kernel use only. * kern_be_args: For kernel use only.
*/ */
struct ctl_lun_req { struct ctl_lun_req {
#define CTL_BE_NAME_LEN 32
char backend[CTL_BE_NAME_LEN]; char backend[CTL_BE_NAME_LEN];
ctl_lunreq_type reqtype; ctl_lunreq_type reqtype;
union ctl_lunreq_data reqdata; union ctl_lunreq_data reqdata;
@ -777,6 +846,8 @@ struct ctl_lun_map {
#define CTL_PORT_REQ _IOWR(CTL_MINOR, 0x26, struct ctl_req) #define CTL_PORT_REQ _IOWR(CTL_MINOR, 0x26, struct ctl_req)
#define CTL_PORT_LIST _IOWR(CTL_MINOR, 0x27, struct ctl_lun_list) #define CTL_PORT_LIST _IOWR(CTL_MINOR, 0x27, struct ctl_lun_list)
#define CTL_LUN_MAP _IOW(CTL_MINOR, 0x28, struct ctl_lun_map) #define CTL_LUN_MAP _IOW(CTL_MINOR, 0x28, struct ctl_lun_map)
#define CTL_GET_LUN_STATS _IOWR(CTL_MINOR, 0x29, struct ctl_get_io_stats)
#define CTL_GET_PORT_STATS _IOWR(CTL_MINOR, 0x2a, struct ctl_get_io_stats)
#endif /* _CTL_IOCTL_H_ */ #endif /* _CTL_IOCTL_H_ */

7
sys/cam/ctl/ctl_private.h
View File

@ -1,6 +1,6 @@
/*- /*-
* Copyright (c) 2003, 2004, 2005, 2008 Silicon Graphics International Corp. * Copyright (c) 2003, 2004, 2005, 2008 Silicon Graphics International Corp.
* Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org> * Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -404,7 +404,10 @@ struct ctl_lun {
struct callout ie_callout; /* INTERVAL TIMER */ struct callout ie_callout; /* INTERVAL TIMER */
struct ctl_mode_pages mode_pages; struct ctl_mode_pages mode_pages;
struct ctl_log_pages log_pages; struct ctl_log_pages log_pages;
struct ctl_lun_io_stats stats; #ifdef CTL_LEGACY_STATS
struct ctl_lun_io_stats legacy_stats;
#endif /* CTL_LEGACY_STATS */
struct ctl_io_stats stats;
uint32_t res_idx; uint32_t res_idx;
uint32_t pr_generation; uint32_t pr_generation;
uint64_t *pr_keys[CTL_MAX_PORTS]; uint64_t *pr_keys[CTL_MAX_PORTS];

3
usr.bin/ctlstat/ctlstat.8
View File

@ -34,7 +34,7 @@
.\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $ .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
.\" $FreeBSD$ .\" $FreeBSD$
.\" .\"
.Dd September 21, 2015 .Dd January 9, 2017
.Dt CTLSTAT 8 .Dt CTLSTAT 8
.Os .Os
.Sh NAME .Sh NAME
@ -120,3 +120,4 @@ every 10 seconds.
.Sh AUTHORS .Sh AUTHORS
.An Ken Merry Aq Mt ken@FreeBSD.org .An Ken Merry Aq Mt ken@FreeBSD.org
.An Will Andrews Aq Mt will@FreeBSD.org .An Will Andrews Aq Mt will@FreeBSD.org
.An Alexander Motin Aq Mt mav@FreeBSD.org

442
usr.bin/ctlstat/ctlstat.c
View File

@ -1,5 +1,6 @@
/*- /*-
* Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp. * Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp.
* Copyright (c) 2017 Alexander Motin <mav@FreeBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -66,17 +67,17 @@ __FBSDID("$FreeBSD$");
#include <cam/ctl/ctl_ioctl.h> #include <cam/ctl/ctl_ioctl.h>
/* /*
* The default amount of space we allocate for LUN storage space. We * The default amount of space we allocate for stats storage space.
* dynamically allocate more if needed. * We dynamically allocate more if needed.
*/ */
#define CTL_STAT_NUM_LUNS 30 #define CTL_STAT_NUM_ITEMS 256
/* /*
* The default number of LUN selection bits we allocate. This is large * The default number of LUN selection bits we allocate. This is large
* because we don't currently increase it if the user specifies a LUN * because we don't currently increase it if the user specifies a LUN
* number of 1024 or larger. * number of 1024 or larger.
*/ */
#define CTL_STAT_LUN_BITS 1024L #define CTL_STAT_BITS 1024L
static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:"; static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:";
static const char *ctlstat_usage = "Usage: ctlstat [-CDdjht] [-l lunnum]" static const char *ctlstat_usage = "Usage: ctlstat [-CDdjht] [-l lunnum]"
@ -101,31 +102,32 @@ typedef enum {
#define CTLSTAT_FLAG_FIRST_RUN (1 << 2) #define CTLSTAT_FLAG_FIRST_RUN (1 << 2)
#define CTLSTAT_FLAG_TOTALS (1 << 3) #define CTLSTAT_FLAG_TOTALS (1 << 3)
#define CTLSTAT_FLAG_DMA_TIME (1 << 4) #define CTLSTAT_FLAG_DMA_TIME (1 << 4)
#define CTLSTAT_FLAG_LUN_TIME_VALID (1 << 5) #define CTLSTAT_FLAG_TIME_VALID (1 << 5)
#define CTLSTAT_FLAG_LUN_MASK (1 << 6) #define CTLSTAT_FLAG_MASK (1 << 6)
#define CTLSTAT_FLAG_PORT_MASK (1 << 7) #define CTLSTAT_FLAG_LUNS (1 << 7)
#define CTLSTAT_FLAG_PORTS (1 << 8)
#define F_CPU(ctx) ((ctx)->flags & CTLSTAT_FLAG_CPU) #define F_CPU(ctx) ((ctx)->flags & CTLSTAT_FLAG_CPU)
#define F_HDR(ctx) ((ctx)->flags & CTLSTAT_FLAG_HEADER) #define F_HDR(ctx) ((ctx)->flags & CTLSTAT_FLAG_HEADER)
#define F_FIRST(ctx) ((ctx)->flags & CTLSTAT_FLAG_FIRST_RUN) #define F_FIRST(ctx) ((ctx)->flags & CTLSTAT_FLAG_FIRST_RUN)
#define F_TOTALS(ctx) ((ctx)->flags & CTLSTAT_FLAG_TOTALS) #define F_TOTALS(ctx) ((ctx)->flags & CTLSTAT_FLAG_TOTALS)
#define F_DMA(ctx) ((ctx)->flags & CTLSTAT_FLAG_DMA_TIME) #define F_DMA(ctx) ((ctx)->flags & CTLSTAT_FLAG_DMA_TIME)
#define F_LUNVAL(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUN_TIME_VALID) #define F_TIMEVAL(ctx) ((ctx)->flags & CTLSTAT_FLAG_TIME_VALID)
#define F_LUNMASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUN_MASK) #define F_MASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_MASK)
#define F_PORTMASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_PORT_MASK) #define F_LUNS(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUNS)
#define F_PORTS(ctx) ((ctx)->flags & CTLSTAT_FLAG_PORTS)
struct ctlstat_context { struct ctlstat_context {
ctlstat_mode_types mode; ctlstat_mode_types mode;
int flags; int flags;
struct ctl_lun_io_stats *cur_lun_stats, *prev_lun_stats, struct ctl_io_stats *cur_stats, *prev_stats;
*tmp_lun_stats; struct ctl_io_stats cur_total_stats[3], prev_total_stats[3];
struct ctl_lun_io_stats cur_total_stats[3], prev_total_stats[3];
struct timespec cur_time, prev_time; struct timespec cur_time, prev_time;
struct ctl_cpu_stats cur_cpu, prev_cpu; struct ctl_cpu_stats cur_cpu, prev_cpu;
uint64_t cur_total_jiffies, prev_total_jiffies; uint64_t cur_total_jiffies, prev_total_jiffies;
uint64_t cur_idle, prev_idle; uint64_t cur_idle, prev_idle;
bitstr_t bit_decl(lun_mask, CTL_STAT_LUN_BITS); bitstr_t bit_decl(item_mask, CTL_STAT_BITS);
bitstr_t bit_decl(port_mask, CTL_MAX_PORTS); int cur_items, prev_items;
int num_luns; int cur_alloc, prev_alloc;
int numdevs; int numdevs;
int header_interval; int header_interval;
}; };
@ -135,12 +137,11 @@ struct ctlstat_context {
#endif #endif
static void usage(int error); static void usage(int error);
static int getstats(int fd, int *num_luns, struct ctl_lun_io_stats **xlun_stats, static int getstats(int fd, int *alloc_items, int *num_items,
struct timespec *cur_time, int *lun_time_valid); struct ctl_io_stats **xstats, struct timespec *cur_time, int *time_valid);
static int getcpu(struct ctl_cpu_stats *cpu_stats); static int getcpu(struct ctl_cpu_stats *cpu_stats);
static void compute_stats(struct ctlstat_context *ctx, static void compute_stats(struct ctl_io_stats *cur_stats,
struct ctl_lun_io_stats *cur_stats, struct ctl_io_stats *prev_stats,
struct ctl_lun_io_stats *prev_stats,
long double etime, long double *mbsec, long double etime, long double *mbsec,
long double *kb_per_transfer, long double *kb_per_transfer,
long double *transfers_per_second, long double *transfers_per_second,
@ -155,64 +156,55 @@ usage(int error)
} }
static int static int
getstats(int fd, int *num_luns, struct ctl_lun_io_stats **xlun_stats, getstats(int fd, int *alloc_items, int *num_items, struct ctl_io_stats **stats,
struct timespec *cur_time, int *flags) struct timespec *cur_time, int *flags)
{ {
struct ctl_lun_io_stats *lun_stats; struct ctl_get_io_stats get_stats;
struct ctl_stats stats; int more_space_count = 0;
int more_space_count;
more_space_count = 0; if (*alloc_items == 0)
*alloc_items = CTL_STAT_NUM_ITEMS;
if (*num_luns == 0)
*num_luns = CTL_STAT_NUM_LUNS;
lun_stats = *xlun_stats;
retry: retry:
if (*stats == NULL)
*stats = malloc(sizeof(**stats) * *alloc_items);
if (lun_stats == NULL) { memset(&get_stats, 0, sizeof(get_stats));
lun_stats = (struct ctl_lun_io_stats *)malloc( get_stats.alloc_len = *alloc_items * sizeof(**stats);
sizeof(*lun_stats) * *num_luns); memset(*stats, 0, get_stats.alloc_len);
} get_stats.stats = *stats;
memset(&stats, 0, sizeof(stats)); if (ioctl(fd, (*flags & CTLSTAT_FLAG_PORTS) ? CTL_GET_PORT_STATS :
stats.alloc_len = *num_luns * sizeof(*lun_stats); CTL_GET_LUN_STATS, &get_stats) == -1)
memset(lun_stats, 0, stats.alloc_len); err(1, "CTL_GET_*_STATS ioctl returned error");
stats.lun_stats = lun_stats;
if (ioctl(fd, CTL_GETSTATS, &stats) == -1) switch (get_stats.status) {
err(1, "error returned from CTL_GETSTATS ioctl");
switch (stats.status) {
case CTL_SS_OK: case CTL_SS_OK:
break; break;
case CTL_SS_ERROR: case CTL_SS_ERROR:
err(1, "CTL_SS_ERROR returned from CTL_GETSTATS ioctl"); err(1, "CTL_GET_*_STATS ioctl returned CTL_SS_ERROR");
break; break;
case CTL_SS_NEED_MORE_SPACE: case CTL_SS_NEED_MORE_SPACE:
if (more_space_count > 0) { if (more_space_count >= 2)
errx(1, "CTL_GETSTATS returned NEED_MORE_SPACE again"); errx(1, "CTL_GET_*_STATS returned NEED_MORE_SPACE again");
} *alloc_items = get_stats.num_items * 5 / 4;
*num_luns = stats.num_luns; free(*stats);
free(lun_stats); *stats = NULL;
lun_stats = NULL;
more_space_count++; more_space_count++;
goto retry; goto retry;
break; /* NOTREACHED */ break; /* NOTREACHED */
default: default:
errx(1, "unknown status %d returned from CTL_GETSTATS ioctl", errx(1, "CTL_GET_*_STATS ioctl returned unknown status %d",
stats.status); get_stats.status);
break; break;
} }
*xlun_stats = lun_stats; *num_items = get_stats.fill_len / sizeof(**stats);
*num_luns = stats.num_luns; cur_time->tv_sec = get_stats.timestamp.tv_sec;
cur_time->tv_sec = stats.timestamp.tv_sec; cur_time->tv_nsec = get_stats.timestamp.tv_nsec;
cur_time->tv_nsec = stats.timestamp.tv_nsec; if (get_stats.flags & CTL_STATS_FLAG_TIME_VALID)
if (stats.flags & CTL_STATS_FLAG_TIME_VALID) *flags |= CTLSTAT_FLAG_TIME_VALID;
*flags |= CTLSTAT_FLAG_LUN_TIME_VALID;
else else
*flags &= ~CTLSTAT_FLAG_LUN_TIME_VALID; *flags &= ~CTLSTAT_FLAG_TIME_VALID;
return (0); return (0);
} }
@ -240,14 +232,13 @@ getcpu(struct ctl_cpu_stats *cpu_stats)
} }
static void static void
compute_stats(struct ctlstat_context *ctx, struct ctl_lun_io_stats *cur_stats, compute_stats(struct ctl_io_stats *cur_stats,
struct ctl_lun_io_stats *prev_stats, long double etime, struct ctl_io_stats *prev_stats, long double etime,
long double *mbsec, long double *kb_per_transfer, long double *mbsec, long double *kb_per_transfer,
long double *transfers_per_second, long double *ms_per_transfer, long double *transfers_per_second, long double *ms_per_transfer,
long double *ms_per_dma, long double *dmas_per_second) long double *ms_per_dma, long double *dmas_per_second)
{ {
uint64_t total_bytes = 0, total_operations = 0, total_dmas = 0; uint64_t total_bytes = 0, total_operations = 0, total_dmas = 0;
uint32_t port;
struct bintime total_time_bt, total_dma_bt; struct bintime total_time_bt, total_dma_bt;
struct timespec total_time_ts, total_dma_ts; struct timespec total_time_ts, total_dma_ts;
int i; int i;
@ -256,31 +247,18 @@ compute_stats(struct ctlstat_context *ctx, struct ctl_lun_io_stats *cur_stats,
bzero(&total_dma_bt, sizeof(total_dma_bt)); bzero(&total_dma_bt, sizeof(total_dma_bt));
bzero(&total_time_ts, sizeof(total_time_ts)); bzero(&total_time_ts, sizeof(total_time_ts));
bzero(&total_dma_ts, sizeof(total_dma_ts)); bzero(&total_dma_ts, sizeof(total_dma_ts));
for (port = 0; port < CTL_MAX_PORTS; port++) { for (i = 0; i < CTL_STATS_NUM_TYPES; i++) {
if (F_PORTMASK(ctx) && total_bytes += cur_stats->bytes[i];
bit_test(ctx->port_mask, port) == 0) total_operations += cur_stats->operations[i];
continue; total_dmas += cur_stats->dmas[i];
for (i = 0; i < CTL_STATS_NUM_TYPES; i++) { bintime_add(&total_time_bt, &cur_stats->time[i]);
total_bytes += cur_stats->ports[port].bytes[i]; bintime_add(&total_dma_bt, &cur_stats->dma_time[i]);
total_operations += if (prev_stats != NULL) {
cur_stats->ports[port].operations[i]; total_bytes -= prev_stats->bytes[i];
total_dmas += cur_stats->ports[port].num_dmas[i]; total_operations -= prev_stats->operations[i];
bintime_add(&total_time_bt, total_dmas -= prev_stats->dmas[i];
&cur_stats->ports[port].time[i]); bintime_sub(&total_time_bt, &prev_stats->time[i]);
bintime_add(&total_dma_bt, bintime_sub(&total_dma_bt, &prev_stats->dma_time[i]);
&cur_stats->ports[port].dma_time[i]);
if (prev_stats != NULL) {
total_bytes -=
prev_stats->ports[port].bytes[i];
total_operations -=
prev_stats->ports[port].operations[i];
total_dmas -=
prev_stats->ports[port].num_dmas[i];
bintime_sub(&total_time_bt,
&prev_stats->ports[port].time[i]);
bintime_sub(&total_dma_bt,
&prev_stats->ports[port].dma_time[i]);
}
} }
} }
@ -340,35 +318,25 @@ compute_stats(struct ctlstat_context *ctx, struct ctl_lun_io_stats *cur_stats,
static const char *iotypes[] = {"NO IO", "READ", "WRITE"}; static const char *iotypes[] = {"NO IO", "READ", "WRITE"};
static void static void
ctlstat_dump(struct ctlstat_context *ctx) { ctlstat_dump(struct ctlstat_context *ctx)
int iotype, lun, port; {
struct ctl_lun_io_stats *stats = ctx->cur_lun_stats; int iotype, i;
struct ctl_io_stats *stats = ctx->cur_stats;
for (lun = 0; lun < ctx->num_luns;lun++) { for (i = 0; i < ctx->cur_items;i++) {
if (F_LUNMASK(ctx) && bit_test(ctx->lun_mask, lun) == 0) if (F_MASK(ctx) && bit_test(ctx->item_mask, i) == 0)
continue; continue;
printf("lun %d\n", lun); printf("%s %d\n", F_PORTS(ctx) ? "port" : "lun", stats[i].item);
for (port = 0; port < CTL_MAX_PORTS; port++) { for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) {
if (F_PORTMASK(ctx) && printf(" io type %d (%s)\n", iotype, iotypes[iotype]);
bit_test(ctx->port_mask, port) == 0) printf(" bytes %ju\n", (uintmax_t)
continue; stats[i].bytes[iotype]);
printf(" port %d\n", printf(" operations %ju\n", (uintmax_t)
stats[lun].ports[port].targ_port); stats[i].operations[iotype]);
for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; printf(" dmas %ju\n", (uintmax_t)
iotype++) { stats[i].dmas[iotype]);
printf(" io type %d (%s)\n", iotype, PRINT_BINTIME(" io time", stats[i].time[iotype]);
iotypes[iotype]); PRINT_BINTIME(" dma time", stats[i].dma_time[iotype]);
printf(" bytes %ju\n", (uintmax_t)
stats[lun].ports[port].bytes[iotype]);
printf(" operations %ju\n", (uintmax_t)
stats[lun].ports[port].operations[iotype]);
PRINT_BINTIME(" io time",
stats[lun].ports[port].time[iotype]);
printf(" num dmas %ju\n", (uintmax_t)
stats[lun].ports[port].num_dmas[iotype]);
PRINT_BINTIME(" dma time",
stats[lun].ports[port].dma_time[iotype]);
}
} }
} }
} }
@ -378,63 +346,49 @@ ctlstat_dump(struct ctlstat_context *ctx) {
(uintmax_t)(((bt).frac >> 32) * 1000000 >> 32)) (uintmax_t)(((bt).frac >> 32) * 1000000 >> 32))
static void static void
ctlstat_json(struct ctlstat_context *ctx) { ctlstat_json(struct ctlstat_context *ctx) {
int iotype, lun, port; int iotype, i;
struct ctl_lun_io_stats *stats = ctx->cur_lun_stats; struct ctl_io_stats *stats = ctx->cur_stats;
printf("{\"luns\":["); printf("{\"%s\":[", F_PORTS(ctx) ? "ports" : "luns");
for (lun = 0; lun < ctx->num_luns; lun++) { for (i = 0; i < ctx->cur_items; i++) {
if (F_LUNMASK(ctx) && bit_test(ctx->lun_mask, lun) == 0) if (F_MASK(ctx) && bit_test(ctx->item_mask, i) == 0)
continue; continue;
printf("{\"ports\":["); printf("{\"num\":%d,\"io\":[",
for (port = 0; port < CTL_MAX_PORTS;port++) { stats[i].item);
if (F_PORTMASK(ctx) && for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) {
bit_test(ctx->port_mask, port) == 0) printf("{\"type\":\"%s\",", iotypes[iotype]);
continue; printf("\"bytes\":%ju,", (uintmax_t)stats[
printf("{\"num\":%d,\"io\":[", i].bytes[iotype]);
stats[lun].ports[port].targ_port); printf("\"operations\":%ju,", (uintmax_t)stats[
for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; i].operations[iotype]);
iotype++) { printf("\"dmas\":%ju}", (uintmax_t)
printf("{\"type\":\"%s\",", iotypes[iotype]); stats[i].dmas[iotype]);
printf("\"bytes\":%ju,", (uintmax_t)stats[ JSON_BINTIME("io time", stats[i].time[iotype]);
lun].ports[port].bytes[iotype]); JSON_BINTIME("dma time", stats[i].dma_time[iotype]);
printf("\"operations\":%ju,", (uintmax_t)stats[ if (iotype < (CTL_STATS_NUM_TYPES - 1))
lun].ports[port].operations[iotype]); printf(","); /* continue io array */
JSON_BINTIME("io time",
stats[lun].ports[port].time[iotype]);
JSON_BINTIME("dma time",
stats[lun].ports[port].dma_time[iotype]);
printf("\"num dmas\":%ju}", (uintmax_t)
stats[lun].ports[port].num_dmas[iotype]);
if (iotype < (CTL_STATS_NUM_TYPES - 1))
printf(","); /* continue io array */
}
printf("]}"); /* close port */
if (port < (CTL_MAX_PORTS - 1))
printf(","); /* continue port array */
} }
printf("]}"); /* close lun */ printf("]}");
if (lun < (ctx->num_luns - 1)) if (i < (ctx->cur_items - 1))
printf(","); /* continue lun array */ printf(","); /* continue lun array */
} }
printf("]}"); /* close luns and toplevel */ printf("]}");
} }
static void static void
ctlstat_standard(struct ctlstat_context *ctx) { ctlstat_standard(struct ctlstat_context *ctx) {
long double etime; long double etime;
uint64_t delta_jiffies, delta_idle; uint64_t delta_jiffies, delta_idle;
uint32_t port;
long double cpu_percentage; long double cpu_percentage;
int i; int i, j;
int j;
cpu_percentage = 0; cpu_percentage = 0;
if (F_CPU(ctx) && (getcpu(&ctx->cur_cpu) != 0)) if (F_CPU(ctx) && (getcpu(&ctx->cur_cpu) != 0))
errx(1, "error returned from getcpu()"); errx(1, "error returned from getcpu()");
etime = ctx->cur_time.tv_sec - ctx->prev_time.tv_sec + etime = ctx->cur_time.tv_sec - ctx->prev_time.tv_sec +
(ctx->prev_time.tv_nsec - ctx->cur_time.tv_nsec) * 1e-9; (ctx->prev_time.tv_nsec - ctx->cur_time.tv_nsec) * 1e-9;
if (F_CPU(ctx)) { if (F_CPU(ctx)) {
ctx->prev_total_jiffies = ctx->cur_total_jiffies; ctx->prev_total_jiffies = ctx->cur_total_jiffies;
@ -465,29 +419,28 @@ ctlstat_standard(struct ctlstat_context *ctx) {
if (F_TOTALS(ctx)) { if (F_TOTALS(ctx)) {
fprintf(stdout, "%s Read %s" fprintf(stdout, "%s Read %s"
" Write %s Total\n", " Write %s Total\n",
(F_LUNVAL(ctx) != 0) ? " " : "", (F_TIMEVAL(ctx) != 0) ? " " : "",
(F_LUNVAL(ctx) != 0) ? " " : "", (F_TIMEVAL(ctx) != 0) ? " " : "",
(F_LUNVAL(ctx) != 0) ? " " : ""); (F_TIMEVAL(ctx) != 0) ? " " : "");
hdr_devs = 3; hdr_devs = 3;
} else { } else {
for (i = 0; i < min(CTL_STAT_LUN_BITS, for (i = 0; i < min(CTL_STAT_BITS,
ctx->num_luns); i++) { ctx->cur_items); i++) {
int lun; int item;
/* /*
* Obviously this won't work with * Obviously this won't work with
* LUN numbers greater than a signed * LUN numbers greater than a signed
* integer. * integer.
*/ */
lun = (int)ctx->cur_lun_stats[i item = (int)ctx->cur_stats[i].item;
].lun_number;
if (F_LUNMASK(ctx) && if (F_MASK(ctx) &&
bit_test(ctx->lun_mask, lun) == 0) bit_test(ctx->item_mask, item) == 0)
continue; continue;
fprintf(stdout, "%15.6s%d %s", fprintf(stdout, "%15.6s%d %s",
"lun", lun, F_PORTS(ctx) ? "port" : "lun", item,
(F_LUNVAL(ctx) != 0) ? " " : ""); (F_TIMEVAL(ctx) != 0) ? " " : "");
hdr_devs++; hdr_devs++;
} }
fprintf(stdout, "\n"); fprintf(stdout, "\n");
@ -496,7 +449,7 @@ ctlstat_standard(struct ctlstat_context *ctx) {
fprintf(stdout, " "); fprintf(stdout, " ");
for (i = 0; i < hdr_devs; i++) for (i = 0; i < hdr_devs; i++)
fprintf(stdout, "%s KB/t %s MB/s", fprintf(stdout, "%s KB/t %s MB/s",
(F_LUNVAL(ctx) != 0) ? " ms" : "", (F_TIMEVAL(ctx) != 0) ? " ms" : "",
(F_DMA(ctx) == 0) ? "tps" : "dps"); (F_DMA(ctx) == 0) ? "tps" : "dps");
fprintf(stdout, "\n"); fprintf(stdout, "\n");
ctx->header_interval = 20; ctx->header_interval = 20;
@ -519,55 +472,48 @@ ctlstat_standard(struct ctlstat_context *ctx) {
memset(&ctx->cur_total_stats, 0, sizeof(ctx->cur_total_stats)); memset(&ctx->cur_total_stats, 0, sizeof(ctx->cur_total_stats));
/* Use macros to make the next loop more readable. */ /* Use macros to make the next loop more readable. */
#define ADD_STATS_BYTES(st, p, i, j) \ #define ADD_STATS_BYTES(st, i, j) \
ctx->cur_total_stats[st].ports[p].bytes[j] += \ ctx->cur_total_stats[st].bytes[j] += \
ctx->cur_lun_stats[i].ports[p].bytes[j] ctx->cur_stats[i].bytes[j]
#define ADD_STATS_OPERATIONS(st, p, i, j) \ #define ADD_STATS_OPERATIONS(st, i, j) \
ctx->cur_total_stats[st].ports[p].operations[j] += \ ctx->cur_total_stats[st].operations[j] += \
ctx->cur_lun_stats[i].ports[p].operations[j] ctx->cur_stats[i].operations[j]
#define ADD_STATS_NUM_DMAS(st, p, i, j) \ #define ADD_STATS_DMAS(st, i, j) \
ctx->cur_total_stats[st].ports[p].num_dmas[j] += \ ctx->cur_total_stats[st].dmas[j] += \
ctx->cur_lun_stats[i].ports[p].num_dmas[j] ctx->cur_stats[i].dmas[j]
#define ADD_STATS_TIME(st, p, i, j) \ #define ADD_STATS_TIME(st, i, j) \
bintime_add(&ctx->cur_total_stats[st].ports[p].time[j], \ bintime_add(&ctx->cur_total_stats[st].time[j], \
&ctx->cur_lun_stats[i].ports[p].time[j]) &ctx->cur_stats[i].time[j])
#define ADD_STATS_DMA_TIME(st, p, i, j) \ #define ADD_STATS_DMA_TIME(st, i, j) \
bintime_add(&ctx->cur_total_stats[st].ports[p].dma_time[j], \ bintime_add(&ctx->cur_total_stats[st].dma_time[j], \
&ctx->cur_lun_stats[i].ports[p].dma_time[j]) &ctx->cur_stats[i].dma_time[j])
for (i = 0; i < ctx->num_luns; i++) { for (i = 0; i < ctx->cur_items; i++) {
if (F_LUNMASK(ctx) && bit_test(ctx->lun_mask, if (F_MASK(ctx) && bit_test(ctx->item_mask,
(int)ctx->cur_lun_stats[i].lun_number) == 0) (int)ctx->cur_stats[i].item) == 0)
continue; continue;
for (port = 0; port < CTL_MAX_PORTS; port++) { for (j = 0; j < CTL_STATS_NUM_TYPES; j++) {
if (F_PORTMASK(ctx) && ADD_STATS_BYTES(2, i, j);
bit_test(ctx->port_mask, port) == 0) ADD_STATS_OPERATIONS(2, i, j);
continue; ADD_STATS_DMAS(2, i, j);
for (j = 0; j < CTL_STATS_NUM_TYPES; j++) { ADD_STATS_TIME(2, i, j);
ADD_STATS_BYTES(2, port, i, j); ADD_STATS_DMA_TIME(2, i, j);
ADD_STATS_OPERATIONS(2, port, i, j);
ADD_STATS_NUM_DMAS(2, port, i, j);
ADD_STATS_TIME(2, port, i, j);
ADD_STATS_DMA_TIME(2, port, i, j);
}
ADD_STATS_BYTES(0, port, i, CTL_STATS_READ);
ADD_STATS_OPERATIONS(0, port, i,
CTL_STATS_READ);
ADD_STATS_NUM_DMAS(0, port, i, CTL_STATS_READ);
ADD_STATS_TIME(0, port, i, CTL_STATS_READ);
ADD_STATS_DMA_TIME(0, port, i, CTL_STATS_READ);
ADD_STATS_BYTES(1, port, i, CTL_STATS_WRITE);
ADD_STATS_OPERATIONS(1, port, i,
CTL_STATS_WRITE);
ADD_STATS_NUM_DMAS(1, port, i, CTL_STATS_WRITE);
ADD_STATS_TIME(1, port, i, CTL_STATS_WRITE);
ADD_STATS_DMA_TIME(1, port, i, CTL_STATS_WRITE);
} }
ADD_STATS_BYTES(0, i, CTL_STATS_READ);
ADD_STATS_OPERATIONS(0, i, CTL_STATS_READ);
ADD_STATS_DMAS(0, i, CTL_STATS_READ);
ADD_STATS_TIME(0, i, CTL_STATS_READ);
ADD_STATS_DMA_TIME(0, i, CTL_STATS_READ);
ADD_STATS_BYTES(1, i, CTL_STATS_WRITE);
ADD_STATS_OPERATIONS(1, i, CTL_STATS_WRITE);
ADD_STATS_DMAS(1, i, CTL_STATS_WRITE);
ADD_STATS_TIME(1, i, CTL_STATS_WRITE);
ADD_STATS_DMA_TIME(1, i, CTL_STATS_WRITE);
} }
for (i = 0; i < 3; i++) { for (i = 0; i < 3; i++) {
compute_stats(ctx, &ctx->cur_total_stats[i], compute_stats(&ctx->cur_total_stats[i],
F_FIRST(ctx) ? NULL : &ctx->prev_total_stats[i], F_FIRST(ctx) ? NULL : &ctx->prev_total_stats[i],
etime, &mbsec[i], &kb_per_transfer[i], etime, &mbsec[i], &kb_per_transfer[i],
&transfers_per_sec[i], &transfers_per_sec[i],
@ -576,7 +522,7 @@ ctlstat_standard(struct ctlstat_context *ctx) {
if (F_DMA(ctx) != 0) if (F_DMA(ctx) != 0)
fprintf(stdout, " %5.1Lf", fprintf(stdout, " %5.1Lf",
ms_per_dma[i]); ms_per_dma[i]);
else if (F_LUNVAL(ctx) != 0) else if (F_TIMEVAL(ctx) != 0)
fprintf(stdout, " %5.1Lf", fprintf(stdout, " %5.1Lf",
ms_per_transfer[i]); ms_per_transfer[i]);
fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf", fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf",
@ -585,25 +531,32 @@ ctlstat_standard(struct ctlstat_context *ctx) {
dmas_per_sec[i], mbsec[i]); dmas_per_sec[i], mbsec[i]);
} }
} else { } else {
for (i = 0; i < min(CTL_STAT_LUN_BITS, ctx->num_luns); i++) { for (i = 0; i < min(CTL_STAT_BITS, ctx->cur_items); i++) {
long double mbsec, kb_per_transfer; long double mbsec, kb_per_transfer;
long double transfers_per_sec; long double transfers_per_sec;
long double ms_per_transfer; long double ms_per_transfer;
long double ms_per_dma; long double ms_per_dma;
long double dmas_per_sec; long double dmas_per_sec;
if (F_LUNMASK(ctx) && bit_test(ctx->lun_mask, if (F_MASK(ctx) && bit_test(ctx->item_mask,
(int)ctx->cur_lun_stats[i].lun_number) == 0) (int)ctx->cur_stats[i].item) == 0)
continue; continue;
compute_stats(ctx, &ctx->cur_lun_stats[i], for (j = 0; j < ctx->prev_items; j++) {
F_FIRST(ctx) ? NULL : &ctx->prev_lun_stats[i], if (ctx->prev_stats[j].item ==
ctx->cur_stats[i].item)
break;
}
if (j >= ctx->prev_items)
j = -1;
compute_stats(&ctx->cur_stats[i],
j >= 0 ? &ctx->prev_stats[j] : NULL,
etime, &mbsec, &kb_per_transfer, etime, &mbsec, &kb_per_transfer,
&transfers_per_sec, &ms_per_transfer, &transfers_per_sec, &ms_per_transfer,
&ms_per_dma, &dmas_per_sec); &ms_per_dma, &dmas_per_sec);
if (F_DMA(ctx)) if (F_DMA(ctx))
fprintf(stdout, " %5.1Lf", fprintf(stdout, " %5.1Lf",
ms_per_dma); ms_per_dma);
else if (F_LUNVAL(ctx) != 0) else if (F_TIMEVAL(ctx) != 0)
fprintf(stdout, " %5.1Lf", fprintf(stdout, " %5.1Lf",
ms_per_transfer); ms_per_transfer);
fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf", fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf",
@ -620,6 +573,7 @@ main(int argc, char **argv)
int count, waittime; int count, waittime;
int fd, retval; int fd, retval;
struct ctlstat_context ctx; struct ctlstat_context ctx;
struct ctl_io_stats *tmp_stats;
/* default values */ /* default values */
retval = 0; retval = 0;
@ -658,15 +612,16 @@ main(int argc, char **argv)
int cur_lun; int cur_lun;
cur_lun = atoi(optarg); cur_lun = atoi(optarg);
if (cur_lun > CTL_STAT_LUN_BITS) if (cur_lun > CTL_STAT_BITS)
errx(1, "Invalid LUN number %d", cur_lun); errx(1, "Invalid LUN number %d", cur_lun);
if (!F_LUNMASK(&ctx)) if (!F_MASK(&ctx))
ctx.numdevs = 1; ctx.numdevs = 1;
else else
ctx.numdevs++; ctx.numdevs++;
bit_set(ctx.lun_mask, cur_lun); bit_set(ctx.item_mask, cur_lun);
ctx.flags |= CTLSTAT_FLAG_LUN_MASK; ctx.flags |= CTLSTAT_FLAG_MASK;
ctx.flags |= CTLSTAT_FLAG_LUNS;
break; break;
} }
case 'n': case 'n':
@ -676,11 +631,16 @@ main(int argc, char **argv)
int cur_port; int cur_port;
cur_port = atoi(optarg); cur_port = atoi(optarg);
if (cur_port > CTL_MAX_PORTS) if (cur_port > CTL_STAT_BITS)
errx(1, "Invalid LUN number %d", cur_port); errx(1, "Invalid port number %d", cur_port);
bit_set(ctx.port_mask, cur_port); if (!F_MASK(&ctx))
ctx.flags |= CTLSTAT_FLAG_PORT_MASK; ctx.numdevs = 1;
else
ctx.numdevs++;
bit_set(ctx.item_mask, cur_port);
ctx.flags |= CTLSTAT_FLAG_MASK;
ctx.flags |= CTLSTAT_FLAG_PORTS;
break; break;
} }
case 't': case 't':
@ -697,29 +657,45 @@ main(int argc, char **argv)
} }
} }
if (!F_TOTALS(&ctx) && !F_LUNMASK(&ctx)) { if (F_LUNS(&ctx) && F_PORTS(&ctx))
errx(1, "Options -p and -l are exclusive.");
if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) {
if (F_TOTALS(&ctx))
ctx.flags |= CTLSTAT_FLAG_PORTS;
else
ctx.flags |= CTLSTAT_FLAG_LUNS;
}
if (!F_TOTALS(&ctx) && !F_MASK(&ctx)) {
/* /*
* Note that this just selects the first N LUNs to display, * Note that this just selects the first N LUNs to display,
* but at this point we have no knoweledge of which LUN * but at this point we have no knoweledge of which LUN
* numbers actually exist. So we may select LUNs that * numbers actually exist. So we may select LUNs that
* aren't there. * aren't there.
*/ */
bit_nset(ctx.lun_mask, 0, min(ctx.numdevs - 1, bit_nset(ctx.item_mask, 0, min(ctx.numdevs - 1,
CTL_STAT_LUN_BITS - 1)); CTL_STAT_BITS - 1));
ctx.flags |= CTLSTAT_FLAG_LUN_MASK; ctx.flags |= CTLSTAT_FLAG_MASK;
} }
if ((fd = open(CTL_DEFAULT_DEV, O_RDWR)) == -1) if ((fd = open(CTL_DEFAULT_DEV, O_RDWR)) == -1)
err(1, "cannot open %s", CTL_DEFAULT_DEV); err(1, "cannot open %s", CTL_DEFAULT_DEV);
for (;count != 0;) { for (;count != 0;) {
ctx.tmp_lun_stats = ctx.prev_lun_stats; tmp_stats = ctx.prev_stats;
ctx.prev_lun_stats = ctx.cur_lun_stats; ctx.prev_stats = ctx.cur_stats;
ctx.cur_lun_stats = ctx.tmp_lun_stats; ctx.cur_stats = tmp_stats;
c = ctx.prev_alloc;
ctx.prev_alloc = ctx.cur_alloc;
ctx.cur_alloc = c;
c = ctx.prev_items;
ctx.prev_items = ctx.cur_items;
ctx.cur_items = c;
ctx.prev_time = ctx.cur_time; ctx.prev_time = ctx.cur_time;
ctx.prev_cpu = ctx.cur_cpu; ctx.prev_cpu = ctx.cur_cpu;
if (getstats(fd, &ctx.num_luns, &ctx.cur_lun_stats, if (getstats(fd, &ctx.cur_alloc, &ctx.cur_items,
&ctx.cur_time, &ctx.flags) != 0) &ctx.cur_stats, &ctx.cur_time, &ctx.flags) != 0)
errx(1, "error returned from getstats()"); errx(1, "error returned from getstats()");
switch(ctx.mode) { switch(ctx.mode) {