ctlstat: add prometheus output
When invoked by inetd, ctlstat -P will now produce output suitable for ingestion into Prometheus. It's a drop-in replacement for https://github.com/Gandi/ctld_exporter, except that it doesn't report the number of initiators per target, and it does report time and dma_time. MFC after: 2 weeks Sponsored by: Axcient Relnotes: yes Reviewed by: bapt, bcr Differential Revision: https://reviews.freebsd.org/D29901
This commit is contained in:
Alan Somers
parent
b9e5884ef7
commit
1a7f22d9c2
@ -5,4 +5,6 @@ MAN= ctlstat.8
|
||||
SDIR= ${SRCTOP}/sys
|
||||
CFLAGS+= -I${SDIR}
|
||||
|
||||
LIBADD= sbuf bsdxml
|
||||
|
||||
.include <bsd.prog.mk>
|
||||
|
||||
@ -34,7 +34,7 @@
|
||||
.\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
|
||||
.\" $FreeBSD$
|
||||
.\"
|
||||
.Dd January 9, 2017
|
||||
.Dd April 22, 2021
|
||||
.Dt CTLSTAT 8
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -48,6 +48,7 @@
|
||||
.Op Fl d
|
||||
.Op Fl D
|
||||
.Op Fl j
|
||||
.Op Fl P
|
||||
.Op Fl l Ar lun
|
||||
.Op Fl n Ar numdevs
|
||||
.Op Fl p Ar port
|
||||
@ -83,6 +84,19 @@ Suppress display of the header.
|
||||
JSON dump mode.
|
||||
Dump statistics every 30 seconds in JavaScript Object Notation (JSON) format.
|
||||
No statistics are computed in this mode, only raw numbers are displayed.
|
||||
.It Fl P
|
||||
Prometheus dump mode.
|
||||
Dump statistics in a format suitable for ingestion into Prometheus.
|
||||
When invoked with this option,
|
||||
.Nm
|
||||
dumps once, regardless of the
|
||||
.Fl t
|
||||
option.
|
||||
This option is especially useful when invoked by
|
||||
.Xr inetd 8 .
|
||||
See the comments in
|
||||
.Pa /etc/inetd.conf
|
||||
for an example configuration.
|
||||
.It Fl l Ar lun
|
||||
Request statistics for the specified LUN.
|
||||
.It Fl n Ar numdevs
|
||||
@ -116,7 +130,13 @@ every 10 seconds.
|
||||
.Xr camcontrol 8 ,
|
||||
.Xr ctladm 8 ,
|
||||
.Xr ctld 8 ,
|
||||
.Xr iostat 8
|
||||
.Xr iostat 8 ,
|
||||
.Lk
|
||||
Prometheus project:
|
||||
.Pa https://prometheus.io/ .
|
||||
.Pp
|
||||
Prometheus exposition formats:
|
||||
.Lk https://prometheus.io/docs/instrumenting/exposition_formats/ .
|
||||
.Sh AUTHORS
|
||||
.An Ken Merry Aq Mt ken@FreeBSD.org
|
||||
.An Will Andrews Aq Mt will@FreeBSD.org
|
||||
|
||||
@ -41,19 +41,23 @@
|
||||
#include <sys/cdefs.h>
|
||||
__FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/param.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/queue.h>
|
||||
#include <sys/callout.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/queue.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/sbuf.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/time.h>
|
||||
#include <bsdxml.h>
|
||||
#include <malloc_np.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <getopt.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
@ -76,8 +80,8 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
static int ctl_stat_bits;
|
||||
|
||||
static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:";
|
||||
static const char *ctlstat_usage = "Usage: ctlstat [-CDdjht] [-l lunnum]"
|
||||
static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:";
|
||||
static const char *ctlstat_usage = "Usage: ctlstat [-CDPdjht] [-l lunnum]"
|
||||
"[-c count] [-n numdevs] [-w wait]\n";
|
||||
|
||||
struct ctl_cpu_stats {
|
||||
@ -92,6 +96,7 @@ typedef enum {
|
||||
CTLSTAT_MODE_STANDARD,
|
||||
CTLSTAT_MODE_DUMP,
|
||||
CTLSTAT_MODE_JSON,
|
||||
CTLSTAT_MODE_PROMETHEUS,
|
||||
} ctlstat_mode_types;
|
||||
|
||||
#define CTLSTAT_FLAG_CPU (1 << 0)
|
||||
@ -129,6 +134,15 @@ struct ctlstat_context {
|
||||
int header_interval;
|
||||
};
|
||||
|
||||
struct cctl_portlist_data {
|
||||
int level;
|
||||
struct sbuf *cur_sb[32];
|
||||
int lun;
|
||||
int ntargets;
|
||||
char *target;
|
||||
char **targets;
|
||||
};
|
||||
|
||||
#ifndef min
|
||||
#define min(x,y) (((x) < (y)) ? (x) : (y))
|
||||
#endif
|
||||
@ -381,6 +395,200 @@ ctlstat_json(struct ctlstat_context *ctx) {
|
||||
printf("]}");
|
||||
}
|
||||
|
||||
#define CTLSTAT_PROMETHEUS_LOOP(field) \
|
||||
for (i = n = 0; i < ctx->cur_items; i++) { \
|
||||
if (F_MASK(ctx) && bit_test(ctx->item_mask, \
|
||||
(int)stats[i].item) == 0) \
|
||||
continue; \
|
||||
for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
|
||||
int lun = stats[i].item; \
|
||||
if (lun >= targdata.ntargets) \
|
||||
errx(1, "LUN %u out of range", lun); \
|
||||
printf("iscsi_target_" #field "{" \
|
||||
"lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
|
||||
"\n", \
|
||||
lun, targdata.targets[lun], iotypes[iotype], \
|
||||
stats[i].field[iotype]); \
|
||||
} \
|
||||
} \
|
||||
|
||||
#define CTLSTAT_PROMETHEUS_TIMELOOP(field) \
|
||||
for (i = n = 0; i < ctx->cur_items; i++) { \
|
||||
if (F_MASK(ctx) && bit_test(ctx->item_mask, \
|
||||
(int)stats[i].item) == 0) \
|
||||
continue; \
|
||||
for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
|
||||
uint64_t us; \
|
||||
struct timespec ts; \
|
||||
int lun = stats[i].item; \
|
||||
if (lun >= targdata.ntargets) \
|
||||
errx(1, "LUN %u out of range", lun); \
|
||||
bintime2timespec(&stats[i].field[iotype], &ts); \
|
||||
us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \
|
||||
printf("iscsi_target_" #field "{" \
|
||||
"lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
|
||||
"\n", \
|
||||
lun, targdata.targets[lun], iotypes[iotype], us); \
|
||||
} \
|
||||
} \
|
||||
|
||||
static void
|
||||
cctl_start_pelement(void *user_data, const char *name, const char **attr __unused)
|
||||
{
|
||||
struct cctl_portlist_data* targdata = user_data;
|
||||
|
||||
targdata->level++;
|
||||
if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) /
|
||||
sizeof(targdata->cur_sb[0])))
|
||||
errx(1, "%s: too many nesting levels, %zd max", __func__,
|
||||
sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0]));
|
||||
|
||||
targdata->cur_sb[targdata->level] = sbuf_new_auto();
|
||||
if (targdata->cur_sb[targdata->level] == NULL)
|
||||
err(1, "%s: Unable to allocate sbuf", __func__);
|
||||
|
||||
if (strcmp(name, "targ_port") == 0) {
|
||||
targdata->lun = -1;
|
||||
free(targdata->target);
|
||||
targdata->target = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
cctl_char_phandler(void *user_data, const XML_Char *str, int len)
|
||||
{
|
||||
struct cctl_portlist_data *targdata = user_data;
|
||||
|
||||
sbuf_bcat(targdata->cur_sb[targdata->level], str, len);
|
||||
}
|
||||
|
||||
static void
|
||||
cctl_end_pelement(void *user_data, const char *name)
|
||||
{
|
||||
struct cctl_portlist_data* targdata = user_data;
|
||||
char *str;
|
||||
|
||||
if (targdata->cur_sb[targdata->level] == NULL)
|
||||
errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
|
||||
targdata->level, name);
|
||||
|
||||
if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0)
|
||||
err(1, "%s: sbuf_finish", __func__);
|
||||
str = strdup(sbuf_data(targdata->cur_sb[targdata->level]));
|
||||
if (str == NULL)
|
||||
err(1, "%s can't allocate %zd bytes for string", __func__,
|
||||
sbuf_len(targdata->cur_sb[targdata->level]));
|
||||
|
||||
sbuf_delete(targdata->cur_sb[targdata->level]);
|
||||
targdata->cur_sb[targdata->level] = NULL;
|
||||
targdata->level--;
|
||||
|
||||
if (strcmp(name, "target") == 0) {
|
||||
free(targdata->target);
|
||||
targdata->target = str;
|
||||
} else if (strcmp(name, "lun") == 0) {
|
||||
targdata->lun = atoi(str);
|
||||
free(str);
|
||||
} else if (strcmp(name, "targ_port") == 0) {
|
||||
if (targdata->lun >= 0 && targdata->target != NULL) {
|
||||
if (targdata->lun >= targdata->ntargets) {
|
||||
/*
|
||||
* This can happen for example if there are
|
||||
* holes in CTL's lunlist.
|
||||
*/
|
||||
targdata->ntargets = MAX(targdata->ntargets * 2,
|
||||
targdata->lun + 1);
|
||||
size_t newsize = targdata->ntargets *
|
||||
sizeof(char*);
|
||||
targdata->targets = rallocx(targdata->targets,
|
||||
newsize, MALLOCX_ZERO);
|
||||
}
|
||||
free(targdata->targets[targdata->lun]);
|
||||
targdata->targets[targdata->lun] = targdata->target;
|
||||
targdata->target = NULL;
|
||||
}
|
||||
free(str);
|
||||
} else {
|
||||
free(str);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
ctlstat_prometheus(int fd, struct ctlstat_context *ctx) {
|
||||
struct ctl_io_stats *stats = ctx->cur_stats;
|
||||
struct ctl_lun_list list;
|
||||
struct cctl_portlist_data targdata;
|
||||
XML_Parser parser;
|
||||
char *port_str = NULL;
|
||||
int iotype, i, n, retval;
|
||||
int port_len = 4096;
|
||||
|
||||
bzero(&targdata, sizeof(targdata));
|
||||
targdata.ntargets = ctx->cur_items;
|
||||
targdata.targets = calloc(targdata.ntargets, sizeof(char*));
|
||||
retry:
|
||||
port_str = (char *)realloc(port_str, port_len);
|
||||
bzero(&list, sizeof(list));
|
||||
list.alloc_len = port_len;
|
||||
list.status = CTL_LUN_LIST_NONE;
|
||||
list.lun_xml = port_str;
|
||||
if (ioctl(fd, CTL_PORT_LIST, &list) == -1)
|
||||
err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__);
|
||||
if (list.status == CTL_LUN_LIST_ERROR) {
|
||||
warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s",
|
||||
__func__, list.error_str);
|
||||
} else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
|
||||
port_len <<= 1;
|
||||
goto retry;
|
||||
}
|
||||
|
||||
parser = XML_ParserCreate(NULL);
|
||||
if (parser == NULL)
|
||||
err(1, "%s: Unable to create XML parser", __func__);
|
||||
XML_SetUserData(parser, &targdata);
|
||||
XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
|
||||
XML_SetCharacterDataHandler(parser, cctl_char_phandler);
|
||||
|
||||
retval = XML_Parse(parser, port_str, strlen(port_str), 1);
|
||||
if (retval != 1) {
|
||||
errx(1, "%s: Unable to parse XML: Error %d", __func__,
|
||||
XML_GetErrorCode(parser));
|
||||
}
|
||||
XML_ParserFree(parser);
|
||||
|
||||
/*
|
||||
* NB: Some clients will print a warning if we don't set Content-Length,
|
||||
* but they still work. And the data still gets into Prometheus.
|
||||
*/
|
||||
printf("HTTP/1.1 200 OK\r\n"
|
||||
"Connection: close\r\n"
|
||||
"Content-Type: text/plain; version=0.0.4\r\n"
|
||||
"\r\n");
|
||||
|
||||
printf("# HELP iscsi_target_bytes Number of bytes\n"
|
||||
"# TYPE iscsi_target_bytes counter\n");
|
||||
CTLSTAT_PROMETHEUS_LOOP(bytes);
|
||||
printf("# HELP iscsi_target_dmas Number of DMA\n"
|
||||
"# TYPE iscsi_target_dmas counter\n");
|
||||
CTLSTAT_PROMETHEUS_LOOP(dmas);
|
||||
printf("# HELP iscsi_target_operations Number of operations\n"
|
||||
"# TYPE iscsi_target_operations counter\n");
|
||||
CTLSTAT_PROMETHEUS_LOOP(operations);
|
||||
printf("# HELP iscsi_target_time Cumulative operation time in us\n"
|
||||
"# TYPE iscsi_target_time counter\n");
|
||||
CTLSTAT_PROMETHEUS_TIMELOOP(time);
|
||||
printf("# HELP iscsi_target_dma_time Cumulative DMA time in us\n"
|
||||
"# TYPE iscsi_target_dma_time counter\n");
|
||||
CTLSTAT_PROMETHEUS_TIMELOOP(dma_time);
|
||||
|
||||
for (i = 0; i < targdata.ntargets; i++)
|
||||
free(targdata.targets[i]);
|
||||
free(targdata.target);
|
||||
free(targdata.targets);
|
||||
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
static void
|
||||
ctlstat_standard(struct ctlstat_context *ctx) {
|
||||
long double etime;
|
||||
@ -659,6 +867,9 @@ main(int argc, char **argv)
|
||||
ctx.flags |= CTLSTAT_FLAG_PORTS;
|
||||
break;
|
||||
}
|
||||
case 'P':
|
||||
ctx.mode = CTLSTAT_MODE_PROMETHEUS;
|
||||
break;
|
||||
case 't':
|
||||
ctx.flags |= CTLSTAT_FLAG_TOTALS;
|
||||
break;
|
||||
@ -676,6 +887,17 @@ main(int argc, char **argv)
|
||||
if (F_LUNS(&ctx) && F_PORTS(&ctx))
|
||||
errx(1, "Options -p and -l are exclusive.");
|
||||
|
||||
if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) {
|
||||
if ((count != -1) ||
|
||||
(waittime != 1) ||
|
||||
/* NB: -P could be compatible with -t in the future */
|
||||
(ctx.flags & CTLSTAT_FLAG_TOTALS))
|
||||
{
|
||||
errx(1, "Option -P is exclusive with -c, -w, and -t");
|
||||
}
|
||||
count = 1;
|
||||
}
|
||||
|
||||
if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) {
|
||||
if (F_TOTALS(&ctx))
|
||||
ctx.flags |= CTLSTAT_FLAG_PORTS;
|
||||
@ -712,6 +934,9 @@ main(int argc, char **argv)
|
||||
case CTLSTAT_MODE_JSON:
|
||||
ctlstat_json(&ctx);
|
||||
break;
|
||||
case CTLSTAT_MODE_PROMETHEUS:
|
||||
ctlstat_prometheus(fd, &ctx);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@ -120,6 +120,9 @@
|
||||
#
|
||||
#prom-sysctl stream tcp nowait nobody /usr/sbin/prometheus_sysctl_exporter prometheus_sysctl_exporter -dgh
|
||||
#
|
||||
# Example entry for the CTL exporter
|
||||
#prom-ctl stream tcp nowait root /usr/bin/ctlstat ctlstat -P
|
||||
#
|
||||
# Example entry for insecure rsync server
|
||||
# This is best combined with encrypted virtual tunnel interfaces, which can be
|
||||
# found with: apropos if_ | grep tunnel
|
||||
|
||||
@ -2000,6 +2000,7 @@ bacula-sd 9103/udp #Bacula Storage Daemon
|
||||
prom-sysctl 9124/tcp #prometheus_sysctl_exporter(8)
|
||||
git 9418/tcp #git pack transfer service
|
||||
git 9418/udp #git pack transfer service
|
||||
prom-ctl 9572/tcp #CTL prometheus
|
||||
odbcpathway 9628/tcp #ODBC Pathway Service
|
||||
odbcpathway 9628/udp #ODBC Pathway Service
|
||||
davsrc 9800/tcp #WebDav Source Port
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user