Add a new 'debug' command tree and 'dump_reqs' command to grab and parse

command and chain frames of in-flight I/O from the driver.

Sponsored by:	Netflix

usr.sbin/mpsutil/Makefile

@@ -1,7 +1,7 @@
 # $FreeBSD$
 
 PROG=	mpsutil
-SRCS=	mps_cmd.c mps_flash.c mps_show.c mpsutil.c
+SRCS=	mps_cmd.c mps_debug.c mps_flash.c mps_show.c mpsutil.c
 MAN=	mpsutil.8
 
 WARNS?= 3

usr.sbin/mpsutil/mps_debug.c

@@ -0,0 +1,190 @@
+/*-
+ * Copyright (c) 2018 Netflix, Inc.
+ * All rights reserved.
+ * Written by: Scott Long <scottl@freebsd.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/errno.h>
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <err.h>
+#include <libutil.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "mpsutil.h"
+
+MPS_TABLE(top, debug);
+
+struct mps_dumpreq_hdr {
+	uint32_t	smid;
+	uint32_t	state;
+	uint32_t	numframes;
+	uint32_t	deschi;
+	uint32_t	desclo;
+};
+
+static int find_sgl(char *);
+static void print_sgl(char *, int, int);
+
+#define MPS_FRAME_LEN 128
+
+static int
+debug_dumpreqs(int ac, char **av)
+{
+	struct mps_dumpreq_hdr *hdr;
+	char *buf, sysctlbuf[128];
+	size_t len;
+	int numframes, error, offset;
+
+	len = 0;
+	buf = NULL;
+	snprintf(sysctlbuf, sizeof(sysctlbuf), "dev.%s.%d.dump_reqs",
+	    is_mps ? "mps" : "mpr", mps_unit);
+
+	error = sysctlbyname(sysctlbuf, NULL, &len, NULL, 0);
+	if (error)
+		return (error);
+
+	if (len == 0)
+		return (0);
+
+	buf = malloc(len);
+	if (buf == NULL)
+		return (ENOMEM);
+
+	error = sysctlbyname(sysctlbuf, buf, &len, NULL, 0);
+	if (error) {
+		printf("len= %zd, error= %d errno= %d\n", len, error, errno);
+		return (error);
+	}
+
+	while (len >= MPS_FRAME_LEN) {
+		hdr = (struct mps_dumpreq_hdr *)buf;
+		numframes = hdr->numframes;
+
+		printf("SMID= %d state= %#x numframes= %d desc.hi= %#08x "
+		    "desc.lo= %#08x\n", hdr->smid, hdr->state,
+		    hdr->numframes, hdr->deschi, hdr->desclo);
+
+		buf += sizeof(struct mps_dumpreq_hdr);
+		len -= sizeof(struct mps_dumpreq_hdr);
+
+		if ((offset = find_sgl(buf)) != -1)
+			print_sgl(buf, offset, numframes);
+
+		buf += MPS_FRAME_LEN * numframes;
+		len -= MPS_FRAME_LEN * numframes;
+	}
+
+	return (error);
+}
+
+static int
+find_sgl(char *buf)
+{
+	MPI2_REQUEST_HEADER *req;
+	MPI2_SCSI_IO_REQUEST *scsi;
+	int offset = 0;
+
+	req = (MPI2_REQUEST_HEADER *)buf;
+
+	switch (req->Function) {
+	case MPI2_FUNCTION_SCSI_IO_REQUEST:
+		scsi = (MPI2_SCSI_IO_REQUEST *)buf;
+		offset = scsi->SGLOffset0;
+		break;
+	default:
+		offset = -1;
+	}
+
+	return (offset);
+}
+
+#define SGL_FLAGS "\10LastElement\7EndOfBuffer\4Local\3Host2IOC\2Addr64\1EndOfList"
+
+static void
+print_sgl(char *buf, int offset, int numframes)
+{
+	MPI2_SGE_SIMPLE64 *sge;
+	MPI2_SGE_CHAIN_UNION *sgc;
+	MPI2_REQUEST_HEADER *req;
+	u_int i = 0, flags;
+	char *frame, tmpbuf[128];
+
+	req = (MPI2_REQUEST_HEADER *)buf;
+	frame = (char *)buf;
+	sge = (MPI2_SGE_SIMPLE64 *)&frame[offset * 4];
+	printf("SGL for command\n");
+
+	hexdump(frame, MPS_FRAME_LEN, NULL, 0);
+	while (frame != NULL) {
+		flags = sge->FlagsLength >> MPI2_SGE_FLAGS_SHIFT;
+		bzero(tmpbuf, sizeof(tmpbuf));
+		mps_parse_flags(flags, SGL_FLAGS, tmpbuf, sizeof(tmpbuf));
+		printf("seg%d flags=%x %s len= 0x%06x addr=0x%016jx\n", i,
+		    flags, tmpbuf, sge->FlagsLength & 0xffffff,
+		    mps_to_u64(&sge->Address));
+		if (flags & (MPI2_SGE_FLAGS_END_OF_LIST |
+		    MPI2_SGE_FLAGS_END_OF_BUFFER))
+			break;
+		sge++;
+		i++;
+		if (flags & MPI2_SGE_FLAGS_LAST_ELEMENT) {
+			sgc = (MPI2_SGE_CHAIN_UNION *)sge;
+			if ((sgc->Flags & MPI2_SGE_FLAGS_CHAIN_ELEMENT) == 0) {
+				printf("Invalid chain element\n");
+				break;
+			}
+			bzero(tmpbuf, sizeof(tmpbuf));
+			mps_parse_flags(sgc->Flags, SGL_FLAGS, tmpbuf,
+			    sizeof(tmpbuf));
+			if (sgc->Flags & MPI2_SGE_FLAGS_64_BIT_ADDRESSING)
+				printf("chain64 flags=0x%x %s len=0x%x "
+				    "Offset=0x%x addr=0x%016jx\n", sgc->Flags,
+				    tmpbuf, sgc->Length, sgc->NextChainOffset,
+				    mps_to_u64(&sgc->u.Address64));
+			else
+				printf("chain32 flags=0x%x %s len=0x%x "
+				    "Offset=0x%x addr=0x%08x\n", sgc->Flags,
+				    tmpbuf, sgc->Length, sgc->NextChainOffset,
+				    sgc->u.Address32);
+			if (--numframes <= 0)
+				break;
+			frame += MPS_FRAME_LEN;
+			sge = (MPI2_SGE_SIMPLE64 *)frame;
+			hexdump(frame, MPS_FRAME_LEN, NULL, 0);
+		}
+	}
+}
+
+MPS_COMMAND(debug, dumpreqs, debug_dumpreqs, "", "Dump the active request queue")

usr.sbin/mpsutil/mps_show.c

@@ -202,6 +202,7 @@ static int
 show_iocfacts(int ac, char **av)
 {
 	MPI2_IOC_FACTS_REPLY *facts;
+	char tmpbuf[128];
 	int error, fd;
 
 	fd = mps_open(mps_unit);
@@ -217,6 +218,14 @@ show_iocfacts(int ac, char **av)
 		return (errno);
 	}
 
+#define IOCCAP "\3ScsiTaskFull" "\4DiagTrace" "\5SnapBuf" "\6ExtBuf" \
+    "\7EEDP" "\10BiDirTarg" "\11Multicast" "\14TransRetry" "\15IR" \
+    "\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc" \
+    "\22FastPath" "\23RDPQArray" "\24AtomicReqDesc" "\25PCIeSRIOV"
+
+	bzero(tmpbuf, sizeof(tmpbuf));
+	mps_parse_flags(facts->IOCCapabilities, IOCCAP, tmpbuf, sizeof(tmpbuf));
+
 	printf("          MsgVersion: %02d.%02d\n",
 	    facts->MsgVersion >> 8, facts->MsgVersion & 0xff);
 	printf("           MsgLength: %d\n", facts->MsgLength);
@@ -236,14 +245,19 @@ show_iocfacts(int ac, char **av)
 	printf("      MaxMSIxVectors: %d\n", facts->MaxMSIxVectors);
 	printf("       RequestCredit: %d\n", facts->RequestCredit);
 	printf("           ProductID: 0x%x\n", facts->ProductID);
-	printf("     IOCCapabilities: 0x%x\n", facts->IOCCapabilities);
+	printf("     IOCCapabilities: 0x%x %s\n", facts->IOCCapabilities,
+	    tmpbuf);
 	printf("           FWVersion: 0x%08x\n", facts->FWVersion.Word);
 	printf(" IOCRequestFrameSize: %d\n", facts->IOCRequestFrameSize);
 	printf("       MaxInitiators: %d\n", facts->MaxInitiators);
 	printf("          MaxTargets: %d\n", facts->MaxTargets);
 	printf("     MaxSasExpanders: %d\n", facts->MaxSasExpanders);
 	printf("       MaxEnclosures: %d\n", facts->MaxEnclosures);
-	printf("       ProtocolFlags: 0x%x\n", facts->ProtocolFlags);
+
+	bzero(tmpbuf, sizeof(tmpbuf));
+	mps_parse_flags(facts->ProtocolFlags,
+	    "\4NvmeDevices\2ScsiTarget\1ScsiInitiator", tmpbuf, sizeof(tmpbuf));
+	printf("       ProtocolFlags: 0x%x %s\n", facts->ProtocolFlags, tmpbuf);
 	printf("  HighPriorityCredit: %d\n", facts->HighPriorityCredit);
 	printf("MaxRepDescPostQDepth: %d\n",
 	    facts->MaxReplyDescriptorPostQueueDepth);

usr.sbin/mpsutil/mpsutil.c

@@ -205,3 +205,32 @@ hexdump(const void *ptr, int length, const char *hdr, int flags)
 		printf("\n");
 	}
 }
+
+#define PCHAR(c) { if (retval < tmpsz) { *outbuf++ = (c); retval++; } }
+
+int
+mps_parse_flags(uintmax_t num, const char *q, char *outbuf, int tmpsz)
+{
+	int n, tmp, retval = 0;
+
+	if (num == 0)
+		return (retval);
+
+	/* %b conversion flag format. */
+	tmp = retval;
+	while (*q) {
+		n = *q++;
+		if (num & (1 << (n - 1))) {
+			PCHAR(retval != tmp ?  ',' : '<');
+			for (; (n = *q) > ' '; ++q)
+				PCHAR(n);
+		} else
+			for (; *q > ' '; ++q)
+				continue;
+	}
+	if (retval != tmp)
+		PCHAR('>');
+
+	return (retval);
+}
+

usr.sbin/mpsutil/mpsutil.h

@@ -42,6 +42,8 @@
 #include <dev/mps/mpi/mpi2_cnfg.h>
 #include <dev/mps/mpi/mpi2_raid.h>
 #include <dev/mps/mpi/mpi2_ioc.h>
+#include <dev/mps/mpi/mpi2_init.h>
+#include <dev/mps/mpi/mpi2_tool.h>
 
 #define MPSUTIL_VERSION	"1.0.0"
 
@@ -102,6 +104,7 @@ void	hexdump(const void *ptr, int length, const char *hdr, int flags);
 #define	HD_OMIT_HEX	(1 << 17)
 #define	HD_OMIT_CHARS	(1 << 18)
 #define HD_REVERSED	(1 << 19)
+int	mps_parse_flags(uintmax_t, const char *, char *, int);
 
 int	mps_open(int unit);
 int	mps_table_handler(struct mpsutil_command **start,
@@ -142,6 +145,13 @@ mps_read_ioc_page(int fd, U8 PageNumber, U16 *IOCStatus)
 	    0, IOCStatus));
 }
 
+static __inline uint64_t
+mps_to_u64(U64 *data)
+{
+
+	return (((uint64_t)(data->High) << 32 ) | data->Low);
+}
+
 MPI2_IOC_FACTS_REPLY * mps_get_iocfacts(int fd);
 
 #endif /* !__MPSUTIL_H__ */