Support array-type of stats in bhyve.

An array-type stat in vmm.ko is defined as follows:
VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");

It is incremented as follows:
vmm_stat_array_incr(vm, vcpuid, IPIS_SENT, array_index, 1);

And output of 'bhyvectl --get-stats' looks like:
ipis sent to vcpu[0]     3114
ipis sent to vcpu[1]     0

Reviewed by:	grehan
Obtained from:	NetApp

sys/amd64/vmm/io/vlapic.c

@@ -430,6 +430,8 @@ vlapic_fire_timer(struct vlapic *vlapic)
 	}
 }
 
+static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
+
 static int
 lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
 {
@@ -466,9 +468,11 @@ lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
 		while ((i = cpusetobj_ffs(&dmask)) != 0) {
 			i--;
 			CPU_CLR(i, &dmask);
-			if (mode == APIC_DELMODE_FIXED)
+			if (mode == APIC_DELMODE_FIXED) {
 				lapic_set_intr(vlapic->vm, i, vec);
-			else
+				vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
+						    IPIS_SENT, i, 1);
+			} else
 				vm_inject_nmi(vlapic->vm, i);
 		}
 

sys/amd64/vmm/vmm_dev.c

@@ -235,18 +235,13 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
 		error = vm_run(sc->vm, vmrun);
 		break;
 	case VM_STAT_DESC: {
-		const char *desc;
 		statdesc = (struct vm_stat_desc *)data;
-		desc = vmm_stat_desc(statdesc->index);
-		if (desc != NULL) {
-			error = 0;
-			strlcpy(statdesc->desc, desc, sizeof(statdesc->desc));
-		} else
-			error = EINVAL;
+		error = vmm_stat_desc_copy(statdesc->index,
+					statdesc->desc, sizeof(statdesc->desc));
 		break;
 	}
 	case VM_STATS: {
-		CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_TYPES);
+		CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_ELEMS);
 		vmstats = (struct vm_stats *)data;
 		getmicrotime(&vmstats->tv);
 		error = vmm_stat_copy(sc->vm, vmstats->cpuid,

sys/amd64/vmm/vmm_stat.c

@@ -39,8 +39,16 @@ __FBSDID("$FreeBSD$");
 #include "vmm_util.h"
 #include "vmm_stat.h"
 
-static int vstnum;
-static struct vmm_stat_type *vsttab[MAX_VMM_STAT_TYPES];
+/*
+ * 'vst_num_elems' is the total number of addressable statistic elements
+ * 'vst_num_types' is the number of unique statistic types
+ *
+ * It is always true that 'vst_num_elems' is greater than or equal to
+ * 'vst_num_types'. This is because a stat type may represent more than
+ * one element (for e.g. VMM_STAT_ARRAY).
+ */
+static int vst_num_elems, vst_num_types;
+static struct vmm_stat_type *vsttab[MAX_VMM_STAT_ELEMS];
 
 static MALLOC_DEFINE(M_VMM_STAT, "vmm stat", "vmm stat");
 
@@ -59,13 +67,15 @@ vmm_stat_init(void *arg)
 	if (vst->scope == VMM_STAT_SCOPE_AMD && !vmm_is_amd())
 		return;
 
-	if (vstnum >= MAX_VMM_STAT_TYPES) {
+	if (vst_num_elems + vst->nelems >= MAX_VMM_STAT_ELEMS) {
 		printf("Cannot accomodate vmm stat type \"%s\"!\n", vst->desc);
 		return;
 	}
 
-	vst->index = vstnum;
-	vsttab[vstnum++] = vst;
+	vst->index = vst_num_elems;
+	vst_num_elems += vst->nelems;
+
+	vsttab[vst_num_types++] = vst;
 }
 
 int
@@ -78,9 +88,9 @@ vmm_stat_copy(struct vm *vm, int vcpu, int *num_stats, uint64_t *buf)
 		return (EINVAL);
 		
 	stats = vcpu_stats(vm, vcpu);
-	for (i = 0; i < vstnum; i++)
+	for (i = 0; i < vst_num_elems; i++)
 		buf[i] = stats[i];
-	*num_stats = vstnum;
+	*num_stats = vst_num_elems;
 	return (0);
 }
 
@@ -89,7 +99,7 @@ vmm_stat_alloc(void)
 {
 	u_long size;
 	
-	size = vstnum * sizeof(uint64_t);
+	size = vst_num_elems * sizeof(uint64_t);
 
 	return (malloc(size, M_VMM_STAT, M_ZERO | M_WAITOK));
 }
@@ -100,14 +110,26 @@ vmm_stat_free(void *vp)
 	free(vp, M_VMM_STAT);
 }
 
-const char *
-vmm_stat_desc(int index)
+int
+vmm_stat_desc_copy(int index, char *buf, int bufsize)
 {
+	int i;
+	struct vmm_stat_type *vst;
 
-	if (index >= 0 && index < vstnum)
-		return (vsttab[index]->desc);
-	else
-		return (NULL);
+	for (i = 0; i < vst_num_types; i++) {
+		vst = vsttab[i];
+		if (index >= vst->index && index < vst->index + vst->nelems) {
+			if (vst->nelems > 1) {
+				snprintf(buf, bufsize, "%s[%d]",
+					 vst->desc, index - vst->index);
+			} else {
+				strlcpy(buf, vst->desc, bufsize);
+			}
+			return (0);	/* found it */
+		}
+	}
+
+	return (EINVAL);
 }
 
 /* global statistics */

sys/amd64/vmm/vmm_stat.h

@@ -34,7 +34,7 @@
 
 struct vm;
 
-#define	MAX_VMM_STAT_TYPES	64		/* arbitrary */
+#define	MAX_VMM_STAT_ELEMS	64		/* arbitrary */
 
 enum vmm_stat_scope {
 	VMM_STAT_SCOPE_ANY,
@@ -44,15 +44,16 @@ enum vmm_stat_scope {
 
 struct vmm_stat_type {
 	int	index;			/* position in the stats buffer */
+	int	nelems;			/* standalone or array */
 	const char *desc;		/* description of statistic */
 	enum vmm_stat_scope scope;
 };
 
 void	vmm_stat_init(void *arg);
 
-#define	VMM_STAT_DEFINE(type, desc, scope)				\
+#define	VMM_STAT_DEFINE(type, nelems, desc, scope)			\
 	struct vmm_stat_type type[1] = {				\
-		{ -1, desc, scope }					\
+		{ -1, nelems, desc, scope }				\
 	};								\
 	SYSINIT(type##_stat, SI_SUB_KLD, SI_ORDER_ANY, vmm_stat_init, type)
 
@@ -60,11 +61,14 @@ void	vmm_stat_init(void *arg);
 	extern struct vmm_stat_type type[1]
 
 #define	VMM_STAT(type, desc)		\
-	VMM_STAT_DEFINE(type, desc, VMM_STAT_SCOPE_ANY)
+	VMM_STAT_DEFINE(type, 1, desc, VMM_STAT_SCOPE_ANY)
 #define	VMM_STAT_INTEL(type, desc)	\
-	VMM_STAT_DEFINE(type, desc, VMM_STAT_SCOPE_INTEL)
+	VMM_STAT_DEFINE(type, 1, desc, VMM_STAT_SCOPE_INTEL)
 #define	VMM_STAT_AMD(type, desc)	\
-	VMM_STAT_DEFINE(type, desc, VMM_STAT_SCOPE_AMD)
+	VMM_STAT_DEFINE(type, 1, desc, VMM_STAT_SCOPE_AMD)
+
+#define	VMM_STAT_ARRAY(type, nelems, desc)	\
+	VMM_STAT_DEFINE(type, nelems, desc, VMM_STAT_SCOPE_ANY)
 
 void	*vmm_stat_alloc(void);
 void 	vmm_stat_free(void *vp);
@@ -73,15 +77,29 @@ void 	vmm_stat_free(void *vp);
  * 'buf' should be at least fit 'MAX_VMM_STAT_TYPES' entries
  */
 int	vmm_stat_copy(struct vm *vm, int vcpu, int *num_stats, uint64_t *buf);
-const char *vmm_stat_desc(int index);
+int	vmm_stat_desc_copy(int index, char *buf, int buflen);
+
+static void __inline
+vmm_stat_array_incr(struct vm *vm, int vcpu, struct vmm_stat_type *vst,
+		    int statidx, uint64_t x)
+{
+#ifdef VMM_KEEP_STATS
+	uint64_t *stats;
+	
+	stats = vcpu_stats(vm, vcpu);
+
+	if (vst->index >= 0 && statidx < vst->nelems)
+		stats[vst->index + statidx] += x;
+#endif
+}
+		   
 
 static void __inline
 vmm_stat_incr(struct vm *vm, int vcpu, struct vmm_stat_type *vst, uint64_t x)
 {
-#ifdef	VMM_KEEP_STATS
-	uint64_t *stats = vcpu_stats(vm, vcpu);
-	if (vst->index >= 0)
-		stats[vst->index] += x;
+
+#ifdef VMM_KEEP_STATS
+	vmm_stat_array_incr(vm, vcpu, vst, 0, x);
 #endif
 }