Don't expose 'vmm_ipinum' as a global.

sys/amd64/include/vmm.h

@@ -47,7 +47,7 @@ struct pmap;
 
 enum x2apic_state;
 
-typedef int	(*vmm_init_func_t)(void);
+typedef int	(*vmm_init_func_t)(int ipinum);
 typedef int	(*vmm_cleanup_func_t)(void);
 typedef void	(*vmm_resume_func_t)(void);
 typedef void *	(*vmi_init_func_t)(struct vm *vm, struct pmap *pmap);

sys/amd64/vmm/amd/amdv.c

@@ -38,7 +38,7 @@ __FBSDID("$FreeBSD$");
 #include "io/iommu.h"
 
 static int
-amdv_init(void)
+amdv_init(int ipinum)
 {
 
 	printf("amdv_init: not implemented\n");

sys/amd64/vmm/intel/ept.c

@@ -77,7 +77,7 @@ SYSCTL_INT(_hw_vmm_ept, OID_AUTO, pmap_flags, CTLFLAG_RD,
     &ept_pmap_flags, 0, NULL);
 
 int
-ept_init(void)
+ept_init(int ipinum)
 {
 	int use_hw_ad_bits, use_superpages, use_exec_only;
 	uint64_t cap;
@@ -99,7 +99,7 @@ ept_init(void)
 	    !INVEPT_ALL_TYPES_SUPPORTED(cap))
 		return (EINVAL);
 
-	ept_pmap_flags = vmm_ipinum & PMAP_NESTED_IPIMASK;
+	ept_pmap_flags = ipinum & PMAP_NESTED_IPIMASK;
 
 	use_superpages = 1;
 	TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages);

sys/amd64/vmm/intel/ept.h

@@ -31,7 +31,7 @@
 
 struct vmx;
 
-int	ept_init(void);
+int	ept_init(int ipinum);
 void	ept_invalidate_mappings(u_long eptp);
 struct vmspace *ept_vmspace_alloc(vm_offset_t min, vm_offset_t max);
 void	ept_vmspace_free(struct vmspace *vmspace);

sys/amd64/vmm/intel/vmx.c

@@ -474,7 +474,7 @@ vmx_restore(void)
 }
 
 static int
-vmx_init(void)
+vmx_init(int ipinum)
 {
 	int error, use_tpr_shadow;
 	uint64_t fixed0, fixed1, feature_control;
@@ -639,7 +639,7 @@ vmx_init(void)
 	}
 
 	/* Initialize EPT */
-	error = ept_init();
+	error = ept_init(ipinum);
 	if (error) {
 		printf("vmx_init: ept initialization failed (%d)\n", error);
 		return (error);

sys/amd64/vmm/io/vlapic.c

@@ -1430,7 +1430,7 @@ vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
 }
 
 void
-vlapic_post_intr(struct vlapic *vlapic, int hostcpu)
+vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
 {
 	/*
 	 * Post an interrupt to the vcpu currently running on 'hostcpu'.
@@ -1444,7 +1444,7 @@ vlapic_post_intr(struct vlapic *vlapic, int hostcpu)
 	if (vlapic->ops.post_intr)
 		(*vlapic->ops.post_intr)(vlapic, hostcpu);
 	else
-		ipi_cpu(hostcpu, vmm_ipinum);
+		ipi_cpu(hostcpu, ipinum);
 }
 
 bool

sys/amd64/vmm/io/vlapic.h

@@ -65,9 +65,9 @@ int vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level);
 /*
  * Post an interrupt to the vcpu running on 'hostcpu'. This will use a
  * hardware assist if available (e.g. Posted Interrupt) or fall back to
- * sending an IPI to interrupt the 'hostcpu'.
+ * sending an 'ipinum' to interrupt the 'hostcpu'.
  */
-void vlapic_post_intr(struct vlapic *vlapic, int hostcpu);
+void vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum);
 
 void vlapic_set_error(struct vlapic *vlapic, uint32_t mask);
 void vlapic_fire_cmci(struct vlapic *vlapic);

sys/amd64/vmm/vmm.c

@@ -130,7 +130,7 @@ struct vm {
 static int vmm_initialized;
 
 static struct vmm_ops *ops;
-#define	VMM_INIT()	(ops != NULL ? (*ops->init)() : 0)
+#define	VMM_INIT(num)	(ops != NULL ? (*ops->init)(num) : 0)
 #define	VMM_CLEANUP()	(ops != NULL ? (*ops->cleanup)() : 0)
 #define	VMM_RESUME()	(ops != NULL ? (*ops->resume)() : 0)
 
@@ -170,6 +170,12 @@ CTASSERT(VMM_MSR_NUM <= 64);	/* msr_mask can keep track of up to 64 msrs */
 /* statistics */
 static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
 
+SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
+
+static int vmm_ipinum;
+SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
+    "IPI vector used for vcpu notifications");
+
 static void
 vcpu_cleanup(struct vm *vm, int i)
 {
@@ -222,7 +228,10 @@ vmm_init(void)
 	int error;
 
 	vmm_host_state_init();
-	vmm_ipi_init();
+
+	vmm_ipinum = vmm_ipi_alloc();
+	if (vmm_ipinum == 0)
+		vmm_ipinum = IPI_AST;
 
 	error = vmm_mem_init();
 	if (error)
@@ -238,7 +247,7 @@ vmm_init(void)
 	vmm_msr_init();
 	vmm_resume_p = vmm_resume;
 
-	return (VMM_INIT());
+	return (VMM_INIT(vmm_ipinum));
 }
 
 static int
@@ -259,7 +268,8 @@ vmm_handler(module_t mod, int what, void *arg)
 		if (error == 0) {
 			vmm_resume_p = NULL;
 			iommu_cleanup();
-			vmm_ipi_cleanup();
+			if (vmm_ipinum != IPI_AST)
+				vmm_ipi_free(vmm_ipinum);
 			error = VMM_CLEANUP();
 			/*
 			 * Something bad happened - prevent new
@@ -294,8 +304,6 @@ static moduledata_t vmm_kmod = {
 DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
 MODULE_VERSION(vmm, 1);
 
-SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
-
 int
 vm_create(const char *name, struct vm **retvm)
 {
@@ -1379,7 +1387,8 @@ vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
 			panic("invalid vcpu state %d", vcpu->state);
 		if (hostcpu != curcpu) {
 			if (lapic_intr)
-				vlapic_post_intr(vcpu->vlapic, hostcpu);
+				vlapic_post_intr(vcpu->vlapic, hostcpu,
+				    vmm_ipinum);
 			else
 				ipi_cpu(hostcpu, vmm_ipinum);
 		}

sys/amd64/vmm/vmm_ipi.c

@@ -44,15 +44,10 @@ __FBSDID("$FreeBSD$");
 
 extern inthand_t IDTVEC(rsvd), IDTVEC(justreturn);
 
-/*
- * The default is to use the IPI_AST to interrupt a vcpu.
- */
-int vmm_ipinum = IPI_AST;
-
 CTASSERT(APIC_SPURIOUS_INT == 255);
 
-void
-vmm_ipi_init(void)
+int
+vmm_ipi_alloc(void)
 {
 	int idx;
 	uintptr_t func;
@@ -72,22 +67,27 @@ vmm_ipi_init(void)
 		ip = &idt[idx];
 		func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset);
 		if (func == (uintptr_t)&IDTVEC(rsvd)) {
-			vmm_ipinum = idx;
-			setidt(vmm_ipinum, IDTVEC(justreturn), SDT_SYSIGT,
+			setidt(idx , IDTVEC(justreturn), SDT_SYSIGT,
 			       SEL_KPL, 0);
-			break;
+			return (idx);
 		}
 	}
-	
-	if (vmm_ipinum != IPI_AST && bootverbose) {
-		printf("vmm_ipi_init: installing ipi handler to interrupt "
-		       "vcpus at vector %d\n", vmm_ipinum);
-	}
+	return (0);
 }
 
 void
-vmm_ipi_cleanup(void)
+vmm_ipi_free(int ipinum)
 {
-	if (vmm_ipinum != IPI_AST)
-		setidt(vmm_ipinum, IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
+	uintptr_t func;
+	struct gate_descriptor *ip;
+
+	KASSERT(ipinum >= APIC_IPI_INTS && ipinum < APIC_SPURIOUS_INT,
+	    ("invalid ipi %d", ipinum));
+
+	ip = &idt[ipinum];
+	func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset);
+	KASSERT(func == (uintptr_t)&IDTVEC(justreturn),
+	    ("invalid ipi %d", ipinum));
+
+	setidt(ipinum, IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
 }

sys/amd64/vmm/vmm_ipi.h

@@ -29,11 +29,7 @@
 #ifndef _VMM_IPI_H_
 #define _VMM_IPI_H_
 
-struct vm;
-
-extern int vmm_ipinum;
-
-void	vmm_ipi_init(void);
-void	vmm_ipi_cleanup(void);
+int	vmm_ipi_alloc(void);
+void	vmm_ipi_free(int num);
 
 #endif