Add a resume hook for bhyve that runs a function on all CPUs during

resume.  For Intel CPUs, invoke vmxon for CPUs that were in VMX mode
at the time of suspend.

Reviewed by:	neel

sys/amd64/amd64/machdep.c

@@ -215,6 +215,8 @@ struct mem_range_softc mem_range_softc;
 
 struct mtx dt_lock;	/* lock for GDT and LDT */
 
+void (*vmm_resume_p)(void);
+
 static void
 cpu_startup(dummy)
 	void *dummy;

sys/amd64/amd64/mp_machdep.c

@@ -1483,6 +1483,8 @@ cpususpend_handler(void)
 
 	if (cpu_ops.cpu_resume)
 		cpu_ops.cpu_resume();
+	if (vmm_resume_p)
+		vmm_resume_p();
 
 	/* Resume MCA and local APIC */
 	mca_resume();

sys/amd64/include/cpu.h

@@ -70,6 +70,9 @@ extern struct	cpu_ops cpu_ops;
 extern char	btext[];
 extern char	etext[];
 
+/* Resume hook for VMM. */
+extern	void (*vmm_resume_p)(void);
+
 void	cpu_halt(void);
 void	cpu_reset(void);
 void	fork_trampoline(void);

sys/amd64/include/vmm.h

@@ -49,6 +49,7 @@ enum x2apic_state;
 
 typedef int	(*vmm_init_func_t)(void);
 typedef int	(*vmm_cleanup_func_t)(void);
+typedef void	(*vmm_resume_func_t)(void);
 typedef void *	(*vmi_init_func_t)(struct vm *vm, struct pmap *pmap);
 typedef int	(*vmi_run_func_t)(void *vmi, int vcpu, register_t rip,
 				  struct pmap *pmap);
@@ -72,6 +73,7 @@ typedef void	(*vmi_vmspace_free)(struct vmspace *vmspace);
 struct vmm_ops {
 	vmm_init_func_t		init;		/* module wide initialization */
 	vmm_cleanup_func_t	cleanup;
+	vmm_resume_func_t	resume;
 
 	vmi_init_func_t		vminit;		/* vm-specific initialization */
 	vmi_run_func_t		vmrun;

sys/amd64/vmm/amd/amdv.c

@@ -53,6 +53,11 @@ amdv_cleanup(void)
 	return (ENXIO);
 }
 
+static void
+amdv_resume(void)
+{
+}
+
 static void *
 amdv_vminit(struct vm *vm, struct pmap *pmap)
 {
@@ -153,6 +158,7 @@ amdv_vmspace_free(struct vmspace *vmspace)
 struct vmm_ops vmm_ops_amd = {
 	amdv_init,
 	amdv_cleanup,
+	amdv_resume,
 	amdv_vminit,
 	amdv_vmrun,
 	amdv_vmcleanup,

sys/amd64/vmm/intel/vmx.c

@@ -524,6 +524,14 @@ vmx_enable(void *arg __unused)
 		vmxon_enabled[curcpu] = 1;
 }
 
+static void
+vmx_restore(void)
+{
+
+	if (vmxon_enabled[curcpu])
+		vmxon(vmxon_region[curcpu]);
+}
+
 static int
 vmx_init(void)
 {
@@ -1958,6 +1966,7 @@ vmx_setcap(void *arg, int vcpu, int type, int val)
 struct vmm_ops vmm_ops_intel = {
 	vmx_init,
 	vmx_cleanup,
+	vmx_restore,
 	vmx_vminit,
 	vmx_run,
 	vmx_vmcleanup,

sys/amd64/vmm/vmm.c

@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 #include <vm/vm_param.h>
 
+#include <machine/cpu.h>
 #include <machine/vm.h>
 #include <machine/pcb.h>
 #include <machine/smp.h>
@@ -131,6 +132,7 @@ static int vmm_initialized;
 static struct vmm_ops *ops;
 #define	VMM_INIT()	(ops != NULL ? (*ops->init)() : 0)
 #define	VMM_CLEANUP()	(ops != NULL ? (*ops->cleanup)() : 0)
+#define	VMM_RESUME()	(ops != NULL ? (*ops->resume)() : 0)
 
 #define	VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL)
 #define	VMRUN(vmi, vcpu, rip, pmap) \
@@ -202,6 +204,12 @@ vm_exitinfo(struct vm *vm, int cpuid)
 	return (&vcpu->exitinfo);
 }
 
+static void
+vmm_resume(void)
+{
+	VMM_RESUME();
+}
+
 static int
 vmm_init(void)
 {
@@ -222,6 +230,7 @@ vmm_init(void)
 		return (ENXIO);
 
 	vmm_msr_init();
+	vmm_resume_p = vmm_resume;
 
 	return (VMM_INIT());
 }
@@ -242,6 +251,7 @@ vmm_handler(module_t mod, int what, void *arg)
 	case MOD_UNLOAD:
 		error = vmmdev_cleanup();
 		if (error == 0) {
+			vmm_resume_p = NULL;
 			iommu_cleanup();
 			vmm_ipi_cleanup();
 			error = VMM_CLEANUP();

sys/x86/acpica/acpi_wakeup.c

@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 
 #include <machine/clock.h>
+#include <machine/cpu.h>
 #include <machine/intr_machdep.h>
 #include <x86/mca.h>
 #include <machine/pcb.h>
@@ -266,6 +267,10 @@ acpi_wakeup_machdep(struct acpi_softc *sc, int state, int sleep_result,
 			restart_cpus(suspcpus);
 #endif
 		mca_resume();
+#ifdef __amd64__
+		if (vmm_resume_p != NULL)
+			vmm_resume_p();
+#endif
 		intr_resume(/*suspend_cancelled*/false);
 
 		AcpiSetFirmwareWakingVector(0);