Allow a virtual machine to be forcibly reset or powered off. This is done

by adding an argument to the VM_SUSPEND ioctl that specifies how the virtual
machine should be suspended, viz. VM_SUSPEND_RESET or VM_SUSPEND_POWEROFF.

The disposition of VM_SUSPEND is also made available to the exit handler
via the 'u.suspended' member of 'struct vm_exit'.

This capability is exposed via the '--force-reset' and '--force-poweroff'
arguments to /usr/sbin/bhyvectl.

Discussed with:	grehan@

lib/libvmmapi/vmmapi.c

@@ -343,10 +343,13 @@ vm_run(struct vmctx *ctx, int vcpu, uint64_t rip, struct vm_exit *vmexit)
 }
 
 int
-vm_suspend(struct vmctx *ctx)
+vm_suspend(struct vmctx *ctx, enum vm_suspend_how how)
 {
+	struct vm_suspend vmsuspend;
 
-	return (ioctl(ctx->fd, VM_SUSPEND, 0));
+	bzero(&vmsuspend, sizeof(vmsuspend));
+	vmsuspend.how = how;
+	return (ioctl(ctx->fd, VM_SUSPEND, &vmsuspend));
 }
 
 static int

lib/libvmmapi/vmmapi.h

@@ -61,7 +61,7 @@ int	vm_set_register(struct vmctx *ctx, int vcpu, int reg, uint64_t val);
 int	vm_get_register(struct vmctx *ctx, int vcpu, int reg, uint64_t *retval);
 int	vm_run(struct vmctx *ctx, int vcpu, uint64_t rip,
 	       struct vm_exit *ret_vmexit);
-int	vm_suspend(struct vmctx *ctx);
+int	vm_suspend(struct vmctx *ctx, enum vm_suspend_how how);
 int	vm_apicid2vcpu(struct vmctx *ctx, int apicid);
 int	vm_inject_exception(struct vmctx *ctx, int vcpu, int vec);
 int	vm_inject_exception2(struct vmctx *ctx, int vcpu, int vec, int errcode);

sys/amd64/include/vmm.h

@@ -29,6 +29,13 @@
 #ifndef _VMM_H_
 #define	_VMM_H_
 
+enum vm_suspend_how {
+	VM_SUSPEND_NONE,
+	VM_SUSPEND_RESET,
+	VM_SUSPEND_POWEROFF,
+	VM_SUSPEND_LAST
+};
+
 #ifdef _KERNEL
 
 #define	VM_MAX_NAMELEN	32
@@ -115,7 +122,7 @@ int vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
 int vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
 		    struct seg_desc *desc);
 int vm_run(struct vm *vm, struct vm_run *vmrun);
-int vm_suspend(struct vm *vm);
+int vm_suspend(struct vm *vm, enum vm_suspend_how how);
 int vm_inject_nmi(struct vm *vm, int vcpu);
 int vm_nmi_pending(struct vm *vm, int vcpuid);
 void vm_nmi_clear(struct vm *vm, int vcpuid);
@@ -134,6 +141,7 @@ int vm_apicid2vcpuid(struct vm *vm, int apicid);
 void vm_activate_cpu(struct vm *vm, int vcpu);
 cpuset_t vm_active_cpus(struct vm *vm);
 struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
+void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip);
 
 /*
  * Rendezvous all vcpus specified in 'dest' and execute 'func(arg)'.
@@ -382,6 +390,9 @@ struct vm_exit {
 		struct {
 			int		vector;
 		} ioapic_eoi;
+		struct {
+			enum vm_suspend_how how;
+		} suspended;
 	} u;
 };
 

sys/amd64/include/vmm_dev.h

@@ -159,6 +159,10 @@ struct vm_hpet_cap {
 	uint32_t	capabilities;	/* lower 32 bits of HPET capabilities */
 };
 
+struct vm_suspend {
+	enum vm_suspend_how how;
+};
+
 enum {
 	/* general routines */
 	IOCNUM_ABIVERS = 0,
@@ -214,7 +218,7 @@ enum {
 #define	VM_RUN		\
 	_IOWR('v', IOCNUM_RUN, struct vm_run)
 #define	VM_SUSPEND	\
-	_IO('v', IOCNUM_SUSPEND)
+	_IOW('v', IOCNUM_SUSPEND, struct vm_suspend)
 #define	VM_MAP_MEMORY	\
 	_IOWR('v', IOCNUM_MAP_MEMORY, struct vm_memory_segment)
 #define	VM_GET_MEMORY_SEG \

sys/amd64/vmm/intel/vmx.c

@@ -2044,16 +2044,6 @@ vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 	return (UNHANDLED);
 }
 
-static __inline int
-vmx_exit_suspended(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
-{
-
-	vmexit->rip = vmcs_guest_rip();
-	vmexit->inst_length = 0;
-	vmexit->exitcode = VM_EXITCODE_SUSPENDED;
-	return (UNHANDLED);
-}
-
 static __inline int
 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
 {
@@ -2173,7 +2163,8 @@ vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
 		disable_intr();
 		if (vcpu_suspended(suspend_cookie)) {
 			enable_intr();
-			handled = vmx_exit_suspended(vmx, vcpu, vmexit);
+			vm_exit_suspended(vmx->vm, vcpu, vmcs_guest_rip());
+			handled = UNHANDLED;
 			break;
 		}
 

sys/amd64/vmm/vmm.c

@@ -1211,16 +1211,45 @@ vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
 }
 
 int
-vm_suspend(struct vm *vm)
+vm_suspend(struct vm *vm, enum vm_suspend_how how)
 {
+	int i;
 
-	if (atomic_cmpset_int(&vm->suspend, 0, 1)) {
+	if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
-		VM_CTR0(vm, "virtual machine suspended");
+		return (EINVAL);
-		return (0);
+
-	} else {
+	if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
-		VM_CTR0(vm, "virtual machine already suspended");
+		VM_CTR2(vm, "virtual machine already suspended %d/%d",
+		    vm->suspend, how);
 		return (EALREADY);
 	}
+
+	VM_CTR1(vm, "virtual machine successfully suspended %d", how);
+
+	/*
+	 * Notify all active vcpus that they are now suspended.
+	 */
+	for (i = 0; i < VM_MAXCPU; i++) {
+		if (CPU_ISSET(i, &vm->active_cpus))
+			vcpu_notify_event(vm, i, false);
+	}
+
+	return (0);
+}
+
+void
+vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
+{
+	struct vm_exit *vmexit;
+
+	KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
+	    ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
+
+	vmexit = vm_exitinfo(vm, vcpuid);
+	vmexit->rip = rip;
+	vmexit->inst_length = 0;
+	vmexit->exitcode = VM_EXITCODE_SUSPENDED;
+	vmexit->u.suspended.how = vm->suspend;
 }
 
 int

sys/amd64/vmm/vmm_dev.c

@@ -166,6 +166,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
 	struct vm_stat_desc *statdesc;
 	struct vm_x2apic *x2apic;
 	struct vm_gpa_pte *gpapte;
+	struct vm_suspend *vmsuspend;
 
 	sc = vmmdev_lookup2(cdev);
 	if (sc == NULL)
@@ -241,7 +242,8 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
 		error = vm_run(sc->vm, vmrun);
 		break;
 	case VM_SUSPEND:
-		error = vm_suspend(sc->vm);
+		vmsuspend = (struct vm_suspend *)data;
+		error = vm_suspend(sc->vm, vmsuspend->how);
 		break;
 	case VM_STAT_DESC: {
 		statdesc = (struct vm_stat_desc *)data;

usr.sbin/bhyve/bhyverun.c

@@ -461,17 +461,18 @@ vmexit_inst_emul(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 
 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
-static int resetcpu = -1;
 
 static int
 vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
+	enum vm_suspend_how how;
 
-	assert(resetcpu != -1);
+	how = vmexit->u.suspended.how;
+	assert(how == VM_SUSPEND_RESET || how == VM_SUSPEND_POWEROFF);
 
 	fbsdrun_deletecpu(ctx, *pvcpu);
 
-	if (*pvcpu != resetcpu) {
+	if (*pvcpu != BSP) {
 		pthread_mutex_lock(&resetcpu_mtx);
 		pthread_cond_signal(&resetcpu_cond);
 		pthread_mutex_unlock(&resetcpu_mtx);
@@ -483,7 +484,12 @@ vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 		pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
 	}
 	pthread_mutex_unlock(&resetcpu_mtx);
+
+	if (how == VM_SUSPEND_RESET)
 		exit(0);
+	if (how == VM_SUSPEND_POWEROFF)
+		exit(1);
+	return (0);	/* NOTREACHED */
 }
 
 static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
@@ -505,6 +511,7 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
 	cpuset_t mask;
 	int error, rc, prevcpu;
 	enum vm_exitcode exitcode;
+	enum vm_suspend_how how;
 
 	if (pincpu >= 0) {
 		CPU_ZERO(&mask);
@@ -538,10 +545,13 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
                         rip = vmexit[vcpu].rip;
 			break;
 		case VMEXIT_RESET:
-			if (vm_suspend(ctx) == 0) {
+		case VMEXIT_POWEROFF:
-				assert(resetcpu == -1);
+			if (rc == VMEXIT_RESET)
-				resetcpu = vcpu;
+				how = VM_SUSPEND_RESET;
-			}
+			else
+				how = VM_SUSPEND_POWEROFF;
+			error = vm_suspend(ctx, how);
+			assert(error == 0 || errno == EALREADY);
                         rip = vmexit[vcpu].rip + vmexit[vcpu].inst_length;
 			break;
 		default:

usr.sbin/bhyvectl/bhyvectl.c

@@ -191,13 +191,16 @@ usage(void)
 	"       [--get-highmem]\n"
 	"       [--get-gpa-pmap]\n"
 	"       [--assert-lapic-lvt=<pin>]\n"
-	"       [--inject-nmi]\n",
+	"       [--inject-nmi]\n"
+	"       [--force-reset]\n"
+	"       [--force-poweroff]\n",
 	progname);
 	exit(1);
 }
 
 static int get_stats, getcap, setcap, capval, get_gpa_pmap;
 static int inject_nmi, assert_lapic_lvt;
+static int force_reset, force_poweroff;
 static const char *capname;
 static int create, destroy, get_lowmem, get_highmem;
 static uint64_t memsize;
@@ -565,6 +568,8 @@ main(int argc, char *argv[])
 		{ "create",	NO_ARG,		&create,	1 },
 		{ "destroy",	NO_ARG,		&destroy,	1 },
 		{ "inject-nmi",	NO_ARG,		&inject_nmi,	1 },
+		{ "force-reset",	NO_ARG,	&force_reset,	1 },
+		{ "force-poweroff", NO_ARG,	&force_poweroff, 1 },
 		{ NULL,		0,		NULL,		0 }
 	};
 
@@ -1535,6 +1540,12 @@ main(int argc, char *argv[])
 			printf("vm_run error %d\n", error);
 	}
 
+	if (!error && force_reset)
+		error = vm_suspend(ctx, VM_SUSPEND_RESET);
+
+	if (!error && force_poweroff)
+		error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
+
 	if (error)
 		printf("errno = %d\n", errno);