d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a way to temporarily suspend and resume virtual CPUs.

Browse Source 
This is used as part of implementing run control in bhyve's debug
server.  The hypervisor now maintains a set of "debugged" CPUs.
Attempting to run a debugged CPU will fail to execute any guest
instructions and will instead report a VM_EXITCODE_DEBUG exit to
the userland hypervisor.  Virtual CPUs are placed into the debugged
state via vm_suspend_cpu() (implemented via a new VM_SUSPEND_CPU ioctl).
Virtual CPUs can be resumed via vm_resume_cpu() (VM_RESUME_CPU ioctl).

The debug server suspends virtual CPUs when it wishes them to stop
executing in the guest (for example, when a debugger attaches to the
server).  The debug server can choose to resume only a subset of CPUs
(for example, when single stepping) or it can choose to resume all
CPUs.  The debug server must explicitly mark a CPU as resumed via
vm_resume_cpu() before the virtual CPU will successfully execute any
guest instructions.

Reviewed by:	avg, grehan
Tested on:	Intel (jhb), AMD (avg)
Differential Revision:	https://reviews.freebsd.org/D14466
This commit is contained in:
jhb 2018-04-06 22:03:43 +00:00
parent 7a2353df98
commit a8d4adc969
8 changed files with 144 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
lib/libvmmapi/vmmapi.c
View File

@ -1373,6 +1373,13 @@ vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus)
return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus));
}
int
vm_debug_cpus(struct vmctx *ctx, cpuset_t *cpus)
{
return (vm_get_cpus(ctx, VM_DEBUG_CPUS, cpus));
}
int
vm_activate_cpu(struct vmctx *ctx, int vcpu)
{
@ -1385,6 +1392,30 @@ vm_activate_cpu(struct vmctx *ctx, int vcpu)
return (error);
}
int
vm_suspend_cpu(struct vmctx *ctx, int vcpu)
{
struct vm_activate_cpu ac;
int error;
bzero(&ac, sizeof(struct vm_activate_cpu));
ac.vcpuid = vcpu;
error = ioctl(ctx->fd, VM_SUSPEND_CPU, &ac);
return (error);
}
int
vm_resume_cpu(struct vmctx *ctx, int vcpu)
{
struct vm_activate_cpu ac;
int error;
bzero(&ac, sizeof(struct vm_activate_cpu));
ac.vcpuid = vcpu;
error = ioctl(ctx->fd, VM_RESUME_CPU, &ac);
return (error);
}
int
vm_get_intinfo(struct vmctx *ctx, int vcpu, uint64_t *info1, uint64_t *info2)
{
@ -1501,7 +1532,8 @@ vm_get_ioctls(size_t *len)
VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE,
VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA,
VM_GLA2GPA_NOFAULT,
VM_ACTIVATE_CPU, VM_GET_CPUS, VM_SET_INTINFO, VM_GET_INTINFO,
VM_ACTIVATE_CPU, VM_GET_CPUS, VM_SUSPEND_CPU, VM_RESUME_CPU,
VM_SET_INTINFO, VM_GET_INTINFO,
VM_RTC_WRITE, VM_RTC_READ, VM_RTC_SETTIME, VM_RTC_GETTIME,
VM_RESTART_INSTRUCTION };

3
lib/libvmmapi/vmmapi.h
View File

@ -216,7 +216,10 @@ int vcpu_reset(struct vmctx *ctx, int vcpu);
int vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus);
int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus);
int vm_debug_cpus(struct vmctx *ctx, cpuset_t *cpus);
int vm_activate_cpu(struct vmctx *ctx, int vcpu);
int vm_suspend_cpu(struct vmctx *ctx, int vcpu);
int vm_resume_cpu(struct vmctx *ctx, int vcpu);
/*
* FreeBSD specific APIs

7
sys/amd64/include/vmm.h
View File

@ -231,8 +231,11 @@ int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state);
int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state);
int vm_apicid2vcpuid(struct vm *vm, int apicid);
int vm_activate_cpu(struct vm *vm, int vcpu);
int vm_suspend_cpu(struct vm *vm, int vcpu);
int vm_resume_cpu(struct vm *vm, int vcpu);
struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip);
void vm_exit_debug(struct vm *vm, int vcpuid, uint64_t rip);
void vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip);
void vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip);
void vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip);
@ -256,6 +259,7 @@ typedef void (*vm_rendezvous_func_t)(struct vm *vm, int vcpuid, void *arg);
void vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
vm_rendezvous_func_t func, void *arg);
cpuset_t vm_active_cpus(struct vm *vm);
cpuset_t vm_debug_cpus(struct vm *vm);
cpuset_t vm_suspended_cpus(struct vm *vm);
#endif /* _SYS__CPUSET_H_ */
@ -280,6 +284,8 @@ vcpu_reqidle(struct vm_eventinfo *info)
return (*info->iptr);
}
int vcpu_debugged(struct vm *vm, int vcpuid);
/*
* Return 1 if device indicated by bus/slot/func is supposed to be a
* pci passthrough device.
@ -540,6 +546,7 @@ enum vm_exitcode {
VM_EXITCODE_MWAIT,
VM_EXITCODE_SVM,
VM_EXITCODE_REQIDLE,
VM_EXITCODE_DEBUG,
VM_EXITCODE_MAX
};

7
sys/amd64/include/vmm_dev.h
View File

@ -209,6 +209,7 @@ struct vm_cpuset {
};
#define VM_ACTIVE_CPUS 0
#define VM_SUSPENDED_CPUS 1
#define VM_DEBUG_CPUS 2
struct vm_intinfo {
int vcpuid;
@ -292,6 +293,8 @@ enum {
/* vm_cpuset */
IOCNUM_ACTIVATE_CPU = 90,
IOCNUM_GET_CPUSET = 91,
IOCNUM_SUSPEND_CPU = 92,
IOCNUM_RESUME_CPU = 93,
/* RTC */
IOCNUM_RTC_READ = 100,
@ -386,6 +389,10 @@ enum {
_IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu)
#define VM_GET_CPUS \
_IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset)
#define VM_SUSPEND_CPU \
_IOW('v', IOCNUM_SUSPEND_CPU, struct vm_activate_cpu)
#define VM_RESUME_CPU \
_IOW('v', IOCNUM_RESUME_CPU, struct vm_activate_cpu)
#define VM_SET_INTINFO \
_IOW('v', IOCNUM_SET_INTINFO, struct vm_intinfo)
#define VM_GET_INTINFO \

6
sys/amd64/vmm/amd/svm.c
View File

@ -2015,6 +2015,12 @@ svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap,
break;
}
if (vcpu_debugged(vm, vcpu)) {
enable_gintr();
vm_exit_debug(vm, vcpu, state->rip);
break;
}
svm_inj_interrupts(svm_sc, vcpu, vlapic);
/* Activate the nested pmap on 'curcpu' */

6
sys/amd64/vmm/intel/vmx.c
View File

@ -2746,6 +2746,12 @@ vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
break;
}
if (vcpu_debugged(vm, vcpu)) {
enable_intr();
vm_exit_debug(vmx->vm, vcpu, rip);
break;
}
vmx_run_trace(vmx, vcpu);
vmx_dr_enter_guest(vmxctx);
rc = vmx_enter_guest(vmxctx, vmx, launched);

72
sys/amd64/vmm/vmm.c
View File

@ -152,6 +152,7 @@ struct vm {
struct vpmtmr *vpmtmr; /* (i) virtual ACPI PM timer */
struct vrtc *vrtc; /* (o) virtual RTC */
volatile cpuset_t active_cpus; /* (i) active vcpus */
volatile cpuset_t debug_cpus; /* (i) vcpus stopped for debug */
int suspend; /* (i) stop VM execution */
volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */
volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */
@ -415,6 +416,7 @@ vm_init(struct vm *vm, bool create)
vm->vrtc = vrtc_init(vm);
CPU_ZERO(&vm->active_cpus);
CPU_ZERO(&vm->debug_cpus);
vm->suspend = 0;
CPU_ZERO(&vm->suspended_cpus);
@ -1283,6 +1285,9 @@ vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
if (vcpu_should_yield(vm, vcpuid))
break;
if (vcpu_debugged(vm, vcpuid))
break;
/*
* Some Linux guests implement "halt" by having all vcpus
* execute HLT with interrupts disabled. 'halted_cpus' keeps
@ -1553,6 +1558,17 @@ vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
vmexit->u.suspended.how = vm->suspend;
}
void
vm_exit_debug(struct vm *vm, int vcpuid, uint64_t rip)
{
struct vm_exit *vmexit;
vmexit = vm_exitinfo(vm, vcpuid);
vmexit->rip = rip;
vmexit->inst_length = 0;
vmexit->exitcode = VM_EXITCODE_DEBUG;
}
void
vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
{
@ -2267,6 +2283,55 @@ vm_activate_cpu(struct vm *vm, int vcpuid)
return (0);
}
int
vm_suspend_cpu(struct vm *vm, int vcpuid)
{
int i;
if (vcpuid < -1 || vcpuid >= VM_MAXCPU)
return (EINVAL);
if (vcpuid == -1) {
vm->debug_cpus = vm->active_cpus;
for (i = 0; i < VM_MAXCPU; i++) {
if (CPU_ISSET(i, &vm->active_cpus))
vcpu_notify_event(vm, i, false);
}
} else {
if (!CPU_ISSET(vcpuid, &vm->active_cpus))
return (EINVAL);
CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus);
vcpu_notify_event(vm, vcpuid, false);
}
return (0);
}
int
vm_resume_cpu(struct vm *vm, int vcpuid)
{
if (vcpuid < -1 || vcpuid >= VM_MAXCPU)
return (EINVAL);
if (vcpuid == -1) {
CPU_ZERO(&vm->debug_cpus);
} else {
if (!CPU_ISSET(vcpuid, &vm->debug_cpus))
return (EINVAL);
CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus);
}
return (0);
}
int
vcpu_debugged(struct vm *vm, int vcpuid)
{
return (CPU_ISSET(vcpuid, &vm->debug_cpus));
}
cpuset_t
vm_active_cpus(struct vm *vm)
{
@ -2274,6 +2339,13 @@ vm_active_cpus(struct vm *vm)
return (vm->active_cpus);
}
cpuset_t
vm_debug_cpus(struct vm *vm)
{
return (vm->debug_cpus);
}
cpuset_t
vm_suspended_cpus(struct vm *vm)
{

10
sys/amd64/vmm/vmm_dev.c
View File

@ -690,12 +690,22 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
*cpuset = vm_active_cpus(sc->vm);
else if (vm_cpuset->which == VM_SUSPENDED_CPUS)
*cpuset = vm_suspended_cpus(sc->vm);
else if (vm_cpuset->which == VM_DEBUG_CPUS)
*cpuset = vm_debug_cpus(sc->vm);
else
error = EINVAL;
if (error == 0)
error = copyout(cpuset, vm_cpuset->cpus, size);
free(cpuset, M_TEMP);
break;
case VM_SUSPEND_CPU:
vac = (struct vm_activate_cpu *)data;
error = vm_suspend_cpu(sc->vm, vac->vcpuid);
break;
case VM_RESUME_CPU:
vac = (struct vm_activate_cpu *)data;
error = vm_resume_cpu(sc->vm, vac->vcpuid);
break;
case VM_SET_INTINFO:
vmii = (struct vm_intinfo *)data;
error = vm_exit_intinfo(sc->vm, vmii->vcpuid, vmii->info1);