Activate vcpus from bhyve(8) using the ioctl VM_ACTIVATE_CPU instead of doing
it implicitly in vmm.ko. Add ioctl VM_GET_CPUS to get the current set of 'active' and 'suspended' cpus and display them via /usr/sbin/bhyvectl using the "--get-active-cpus" and "--get-suspended-cpus" options. This is in preparation for being able to reset virtual machine state without having to destroy and recreate it.
This commit is contained in:
parent
2ceda70233
commit
95ebc360ef
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=266933
@ -29,11 +29,12 @@
|
||||
#include <sys/cdefs.h>
|
||||
__FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/param.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/_iovec.h>
|
||||
#include <sys/cpuset.h>
|
||||
|
||||
#include <machine/specialreg.h>
|
||||
#include <machine/param.h>
|
||||
@ -1043,3 +1044,44 @@ vm_copyout(struct vmctx *ctx, int vcpu, const void *vp, struct iovec *iov,
|
||||
len -= n;
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
vm_get_cpus(struct vmctx *ctx, int which, cpuset_t *cpus)
|
||||
{
|
||||
struct vm_cpuset vm_cpuset;
|
||||
int error;
|
||||
|
||||
bzero(&vm_cpuset, sizeof(struct vm_cpuset));
|
||||
vm_cpuset.which = which;
|
||||
vm_cpuset.cpusetsize = sizeof(cpuset_t);
|
||||
vm_cpuset.cpus = cpus;
|
||||
|
||||
error = ioctl(ctx->fd, VM_GET_CPUS, &vm_cpuset);
|
||||
return (error);
|
||||
}
|
||||
|
||||
int
|
||||
vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus)
|
||||
{
|
||||
|
||||
return (vm_get_cpus(ctx, VM_ACTIVE_CPUS, cpus));
|
||||
}
|
||||
|
||||
int
|
||||
vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus)
|
||||
{
|
||||
|
||||
return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus));
|
||||
}
|
||||
|
||||
int
|
||||
vm_activate_cpu(struct vmctx *ctx, int vcpu)
|
||||
{
|
||||
struct vm_activate_cpu ac;
|
||||
int error;
|
||||
|
||||
bzero(&ac, sizeof(struct vm_activate_cpu));
|
||||
ac.vcpuid = vcpu;
|
||||
error = ioctl(ctx->fd, VM_ACTIVATE_CPU, &ac);
|
||||
return (error);
|
||||
}
|
||||
|
@ -29,6 +29,9 @@
|
||||
#ifndef _VMMAPI_H_
|
||||
#define _VMMAPI_H_
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/cpuset.h>
|
||||
|
||||
struct iovec;
|
||||
struct vmctx;
|
||||
enum x2apic_state;
|
||||
@ -125,6 +128,10 @@ void vm_copyout(struct vmctx *ctx, int vcpu, const void *host_src,
|
||||
/* Reset vcpu register state */
|
||||
int vcpu_reset(struct vmctx *ctx, int vcpu);
|
||||
|
||||
int vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus);
|
||||
int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus);
|
||||
int vm_activate_cpu(struct vmctx *ctx, int vcpu);
|
||||
|
||||
/*
|
||||
* FreeBSD specific APIs
|
||||
*/
|
||||
|
@ -140,8 +140,9 @@ int vm_set_capability(struct vm *vm, int vcpu, int type, int val);
|
||||
int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state);
|
||||
int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state);
|
||||
int vm_apicid2vcpuid(struct vm *vm, int apicid);
|
||||
void vm_activate_cpu(struct vm *vm, int vcpu);
|
||||
int vm_activate_cpu(struct vm *vm, int vcpu);
|
||||
cpuset_t vm_active_cpus(struct vm *vm);
|
||||
cpuset_t vm_suspended_cpus(struct vm *vm);
|
||||
struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
|
||||
void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip);
|
||||
|
||||
|
@ -177,6 +177,18 @@ struct vm_gla2gpa {
|
||||
uint64_t gpa;
|
||||
};
|
||||
|
||||
struct vm_activate_cpu {
|
||||
int vcpuid;
|
||||
};
|
||||
|
||||
struct vm_cpuset {
|
||||
int which;
|
||||
int cpusetsize;
|
||||
cpuset_t *cpus;
|
||||
};
|
||||
#define VM_ACTIVE_CPUS 0
|
||||
#define VM_SUSPENDED_CPUS 1
|
||||
|
||||
enum {
|
||||
/* general routines */
|
||||
IOCNUM_ABIVERS = 0,
|
||||
@ -229,6 +241,10 @@ enum {
|
||||
IOCNUM_ISA_DEASSERT_IRQ = 81,
|
||||
IOCNUM_ISA_PULSE_IRQ = 82,
|
||||
IOCNUM_ISA_SET_IRQ_TRIGGER = 83,
|
||||
|
||||
/* vm_cpuset */
|
||||
IOCNUM_ACTIVATE_CPU = 90,
|
||||
IOCNUM_GET_CPUSET = 91,
|
||||
};
|
||||
|
||||
#define VM_RUN \
|
||||
@ -301,4 +317,8 @@ enum {
|
||||
_IOWR('v', IOCNUM_GET_GPA_PMAP, struct vm_gpa_pte)
|
||||
#define VM_GLA2GPA \
|
||||
_IOWR('v', IOCNUM_GLA2GPA, struct vm_gla2gpa)
|
||||
#define VM_ACTIVATE_CPU \
|
||||
_IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu)
|
||||
#define VM_GET_CPUS \
|
||||
_IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset)
|
||||
#endif
|
||||
|
@ -1004,11 +1004,7 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
|
||||
if (vlapic2->boot_state != BS_SIPI)
|
||||
return (0);
|
||||
|
||||
/*
|
||||
* XXX this assumes that the startup IPI always succeeds
|
||||
*/
|
||||
vlapic2->boot_state = BS_RUNNING;
|
||||
vm_activate_cpu(vlapic2->vm, dest);
|
||||
|
||||
*retu = true;
|
||||
vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
|
||||
|
@ -342,8 +342,6 @@ vm_create(const char *name, struct vm **retvm)
|
||||
struct vm *vm;
|
||||
struct vmspace *vmspace;
|
||||
|
||||
const int BSP = 0;
|
||||
|
||||
/*
|
||||
* If vmm.ko could not be successfully initialized then don't attempt
|
||||
* to create the virtual machine.
|
||||
@ -373,8 +371,6 @@ vm_create(const char *name, struct vm **retvm)
|
||||
guest_msrs_init(vm, i);
|
||||
}
|
||||
|
||||
vm_activate_cpu(vm, BSP);
|
||||
|
||||
*retvm = vm;
|
||||
return (0);
|
||||
}
|
||||
@ -1294,6 +1290,12 @@ vm_run(struct vm *vm, struct vm_run *vmrun)
|
||||
if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
|
||||
return (EINVAL);
|
||||
|
||||
if (!CPU_ISSET(vcpuid, &vm->active_cpus))
|
||||
return (EINVAL);
|
||||
|
||||
if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
|
||||
return (EINVAL);
|
||||
|
||||
rptr = &vm->rendezvous_func;
|
||||
sptr = &vm->suspend;
|
||||
pmap = vmspace_pmap(vm->vmspace);
|
||||
@ -1708,17 +1710,19 @@ vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
|
||||
return (state);
|
||||
}
|
||||
|
||||
void
|
||||
int
|
||||
vm_activate_cpu(struct vm *vm, int vcpuid)
|
||||
{
|
||||
|
||||
KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU,
|
||||
("vm_activate_cpu: invalid vcpuid %d", vcpuid));
|
||||
KASSERT(!CPU_ISSET(vcpuid, &vm->active_cpus),
|
||||
("vm_activate_cpu: vcpuid %d is already active", vcpuid));
|
||||
if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
|
||||
return (EINVAL);
|
||||
|
||||
if (CPU_ISSET(vcpuid, &vm->active_cpus))
|
||||
return (EBUSY);
|
||||
|
||||
VCPU_CTR0(vm, vcpuid, "activated");
|
||||
CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
|
||||
return (0);
|
||||
}
|
||||
|
||||
cpuset_t
|
||||
@ -1728,6 +1732,13 @@ vm_active_cpus(struct vm *vm)
|
||||
return (vm->active_cpus);
|
||||
}
|
||||
|
||||
cpuset_t
|
||||
vm_suspended_cpus(struct vm *vm)
|
||||
{
|
||||
|
||||
return (vm->suspended_cpus);
|
||||
}
|
||||
|
||||
void *
|
||||
vcpu_stats(struct vm *vm, int vcpuid)
|
||||
{
|
||||
|
@ -146,7 +146,8 @@ static int
|
||||
vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
|
||||
struct thread *td)
|
||||
{
|
||||
int error, vcpu, state_changed;
|
||||
int error, vcpu, state_changed, size;
|
||||
cpuset_t *cpuset;
|
||||
struct vmmdev_softc *sc;
|
||||
struct vm_memory_segment *seg;
|
||||
struct vm_register *vmreg;
|
||||
@ -170,6 +171,8 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
|
||||
struct vm_gpa_pte *gpapte;
|
||||
struct vm_suspend *vmsuspend;
|
||||
struct vm_gla2gpa *gg;
|
||||
struct vm_activate_cpu *vac;
|
||||
struct vm_cpuset *vm_cpuset;
|
||||
|
||||
sc = vmmdev_lookup2(cdev);
|
||||
if (sc == NULL)
|
||||
@ -195,6 +198,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
|
||||
case VM_PPTDEV_MSIX:
|
||||
case VM_SET_X2APIC_STATE:
|
||||
case VM_GLA2GPA:
|
||||
case VM_ACTIVATE_CPU:
|
||||
/*
|
||||
* XXX fragile, handle with care
|
||||
* Assumes that the first field of the ioctl data is the vcpu.
|
||||
@ -439,6 +443,29 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
|
||||
}
|
||||
break;
|
||||
}
|
||||
case VM_ACTIVATE_CPU:
|
||||
vac = (struct vm_activate_cpu *)data;
|
||||
error = vm_activate_cpu(sc->vm, vac->vcpuid);
|
||||
break;
|
||||
case VM_GET_CPUS:
|
||||
error = 0;
|
||||
vm_cpuset = (struct vm_cpuset *)data;
|
||||
size = vm_cpuset->cpusetsize;
|
||||
if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) {
|
||||
error = ERANGE;
|
||||
break;
|
||||
}
|
||||
cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO);
|
||||
if (vm_cpuset->which == VM_ACTIVE_CPUS)
|
||||
*cpuset = vm_active_cpus(sc->vm);
|
||||
else if (vm_cpuset->which == VM_SUSPENDED_CPUS)
|
||||
*cpuset = vm_suspended_cpus(sc->vm);
|
||||
else
|
||||
error = EINVAL;
|
||||
if (error == 0)
|
||||
error = copyout(cpuset, vm_cpuset->cpus, size);
|
||||
free(cpuset, M_TEMP);
|
||||
break;
|
||||
default:
|
||||
error = ENOTTY;
|
||||
break;
|
||||
|
@ -242,6 +242,15 @@ fbsdrun_addcpu(struct vmctx *ctx, int fromcpu, int newcpu, uint64_t rip)
|
||||
|
||||
assert(fromcpu == BSP);
|
||||
|
||||
/*
|
||||
* The 'newcpu' must be activated in the context of 'fromcpu'. If
|
||||
* vm_activate_cpu() is delayed until newcpu's pthread starts running
|
||||
* then vmm.ko is out-of-sync with bhyve and this can create a race
|
||||
* with vm_suspend().
|
||||
*/
|
||||
error = vm_activate_cpu(ctx, newcpu);
|
||||
assert(error == 0);
|
||||
|
||||
CPU_SET_ATOMIC(newcpu, &cpumask);
|
||||
|
||||
/*
|
||||
@ -532,6 +541,7 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
|
||||
int error, rc, prevcpu;
|
||||
enum vm_exitcode exitcode;
|
||||
enum vm_suspend_how how;
|
||||
cpuset_t active_cpus;
|
||||
|
||||
if (vcpumap[vcpu] != NULL) {
|
||||
error = pthread_setaffinity_np(pthread_self(),
|
||||
@ -539,6 +549,9 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
|
||||
assert(error == 0);
|
||||
}
|
||||
|
||||
error = vm_active_cpus(ctx, &active_cpus);
|
||||
assert(CPU_ISSET(vcpu, &active_cpus));
|
||||
|
||||
while (1) {
|
||||
error = vm_run(ctx, vcpu, rip, &vmexit[vcpu]);
|
||||
if (error != 0)
|
||||
|
@ -32,7 +32,6 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <machine/vmm.h>
|
||||
#include <machine/vmm_dev.h>
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
@ -193,7 +193,9 @@ usage(void)
|
||||
" [--assert-lapic-lvt=<pin>]\n"
|
||||
" [--inject-nmi]\n"
|
||||
" [--force-reset]\n"
|
||||
" [--force-poweroff]\n",
|
||||
" [--force-poweroff]\n"
|
||||
" [--get-active-cpus]\n"
|
||||
" [--get-suspended-cpus]\n",
|
||||
progname);
|
||||
exit(1);
|
||||
}
|
||||
@ -203,6 +205,7 @@ static int inject_nmi, assert_lapic_lvt;
|
||||
static int force_reset, force_poweroff;
|
||||
static const char *capname;
|
||||
static int create, destroy, get_lowmem, get_highmem;
|
||||
static int get_active_cpus, get_suspended_cpus;
|
||||
static uint64_t memsize;
|
||||
static int set_cr0, get_cr0, set_cr3, get_cr3, set_cr4, get_cr4;
|
||||
static int set_efer, get_efer;
|
||||
@ -390,6 +393,25 @@ enum {
|
||||
ASSERT_LAPIC_LVT,
|
||||
};
|
||||
|
||||
static void
|
||||
print_cpus(const char *banner, const cpuset_t *cpus)
|
||||
{
|
||||
int i, first;
|
||||
|
||||
first = 1;
|
||||
printf("%s:\t", banner);
|
||||
if (!CPU_EMPTY(cpus)) {
|
||||
for (i = 0; i < CPU_SETSIZE; i++) {
|
||||
if (CPU_ISSET(i, cpus)) {
|
||||
printf("%s%d", first ? " " : ", ", i);
|
||||
first = 0;
|
||||
}
|
||||
}
|
||||
} else
|
||||
printf(" (none)");
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char *argv[])
|
||||
{
|
||||
@ -401,6 +423,7 @@ main(int argc, char *argv[])
|
||||
uint64_t ctl, eptp, bm, addr, u64, pteval[4], *pte;
|
||||
struct vmctx *ctx;
|
||||
int wired;
|
||||
cpuset_t cpus;
|
||||
|
||||
uint64_t cr0, cr3, cr4, dr7, rsp, rip, rflags, efer, pat;
|
||||
uint64_t rax, rbx, rcx, rdx, rsi, rdi, rbp;
|
||||
@ -570,6 +593,8 @@ main(int argc, char *argv[])
|
||||
{ "inject-nmi", NO_ARG, &inject_nmi, 1 },
|
||||
{ "force-reset", NO_ARG, &force_reset, 1 },
|
||||
{ "force-poweroff", NO_ARG, &force_poweroff, 1 },
|
||||
{ "get-active-cpus", NO_ARG, &get_active_cpus, 1 },
|
||||
{ "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 },
|
||||
{ NULL, 0, NULL, 0 }
|
||||
};
|
||||
|
||||
@ -1529,6 +1554,18 @@ main(int argc, char *argv[])
|
||||
}
|
||||
}
|
||||
|
||||
if (!error && (get_active_cpus || get_all)) {
|
||||
error = vm_active_cpus(ctx, &cpus);
|
||||
if (!error)
|
||||
print_cpus("active cpus", &cpus);
|
||||
}
|
||||
|
||||
if (!error && (get_suspended_cpus || get_all)) {
|
||||
error = vm_suspended_cpus(ctx, &cpus);
|
||||
if (!error)
|
||||
print_cpus("suspended cpus", &cpus);
|
||||
}
|
||||
|
||||
if (!error && run) {
|
||||
error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip);
|
||||
assert(error == 0);
|
||||
|
Loading…
Reference in New Issue
Block a user