Add an option "-a" to present the local apic in the XAPIC mode instead of the

default X2APIC mode to the guest.

sys/amd64/vmm/io/vlapic.c

@@ -823,14 +823,8 @@ static struct io_region vlapic_mmio[VM_MAXCPU];
 struct vlapic *
 vlapic_init(struct vm *vm, int vcpuid)
 {
-	int err;
-	enum x2apic_state state;
 	struct vlapic 		*vlapic;
 
-	err = vm_get_x2apic_state(vm, vcpuid, &state);
-	if (err)
-		panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
-
 	vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
 	vlapic->vm = vm;
 	vlapic->vcpuid = vcpuid;
@@ -840,9 +834,6 @@ vlapic_init(struct vm *vm, int vcpuid)
 	if (vcpuid == 0)
 		vlapic->msr_apicbase |= APICBASE_BSP;
 
-	if (state == X2APIC_ENABLED)
-		vlapic->msr_apicbase |= APICBASE_X2APIC;
-
 	vlapic->ops = &vlapic_dev_ops;
 
 	vlapic->mmio = vlapic_mmio + vcpuid;
@@ -888,3 +879,16 @@ vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
 
 	vlapic->msr_apicbase = val;
 }
+
+void
+vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
+{
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, vcpuid);
+
+	if (state == X2APIC_ENABLED)
+		vlapic->msr_apicbase |= APICBASE_X2APIC;
+	else
+		vlapic->msr_apicbase &= ~APICBASE_X2APIC;
+}

sys/amd64/vmm/io/vlapic.h

@@ -88,6 +88,8 @@ struct vm;
  */
 #define	ISRVEC_STK_SIZE		(16 + 1)
 
+enum x2apic_state;
+
 struct vlapic *vlapic_init(struct vm *vm, int vcpuid);
 void vlapic_cleanup(struct vlapic *vlapic);
 
@@ -104,5 +106,6 @@ void vlapic_timer_tick(struct vlapic *vlapic);
 
 uint64_t vlapic_get_apicbase(struct vlapic *vlapic);
 void vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val);
+void vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state s);
 
 #endif	/* _VLAPIC_H_ */

sys/amd64/vmm/vmm.c

@@ -160,8 +160,8 @@ vcpu_init(struct vm *vm, uint32_t vcpu_id)
 
 	vcpu->hostcpu = -1;
 	vcpu->vcpuid = vcpu_id;
-	vcpu->x2apic_state = X2APIC_ENABLED;
 	vcpu->vlapic = vlapic_init(vm, vcpu_id);
+	vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED);
 	vcpu->guestfpu = fpu_save_area_alloc();
 	fpu_save_area_reset(vcpu->guestfpu);
 	vcpu->stats = vmm_stat_alloc();
@@ -770,5 +770,7 @@ vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
 
 	vm->vcpu[vcpuid].x2apic_state = state;
 
+	vlapic_set_x2apic_state(vm, vcpuid, state);
+
 	return (0);
 }

usr.sbin/bhyve/fbsdrun.c

@@ -84,7 +84,7 @@ int guest_ncpus;
 
 static int pincpu = -1;
 static int guest_vcpu_mux;
-static int guest_vmexit_on_hlt, guest_vmexit_on_pause;
+static int guest_vmexit_on_hlt, guest_vmexit_on_pause, disable_x2apic;
 
 static int foundcpus;
 
@@ -128,8 +128,9 @@ usage(int code)
 {
 
         fprintf(stderr,
-                "Usage: %s [-ehBHIP][-g <gdb port>][-z <hz>][-s <pci>]"
+                "Usage: %s [-aehBHIP][-g <gdb port>][-z <hz>][-s <pci>]"
 		"[-S <pci>][-p pincpu][-n <pci>][-m lowmem][-M highmem] <vm>\n"
+		"       -a: local apic is in XAPIC mode (default is X2APIC)\n"
 		"       -g: gdb port (default is %d and 0 means don't open)\n"
 		"       -c: # cpus (default 1)\n"
 		"       -p: pin vcpu 'n' to host cpu 'pincpu + n'\n"
@@ -173,6 +174,13 @@ fbsdrun_add_oemtbl(void *tbl, int tblsz)
 	oem_tbl_size = tblsz;
 }
 
+int
+fbsdrun_disable_x2apic(void)
+{
+
+	return (disable_x2apic);
+}
+
 int
 fbsdrun_vmexit_on_pause(void)
 {
@@ -553,8 +561,11 @@ main(int argc, char *argv[])
 	guest_ncpus = 1;
 	ioapic = 0;
 
-	while ((c = getopt(argc, argv, "ehBHIPxp:g:c:z:s:S:n:m:M:")) != -1) {
+	while ((c = getopt(argc, argv, "aehBHIPxp:g:c:z:s:S:n:m:M:")) != -1) {
 		switch (c) {
+		case 'a':
+			disable_x2apic = 1;
+			break;
 		case 'B':
 			inject_bkpt = 1;
 			break;
@@ -656,6 +667,16 @@ main(int argc, char *argv[])
 		handler[VM_EXITCODE_PAUSE] = vmexit_pause;
         }
 
+	if (fbsdrun_disable_x2apic())
+		err = vm_set_x2apic_state(ctx, BSP, X2APIC_DISABLED);
+	else
+		err = vm_set_x2apic_state(ctx, BSP, X2APIC_ENABLED);
+
+	if (err) {
+		printf("Unable to set x2apic state (%d)\n", err);
+		exit(1);
+	}
+
 	if (lomem_sz != 0) {
 		lomem_addr = vm_map_memory(ctx, 0, lomem_sz);
 		if (lomem_addr == (char *) MAP_FAILED) {

usr.sbin/bhyve/fbsdrun.h

@@ -50,4 +50,5 @@ void fbsdrun_add_oemtbl(void *tbl, int tblsz);
 int  fbsdrun_muxed(void);
 int  fbsdrun_vmexit_on_hlt(void);
 int  fbsdrun_vmexit_on_pause(void);
+int  fbsdrun_disable_x2apic(void);
 #endif

usr.sbin/bhyve/spinup_ap.c

@@ -156,6 +156,12 @@ spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
 		assert(error == 0);
 	}
 
+	if (fbsdrun_disable_x2apic())
+		error = vm_set_x2apic_state(ctx, newcpu, X2APIC_DISABLED);
+	else
+		error = vm_set_x2apic_state(ctx, newcpu, X2APIC_ENABLED);
+	assert(error == 0);
+
 	/*
 	 * There are 2 startup modes possible here:
 	 *  - if the CPU supports 'unrestricted guest' mode, the spinup can