xen: remove direct usage of HYPERVISOR_start_info

HYPERVISOR_start_info is only available to PV and PVHv1 guests, HVM
and PVHv2 guests get this data from HVM parameters that are fetched
using a hypercall.

Instead provide a set of helper functions that should be used to fetch
this data. The helper functions have different implementations
depending on whether FreeBSD is running as PVHv1 or HVM/PVHv2 guest
type.

This helps to cleanup generic Xen code by removing quite a lot of
xen_pv_domain and xen_hvm_domain macro usages.

Sponsored by:	Citrix Systems R&D

sys/dev/xen/balloon/balloon.c

@@ -382,8 +382,7 @@ xenballoon_attach(device_t dev)
 
 	mtx_init(&balloon_mutex, "balloon_mutex", NULL, MTX_DEF);
 
-	bs.current_pages = xen_pv_domain() ?
-	    HYPERVISOR_start_info->nr_pages : realmem;
+	bs.current_pages = realmem;
 	bs.target_pages  = bs.current_pages;
 	bs.balloon_low   = 0;
 	bs.balloon_high  = 0;

sys/dev/xen/console/xen_console.c

@@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/stdarg.h>
 
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
 #include <xen/xen_intr.h>
@@ -129,12 +132,6 @@ static struct xencons_priv main_cons;
 
 #define XC_POLLTIME 	(hz/10)
 
-/*
- * Virtual address of the shared console page (only for PV guest)
- * TODO: Introduce a function to set it
- */
-char *console_page;
-
 /*----------------------------- Debug function ------------------------------*/
 struct putchar_arg {
 	char	*buf;
@@ -273,9 +270,9 @@ static const struct xencons_ops xencons_hypervisor_ops = {
 static void
 xencons_early_init_ring(struct xencons_priv *cons)
 {
-	/* The shared page for PV is already mapped by the boot code */
-	cons->intf = (struct xencons_interface *)console_page;
-	cons->evtchn = HYPERVISOR_start_info->console.domU.evtchn;
+	cons->intf = pmap_mapdev_attr(ptoa(xen_get_console_mfn()), PAGE_SIZE,
+	    PAT_WRITE_BACK);
+	cons->evtchn = xen_get_console_evtchn();
 }
 
 static int

sys/dev/xen/xenstore/xenstore.c

@@ -115,7 +115,7 @@ MALLOC_DEFINE(M_XENSTORE, "xenstore", "XenStore data and results");
  * to get the guest frame number for the shared page and then map it
  * into kva.  See xs_init() for details.
  */
-struct xenstore_domain_interface *xen_store;
+static struct xenstore_domain_interface *xen_store;
 
 /*-------------------------- Private Data Structures ------------------------*/
 
@@ -1103,38 +1103,30 @@ xs_attach(device_t dev)
 	struct proc *p;
 
 	xs.initialized = false;
-	if (xen_hvm_domain()) {
-		xs.evtchn = hvm_get_parameter(HVM_PARAM_STORE_EVTCHN);
-		xs.gpfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
-		xen_store = pmap_mapdev(xs.gpfn * PAGE_SIZE, PAGE_SIZE);
-		xs.initialized = true;
-	} else if (xen_pv_domain()) {
-		if (HYPERVISOR_start_info->store_evtchn == 0) {
-			struct evtchn_alloc_unbound alloc_unbound;
+	xs.evtchn = xen_get_xenstore_evtchn();
+	if (xs.evtchn == 0) {
+		struct evtchn_alloc_unbound alloc_unbound;
 
-			/* Allocate a local event channel for xenstore */
-			alloc_unbound.dom = DOMID_SELF;
-			alloc_unbound.remote_dom = DOMID_SELF;
-			error = HYPERVISOR_event_channel_op(
-			    EVTCHNOP_alloc_unbound, &alloc_unbound);
-			if (error != 0)
-				panic(
-				   "unable to alloc event channel for Dom0: %d",
-				    error);
+		/* Allocate a local event channel for xenstore */
+		alloc_unbound.dom = DOMID_SELF;
+		alloc_unbound.remote_dom = DOMID_SELF;
+		error = HYPERVISOR_event_channel_op(
+		    EVTCHNOP_alloc_unbound, &alloc_unbound);
+		if (error != 0)
+			panic(
+			   "unable to alloc event channel for Dom0: %d",
+			    error);
 
-			HYPERVISOR_start_info->store_evtchn =
-			    alloc_unbound.port;
-			xs.evtchn = alloc_unbound.port;
+		xs.evtchn = alloc_unbound.port;
 
-			/* Allocate memory for the xs shared ring */
-			xen_store = malloc(PAGE_SIZE, M_XENSTORE,
-			    M_WAITOK | M_ZERO);
-		} else {
-			xs.evtchn = HYPERVISOR_start_info->store_evtchn;
-			xs.initialized = true;
-		}
+		/* Allocate memory for the xs shared ring */
+		xen_store = malloc(PAGE_SIZE, M_XENSTORE, M_WAITOK | M_ZERO);
+		xs.gpfn = atop(pmap_kextract((vm_offset_t)xen_store));
 	} else {
-		panic("Unknown domain type, cannot initialize xenstore.");
+		xs.gpfn = xen_get_xenstore_mfn();
+		xen_store = pmap_mapdev_attr(ptoa(xs.gpfn), PAGE_SIZE,
+		    PAT_WRITE_BACK);
+		xs.initialized = true;
 	}
 
 	TAILQ_INIT(&xs.reply_list);
@@ -1255,6 +1247,27 @@ SYSCTL_ULONG(_dev_xen, OID_AUTO, xsd_kva, CTLFLAG_RD, (u_long *) &xen_store, 0,
 
 /*-------------------------------- Public API --------------------------------*/
 /*------- API comments for these methods can be found in xenstorevar.h -------*/
+bool
+xs_initialized(void)
+{
+
+	return (xs.initialized);
+}
+
+evtchn_port_t
+xs_evtchn(void)
+{
+
+    return (xs.evtchn);
+}
+
+vm_paddr_t
+xs_address(void)
+{
+
+    return (ptoa(xs.gpfn));
+}
+
 int
 xs_directory(struct xs_transaction t, const char *dir, const char *node,
     u_int *num, const char ***result)

sys/dev/xen/xenstore/xenstored_dev.c

@@ -43,7 +43,6 @@ __FBSDID("$FreeBSD$");
 
 #include <xen/hypervisor.h>
 #include <xen/xenstore/xenstorevar.h>
-#include <xen/xenstore/xenstore_internal.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
@@ -68,8 +67,7 @@ xsd_dev_read(struct cdev *dev, struct uio *uio, int ioflag)
 	char evtchn[XSD_READ_SIZE];
 	int error, len;
 
-	len = snprintf(evtchn, sizeof(evtchn), "%u",
-	    HYPERVISOR_start_info->store_evtchn);
+	len = snprintf(evtchn, sizeof(evtchn), "%u", xs_evtchn());
 	if (len < 0 || len > uio->uio_resid)
 		return (EINVAL);
 
@@ -88,7 +86,7 @@ xsd_dev_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr,
 	if (offset != 0)
 		return (EINVAL);
 
-	*paddr = pmap_kextract((vm_offset_t)xen_store);
+	*paddr = xs_address();
 
 	return (0);
 }
@@ -104,9 +102,7 @@ static void
 xsd_dev_identify(driver_t *driver __unused, device_t parent)
 {
 
-	if (!xen_pv_domain())
-		return;
-	if (HYPERVISOR_start_info->store_mfn != 0)
+	if (!xen_domain() || xs_initialized())
 		return;
 
 	/*

sys/x86/xen/hvm.c

@@ -94,8 +94,6 @@ DPCPU_DEFINE(struct vcpu_info *, vcpu_info);
 
 /*------------------ Hypervisor Access Shared Memory Regions -----------------*/
 shared_info_t *HYPERVISOR_shared_info;
-start_info_t *HYPERVISOR_start_info;
-
 
 /*------------------------------ Sysctl tunables -----------------------------*/
 int xen_disable_pv_disks = 0;
@@ -425,3 +423,47 @@ xen_hvm_cpu_init(void)
 		DPCPU_SET(vcpu_info, vcpu_info);
 }
 SYSINIT(xen_hvm_cpu_init, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_cpu_init, NULL);
+
+/* HVM/PVH start_info accessors */
+static vm_paddr_t
+hvm_get_xenstore_mfn(void)
+{
+
+	return (hvm_get_parameter(HVM_PARAM_STORE_PFN));
+}
+
+static evtchn_port_t
+hvm_get_xenstore_evtchn(void)
+{
+
+	return (hvm_get_parameter(HVM_PARAM_STORE_EVTCHN));
+}
+
+static vm_paddr_t
+hvm_get_console_mfn(void)
+{
+
+	return (hvm_get_parameter(HVM_PARAM_CONSOLE_PFN));
+}
+
+static evtchn_port_t
+hvm_get_console_evtchn(void)
+{
+
+	return (hvm_get_parameter(HVM_PARAM_CONSOLE_EVTCHN));
+}
+
+static uint32_t
+hvm_get_start_flags(void)
+{
+
+	return (0);
+}
+
+struct hypervisor_info hypervisor_info = {
+	.get_xenstore_mfn		= hvm_get_xenstore_mfn,
+	.get_xenstore_evtchn		= hvm_get_xenstore_evtchn,
+	.get_console_mfn		= hvm_get_console_mfn,
+	.get_console_evtchn		= hvm_get_console_evtchn,
+	.get_start_flags		= hvm_get_start_flags,
+};

sys/x86/xen/pv.c

@@ -125,9 +125,55 @@ struct init_ops xen_init_ops = {
 
 static struct bios_smap xen_smap[MAX_E820_ENTRIES];
 
+static start_info_t *legacy_start_info;
+
+/*----------------------- Legacy PVH start_info accessors --------------------*/
+static vm_paddr_t
+legacy_get_xenstore_mfn(void)
+{
+
+	return (legacy_start_info->store_mfn);
+}
+
+static evtchn_port_t
+legacy_get_xenstore_evtchn(void)
+{
+
+	return (legacy_start_info->store_evtchn);
+}
+
+static vm_paddr_t
+legacy_get_console_mfn(void)
+{
+
+	return (legacy_start_info->console.domU.mfn);
+}
+
+static evtchn_port_t
+legacy_get_console_evtchn(void)
+{
+
+	return (legacy_start_info->console.domU.evtchn);
+}
+
+static uint32_t
+legacy_get_start_flags(void)
+{
+
+	return (legacy_start_info->flags);
+}
+
+struct hypervisor_info legacy_info = {
+	.get_xenstore_mfn		= legacy_get_xenstore_mfn,
+	.get_xenstore_evtchn		= legacy_get_xenstore_evtchn,
+	.get_console_mfn		= legacy_get_console_mfn,
+	.get_console_evtchn		= legacy_get_console_evtchn,
+	.get_start_flags		= legacy_get_start_flags,
+};
+
 /*-------------------------------- Xen PV init -------------------------------*/
 /*
- * First function called by the Xen PVH boot sequence.
+ * First function called by the Xen legacy PVH boot sequence.
  *
  * Set some Xen global variables and prepare the environment so it is
  * as similar as possible to what native FreeBSD init function expects.
@@ -155,21 +201,10 @@ hammer_time_xen(start_info_t *si, uint64_t xenstack)
 	physfree = xenstack + 3 * PAGE_SIZE - KERNBASE;
 
 	/* Setup Xen global variables */
-	HYPERVISOR_start_info = si;
+	legacy_start_info = si;
 	HYPERVISOR_shared_info =
 	    (shared_info_t *)(si->shared_info + KERNBASE);
 
-	/*
-	 * Setup some misc global variables for Xen devices
-	 *
-	 * XXX: Devices that need these specific variables should
-	 *      be rewritten to fetch this info by themselves from the
-	 *      start_info page.
-	 */
-	xen_store = (struct xenstore_domain_interface *)
-	    (ptoa(si->store_mfn) + KERNBASE);
-	console_page = (char *)(ptoa(si->console.domU.mfn) + KERNBASE);
-
 	/*
 	 * Use the stack Xen gives us to build the page tables
 	 * as native FreeBSD expects to find them (created
@@ -202,6 +237,7 @@ hammer_time_xen(start_info_t *si, uint64_t xenstack)
 	/* Set the hooks for early functions that diverge from bare metal */
 	init_ops = xen_init_ops;
 	apic_ops = xen_apic_ops;
+	hypervisor_info = legacy_info;
 
 	/* Now we can jump into the native init function */
 	return (hammer_time(0, physfree));
@@ -291,8 +327,8 @@ xen_pv_set_env(void)
 	char *cmd_line_next, *cmd_line;
 	size_t env_size;
 
-	cmd_line = HYPERVISOR_start_info->cmd_line;
-	env_size = sizeof(HYPERVISOR_start_info->cmd_line);
+	cmd_line = legacy_start_info->cmd_line;
+	env_size = sizeof(legacy_start_info->cmd_line);
 
 	/* Skip leading spaces */
 	for (; isspace(*cmd_line) && (env_size != 0); cmd_line++)
@@ -322,9 +358,8 @@ xen_pv_parse_symtab(void)
 	int i, j;
 
 	size = end;
-	sym_end = HYPERVISOR_start_info->mod_start != 0 ?
-	    HYPERVISOR_start_info->mod_start :
-	    HYPERVISOR_start_info->mfn_list;
+	sym_end = legacy_start_info->mod_start != 0 ?
+	    legacy_start_info->mod_start : legacy_start_info->mfn_list;
 
 	/*
 	 * Make sure the size is right headed, sym_end is just a
@@ -375,8 +410,8 @@ xen_pv_parse_preload_data(u_int64_t modulep)
 	vm_paddr_t	 metadata;
 	char             *envp;
 
-	if (HYPERVISOR_start_info->mod_start != 0) {
-		preload_metadata = (caddr_t)(HYPERVISOR_start_info->mod_start);
+	if (legacy_start_info->mod_start != 0) {
+		preload_metadata = (caddr_t)legacy_start_info->mod_start;
 
 		kmdp = preload_search_by_type("elf kernel");
 		if (kmdp == NULL)
@@ -391,7 +426,7 @@ xen_pv_parse_preload_data(u_int64_t modulep)
 		 * which contains the relocated modulep address.
 		 */
 		metadata = MD_FETCH(kmdp, MODINFOMD_MODULEP, vm_paddr_t);
-		off = HYPERVISOR_start_info->mod_start - metadata;
+		off = legacy_start_info->mod_start - metadata;
 
 		preload_bootstrap_relocate(off);
 

sys/xen/xen-os.h

@@ -40,18 +40,62 @@
 #define __ASSEMBLY__
 #endif
 
-#include <machine/xen/xen-os.h>
-
 #include <xen/interface/xen.h>
 
+#ifndef __ASSEMBLY__
+#include <xen/interface/event_channel.h>
+
+struct hypervisor_info {
+	vm_paddr_t (*get_xenstore_mfn)(void);
+	evtchn_port_t (*get_xenstore_evtchn)(void);
+	vm_paddr_t (*get_console_mfn)(void);
+	evtchn_port_t (*get_console_evtchn)(void);
+	uint32_t (*get_start_flags)(void);
+};
+extern struct hypervisor_info hypervisor_info;
+
+static inline vm_paddr_t
+xen_get_xenstore_mfn(void)
+{
+
+	return (hypervisor_info.get_xenstore_mfn());
+}
+
+static inline evtchn_port_t
+xen_get_xenstore_evtchn(void)
+{
+
+	return (hypervisor_info.get_xenstore_evtchn());
+}
+
+static inline vm_paddr_t
+xen_get_console_mfn(void)
+{
+
+	return (hypervisor_info.get_console_mfn());
+}
+
+static inline evtchn_port_t
+xen_get_console_evtchn(void)
+{
+
+	return (hypervisor_info.get_console_evtchn());
+}
+
+static inline uint32_t
+xen_get_start_flags(void)
+{
+
+	return (hypervisor_info.get_start_flags());
+}
+#endif
+
+#include <machine/xen/xen-os.h>
+
 /* Everything below this point is not included by assembler (.S) files. */
 #ifndef __ASSEMBLY__
 
 extern shared_info_t *HYPERVISOR_shared_info;
-extern start_info_t *HYPERVISOR_start_info;
-
-/* XXX: we need to get rid of this and use HYPERVISOR_start_info directly */
-extern char *console_page;
 
 extern int xen_disable_pv_disks;
 extern int xen_disable_pv_nics;
@@ -87,8 +131,8 @@ xen_hvm_domain(void)
 static inline bool
 xen_initial_domain(void)
 {
-	return (xen_domain() && HYPERVISOR_start_info != NULL &&
-	    (HYPERVISOR_start_info->flags & SIF_INITDOMAIN) != 0);
+
+	return (xen_domain() && (xen_get_start_flags() & SIF_INITDOMAIN) != 0);
 }
 
 /*

sys/xen/xenstore/xenstorevar.h

@@ -52,9 +52,7 @@
 /* XenStore allocations including XenStore data returned to clients. */
 MALLOC_DECLARE(M_XENSTORE);
 
-struct xenstore_domain_interface;
 struct xs_watch;
-extern struct xenstore_domain_interface *xen_store;
 
 typedef	void (xs_watch_cb_t)(struct xs_watch *, const char **vec,
     unsigned int len);
@@ -84,6 +82,27 @@ struct xs_transaction
 
 #define XST_NIL ((struct xs_transaction) { 0 })
 
+/**
+ * Check if Xenstore is initialized.
+ *
+ * \return  True if initialized, false otherwise.
+ */
+bool xs_initialized(void);
+
+/**
+ * Return xenstore event channel port.
+ *
+ * \return event channel port.
+ */
+evtchn_port_t xs_evtchn(void);
+
+/**
+ * Return xenstore page physical memory address.
+ *
+ * \return xenstore page physical address.
+ */
+vm_paddr_t xs_address(void);
+
 /**
  * Fetch the contents of a directory in the XenStore.
  *