hyperv: Implement userspace gettimeofday(2) with Hyper-V reference TSC

This 6 times gettimeofday performance, as measured by
tools/tools/syscall_timing

Reviewed by:	kib
MFC after:	1 week
Sponsored by:	Microsoft
Differential Revision:	https://reviews.freebsd.org/D8789

lib/libc/x86/sys/__vdso_gettc.c

@@ -45,6 +45,10 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpufunc.h>
 #include <machine/specialreg.h>
 #include <dev/acpica/acpi_hpet.h>
+#ifdef __amd64__
+#include <machine/atomic.h>
+#include <dev/hyperv/hyperv.h>
+#endif
 #include "libc_private.h"
 
 static void
@@ -144,6 +148,67 @@ __vdso_init_hpet(uint32_t u)
 	_close(fd);
 }
 
+#ifdef __amd64__
+
+#define HYPERV_REFTSC_DEVPATH	"/dev/" HYPERV_REFTSC_DEVNAME
+
+/*
+ * NOTE:
+ * We use 'NULL' for this variable to indicate that initialization
+ * is required.  And if this variable is 'MAP_FAILED', then Hyper-V
+ * reference TSC can not be used, e.g. in misconfigured jail.
+ */
+static struct hyperv_reftsc *hyperv_ref_tsc;
+
+static void
+__vdso_init_hyperv_tsc(void)
+{
+	int fd;
+
+	fd = _open(HYPERV_REFTSC_DEVPATH, O_RDONLY);
+	if (fd < 0) {
+		/* Prevent the caller from re-entering. */
+		hyperv_ref_tsc = MAP_FAILED;
+		return;
+	}
+	hyperv_ref_tsc = mmap(NULL, sizeof(*hyperv_ref_tsc), PROT_READ,
+	    MAP_SHARED, fd, 0);
+	_close(fd);
+}
+
+static int
+__vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
+{
+	uint64_t disc, ret, tsc, scale;
+	uint32_t seq;
+	int64_t ofs;
+
+	while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
+		scale = tsc_ref->tsc_scale;
+		ofs = tsc_ref->tsc_ofs;
+
+		lfence_mb();
+		tsc = rdtsc();
+
+		/* ret = ((tsc * scale) >> 64) + ofs */
+		__asm__ __volatile__ ("mulq %3" :
+		    "=d" (ret), "=a" (disc) :
+		    "a" (tsc), "r" (scale));
+		ret += ofs;
+
+		atomic_thread_fence_acq();
+		if (tsc_ref->tsc_seq == seq) {
+			*tc = ret;
+			return (0);
+		}
+
+		/* Sequence changed; re-sync. */
+	}
+	return (ENOSYS);
+}
+
+#endif	/* __amd64__ */
+
 #pragma weak __vdso_gettc
 int
 __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
@@ -165,6 +230,14 @@ __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
 			return (ENOSYS);
 		*tc = *(volatile uint32_t *)(hpet_dev_map + HPET_MAIN_COUNTER);
 		return (0);
+#ifdef __amd64__
+	case VDSO_TH_ALGO_X86_HVTSC:
+		if (hyperv_ref_tsc == NULL)
+			__vdso_init_hyperv_tsc();
+		if (hyperv_ref_tsc == MAP_FAILED)
+			return (ENOSYS);
+		return (__vdso_hyperv_tsc(hyperv_ref_tsc, tc));
+#endif
 	default:
 		return (ENOSYS);
 	}

sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c

@@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/systm.h>
 #include <sys/timetc.h>
+#include <sys/vdso.h>
 
 #include <machine/cpufunc.h>
 #include <machine/cputypes.h>
@@ -52,18 +53,20 @@ struct hyperv_reftsc_ctx {
 	struct hyperv_dma	tsc_ref_dma;
 };
 
+static uint32_t			hyperv_tsc_vdso_timehands(
+				    struct vdso_timehands *,
+				    struct timecounter *);
+
 static d_open_t			hyperv_tsc_open;
 static d_mmap_t			hyperv_tsc_mmap;
 
 static struct timecounter	hyperv_tsc_timecounter = {
 	.tc_get_timecount	= NULL,	/* based on CPU vendor. */
-	.tc_poll_pps		= NULL,
 	.tc_counter_mask	= 0xffffffff,
 	.tc_frequency		= HYPERV_TIMER_FREQ,
 	.tc_name		= "Hyper-V-TSC",
 	.tc_quality		= 3000,
-	.tc_flags		= 0,
+	.tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands,
-	.tc_priv		= NULL
 };
 
 static struct cdevsw		hyperv_tsc_cdevsw = {
@@ -117,6 +120,18 @@ hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
 	return (0);
 }
 
+static uint32_t
+hyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
+    struct timecounter *tc __unused)
+{
+
+	vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC;
+	vdso_th->th_x86_shift = 0;
+	vdso_th->th_x86_hpet_idx = 0;
+	bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
+	return (1);
+}
+
 #define HYPERV_TSC_TIMECOUNT(fence)					\
 static u_int								\
 hyperv_tsc_timecount_##fence(struct timecounter *tc)			\

sys/sys/vdso.h

@@ -54,6 +54,8 @@ struct vdso_timekeep {
 #define	VDSO_TK_VER_CURR	VDSO_TK_VER_1
 #define	VDSO_TH_ALGO_1		0x1
 #define	VDSO_TH_ALGO_2		0x2
+#define	VDSO_TH_ALGO_3		0x3
+#define	VDSO_TH_ALGO_4		0x4
 
 #ifndef _KERNEL
 

sys/x86/include/vdso.h

@@ -39,6 +39,7 @@
 
 #define	VDSO_TH_ALGO_X86_TSC	VDSO_TH_ALGO_1
 #define	VDSO_TH_ALGO_X86_HPET	VDSO_TH_ALGO_2
+#define	VDSO_TH_ALGO_X86_HVTSC	VDSO_TH_ALGO_3	/* Hyper-V ref. TSC */
 
 #ifdef _KERNEL
 #ifdef COMPAT_FREEBSD32