freebsd-dev/sys/compat/linux/linux_vdso_gtod.inc

/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
 * Copyright (c) 2021 Dmitry Chagin <dchagin@FreeBSD.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

static int
fls(int mask)
{

	if (mask == 0)
		return (0);
	return ((__builtin_clz(mask) ^ 0x1f) + 1);
}

#ifdef _LP64
static int
flsl(long mask)
{
	int bit;

	if (mask == 0)
		return (0);
	for (bit = 1; mask != 1; bit++)
		mask = (unsigned long)mask >> 1;
	return (bit);
}
#else
static int
flsll(long long mask)
{
	int bit;

	if (mask == 0)
		return (0);
	for (bit = 1; mask != 1; bit++)
		mask = (unsigned long long)mask >> 1;
	return (bit);
}
#endif

static int
__vdso_native_to_linux_timespec(struct l_timespec *lts,
    struct timespec *nts)
{

#ifdef COMPAT_LINUX32
	if (nts->tv_sec > INT_MAX || nts->tv_sec < INT_MIN)
		return (LINUX_EOVERFLOW);
#endif
	lts->tv_sec = nts->tv_sec;
	lts->tv_nsec = nts->tv_nsec;
	return (0);
}

static int
__vdso_native_to_linux_timeval(l_timeval *ltv,
    struct timeval *ntv)
{

#ifdef COMPAT_LINUX32
	if (ntv->tv_sec > INT_MAX || ntv->tv_sec < INT_MIN)
		return (LINUX_EOVERFLOW);
#endif
	ltv->tv_sec = ntv->tv_sec;
	ltv->tv_usec = ntv->tv_usec;
	return (0);
}


#if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32))
static int
__vdso_native_to_linux_timespec64(struct l_timespec64 *lts,
    struct timespec *nts)
{

	lts->tv_sec = nts->tv_sec;
	lts->tv_nsec = nts->tv_nsec;
	return (0);
}
#endif

static int
__vdso_linux_to_native_clockid(clockid_t *n, clockid_t l)
{

	switch (l) {
	case LINUX_CLOCK_REALTIME:
		*n = CLOCK_REALTIME;
		break;
	case LINUX_CLOCK_MONOTONIC:
		*n = CLOCK_MONOTONIC;
		break;
	case LINUX_CLOCK_REALTIME_COARSE:
		*n = CLOCK_REALTIME_FAST;
		break;
	case LINUX_CLOCK_MONOTONIC_COARSE:
	case LINUX_CLOCK_MONOTONIC_RAW:
		*n = CLOCK_MONOTONIC_FAST;
		break;
	case LINUX_CLOCK_BOOTTIME:
		*n = CLOCK_UPTIME;
		break;
	default:
		return (LINUX_EINVAL);
	}
	return (0);
}

/*
 * The code below adapted from
 * lib/libc/sys/__vdso_gettimeofday.c
 */

static inline void
__vdso_gettimekeep(struct vdso_timekeep **tk)
{

	*tk = (struct vdso_timekeep *)kern_timekeep_base;
}

static int
tc_delta(const struct vdso_timehands *th, u_int *delta)
{
	int error;
	u_int tc;

	error = __vdso_gettc(th, &tc);
	if (error == 0)
		*delta = (tc - th->th_offset_count) & th->th_counter_mask;
	return (error);
}

/*
 * Calculate the absolute or boot-relative time from the
 * machine-specific fast timecounter and the published timehands
 * structure read from the shared page.
 *
 * The lockless reading scheme is similar to the one used to read the
 * in-kernel timehands, see sys/kern/kern_tc.c:binuptime().  This code
 * is based on the kernel implementation.
 */
static int
freebsd_binuptime(struct bintime *bt, struct vdso_timekeep *tk, bool abs)
{
	struct vdso_timehands *th;
	uint32_t curr, gen;
	uint64_t scale, x;
	u_int delta, scale_bits;
	int error;

	do {
		if (!tk->tk_enabled)
			return (ENOSYS);

		curr = atomic_load_acq_32(&tk->tk_current);
		th = &tk->tk_th[curr];
		gen = atomic_load_acq_32(&th->th_gen);
		*bt = th->th_offset;
		error = tc_delta(th, &delta);
		if (error == EAGAIN)
			continue;
		if (error != 0)
			return (error);
		scale = th->th_scale;
#ifdef _LP64
		scale_bits = flsl(scale);
#else
		scale_bits = flsll(scale);
#endif
		if (__predict_false(scale_bits + fls(delta) > 63)) {
			x = (scale >> 32) * delta;
			scale &= 0xffffffff;
			bt->sec += x >> 32;
			bintime_addx(bt, x << 32);
		}
		bintime_addx(bt, scale * delta);
		if (abs)
			bintime_add(bt, &th->th_boottime);

		/*
		 * Ensure that the load of th_offset is completed
		 * before the load of th_gen.
		 */
		atomic_thread_fence_acq();
	} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
	return (0);
}

static int
freebsd_getnanouptime(struct bintime *bt, struct vdso_timekeep *tk)
{
	struct vdso_timehands *th;
	uint32_t curr, gen;

	do {
		if (!tk->tk_enabled)
			return (ENOSYS);

		curr = atomic_load_acq_32(&tk->tk_current);
		th = &tk->tk_th[curr];
		gen = atomic_load_acq_32(&th->th_gen);
		*bt = th->th_offset;

		/*
		 * Ensure that the load of th_offset is completed
		 * before the load of th_gen.
		 */
		atomic_thread_fence_acq();
	} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
	return (0);
}

static int
freebsd_gettimeofday(struct timeval *tv, struct timezone *tz)
{
	struct vdso_timekeep *tk;
	struct bintime bt;
	int error;

	if (tz != NULL)
		return (ENOSYS);
	__vdso_gettimekeep(&tk);
	if (tk == NULL)
		return (ENOSYS);
	if (tk->tk_ver != VDSO_TK_VER_CURR)
		return (ENOSYS);
	error = freebsd_binuptime(&bt, tk, true);
	if (error == 0)
		bintime2timeval(&bt, tv);
	return (error);
}

static int
freebsd_clock_gettime(clockid_t clock_id, struct timespec *ts)
{
	struct vdso_timekeep *tk;
	struct bintime bt;
	int error;

	__vdso_gettimekeep(&tk);
	if (tk == NULL)
		return (ENOSYS);
	if (tk->tk_ver != VDSO_TK_VER_CURR)
		return (ENOSYS);
	switch (clock_id) {
	case CLOCK_REALTIME:
	case CLOCK_REALTIME_PRECISE:
	case CLOCK_REALTIME_FAST:
		error = freebsd_binuptime(&bt, tk, true);
		break;
	case CLOCK_MONOTONIC:
	case CLOCK_MONOTONIC_PRECISE:
	case CLOCK_UPTIME:
	case CLOCK_UPTIME_PRECISE:
		error = freebsd_binuptime(&bt, tk, false);
		break;
	case CLOCK_MONOTONIC_FAST:
	case CLOCK_UPTIME_FAST:
		error = freebsd_getnanouptime(&bt, tk);
		break;
	default:
		error = ENOSYS;
		break;
	}
	if (error == 0)
		bintime2timespec(&bt, ts);
	return (error);
}

/*
 * Linux vDSO interfaces
 *
 */
int
__vdso_clock_gettime(clockid_t clock_id, struct l_timespec *lts)
{
	struct timespec ts;
	clockid_t which;
	int error;

	error = __vdso_linux_to_native_clockid(&which, clock_id);
	if (error != 0)
		return (__vdso_clock_gettime_fallback(clock_id, lts));
	error = freebsd_clock_gettime(which, &ts);
	if (error == 0)
		return (-__vdso_native_to_linux_timespec(lts, &ts));
	else
		return (__vdso_clock_gettime_fallback(clock_id, lts));
}

int
__vdso_gettimeofday(l_timeval *ltv, struct timezone *tz)
{
	struct timeval tv;
	int error;

	error = freebsd_gettimeofday(&tv, tz);
	if (error != 0)
		return (__vdso_gettimeofday_fallback(ltv, tz));
	return (-__vdso_native_to_linux_timeval(ltv, &tv));
}

int
__vdso_clock_getres(clockid_t clock_id, struct l_timespec *lts)
{

	return (__vdso_clock_getres_fallback(clock_id, lts));
}

#if defined(__i386__) || defined(COMPAT_LINUX32)
int
__vdso_clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts)
{
	struct timespec ts;
	clockid_t which;
	int error;

	error = __vdso_linux_to_native_clockid(&which, clock_id);
	if (error != 0)
		return (__vdso_clock_gettime64_fallback(clock_id, lts));
	error = freebsd_clock_gettime(which, &ts);
	if (error == 0)
		return(-__vdso_native_to_linux_timespec64(lts, &ts));
	else
		return(__vdso_clock_gettime64_fallback(clock_id, lts));
}

int clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts)
    __attribute__((weak, alias("__vdso_clock_gettime64")));
#endif

#if defined(__i386__) || defined(__amd64__)
int
__vdso_getcpu(uint32_t *cpu, uint32_t *node, void *cache)
{
	int ret;

	if (node != NULL)
		return (__vdso_getcpu_fallback(cpu, node, cache));
	ret = __vdso_getcpu_try();
	if (ret < 0)
		return (__vdso_getcpu_fallback(cpu, node, cache));
	*cpu = ret;
	return (0);
}
#endif

#if defined(__i386__) || defined(__amd64__)
int
__vdso_time(long *tm)
{
	struct timeval tv;
	int error;

	error = freebsd_gettimeofday(&tv, NULL);
	if (error != 0)
		return (__vdso_time_fallback(tm));
	if (tm != NULL)
		*tm = tv.tv_sec;
	return (tv.tv_sec);
}
#endif