d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
67d3974849
freebsd-dev/sys/compat/linux/linux_futex.c
Dmitry Chagin 81338031c4 Switch linuxulator to use the native 1:1 threads. 
The reasons:
1. Get rid of the stubs/quirks with process dethreading,
   process reparent when the process group leader exits and close
   to this problems on wait(), waitpid(), etc.
2. Reuse our kernel code instead of writing excessive thread
   managment routines in Linuxulator.

Implementation details:

1. The thread is created via kern_thr_new() in the clone() call with
   the CLONE_THREAD parameter. Thus, everything else is a process.
2. The test that the process has a threads is done via P_HADTHREADS
   bit p_flag of struct proc.
3. Per thread emulator state data structure is now located in the
   struct thread and freed in the thread_dtor() hook.
   Mandatory holdig of the p_mtx required when referencing emuldata
   from the other threads.
4. PID mangling has changed. Now Linux pid is the native tid
   and Linux tgid is the native pid, with the exception of the first
   thread in the process where tid and pid are one and the same.

Ugliness:

   In case when the Linux thread is the initial thread in the thread
   group thread id is equal to the process id. Glibc depends on this
   magic (assert in pthread_getattr_np.c). So for system calls that
   take thread id as a parameter we should use the special method
   to reference struct thread.

Differential Revision:	https://reviews.freebsd.org/D1039
2015-05-24 14:53:16 +00:00

1237 lines
35 KiB
C
Raw Blame History

/* $NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $ */
/*-
* Copyright (c) 2005 Emmanuel Dreyfus, all rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by Emmanuel Dreyfus
* 4. The name of the author may not be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE THE AUTHOR AND CONTRIBUTORS ``AS IS''
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#if 0
__KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $");
#endif
#include "opt_compat.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/imgact.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/sx.h>
#include <sys/umtx.h>
#ifdef COMPAT_LINUX32
#include <machine/../linux32/linux.h>
#include <machine/../linux32/linux32_proto.h>
#else
#include <machine/../linux/linux.h>
#include <machine/../linux/linux_proto.h>
#endif
#include <compat/linux/linux_dtrace.h>
#include <compat/linux/linux_emul.h>
#include <compat/linux/linux_futex.h>
#include <compat/linux/linux_util.h>
/* DTrace init */
LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
/**
* Futex part for the special DTrace module "locks".
*/
LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, locked, "struct mtx *");
LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, unlock, "struct mtx *");
/**
* Per futex probes.
*/
LIN_SDT_PROBE_DEFINE1(futex, futex, create, "struct sx *");
LIN_SDT_PROBE_DEFINE1(futex, futex, destroy, "struct sx *");
/**
* DTrace probes in this module.
*/
LIN_SDT_PROBE_DEFINE2(futex, futex_put, entry, "struct futex *",
"struct waiting_proc *");
LIN_SDT_PROBE_DEFINE3(futex, futex_put, destroy, "uint32_t *", "uint32_t",
"int");
LIN_SDT_PROBE_DEFINE3(futex, futex_put, unlock, "uint32_t *", "uint32_t",
"int");
LIN_SDT_PROBE_DEFINE0(futex, futex_put, return);
LIN_SDT_PROBE_DEFINE3(futex, futex_get0, entry, "uint32_t *", "struct futex **",
"uint32_t");
LIN_SDT_PROBE_DEFINE1(futex, futex_get0, umtx_key_get_error, "int");
LIN_SDT_PROBE_DEFINE3(futex, futex_get0, shared, "uint32_t *", "uint32_t",
"int");
LIN_SDT_PROBE_DEFINE1(futex, futex_get0, null, "uint32_t *");
LIN_SDT_PROBE_DEFINE3(futex, futex_get0, new, "uint32_t *", "uint32_t", "int");
LIN_SDT_PROBE_DEFINE1(futex, futex_get0, return, "int");
LIN_SDT_PROBE_DEFINE3(futex, futex_get, entry, "uint32_t *",
"struct waiting_proc **", "struct futex **");
LIN_SDT_PROBE_DEFINE0(futex, futex_get, error);
LIN_SDT_PROBE_DEFINE1(futex, futex_get, return, "int");
LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, entry, "struct futex *",
"struct waiting_proc **", "int");
LIN_SDT_PROBE_DEFINE5(futex, futex_sleep, requeue_error, "int", "uint32_t *",
"struct waiting_proc *", "uint32_t *", "uint32_t");
LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, sleep_error, "int", "uint32_t *",
"struct waiting_proc *");
LIN_SDT_PROBE_DEFINE1(futex, futex_sleep, return, "int");
LIN_SDT_PROBE_DEFINE3(futex, futex_wake, entry, "struct futex *", "int",
"uint32_t");
LIN_SDT_PROBE_DEFINE3(futex, futex_wake, iterate, "uint32_t",
"struct waiting_proc *", "uint32_t");
LIN_SDT_PROBE_DEFINE1(futex, futex_wake, wakeup, "struct waiting_proc *");
LIN_SDT_PROBE_DEFINE1(futex, futex_wake, return, "int");
LIN_SDT_PROBE_DEFINE4(futex, futex_requeue, entry, "struct futex *", "int",
"struct futex *", "int");
LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, wakeup, "struct waiting_proc *");
LIN_SDT_PROBE_DEFINE3(futex, futex_requeue, requeue, "uint32_t *",
"struct waiting_proc *", "uint32_t");
LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, return, "int");
LIN_SDT_PROBE_DEFINE4(futex, futex_wait, entry, "struct futex *",
"struct waiting_proc **", "int", "uint32_t");
LIN_SDT_PROBE_DEFINE1(futex, futex_wait, sleep_error, "int");
LIN_SDT_PROBE_DEFINE1(futex, futex_wait, return, "int");
LIN_SDT_PROBE_DEFINE3(futex, futex_atomic_op, entry, "struct thread *",
"int", "uint32_t");
LIN_SDT_PROBE_DEFINE4(futex, futex_atomic_op, decoded_op, "int", "int", "int",
"int");
LIN_SDT_PROBE_DEFINE0(futex, futex_atomic_op, missing_access_check);
LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_op, "int");
LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_cmp, "int");
LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, return, "int");
LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, entry, "struct thread *",
"struct linux_sys_futex_args *");
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_clockswitch);
LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, itimerfix_error, "int");
LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, copyin_error, "int");
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, invalid_cmp_requeue_use);
LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wait, "uint32_t *",
"uint32_t", "uint32_t");
LIN_SDT_PROBE_DEFINE4(futex, linux_sys_futex, debug_wait_value_neq,
"uint32_t *", "uint32_t", "int", "uint32_t");
LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wake, "uint32_t *",
"uint32_t", "uint32_t");
LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_cmp_requeue, "uint32_t *",
"uint32_t", "uint32_t", "uint32_t *", "struct l_timespec *");
LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, debug_cmp_requeue_value_neq,
"uint32_t", "int");
LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_wake_op, "uint32_t *",
"int", "uint32_t", "uint32_t *", "uint32_t");
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unhandled_efault);
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_lock_pi);
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_unlock_pi);
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_trylock_pi);
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, deprecated_requeue);
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_wait_requeue_pi);
LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_cmp_requeue_pi);
LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, unknown_operation, "int");
LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, return, "int");
LIN_SDT_PROBE_DEFINE2(futex, linux_set_robust_list, entry, "struct thread *",
"struct linux_set_robust_list_args *");
LIN_SDT_PROBE_DEFINE0(futex, linux_set_robust_list, size_error);
LIN_SDT_PROBE_DEFINE1(futex, linux_set_robust_list, return, "int");
LIN_SDT_PROBE_DEFINE2(futex, linux_get_robust_list, entry, "struct thread *",
"struct linux_get_robust_list_args *");
LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, copyout_error, "int");
LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, return, "int");
LIN_SDT_PROBE_DEFINE3(futex, handle_futex_death, entry,
"struct linux_emuldata *", "uint32_t *", "unsigned int");
LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, copyin_error, "int");
LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, return, "int");
LIN_SDT_PROBE_DEFINE3(futex, fetch_robust_entry, entry,
"struct linux_robust_list **", "struct linux_robust_list **",
"unsigned int *");
LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, copyin_error, "int");
LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, return, "int");
LIN_SDT_PROBE_DEFINE2(futex, release_futexes, entry, "struct thread *",
"struct linux_emuldata *");
LIN_SDT_PROBE_DEFINE1(futex, release_futexes, copyin_error, "int");
LIN_SDT_PROBE_DEFINE0(futex, release_futexes, return);
static MALLOC_DEFINE(M_FUTEX, "futex", "Linux futexes");
static MALLOC_DEFINE(M_FUTEX_WP, "futex wp", "Linux futexes wp");
struct futex;
struct waiting_proc {
uint32_t wp_flags;
struct futex *wp_futex;
TAILQ_ENTRY(waiting_proc) wp_list;
};
struct futex {
struct sx f_lck;
uint32_t *f_uaddr; /* user-supplied value, for debug */
struct umtx_key f_key;
uint32_t f_refcount;
uint32_t f_bitset;
LIST_ENTRY(futex) f_list;
TAILQ_HEAD(lf_waiting_proc, waiting_proc) f_waiting_proc;
};
struct futex_list futex_list;
#define FUTEX_LOCK(f) sx_xlock(&(f)->f_lck)
#define FUTEX_UNLOCK(f) sx_xunlock(&(f)->f_lck)
#define FUTEX_INIT(f) do { \
sx_init_flags(&(f)->f_lck, "ftlk", \
SX_DUPOK); \
LIN_SDT_PROBE1(futex, futex, create, \
&(f)->f_lck); \
} while (0)
#define FUTEX_DESTROY(f) do { \
LIN_SDT_PROBE1(futex, futex, destroy, \
&(f)->f_lck); \
sx_destroy(&(f)->f_lck); \
} while (0)
#define FUTEX_ASSERT_LOCKED(f) sx_assert(&(f)->f_lck, SA_XLOCKED)
struct mtx futex_mtx; /* protects the futex list */
#define FUTEXES_LOCK do { \
mtx_lock(&futex_mtx); \
LIN_SDT_PROBE1(locks, futex_mtx, \
locked, &futex_mtx); \
} while (0)
#define FUTEXES_UNLOCK do { \
LIN_SDT_PROBE1(locks, futex_mtx, \
unlock, &futex_mtx); \
mtx_unlock(&futex_mtx); \
} while (0)
/* flags for futex_get() */
#define FUTEX_CREATE_WP 0x1 /* create waiting_proc */
#define FUTEX_DONTCREATE 0x2 /* don't create futex if not exists */
#define FUTEX_DONTEXISTS 0x4 /* return EINVAL if futex exists */
#define FUTEX_SHARED 0x8 /* shared futex */
/* wp_flags */
#define FUTEX_WP_REQUEUED 0x1 /* wp requeued - wp moved from wp_list
* of futex where thread sleep to wp_list
* of another futex.
*/
#define FUTEX_WP_REMOVED 0x2 /* wp is woken up and removed from futex
* wp_list to prevent double wakeup.
*/
/* support.s */
int futex_xchgl(int oparg, uint32_t *uaddr, int *oldval);
int futex_addl(int oparg, uint32_t *uaddr, int *oldval);
int futex_orl(int oparg, uint32_t *uaddr, int *oldval);
int futex_andl(int oparg, uint32_t *uaddr, int *oldval);
int futex_xorl(int oparg, uint32_t *uaddr, int *oldval);
static void
futex_put(struct futex *f, struct waiting_proc *wp)
{
LIN_SDT_PROBE2(futex, futex_put, entry, f, wp);
FUTEX_ASSERT_LOCKED(f);
if (wp != NULL) {
if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0)
TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
free(wp, M_FUTEX_WP);
}
FUTEXES_LOCK;
if (--f->f_refcount == 0) {
LIST_REMOVE(f, f_list);
FUTEXES_UNLOCK;
FUTEX_UNLOCK(f);
LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr,
f->f_refcount, f->f_key.shared);
LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d "
"shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared);
umtx_key_release(&f->f_key);
FUTEX_DESTROY(f);
free(f, M_FUTEX);
LIN_SDT_PROBE0(futex, futex_put, return);
return;
}
LIN_SDT_PROBE3(futex, futex_put, unlock, f->f_uaddr, f->f_refcount,
f->f_key.shared);
LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d",
f->f_uaddr, f->f_refcount, f->f_key.shared);
FUTEXES_UNLOCK;
FUTEX_UNLOCK(f);
LIN_SDT_PROBE0(futex, futex_put, return);
}
static int
futex_get0(uint32_t *uaddr, struct futex **newf, uint32_t flags)
{
struct futex *f, *tmpf;
struct umtx_key key;
int error;
LIN_SDT_PROBE3(futex, futex_get0, entry, uaddr, newf, flags);
*newf = tmpf = NULL;
error = umtx_key_get(uaddr, TYPE_FUTEX, (flags & FUTEX_SHARED) ?
AUTO_SHARE : THREAD_SHARE, &key);
if (error) {
LIN_SDT_PROBE1(futex, futex_get0, umtx_key_get_error, error);
LIN_SDT_PROBE1(futex, futex_get0, return, error);
return (error);
}
retry:
FUTEXES_LOCK;
LIST_FOREACH(f, &futex_list, f_list) {
if (umtx_key_match(&f->f_key, &key)) {
if (tmpf != NULL) {
FUTEX_UNLOCK(tmpf);
FUTEX_DESTROY(tmpf);
free(tmpf, M_FUTEX);
}
if (flags & FUTEX_DONTEXISTS) {
FUTEXES_UNLOCK;
umtx_key_release(&key);
LIN_SDT_PROBE1(futex, futex_get0, return,
EINVAL);
return (EINVAL);
}
/*
* Increment refcount of the found futex to
* prevent it from deallocation before FUTEX_LOCK()
*/
++f->f_refcount;
FUTEXES_UNLOCK;
umtx_key_release(&key);
FUTEX_LOCK(f);
*newf = f;
LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr,
f->f_refcount, f->f_key.shared);
LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d",
uaddr, f->f_refcount, f->f_key.shared);
LIN_SDT_PROBE1(futex, futex_get0, return, 0);
return (0);
}
}
if (flags & FUTEX_DONTCREATE) {
FUTEXES_UNLOCK;
umtx_key_release(&key);
LIN_SDT_PROBE1(futex, futex_get0, null, uaddr);
LINUX_CTR1(sys_futex, "futex_get uaddr %p null", uaddr);
LIN_SDT_PROBE1(futex, futex_get0, return, 0);
return (0);
}
if (tmpf == NULL) {
FUTEXES_UNLOCK;
tmpf = malloc(sizeof(*tmpf), M_FUTEX, M_WAITOK | M_ZERO);
tmpf->f_uaddr = uaddr;
tmpf->f_key = key;
tmpf->f_refcount = 1;
tmpf->f_bitset = FUTEX_BITSET_MATCH_ANY;
FUTEX_INIT(tmpf);
TAILQ_INIT(&tmpf->f_waiting_proc);
/*
* Lock the new futex before an insert into the futex_list
* to prevent futex usage by other.
*/
FUTEX_LOCK(tmpf);
goto retry;
}
LIST_INSERT_HEAD(&futex_list, tmpf, f_list);
FUTEXES_UNLOCK;
LIN_SDT_PROBE3(futex, futex_get0, new, uaddr, tmpf->f_refcount,
tmpf->f_key.shared);
LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d new",
uaddr, tmpf->f_refcount, tmpf->f_key.shared);
*newf = tmpf;
LIN_SDT_PROBE1(futex, futex_get0, return, 0);
return (0);
}
static int
futex_get(uint32_t *uaddr, struct waiting_proc **wp, struct futex **f,
uint32_t flags)
{
int error;
LIN_SDT_PROBE3(futex, futex_get, entry, uaddr, wp, f);
if (flags & FUTEX_CREATE_WP) {
*wp = malloc(sizeof(struct waiting_proc), M_FUTEX_WP, M_WAITOK);
(*wp)->wp_flags = 0;
}
error = futex_get0(uaddr, f, flags);
if (error) {
LIN_SDT_PROBE0(futex, futex_get, error);
if (flags & FUTEX_CREATE_WP)
free(*wp, M_FUTEX_WP);
LIN_SDT_PROBE1(futex, futex_get, return, error);
return (error);
}
if (flags & FUTEX_CREATE_WP) {
TAILQ_INSERT_HEAD(&(*f)->f_waiting_proc, *wp, wp_list);
(*wp)->wp_futex = *f;
}
LIN_SDT_PROBE1(futex, futex_get, return, error);
return (error);
}
static int
futex_sleep(struct futex *f, struct waiting_proc *wp, int timeout)
{
int error;
FUTEX_ASSERT_LOCKED(f);
LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, timeout);
LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %d ref %d",
f->f_uaddr, wp, timeout, f->f_refcount);
error = sx_sleep(wp, &f->f_lck, PCATCH, "futex", timeout);
if (wp->wp_flags & FUTEX_WP_REQUEUED) {
KASSERT(f != wp->wp_futex, ("futex != wp_futex"));
if (error) {
LIN_SDT_PROBE5(futex, futex_sleep, requeue_error, error,
f->f_uaddr, wp, wp->wp_futex->f_uaddr,
wp->wp_futex->f_refcount);
}
LINUX_CTR5(sys_futex, "futex_sleep out error %d uaddr %p wp"
" %p requeued uaddr %p ref %d",
error, f->f_uaddr, wp, wp->wp_futex->f_uaddr,
wp->wp_futex->f_refcount);
futex_put(f, NULL);
f = wp->wp_futex;
FUTEX_LOCK(f);
} else {
if (error) {
LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error,
f->f_uaddr, wp);
}
LINUX_CTR3(sys_futex, "futex_sleep out error %d uaddr %p wp %p",
error, f->f_uaddr, wp);
}
futex_put(f, wp);
LIN_SDT_PROBE1(futex, futex_sleep, return, error);
return (error);
}
static int
futex_wake(struct futex *f, int n, uint32_t bitset)
{
struct waiting_proc *wp, *wpt;
int count = 0;
LIN_SDT_PROBE3(futex, futex_wake, entry, f, n, bitset);
if (bitset == 0) {
LIN_SDT_PROBE1(futex, futex_wake, return, EINVAL);
return (EINVAL);
}
FUTEX_ASSERT_LOCKED(f);
TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
LIN_SDT_PROBE3(futex, futex_wake, iterate, f->f_uaddr, wp,
f->f_refcount);
LINUX_CTR3(sys_futex, "futex_wake uaddr %p wp %p ref %d",
f->f_uaddr, wp, f->f_refcount);
/*
* Unless we find a matching bit in
* the bitset, continue searching.
*/
if (!(wp->wp_futex->f_bitset & bitset))
continue;
wp->wp_flags |= FUTEX_WP_REMOVED;
TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
LIN_SDT_PROBE1(futex, futex_wake, wakeup, wp);
wakeup_one(wp);
if (++count == n)
break;
}
LIN_SDT_PROBE1(futex, futex_wake, return, count);
return (count);
}
static int
futex_requeue(struct futex *f, int n, struct futex *f2, int n2)
{
struct waiting_proc *wp, *wpt;
int count = 0;
LIN_SDT_PROBE4(futex, futex_requeue, entry, f, n, f2, n2);
FUTEX_ASSERT_LOCKED(f);
FUTEX_ASSERT_LOCKED(f2);
TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
if (++count <= n) {
LINUX_CTR2(sys_futex, "futex_req_wake uaddr %p wp %p",
f->f_uaddr, wp);
wp->wp_flags |= FUTEX_WP_REMOVED;
TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
LIN_SDT_PROBE1(futex, futex_requeue, wakeup, wp);
wakeup_one(wp);
} else {
LIN_SDT_PROBE3(futex, futex_requeue, requeue,
f->f_uaddr, wp, f2->f_uaddr);
LINUX_CTR3(sys_futex, "futex_requeue uaddr %p wp %p to %p",
f->f_uaddr, wp, f2->f_uaddr);
wp->wp_flags |= FUTEX_WP_REQUEUED;
/* Move wp to wp_list of f2 futex */
TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
TAILQ_INSERT_HEAD(&f2->f_waiting_proc, wp, wp_list);
/*
* Thread which sleeps on wp after waking should
* acquire f2 lock, so increment refcount of f2 to
* prevent it from premature deallocation.
*/
wp->wp_futex = f2;
FUTEXES_LOCK;
++f2->f_refcount;
FUTEXES_UNLOCK;
if (count - n >= n2)
break;
}
}
LIN_SDT_PROBE1(futex, futex_requeue, return, count);
return (count);
}
static int
futex_wait(struct futex *f, struct waiting_proc *wp, int timeout_hz,
uint32_t bitset)
{
int error;
LIN_SDT_PROBE4(futex, futex_wait, entry, f, wp, timeout_hz, bitset);
if (bitset == 0) {
LIN_SDT_PROBE1(futex, futex_wait, return, EINVAL);
return (EINVAL);
}
f->f_bitset = bitset;
error = futex_sleep(f, wp, timeout_hz);
if (error)
LIN_SDT_PROBE1(futex, futex_wait, sleep_error, error);
if (error == EWOULDBLOCK)
error = ETIMEDOUT;
LIN_SDT_PROBE1(futex, futex_wait, return, error);
return (error);
}
static int
futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr)
{
int op = (encoded_op >> 28) & 7;
int cmp = (encoded_op >> 24) & 15;
int oparg = (encoded_op << 8) >> 20;
int cmparg = (encoded_op << 20) >> 20;
int oldval = 0, ret;
LIN_SDT_PROBE3(futex, futex_atomic_op, entry, td, encoded_op, uaddr);
if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
oparg = 1 << oparg;
LIN_SDT_PROBE4(futex, futex_atomic_op, decoded_op, op, cmp, oparg,
cmparg);
/* XXX: Linux verifies access here and returns EFAULT */
LIN_SDT_PROBE0(futex, futex_atomic_op, missing_access_check);
switch (op) {
case FUTEX_OP_SET:
ret = futex_xchgl(oparg, uaddr, &oldval);
break;
case FUTEX_OP_ADD:
ret = futex_addl(oparg, uaddr, &oldval);
break;
case FUTEX_OP_OR:
ret = futex_orl(oparg, uaddr, &oldval);
break;
case FUTEX_OP_ANDN:
ret = futex_andl(~oparg, uaddr, &oldval);
break;
case FUTEX_OP_XOR:
ret = futex_xorl(oparg, uaddr, &oldval);
break;
default:
LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_op, op);
ret = -ENOSYS;
break;
}
if (ret) {
LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
return (ret);
}
switch (cmp) {
case FUTEX_OP_CMP_EQ:
ret = (oldval == cmparg);
break;
case FUTEX_OP_CMP_NE:
ret = (oldval != cmparg);
break;
case FUTEX_OP_CMP_LT:
ret = (oldval < cmparg);
break;
case FUTEX_OP_CMP_GE:
ret = (oldval >= cmparg);
break;
case FUTEX_OP_CMP_LE:
ret = (oldval <= cmparg);
break;
case FUTEX_OP_CMP_GT:
ret = (oldval > cmparg);
break;
default:
LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_cmp, cmp);
ret = -ENOSYS;
}
LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
return (ret);
}
int
linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args)
{
int clockrt, nrwake, op_ret, ret;
struct linux_emuldata *em;
struct waiting_proc *wp;
struct futex *f, *f2;
struct l_timespec timeout;
struct timeval utv, ctv;
int timeout_hz;
int error;
uint32_t flags, val;
LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args);
if (args->op & LINUX_FUTEX_PRIVATE_FLAG) {
flags = 0;
args->op &= ~LINUX_FUTEX_PRIVATE_FLAG;
} else
flags = FUTEX_SHARED;
/*
* Currently support for switching between CLOCK_MONOTONIC and
* CLOCK_REALTIME is not present. However Linux forbids the use of
* FUTEX_CLOCK_REALTIME with any op except FUTEX_WAIT_BITSET and
* FUTEX_WAIT_REQUEUE_PI.
*/
clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME;
args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME;
if (clockrt && args->op != LINUX_FUTEX_WAIT_BITSET &&
args->op != LINUX_FUTEX_WAIT_REQUEUE_PI) {
LIN_SDT_PROBE0(futex, linux_sys_futex,
unimplemented_clockswitch);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
}
error = 0;
f = f2 = NULL;
switch (args->op) {
case LINUX_FUTEX_WAIT:
args->val3 = FUTEX_BITSET_MATCH_ANY;
/* FALLTHROUGH */
case LINUX_FUTEX_WAIT_BITSET:
LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wait, args->uaddr,
args->val, args->val3);
LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x",
args->uaddr, args->val, args->val3);
error = futex_get(args->uaddr, &wp, &f,
flags | FUTEX_CREATE_WP);
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
error = copyin(args->uaddr, &val, sizeof(val));
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
error);
LINUX_CTR1(sys_futex, "WAIT copyin failed %d",
error);
futex_put(f, wp);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
if (val != args->val) {
LIN_SDT_PROBE4(futex, linux_sys_futex,
debug_wait_value_neq, args->uaddr, args->val, val,
args->val3);
LINUX_CTR3(sys_futex,
"WAIT uaddr %p val 0x%x != uval 0x%x",
args->uaddr, args->val, val);
futex_put(f, wp);
LIN_SDT_PROBE1(futex, linux_sys_futex, return,
EWOULDBLOCK);
return (EWOULDBLOCK);
}
if (args->timeout != NULL) {
error = copyin(args->timeout, &timeout, sizeof(timeout));
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
error);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
futex_put(f, wp);
return (error);
}
TIMESPEC_TO_TIMEVAL(&utv, &timeout);
error = itimerfix(&utv);
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, itimerfix_error,
error);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
futex_put(f, wp);
return (error);
}
if (clockrt) {
microtime(&ctv);
timevalsub(&utv, &ctv);
} else if (args->op == LINUX_FUTEX_WAIT_BITSET) {
microuptime(&ctv);
timevalsub(&utv, &ctv);
}
if (utv.tv_sec < 0)
timevalclear(&utv);
timeout_hz = tvtohz(&utv);
} else
timeout_hz = 0;
error = futex_wait(f, wp, timeout_hz, args->val3);
break;
case LINUX_FUTEX_WAKE:
args->val3 = FUTEX_BITSET_MATCH_ANY;
/* FALLTHROUGH */
case LINUX_FUTEX_WAKE_BITSET:
LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wake, args->uaddr,
args->val, args->val3);
LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x",
args->uaddr, args->val, args->val3);
error = futex_get(args->uaddr, NULL, &f,
flags | FUTEX_DONTCREATE);
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
if (f == NULL) {
td->td_retval[0] = 0;
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
td->td_retval[0] = futex_wake(f, args->val, args->val3);
futex_put(f, NULL);
break;
case LINUX_FUTEX_CMP_REQUEUE:
LIN_SDT_PROBE5(futex, linux_sys_futex, debug_cmp_requeue,
args->uaddr, args->val, args->val3, args->uaddr2,
args->timeout);
LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p "
"nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x",
args->uaddr, args->val, args->val3, args->uaddr2,
args->timeout);
/*
* Linux allows this, we would not, it is an incorrect
* usage of declared ABI, so return EINVAL.
*/
if (args->uaddr == args->uaddr2) {
LIN_SDT_PROBE0(futex, linux_sys_futex,
invalid_cmp_requeue_use);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
return (EINVAL);
}
error = futex_get(args->uaddr, NULL, &f, flags);
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
/*
* To avoid deadlocks return EINVAL if second futex
* exists at this time.
*
* Glibc fall back to FUTEX_WAKE in case of any error
* returned by FUTEX_CMP_REQUEUE.
*/
error = futex_get(args->uaddr2, NULL, &f2,
flags | FUTEX_DONTEXISTS);
if (error) {
futex_put(f, NULL);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
error = copyin(args->uaddr, &val, sizeof(val));
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
error);
LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d",
error);
futex_put(f2, NULL);
futex_put(f, NULL);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
if (val != args->val3) {
LIN_SDT_PROBE2(futex, linux_sys_futex,
debug_cmp_requeue_value_neq, args->val, val);
LINUX_CTR2(sys_futex, "CMP_REQUEUE val 0x%x != uval 0x%x",
args->val, val);
futex_put(f2, NULL);
futex_put(f, NULL);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, EAGAIN);
return (EAGAIN);
}
nrwake = (int)(unsigned long)args->timeout;
td->td_retval[0] = futex_requeue(f, args->val, f2, nrwake);
futex_put(f2, NULL);
futex_put(f, NULL);
break;
case LINUX_FUTEX_WAKE_OP:
LIN_SDT_PROBE5(futex, linux_sys_futex, debug_wake_op,
args->uaddr, args->op, args->val, args->uaddr2, args->val3);
LINUX_CTR5(sys_futex, "WAKE_OP "
"uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x",
args->uaddr, args->val, args->uaddr2, args->val3,
args->timeout);
error = futex_get(args->uaddr, NULL, &f, flags);
if (error) {
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
if (args->uaddr != args->uaddr2)
error = futex_get(args->uaddr2, NULL, &f2, flags);
if (error) {
futex_put(f, NULL);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
/*
* This function returns positive number as results and
* negative as errors
*/
op_ret = futex_atomic_op(td, args->val3, args->uaddr2);
LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x",
args->uaddr, op_ret);
if (op_ret < 0) {
/* XXX: We don't handle the EFAULT yet. */
if (op_ret != -EFAULT) {
if (f2 != NULL)
futex_put(f2, NULL);
futex_put(f, NULL);
LIN_SDT_PROBE1(futex, linux_sys_futex, return,
-op_ret);
return (-op_ret);
} else {
LIN_SDT_PROBE0(futex, linux_sys_futex,
unhandled_efault);
}
if (f2 != NULL)
futex_put(f2, NULL);
futex_put(f, NULL);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, EFAULT);
return (EFAULT);
}
ret = futex_wake(f, args->val, args->val3);
if (op_ret > 0) {
op_ret = 0;
nrwake = (int)(unsigned long)args->timeout;
if (f2 != NULL)
op_ret += futex_wake(f2, nrwake, args->val3);
else
op_ret += futex_wake(f, nrwake, args->val3);
ret += op_ret;
}
if (f2 != NULL)
futex_put(f2, NULL);
futex_put(f, NULL);
td->td_retval[0] = ret;
break;
case LINUX_FUTEX_LOCK_PI:
/* not yet implemented */
linux_msg(td,
"linux_sys_futex: "
"op LINUX_FUTEX_LOCK_PI not implemented\n");
LIN_SDT_PROBE0(futex, linux_sys_futex, unimplemented_lock_pi);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
case LINUX_FUTEX_UNLOCK_PI:
/* not yet implemented */
linux_msg(td,
"linux_sys_futex: "
"op LINUX_FUTEX_UNLOCK_PI not implemented\n");
LIN_SDT_PROBE0(futex, linux_sys_futex, unimplemented_unlock_pi);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
case LINUX_FUTEX_TRYLOCK_PI:
/* not yet implemented */
linux_msg(td,
"linux_sys_futex: "
"op LINUX_FUTEX_TRYLOCK_PI not implemented\n");
LIN_SDT_PROBE0(futex, linux_sys_futex,
unimplemented_trylock_pi);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
case LINUX_FUTEX_REQUEUE:
/*
* Glibc does not use this operation since version 2.3.3,
* as it is racy and replaced by FUTEX_CMP_REQUEUE operation.
* Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when
* FUTEX_REQUEUE returned EINVAL.
*/
em = em_find(td);
if ((em->flags & LINUX_XDEPR_REQUEUEOP) == 0) {
linux_msg(td,
"linux_sys_futex: "
"unsupported futex_requeue op\n");
em->flags |= LINUX_XDEPR_REQUEUEOP;
LIN_SDT_PROBE0(futex, linux_sys_futex,
deprecated_requeue);
}
LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
return (EINVAL);
case LINUX_FUTEX_WAIT_REQUEUE_PI:
/* not yet implemented */
linux_msg(td,
"linux_sys_futex: "
"op FUTEX_WAIT_REQUEUE_PI not implemented\n");
LIN_SDT_PROBE0(futex, linux_sys_futex,
unimplemented_wait_requeue_pi);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
case LINUX_FUTEX_CMP_REQUEUE_PI:
/* not yet implemented */
linux_msg(td,
"linux_sys_futex: "
"op LINUX_FUTEX_CMP_REQUEUE_PI not implemented\n");
LIN_SDT_PROBE0(futex, linux_sys_futex,
unimplemented_cmp_requeue_pi);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
default:
linux_msg(td,
"linux_sys_futex: unknown op %d\n", args->op);
LIN_SDT_PROBE1(futex, linux_sys_futex, unknown_operation,
args->op);
LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
return (ENOSYS);
}
LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
return (error);
}
int
linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args)
{
struct linux_emuldata *em;
LIN_SDT_PROBE2(futex, linux_set_robust_list, entry, td, args);
if (args->len != sizeof(struct linux_robust_list_head)) {
LIN_SDT_PROBE0(futex, linux_set_robust_list, size_error);
LIN_SDT_PROBE1(futex, linux_set_robust_list, return, EINVAL);
return (EINVAL);
}
em = em_find(td);
em->robust_futexes = args->head;
LIN_SDT_PROBE1(futex, linux_set_robust_list, return, 0);
return (0);
}
int
linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args)
{
struct linux_emuldata *em;
struct linux_robust_list_head *head;
l_size_t len = sizeof(struct linux_robust_list_head);
struct thread *td2;
int error = 0;
LIN_SDT_PROBE2(futex, linux_get_robust_list, entry, td, args);
if (!args->pid) {
em = em_find(td);
KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
head = em->robust_futexes;
} else {
td2 = tdfind(args->pid, -1);
if (td2 == NULL) {
LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
ESRCH);
return (ESRCH);
}
em = em_find(td2);
KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
/* XXX: ptrace? */
if (priv_check(td, PRIV_CRED_SETUID) ||
priv_check(td, PRIV_CRED_SETEUID) ||
p_candebug(td, td2->td_proc)) {
PROC_UNLOCK(td2->td_proc);
LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
EPERM);
return (EPERM);
}
head = em->robust_futexes;
PROC_UNLOCK(td2->td_proc);
}
error = copyout(&len, args->len, sizeof(l_size_t));
if (error) {
LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
error);
LIN_SDT_PROBE1(futex, linux_get_robust_list, return, EFAULT);
return (EFAULT);
}
error = copyout(head, args->head, sizeof(struct linux_robust_list_head));
if (error) {
LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
error);
}
LIN_SDT_PROBE1(futex, linux_get_robust_list, return, error);
return (error);
}
static int
handle_futex_death(struct linux_emuldata *em, uint32_t *uaddr,
unsigned int pi)
{
uint32_t uval, nval, mval;
struct futex *f;
int error;
LIN_SDT_PROBE3(futex, handle_futex_death, entry, em, uaddr, pi);
retry:
error = copyin(uaddr, &uval, 4);
if (error) {
LIN_SDT_PROBE1(futex, handle_futex_death, copyin_error, error);
LIN_SDT_PROBE1(futex, handle_futex_death, return, EFAULT);
return (EFAULT);
}
if ((uval & FUTEX_TID_MASK) == em->em_tid) {
mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
nval = casuword32(uaddr, uval, mval);
if (nval == -1) {
LIN_SDT_PROBE1(futex, handle_futex_death, return,
EFAULT);
return (EFAULT);
}
if (nval != uval)
goto retry;
if (!pi && (uval & FUTEX_WAITERS)) {
error = futex_get(uaddr, NULL, &f,
FUTEX_DONTCREATE | FUTEX_SHARED);
if (error) {
LIN_SDT_PROBE1(futex, handle_futex_death,
return, error);
return (error);
}
if (f != NULL) {
futex_wake(f, 1, FUTEX_BITSET_MATCH_ANY);
futex_put(f, NULL);
}
}
}
LIN_SDT_PROBE1(futex, handle_futex_death, return, 0);
return (0);
}
static int
fetch_robust_entry(struct linux_robust_list **entry,
struct linux_robust_list **head, unsigned int *pi)
{
l_ulong uentry;
int error;
LIN_SDT_PROBE3(futex, fetch_robust_entry, entry, entry, head, pi);
error = copyin((const void *)head, &uentry, sizeof(l_ulong));
if (error) {
LIN_SDT_PROBE1(futex, fetch_robust_entry, copyin_error, error);
LIN_SDT_PROBE1(futex, fetch_robust_entry, return, EFAULT);
return (EFAULT);
}
*entry = (void *)(uentry & ~1UL);
*pi = uentry & 1;
LIN_SDT_PROBE1(futex, fetch_robust_entry, return, 0);
return (0);
}
/* This walks the list of robust futexes releasing them. */
void
release_futexes(struct thread *td, struct linux_emuldata *em)
{
struct linux_robust_list_head *head = NULL;
struct linux_robust_list *entry, *next_entry, *pending;
unsigned int limit = 2048, pi, next_pi, pip;
l_long futex_offset;
int rc, error;
LIN_SDT_PROBE2(futex, release_futexes, entry, td, em);
head = em->robust_futexes;
if (head == NULL) {
LIN_SDT_PROBE0(futex, release_futexes, return);
return;
}
if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) {
LIN_SDT_PROBE0(futex, release_futexes, return);
return;
}
error = copyin(&head->futex_offset, &futex_offset,
sizeof(futex_offset));
if (error) {
LIN_SDT_PROBE1(futex, release_futexes, copyin_error, error);
LIN_SDT_PROBE0(futex, release_futexes, return);
return;
}
if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) {
LIN_SDT_PROBE0(futex, release_futexes, return);
return;
}
while (entry != &head->list) {
rc = fetch_robust_entry(&next_entry, PTRIN(&entry->next), &next_pi);
if (entry != pending)
if (handle_futex_death(em,
(uint32_t *)((caddr_t)entry + futex_offset), pi)) {
LIN_SDT_PROBE0(futex, release_futexes, return);
return;
}
if (rc) {
LIN_SDT_PROBE0(futex, release_futexes, return);
return;
}
entry = next_entry;
pi = next_pi;
if (!--limit)
break;
sched_relinquish(curthread);
}
if (pending)
handle_futex_death(em, (uint32_t *)((caddr_t)pending + futex_offset), pip);
LIN_SDT_PROBE0(futex, release_futexes, return);
}
Reference in New Issue View Git Blame Copy Permalink