9b2bbf0c4f
directly. This improves API compliance, asserts, etc. Reviewed by: kib, markj Differential Revision: https://reviews.freebsd.org/D23283
580 lines
15 KiB
C
580 lines
15 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
*
|
|
* Copyright (C) 2018 The FreeBSD Foundation. All rights reserved.
|
|
* Copyright (C) 2018, 2019 Andrew Turner
|
|
*
|
|
* This software was developed by Mitchell Horne under sponsorship of
|
|
* the FreeBSD Foundation.
|
|
*
|
|
* This software was developed by SRI International and the University of
|
|
* Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
|
|
* ("CTSRD"), as part of the DARPA CRASH research programme.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*
|
|
* $FreeBSD$
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/conf.h>
|
|
#include <sys/eventhandler.h>
|
|
#include <sys/kcov.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/limits.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/rwlock.h>
|
|
#include <sys/sysctl.h>
|
|
|
|
#include <vm/vm.h>
|
|
#include <vm/pmap.h>
|
|
#include <vm/vm_extern.h>
|
|
#include <vm/vm_object.h>
|
|
#include <vm/vm_page.h>
|
|
#include <vm/vm_pager.h>
|
|
#include <vm/vm_param.h>
|
|
|
|
MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type");
|
|
|
|
#define KCOV_ELEMENT_SIZE sizeof(uint64_t)
|
|
|
|
/*
|
|
* To know what the code can safely perform at any point in time we use a
|
|
* state machine. In the normal case the state transitions are:
|
|
*
|
|
* OPEN -> READY -> RUNNING -> DYING
|
|
* | | ^ | ^ ^
|
|
* | | +--------+ | |
|
|
* | +-------------------+ |
|
|
* +-----------------------------+
|
|
*
|
|
* The states are:
|
|
* OPEN: The kcov fd has been opened, but no buffer is available to store
|
|
* coverage data.
|
|
* READY: The buffer to store coverage data has been allocated. Userspace
|
|
* can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When
|
|
* this has been set the buffer can be written to by the kernel,
|
|
* and mmaped by userspace.
|
|
* RUNNING: The coverage probes are able to store coverage data in the buffer.
|
|
* This is entered with ioctl(fd, KIOENABLE, mode);. The READY state
|
|
* can be exited by ioctl(fd, KIODISABLE); or exiting the thread to
|
|
* return to the READY state to allow tracing to be reused, or by
|
|
* closing the kcov fd to enter the DYING state.
|
|
* DYING: The fd has been closed. All states can enter into this state when
|
|
* userspace closes the kcov fd.
|
|
*
|
|
* We need to be careful when moving into and out of the RUNNING state. As
|
|
* an interrupt may happen while this is happening the ordering of memory
|
|
* operations is important so struct kcov_info is valid for the tracing
|
|
* functions.
|
|
*
|
|
* When moving into the RUNNING state prior stores to struct kcov_info need
|
|
* to be observed before the state is set. This allows for interrupts that
|
|
* may call into one of the coverage functions to fire at any point while
|
|
* being enabled and see a consistent struct kcov_info.
|
|
*
|
|
* When moving out of the RUNNING state any later stores to struct kcov_info
|
|
* need to be observed after the state is set. As with entering this is to
|
|
* present a consistent struct kcov_info to interrupts.
|
|
*/
|
|
typedef enum {
|
|
KCOV_STATE_INVALID,
|
|
KCOV_STATE_OPEN, /* The device is open, but with no buffer */
|
|
KCOV_STATE_READY, /* The buffer has been allocated */
|
|
KCOV_STATE_RUNNING, /* Recording trace data */
|
|
KCOV_STATE_DYING, /* The fd was closed */
|
|
} kcov_state_t;
|
|
|
|
/*
|
|
* (l) Set while holding the kcov_lock mutex and not in the RUNNING state.
|
|
* (o) Only set once while in the OPEN state. Cleaned up while in the DYING
|
|
* state, and with no thread associated with the struct kcov_info.
|
|
* (s) Set atomically to enter or exit the RUNNING state, non-atomically
|
|
* otherwise. See above for a description of the other constraints while
|
|
* moving into or out of the RUNNING state.
|
|
*/
|
|
struct kcov_info {
|
|
struct thread *thread; /* (l) */
|
|
vm_object_t bufobj; /* (o) */
|
|
vm_offset_t kvaddr; /* (o) */
|
|
size_t entries; /* (o) */
|
|
size_t bufsize; /* (o) */
|
|
kcov_state_t state; /* (s) */
|
|
int mode; /* (l) */
|
|
};
|
|
|
|
/* Prototypes */
|
|
static d_open_t kcov_open;
|
|
static d_close_t kcov_close;
|
|
static d_mmap_single_t kcov_mmap_single;
|
|
static d_ioctl_t kcov_ioctl;
|
|
|
|
static int kcov_alloc(struct kcov_info *info, size_t entries);
|
|
static void kcov_free(struct kcov_info *info);
|
|
static void kcov_init(const void *unused);
|
|
|
|
static struct cdevsw kcov_cdevsw = {
|
|
.d_version = D_VERSION,
|
|
.d_open = kcov_open,
|
|
.d_close = kcov_close,
|
|
.d_mmap_single = kcov_mmap_single,
|
|
.d_ioctl = kcov_ioctl,
|
|
.d_name = "kcov",
|
|
};
|
|
|
|
SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW, 0, "Kernel coverage");
|
|
|
|
static u_int kcov_max_entries = KCOV_MAXENTRIES;
|
|
SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW,
|
|
&kcov_max_entries, 0,
|
|
"Maximum number of entries in the kcov buffer");
|
|
|
|
static struct mtx kcov_lock;
|
|
static int active_count;
|
|
|
|
static struct kcov_info *
|
|
get_kinfo(struct thread *td)
|
|
{
|
|
struct kcov_info *info;
|
|
|
|
/* We might have a NULL thread when releasing the secondary CPUs */
|
|
if (td == NULL)
|
|
return (NULL);
|
|
|
|
/*
|
|
* We are in an interrupt, stop tracing as it is not explicitly
|
|
* part of a syscall.
|
|
*/
|
|
if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL)
|
|
return (NULL);
|
|
|
|
/*
|
|
* If info is NULL or the state is not running we are not tracing.
|
|
*/
|
|
info = td->td_kcov_info;
|
|
if (info == NULL ||
|
|
atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING)
|
|
return (NULL);
|
|
|
|
return (info);
|
|
}
|
|
|
|
static void
|
|
trace_pc(uintptr_t ret)
|
|
{
|
|
struct thread *td;
|
|
struct kcov_info *info;
|
|
uint64_t *buf, index;
|
|
|
|
td = curthread;
|
|
info = get_kinfo(td);
|
|
if (info == NULL)
|
|
return;
|
|
|
|
/*
|
|
* Check we are in the PC-trace mode.
|
|
*/
|
|
if (info->mode != KCOV_MODE_TRACE_PC)
|
|
return;
|
|
|
|
KASSERT(info->kvaddr != 0,
|
|
("__sanitizer_cov_trace_pc: NULL buf while running"));
|
|
|
|
buf = (uint64_t *)info->kvaddr;
|
|
|
|
/* The first entry of the buffer holds the index */
|
|
index = buf[0];
|
|
if (index + 2 > info->entries)
|
|
return;
|
|
|
|
buf[index + 1] = ret;
|
|
buf[0] = index + 1;
|
|
}
|
|
|
|
static bool
|
|
trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret)
|
|
{
|
|
struct thread *td;
|
|
struct kcov_info *info;
|
|
uint64_t *buf, index;
|
|
|
|
td = curthread;
|
|
info = get_kinfo(td);
|
|
if (info == NULL)
|
|
return (false);
|
|
|
|
/*
|
|
* Check we are in the comparison-trace mode.
|
|
*/
|
|
if (info->mode != KCOV_MODE_TRACE_CMP)
|
|
return (false);
|
|
|
|
KASSERT(info->kvaddr != 0,
|
|
("__sanitizer_cov_trace_pc: NULL buf while running"));
|
|
|
|
buf = (uint64_t *)info->kvaddr;
|
|
|
|
/* The first entry of the buffer holds the index */
|
|
index = buf[0];
|
|
|
|
/* Check we have space to store all elements */
|
|
if (index * 4 + 4 + 1 > info->entries)
|
|
return (false);
|
|
|
|
while (1) {
|
|
buf[index * 4 + 1] = type;
|
|
buf[index * 4 + 2] = arg1;
|
|
buf[index * 4 + 3] = arg2;
|
|
buf[index * 4 + 4] = ret;
|
|
|
|
if (atomic_cmpset_64(&buf[0], index, index + 1))
|
|
break;
|
|
buf[0] = index;
|
|
}
|
|
|
|
return (true);
|
|
}
|
|
|
|
/*
|
|
* The fd is being closed, cleanup everything we can.
|
|
*/
|
|
static void
|
|
kcov_mmap_cleanup(void *arg)
|
|
{
|
|
struct kcov_info *info = arg;
|
|
struct thread *thread;
|
|
|
|
mtx_lock_spin(&kcov_lock);
|
|
/*
|
|
* Move to KCOV_STATE_DYING to stop adding new entries.
|
|
*
|
|
* If the thread is running we need to wait until thread exit to
|
|
* clean up as it may currently be adding a new entry. If this is
|
|
* the case being in KCOV_STATE_DYING will signal that the buffer
|
|
* needs to be cleaned up.
|
|
*/
|
|
atomic_store_int(&info->state, KCOV_STATE_DYING);
|
|
atomic_thread_fence_seq_cst();
|
|
thread = info->thread;
|
|
mtx_unlock_spin(&kcov_lock);
|
|
|
|
if (thread != NULL)
|
|
return;
|
|
|
|
/*
|
|
* We can safely clean up the info struct as it is in the
|
|
* KCOV_STATE_DYING state with no thread associated.
|
|
*
|
|
* The KCOV_STATE_DYING stops new threads from using it.
|
|
* The lack of a thread means nothing is currently using the buffers.
|
|
*/
|
|
kcov_free(info);
|
|
}
|
|
|
|
static int
|
|
kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
|
|
{
|
|
struct kcov_info *info;
|
|
int error;
|
|
|
|
info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK);
|
|
info->state = KCOV_STATE_OPEN;
|
|
info->thread = NULL;
|
|
info->mode = -1;
|
|
|
|
if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0)
|
|
kcov_mmap_cleanup(info);
|
|
|
|
return (error);
|
|
}
|
|
|
|
static int
|
|
kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
|
|
{
|
|
struct kcov_info *info;
|
|
int error;
|
|
|
|
|
|
if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
|
|
return (error);
|
|
|
|
KASSERT(info != NULL, ("kcov_close with no kcov_info structure"));
|
|
|
|
/* Trying to close, but haven't disabled */
|
|
if (info->state == KCOV_STATE_RUNNING)
|
|
return (EBUSY);
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size,
|
|
struct vm_object **object, int nprot)
|
|
{
|
|
struct kcov_info *info;
|
|
int error;
|
|
|
|
if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) !=
|
|
(PROT_READ | PROT_WRITE))
|
|
return (EINVAL);
|
|
|
|
if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
|
|
return (error);
|
|
|
|
if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries)
|
|
return (EINVAL);
|
|
|
|
vm_object_reference(info->bufobj);
|
|
*offset = 0;
|
|
*object = info->bufobj;
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
kcov_alloc(struct kcov_info *info, size_t entries)
|
|
{
|
|
size_t n, pages;
|
|
vm_page_t m;
|
|
|
|
KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer"));
|
|
KASSERT(info->state == KCOV_STATE_OPEN,
|
|
("kcov_alloc: Not in open state (%x)", info->state));
|
|
|
|
if (entries < 2 || entries > kcov_max_entries)
|
|
return (EINVAL);
|
|
|
|
/* Align to page size so mmap can't access other kernel memory */
|
|
info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE);
|
|
pages = info->bufsize / PAGE_SIZE;
|
|
|
|
if ((info->kvaddr = kva_alloc(info->bufsize)) == 0)
|
|
return (ENOMEM);
|
|
|
|
info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize,
|
|
PROT_READ | PROT_WRITE, 0, curthread->td_ucred);
|
|
|
|
VM_OBJECT_WLOCK(info->bufobj);
|
|
for (n = 0; n < pages; n++) {
|
|
m = vm_page_grab(info->bufobj, n,
|
|
VM_ALLOC_ZERO | VM_ALLOC_WIRED);
|
|
vm_page_valid(m);
|
|
vm_page_xunbusy(m);
|
|
pmap_qenter(info->kvaddr + n * PAGE_SIZE, &m, 1);
|
|
}
|
|
VM_OBJECT_WUNLOCK(info->bufobj);
|
|
|
|
info->entries = entries;
|
|
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
kcov_free(struct kcov_info *info)
|
|
{
|
|
vm_page_t m;
|
|
size_t i;
|
|
|
|
if (info->kvaddr != 0) {
|
|
pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE);
|
|
kva_free(info->kvaddr, info->bufsize);
|
|
}
|
|
if (info->bufobj != NULL) {
|
|
VM_OBJECT_WLOCK(info->bufobj);
|
|
m = vm_page_lookup(info->bufobj, 0);
|
|
for (i = 0; i < info->bufsize / PAGE_SIZE; i++) {
|
|
vm_page_unwire_noq(m);
|
|
m = vm_page_next(m);
|
|
}
|
|
VM_OBJECT_WUNLOCK(info->bufobj);
|
|
vm_object_deallocate(info->bufobj);
|
|
}
|
|
free(info, M_KCOV_INFO);
|
|
}
|
|
|
|
static int
|
|
kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused,
|
|
struct thread *td)
|
|
{
|
|
struct kcov_info *info;
|
|
int mode, error;
|
|
|
|
if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
|
|
return (error);
|
|
|
|
if (cmd == KIOSETBUFSIZE) {
|
|
/*
|
|
* Set the size of the coverage buffer. Should be called
|
|
* before enabling coverage collection for that thread.
|
|
*/
|
|
if (info->state != KCOV_STATE_OPEN) {
|
|
return (EBUSY);
|
|
}
|
|
error = kcov_alloc(info, *(u_int *)data);
|
|
if (error == 0)
|
|
info->state = KCOV_STATE_READY;
|
|
return (error);
|
|
}
|
|
|
|
mtx_lock_spin(&kcov_lock);
|
|
switch (cmd) {
|
|
case KIOENABLE:
|
|
if (info->state != KCOV_STATE_READY) {
|
|
error = EBUSY;
|
|
break;
|
|
}
|
|
if (td->td_kcov_info != NULL) {
|
|
error = EINVAL;
|
|
break;
|
|
}
|
|
mode = *(int *)data;
|
|
if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) {
|
|
error = EINVAL;
|
|
break;
|
|
}
|
|
|
|
/* Lets hope nobody opens this 2 billion times */
|
|
KASSERT(active_count < INT_MAX,
|
|
("%s: Open too many times", __func__));
|
|
active_count++;
|
|
if (active_count == 1) {
|
|
cov_register_pc(&trace_pc);
|
|
cov_register_cmp(&trace_cmp);
|
|
}
|
|
|
|
KASSERT(info->thread == NULL,
|
|
("Enabling kcov when already enabled"));
|
|
info->thread = td;
|
|
info->mode = mode;
|
|
/*
|
|
* Ensure the mode has been set before starting coverage
|
|
* tracing.
|
|
*/
|
|
atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING);
|
|
td->td_kcov_info = info;
|
|
break;
|
|
case KIODISABLE:
|
|
/* Only the currently enabled thread may disable itself */
|
|
if (info->state != KCOV_STATE_RUNNING ||
|
|
info != td->td_kcov_info) {
|
|
error = EINVAL;
|
|
break;
|
|
}
|
|
KASSERT(active_count > 0, ("%s: Open count is zero", __func__));
|
|
active_count--;
|
|
if (active_count == 0) {
|
|
cov_unregister_pc();
|
|
cov_unregister_cmp();
|
|
}
|
|
|
|
td->td_kcov_info = NULL;
|
|
atomic_store_int(&info->state, KCOV_STATE_READY);
|
|
/*
|
|
* Ensure we have exited the READY state before clearing the
|
|
* rest of the info struct.
|
|
*/
|
|
atomic_thread_fence_rel();
|
|
info->mode = -1;
|
|
info->thread = NULL;
|
|
break;
|
|
default:
|
|
error = EINVAL;
|
|
break;
|
|
}
|
|
mtx_unlock_spin(&kcov_lock);
|
|
|
|
return (error);
|
|
}
|
|
|
|
static void
|
|
kcov_thread_dtor(void *arg __unused, struct thread *td)
|
|
{
|
|
struct kcov_info *info;
|
|
|
|
info = td->td_kcov_info;
|
|
if (info == NULL)
|
|
return;
|
|
|
|
mtx_lock_spin(&kcov_lock);
|
|
KASSERT(active_count > 0, ("%s: Open count is zero", __func__));
|
|
active_count--;
|
|
if (active_count == 0) {
|
|
cov_unregister_pc();
|
|
cov_unregister_cmp();
|
|
}
|
|
td->td_kcov_info = NULL;
|
|
if (info->state != KCOV_STATE_DYING) {
|
|
/*
|
|
* The kcov file is still open. Mark it as unused and
|
|
* wait for it to be closed before cleaning up.
|
|
*/
|
|
atomic_store_int(&info->state, KCOV_STATE_READY);
|
|
atomic_thread_fence_seq_cst();
|
|
/* This info struct is unused */
|
|
info->thread = NULL;
|
|
mtx_unlock_spin(&kcov_lock);
|
|
return;
|
|
}
|
|
mtx_unlock_spin(&kcov_lock);
|
|
|
|
/*
|
|
* We can safely clean up the info struct as it is in the
|
|
* KCOV_STATE_DYING state where the info struct is associated with
|
|
* the current thread that's about to exit.
|
|
*
|
|
* The KCOV_STATE_DYING stops new threads from using it.
|
|
* It also stops the current thread from trying to use the info struct.
|
|
*/
|
|
kcov_free(info);
|
|
}
|
|
|
|
static void
|
|
kcov_init(const void *unused)
|
|
{
|
|
struct make_dev_args args;
|
|
struct cdev *dev;
|
|
|
|
mtx_init(&kcov_lock, "kcov lock", NULL, MTX_SPIN);
|
|
|
|
make_dev_args_init(&args);
|
|
args.mda_devsw = &kcov_cdevsw;
|
|
args.mda_uid = UID_ROOT;
|
|
args.mda_gid = GID_WHEEL;
|
|
args.mda_mode = 0600;
|
|
if (make_dev_s(&args, &dev, "kcov") != 0) {
|
|
printf("%s", "Failed to create kcov device");
|
|
return;
|
|
}
|
|
|
|
EVENTHANDLER_REGISTER(thread_dtor, kcov_thread_dtor, NULL,
|
|
EVENTHANDLER_PRI_ANY);
|
|
}
|
|
|
|
SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL);
|