d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

epoch(9): cleanups, additional debug checks, and add global_epoch

Browse Source 
- GC the _nopreempt routines
    - to really benefit we'd need a separate routine
    - they're not currently in use
    - they complicate the API for no benefit at this time

- check that we're actually in a epoch section at exit

- handle epoch_call() early in boot

- Fix copyright declaration language

Approved by:	sbruno@
This commit is contained in:
Matt Macy 2018-05-13 23:24:48 +00:00
parent 33346ed65c
commit 1f4beb6312
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=333591
2 changed files with 56 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
sys/kern/subr_epoch.c
View File

@ -1,27 +1,29 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Neither the name of Matthew Macy nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
@ -49,7 +51,7 @@ __FBSDID("$FreeBSD$");
#include <ck_epoch.h>
MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation");
static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation");
/* arbitrary --- needs benchmarking */
#define MAX_ADAPTIVE_SPIN 5000
@ -116,6 +118,7 @@ struct epoch {
static __read_mostly int domcount[MAXMEMDOM];
static __read_mostly int domoffsets[MAXMEMDOM];
static __read_mostly int inited;
__read_mostly epoch_t global_epoch;
static void epoch_call_task(void *context);
@ -136,10 +139,8 @@ epoch_init(void *arg __unused)
migrate_count = counter_u64_alloc(M_WAITOK);
turnstile_count = counter_u64_alloc(M_WAITOK);
switch_count = counter_u64_alloc(M_WAITOK);
if (usedomains == false) {
inited = 1;
return;
}
if (usedomains == false)
goto done;
count = domain = 0;
domoffsets[0] = 0;
for (domain = 0; domain < vm_ndomains; domain++) {
@ -156,9 +157,11 @@ epoch_init(void *arg __unused)
break;
}
}
done:
inited = 1;
global_epoch = epoch_alloc();
}
SYSINIT(epoch, SI_SUB_CPU + 1, SI_ORDER_FIRST, epoch_init, NULL);
SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL);
static void
epoch_init_numa(epoch_t epoch)
@ -310,19 +313,6 @@ epoch_enter(epoch_t epoch)
critical_exit();
}
void
epoch_enter_nopreempt(epoch_t epoch)
{
struct epoch_pcpu_state *eps;
INIT_CHECK(epoch);
critical_enter();
eps = epoch->e_pcpu[curcpu];
curthread->td_epochnest++;
MPASS(curthread->td_epochnest < UCHAR_MAX - 2);
ck_epoch_begin(&eps->eps_record.er_record, NULL);
}
void
epoch_exit(epoch_t epoch)
{
@ -331,6 +321,7 @@ epoch_exit(epoch_t epoch)
td = curthread;
INIT_CHECK(epoch);
MPASS(td->td_epochnest);
critical_enter();
eps = epoch->e_pcpu[curcpu];
sched_unpin();
@ -342,19 +333,6 @@ epoch_exit(epoch_t epoch)
critical_exit();
}
void
epoch_exit_nopreempt(epoch_t epoch)
{
struct epoch_pcpu_state *eps;
INIT_CHECK(epoch);
MPASS(curthread->td_critnest);
eps = epoch->e_pcpu[curcpu];
ck_epoch_end(&eps->eps_record.er_record, NULL);
curthread->td_epochnest--;
critical_exit();
}
/*
* epoch_block_handler is a callback from the ck code when another thread is
* currently in an epoch section.
@ -541,12 +519,17 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t
cb = (void *)ctx;
MPASS(callback);
/* too early in boot to have epoch set up */
if (__predict_false(epoch == NULL)) {
callback(ctx);
return;
}
MPASS(cb->ec_callback == NULL);
MPASS(cb->ec_link.stqe_next == NULL);
MPASS(epoch);
MPASS(callback);
cb->ec_callback = callback;
counter_u64_add(epoch->e_frees, 1);
critical_enter();
eps = epoch->e_pcpu[curcpu];
STAILQ_INSERT_HEAD(&eps->eps_cblist, cb, ec_link);

41
sys/sys/epoch.h
View File

@ -1,27 +1,28 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Neither the name of Matthew Macy nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
@ -32,6 +33,8 @@
struct epoch;
typedef struct epoch *epoch_t;
extern epoch_t global_epoch;
struct epoch_context {
void *data[2];
} __aligned(sizeof(void *));
@ -42,8 +45,6 @@ epoch_t epoch_alloc(void);
void epoch_free(epoch_t epoch);
void epoch_enter(epoch_t epoch);
void epoch_exit(epoch_t epoch);
void epoch_enter_nopreempt(epoch_t epoch);
void epoch_exit_nopreempt(epoch_t epoch);
void epoch_wait(epoch_t epoch);
void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t));
int in_epoch(void);