Split RPC pool threads into number of smaller semi-isolated groups.
Old design with unified thread pool was good from the point of thread utilization. But single pool-wide mutex became huge congestion point for systems with many CPUs. To reduce the congestion create several thread groups within a pool (one group for every 6 CPUs and 12 threads), each group with own mutex. Each connection during its registration is assigned to one of the groups in round-robin fashion. File affinify code may still move requests between the groups, but otherwise groups are self-contained. MFC after: 2 weeks Sponsored by: iXsystems, Inc.
This commit is contained in:
parent
a288c757d4
commit
b563304c50
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=267228
351
sys/rpc/svc.c
351
sys/rpc/svc.c
@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/queue.h>
|
||||
#include <sys/socketvar.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/smp.h>
|
||||
#include <sys/sx.h>
|
||||
#include <sys/ucred.h>
|
||||
|
||||
@ -70,7 +71,7 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
static struct svc_callout *svc_find(SVCPOOL *pool, rpcprog_t, rpcvers_t,
|
||||
char *);
|
||||
static void svc_new_thread(SVCPOOL *pool);
|
||||
static void svc_new_thread(SVCGROUP *grp);
|
||||
static void xprt_unregister_locked(SVCXPRT *xprt);
|
||||
static void svc_change_space_used(SVCPOOL *pool, int delta);
|
||||
static bool_t svc_request_space_available(SVCPOOL *pool);
|
||||
@ -79,11 +80,14 @@ static bool_t svc_request_space_available(SVCPOOL *pool);
|
||||
|
||||
static int svcpool_minthread_sysctl(SYSCTL_HANDLER_ARGS);
|
||||
static int svcpool_maxthread_sysctl(SYSCTL_HANDLER_ARGS);
|
||||
static int svcpool_threads_sysctl(SYSCTL_HANDLER_ARGS);
|
||||
|
||||
SVCPOOL*
|
||||
svcpool_create(const char *name, struct sysctl_oid_list *sysctl_base)
|
||||
{
|
||||
SVCPOOL *pool;
|
||||
SVCGROUP *grp;
|
||||
int g;
|
||||
|
||||
pool = malloc(sizeof(SVCPOOL), M_RPC, M_WAITOK|M_ZERO);
|
||||
|
||||
@ -91,15 +95,22 @@ svcpool_create(const char *name, struct sysctl_oid_list *sysctl_base)
|
||||
pool->sp_name = name;
|
||||
pool->sp_state = SVCPOOL_INIT;
|
||||
pool->sp_proc = NULL;
|
||||
TAILQ_INIT(&pool->sp_xlist);
|
||||
TAILQ_INIT(&pool->sp_active);
|
||||
TAILQ_INIT(&pool->sp_callouts);
|
||||
TAILQ_INIT(&pool->sp_lcallouts);
|
||||
LIST_INIT(&pool->sp_threads);
|
||||
LIST_INIT(&pool->sp_idlethreads);
|
||||
pool->sp_minthreads = 1;
|
||||
pool->sp_maxthreads = 1;
|
||||
pool->sp_threadcount = 0;
|
||||
pool->sp_groupcount = 1;
|
||||
for (g = 0; g < SVC_MAXGROUPS; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_init(&grp->sg_lock, "sg_lock", NULL, MTX_DEF);
|
||||
grp->sg_pool = pool;
|
||||
grp->sg_state = SVCPOOL_ACTIVE;
|
||||
TAILQ_INIT(&grp->sg_xlist);
|
||||
TAILQ_INIT(&grp->sg_active);
|
||||
LIST_INIT(&grp->sg_idlethreads);
|
||||
grp->sg_minthreads = 1;
|
||||
grp->sg_maxthreads = 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Don't use more than a quarter of mbuf clusters or more than
|
||||
@ -114,12 +125,19 @@ svcpool_create(const char *name, struct sysctl_oid_list *sysctl_base)
|
||||
if (sysctl_base) {
|
||||
SYSCTL_ADD_PROC(&pool->sp_sysctl, sysctl_base, OID_AUTO,
|
||||
"minthreads", CTLTYPE_INT | CTLFLAG_RW,
|
||||
pool, 0, svcpool_minthread_sysctl, "I", "");
|
||||
pool, 0, svcpool_minthread_sysctl, "I",
|
||||
"Minimal number of threads");
|
||||
SYSCTL_ADD_PROC(&pool->sp_sysctl, sysctl_base, OID_AUTO,
|
||||
"maxthreads", CTLTYPE_INT | CTLFLAG_RW,
|
||||
pool, 0, svcpool_maxthread_sysctl, "I", "");
|
||||
pool, 0, svcpool_maxthread_sysctl, "I",
|
||||
"Maximal number of threads");
|
||||
SYSCTL_ADD_PROC(&pool->sp_sysctl, sysctl_base, OID_AUTO,
|
||||
"threads", CTLTYPE_INT | CTLFLAG_RD,
|
||||
pool, 0, svcpool_threads_sysctl, "I",
|
||||
"Current number of threads");
|
||||
SYSCTL_ADD_INT(&pool->sp_sysctl, sysctl_base, OID_AUTO,
|
||||
"threads", CTLFLAG_RD, &pool->sp_threadcount, 0, "");
|
||||
"groups", CTLFLAG_RD, &pool->sp_groupcount, 0,
|
||||
"Number of thread groups");
|
||||
|
||||
SYSCTL_ADD_UINT(&pool->sp_sysctl, sysctl_base, OID_AUTO,
|
||||
"request_space_used", CTLFLAG_RD,
|
||||
@ -158,20 +176,29 @@ svcpool_create(const char *name, struct sysctl_oid_list *sysctl_base)
|
||||
void
|
||||
svcpool_destroy(SVCPOOL *pool)
|
||||
{
|
||||
SVCGROUP *grp;
|
||||
SVCXPRT *xprt, *nxprt;
|
||||
struct svc_callout *s;
|
||||
struct svc_loss_callout *sl;
|
||||
struct svcxprt_list cleanup;
|
||||
int g;
|
||||
|
||||
TAILQ_INIT(&cleanup);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
|
||||
while (TAILQ_FIRST(&pool->sp_xlist)) {
|
||||
xprt = TAILQ_FIRST(&pool->sp_xlist);
|
||||
xprt_unregister_locked(xprt);
|
||||
TAILQ_INSERT_TAIL(&cleanup, xprt, xp_link);
|
||||
for (g = 0; g < SVC_MAXGROUPS; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_lock(&grp->sg_lock);
|
||||
while ((xprt = TAILQ_FIRST(&grp->sg_xlist)) != NULL) {
|
||||
xprt_unregister_locked(xprt);
|
||||
TAILQ_INSERT_TAIL(&cleanup, xprt, xp_link);
|
||||
}
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
TAILQ_FOREACH_SAFE(xprt, &cleanup, xp_link, nxprt) {
|
||||
SVC_RELEASE(xprt);
|
||||
}
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
while ((s = TAILQ_FIRST(&pool->sp_callouts)) != NULL) {
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
svc_unreg(pool, s->sc_prog, s->sc_vers);
|
||||
@ -184,10 +211,10 @@ svcpool_destroy(SVCPOOL *pool)
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
|
||||
TAILQ_FOREACH_SAFE(xprt, &cleanup, xp_link, nxprt) {
|
||||
SVC_RELEASE(xprt);
|
||||
for (g = 0; g < SVC_MAXGROUPS; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_destroy(&grp->sg_lock);
|
||||
}
|
||||
|
||||
mtx_destroy(&pool->sp_lock);
|
||||
|
||||
if (pool->sp_rcache)
|
||||
@ -197,14 +224,23 @@ svcpool_destroy(SVCPOOL *pool)
|
||||
free(pool, M_RPC);
|
||||
}
|
||||
|
||||
static bool_t
|
||||
svcpool_active(SVCPOOL *pool)
|
||||
/*
|
||||
* Sysctl handler to get the present thread count on a pool
|
||||
*/
|
||||
static int
|
||||
svcpool_threads_sysctl(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
enum svcpool_state state = pool->sp_state;
|
||||
SVCPOOL *pool;
|
||||
int threads, error, g;
|
||||
|
||||
if (state == SVCPOOL_INIT || state == SVCPOOL_CLOSING)
|
||||
return (FALSE);
|
||||
return (TRUE);
|
||||
pool = oidp->oid_arg1;
|
||||
threads = 0;
|
||||
mtx_lock(&pool->sp_lock);
|
||||
for (g = 0; g < pool->sp_groupcount; g++)
|
||||
threads += pool->sp_groups[g].sg_threadcount;
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
error = sysctl_handle_int(oidp, &threads, 0, req);
|
||||
return (error);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -214,7 +250,7 @@ static int
|
||||
svcpool_minthread_sysctl(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
SVCPOOL *pool;
|
||||
int newminthreads, error, n;
|
||||
int newminthreads, error, g;
|
||||
|
||||
pool = oidp->oid_arg1;
|
||||
newminthreads = pool->sp_minthreads;
|
||||
@ -223,21 +259,11 @@ svcpool_minthread_sysctl(SYSCTL_HANDLER_ARGS)
|
||||
if (newminthreads > pool->sp_maxthreads)
|
||||
return (EINVAL);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
if (newminthreads > pool->sp_minthreads
|
||||
&& svcpool_active(pool)) {
|
||||
/*
|
||||
* If the pool is running and we are
|
||||
* increasing, create some more threads now.
|
||||
*/
|
||||
n = newminthreads - pool->sp_threadcount;
|
||||
if (n > 0) {
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
while (n--)
|
||||
svc_new_thread(pool);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
}
|
||||
}
|
||||
pool->sp_minthreads = newminthreads;
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
pool->sp_groups[g].sg_minthreads = max(1,
|
||||
pool->sp_minthreads / pool->sp_groupcount);
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
}
|
||||
return (error);
|
||||
@ -250,8 +276,7 @@ static int
|
||||
svcpool_maxthread_sysctl(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
SVCPOOL *pool;
|
||||
SVCTHREAD *st;
|
||||
int newmaxthreads, error;
|
||||
int newmaxthreads, error, g;
|
||||
|
||||
pool = oidp->oid_arg1;
|
||||
newmaxthreads = pool->sp_maxthreads;
|
||||
@ -260,17 +285,11 @@ svcpool_maxthread_sysctl(SYSCTL_HANDLER_ARGS)
|
||||
if (newmaxthreads < pool->sp_minthreads)
|
||||
return (EINVAL);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
if (newmaxthreads < pool->sp_maxthreads
|
||||
&& svcpool_active(pool)) {
|
||||
/*
|
||||
* If the pool is running and we are
|
||||
* decreasing, wake up some idle threads to
|
||||
* encourage them to exit.
|
||||
*/
|
||||
LIST_FOREACH(st, &pool->sp_idlethreads, st_ilink)
|
||||
cv_signal(&st->st_cond);
|
||||
}
|
||||
pool->sp_maxthreads = newmaxthreads;
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
pool->sp_groups[g].sg_maxthreads = max(1,
|
||||
pool->sp_maxthreads / pool->sp_groupcount);
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
}
|
||||
return (error);
|
||||
@ -283,13 +302,17 @@ void
|
||||
xprt_register(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp;
|
||||
int g;
|
||||
|
||||
SVC_ACQUIRE(xprt);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
g = atomic_fetchadd_int(&pool->sp_nextgroup, 1) % pool->sp_groupcount;
|
||||
xprt->xp_group = grp = &pool->sp_groups[g];
|
||||
mtx_lock(&grp->sg_lock);
|
||||
xprt->xp_registered = TRUE;
|
||||
xprt->xp_active = FALSE;
|
||||
TAILQ_INSERT_TAIL(&pool->sp_xlist, xprt, xp_link);
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
TAILQ_INSERT_TAIL(&grp->sg_xlist, xprt, xp_link);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -300,29 +323,29 @@ xprt_register(SVCXPRT *xprt)
|
||||
static void
|
||||
xprt_unregister_locked(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp = xprt->xp_group;
|
||||
|
||||
mtx_assert(&pool->sp_lock, MA_OWNED);
|
||||
mtx_assert(&grp->sg_lock, MA_OWNED);
|
||||
KASSERT(xprt->xp_registered == TRUE,
|
||||
("xprt_unregister_locked: not registered"));
|
||||
xprt_inactive_locked(xprt);
|
||||
TAILQ_REMOVE(&pool->sp_xlist, xprt, xp_link);
|
||||
TAILQ_REMOVE(&grp->sg_xlist, xprt, xp_link);
|
||||
xprt->xp_registered = FALSE;
|
||||
}
|
||||
|
||||
void
|
||||
xprt_unregister(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp = xprt->xp_group;
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
if (xprt->xp_registered == FALSE) {
|
||||
/* Already unregistered by another thread */
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
return;
|
||||
}
|
||||
xprt_unregister_locked(xprt);
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
|
||||
SVC_RELEASE(xprt);
|
||||
}
|
||||
@ -333,11 +356,11 @@ xprt_unregister(SVCXPRT *xprt)
|
||||
static int
|
||||
xprt_assignthread(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp = xprt->xp_group;
|
||||
SVCTHREAD *st;
|
||||
|
||||
mtx_assert(&pool->sp_lock, MA_OWNED);
|
||||
st = LIST_FIRST(&pool->sp_idlethreads);
|
||||
mtx_assert(&grp->sg_lock, MA_OWNED);
|
||||
st = LIST_FIRST(&grp->sg_idlethreads);
|
||||
if (st) {
|
||||
LIST_REMOVE(st, st_ilink);
|
||||
SVC_ACQUIRE(xprt);
|
||||
@ -354,10 +377,10 @@ xprt_assignthread(SVCXPRT *xprt)
|
||||
* from a socket upcall). Don't create more
|
||||
* than one thread per second.
|
||||
*/
|
||||
if (pool->sp_state == SVCPOOL_ACTIVE
|
||||
&& pool->sp_lastcreatetime < time_uptime
|
||||
&& pool->sp_threadcount < pool->sp_maxthreads) {
|
||||
pool->sp_state = SVCPOOL_THREADWANTED;
|
||||
if (grp->sg_state == SVCPOOL_ACTIVE
|
||||
&& grp->sg_lastcreatetime < time_uptime
|
||||
&& grp->sg_threadcount < grp->sg_maxthreads) {
|
||||
grp->sg_state = SVCPOOL_THREADWANTED;
|
||||
}
|
||||
}
|
||||
return (FALSE);
|
||||
@ -366,40 +389,40 @@ xprt_assignthread(SVCXPRT *xprt)
|
||||
void
|
||||
xprt_active(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp = xprt->xp_group;
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
|
||||
if (!xprt->xp_registered) {
|
||||
/*
|
||||
* Race with xprt_unregister - we lose.
|
||||
*/
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!xprt->xp_active) {
|
||||
xprt->xp_active = TRUE;
|
||||
if (xprt->xp_thread == NULL) {
|
||||
if (!svc_request_space_available(pool) ||
|
||||
if (!svc_request_space_available(xprt->xp_pool) ||
|
||||
!xprt_assignthread(xprt))
|
||||
TAILQ_INSERT_TAIL(&pool->sp_active, xprt,
|
||||
TAILQ_INSERT_TAIL(&grp->sg_active, xprt,
|
||||
xp_alink);
|
||||
}
|
||||
}
|
||||
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
void
|
||||
xprt_inactive_locked(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp = xprt->xp_group;
|
||||
|
||||
mtx_assert(&pool->sp_lock, MA_OWNED);
|
||||
mtx_assert(&grp->sg_lock, MA_OWNED);
|
||||
if (xprt->xp_active) {
|
||||
if (xprt->xp_thread == NULL)
|
||||
TAILQ_REMOVE(&pool->sp_active, xprt, xp_alink);
|
||||
TAILQ_REMOVE(&grp->sg_active, xprt, xp_alink);
|
||||
xprt->xp_active = FALSE;
|
||||
}
|
||||
}
|
||||
@ -407,11 +430,11 @@ xprt_inactive_locked(SVCXPRT *xprt)
|
||||
void
|
||||
xprt_inactive(SVCXPRT *xprt)
|
||||
{
|
||||
SVCPOOL *pool = xprt->xp_pool;
|
||||
SVCGROUP *grp = xprt->xp_group;
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
xprt_inactive_locked(xprt);
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -991,14 +1014,14 @@ svc_executereq(struct svc_req *rqstp)
|
||||
}
|
||||
|
||||
static void
|
||||
svc_checkidle(SVCPOOL *pool)
|
||||
svc_checkidle(SVCGROUP *grp)
|
||||
{
|
||||
SVCXPRT *xprt, *nxprt;
|
||||
time_t timo;
|
||||
struct svcxprt_list cleanup;
|
||||
|
||||
TAILQ_INIT(&cleanup);
|
||||
TAILQ_FOREACH_SAFE(xprt, &pool->sp_xlist, xp_link, nxprt) {
|
||||
TAILQ_FOREACH_SAFE(xprt, &grp->sg_xlist, xp_link, nxprt) {
|
||||
/*
|
||||
* Only some transports have idle timers. Don't time
|
||||
* something out which is just waking up.
|
||||
@ -1013,27 +1036,31 @@ svc_checkidle(SVCPOOL *pool)
|
||||
}
|
||||
}
|
||||
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
TAILQ_FOREACH_SAFE(xprt, &cleanup, xp_link, nxprt) {
|
||||
SVC_RELEASE(xprt);
|
||||
}
|
||||
mtx_lock(&pool->sp_lock);
|
||||
|
||||
mtx_lock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
svc_assign_waiting_sockets(SVCPOOL *pool)
|
||||
{
|
||||
SVCGROUP *grp;
|
||||
SVCXPRT *xprt;
|
||||
int g;
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
while ((xprt = TAILQ_FIRST(&pool->sp_active)) != NULL) {
|
||||
if (xprt_assignthread(xprt))
|
||||
TAILQ_REMOVE(&pool->sp_active, xprt, xp_alink);
|
||||
else
|
||||
break;
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_lock(&grp->sg_lock);
|
||||
while ((xprt = TAILQ_FIRST(&grp->sg_active)) != NULL) {
|
||||
if (xprt_assignthread(xprt))
|
||||
TAILQ_REMOVE(&grp->sg_active, xprt, xp_alink);
|
||||
else
|
||||
break;
|
||||
}
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -1067,8 +1094,9 @@ svc_request_space_available(SVCPOOL *pool)
|
||||
}
|
||||
|
||||
static void
|
||||
svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
svc_run_internal(SVCGROUP *grp, bool_t ismaster)
|
||||
{
|
||||
SVCPOOL *pool = grp->sg_pool;
|
||||
SVCTHREAD *st, *stpref;
|
||||
SVCXPRT *xprt;
|
||||
enum xprt_stat stat;
|
||||
@ -1083,35 +1111,34 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
STAILQ_INIT(&st->st_reqs);
|
||||
cv_init(&st->st_cond, "rpcsvc");
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
LIST_INSERT_HEAD(&pool->sp_threads, st, st_link);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
|
||||
/*
|
||||
* If we are a new thread which was spawned to cope with
|
||||
* increased load, set the state back to SVCPOOL_ACTIVE.
|
||||
*/
|
||||
if (pool->sp_state == SVCPOOL_THREADSTARTING)
|
||||
pool->sp_state = SVCPOOL_ACTIVE;
|
||||
if (grp->sg_state == SVCPOOL_THREADSTARTING)
|
||||
grp->sg_state = SVCPOOL_ACTIVE;
|
||||
|
||||
while (pool->sp_state != SVCPOOL_CLOSING) {
|
||||
while (grp->sg_state != SVCPOOL_CLOSING) {
|
||||
/*
|
||||
* Create new thread if requested.
|
||||
*/
|
||||
if (pool->sp_state == SVCPOOL_THREADWANTED) {
|
||||
pool->sp_state = SVCPOOL_THREADSTARTING;
|
||||
pool->sp_lastcreatetime = time_uptime;
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
svc_new_thread(pool);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
if (grp->sg_state == SVCPOOL_THREADWANTED) {
|
||||
grp->sg_state = SVCPOOL_THREADSTARTING;
|
||||
grp->sg_lastcreatetime = time_uptime;
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
svc_new_thread(grp);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check for idle transports once per second.
|
||||
*/
|
||||
if (time_uptime > pool->sp_lastidlecheck) {
|
||||
pool->sp_lastidlecheck = time_uptime;
|
||||
svc_checkidle(pool);
|
||||
if (time_uptime > grp->sg_lastidlecheck) {
|
||||
grp->sg_lastidlecheck = time_uptime;
|
||||
svc_checkidle(grp);
|
||||
}
|
||||
|
||||
xprt = st->st_xprt;
|
||||
@ -1119,7 +1146,7 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
/*
|
||||
* Enforce maxthreads count.
|
||||
*/
|
||||
if (pool->sp_threadcount > pool->sp_maxthreads)
|
||||
if (grp->sg_threadcount > grp->sg_maxthreads)
|
||||
break;
|
||||
|
||||
/*
|
||||
@ -1128,22 +1155,22 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
* by a thread.
|
||||
*/
|
||||
if (svc_request_space_available(pool) &&
|
||||
(xprt = TAILQ_FIRST(&pool->sp_active)) != NULL) {
|
||||
TAILQ_REMOVE(&pool->sp_active, xprt, xp_alink);
|
||||
(xprt = TAILQ_FIRST(&grp->sg_active)) != NULL) {
|
||||
TAILQ_REMOVE(&grp->sg_active, xprt, xp_alink);
|
||||
SVC_ACQUIRE(xprt);
|
||||
xprt->xp_thread = st;
|
||||
st->st_xprt = xprt;
|
||||
continue;
|
||||
}
|
||||
|
||||
LIST_INSERT_HEAD(&pool->sp_idlethreads, st, st_ilink);
|
||||
LIST_INSERT_HEAD(&grp->sg_idlethreads, st, st_ilink);
|
||||
if (ismaster || (!ismaster &&
|
||||
pool->sp_threadcount > pool->sp_minthreads))
|
||||
grp->sg_threadcount > grp->sg_minthreads))
|
||||
error = cv_timedwait_sig(&st->st_cond,
|
||||
&pool->sp_lock, 5 * hz);
|
||||
&grp->sg_lock, 5 * hz);
|
||||
else
|
||||
error = cv_wait_sig(&st->st_cond,
|
||||
&pool->sp_lock);
|
||||
&grp->sg_lock);
|
||||
if (st->st_xprt == NULL)
|
||||
LIST_REMOVE(st, st_ilink);
|
||||
|
||||
@ -1152,19 +1179,19 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
*/
|
||||
if (error == EWOULDBLOCK) {
|
||||
if (!ismaster
|
||||
&& (pool->sp_threadcount
|
||||
> pool->sp_minthreads)
|
||||
&& (grp->sg_threadcount
|
||||
> grp->sg_minthreads)
|
||||
&& !st->st_xprt)
|
||||
break;
|
||||
} else if (error) {
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
svc_exit(pool);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
|
||||
/*
|
||||
* Drain the transport socket and queue up any RPCs.
|
||||
@ -1194,7 +1221,7 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
rqstp, rq_link);
|
||||
}
|
||||
} while (rqstp == NULL && stat == XPRT_MOREREQS
|
||||
&& pool->sp_state != SVCPOOL_CLOSING);
|
||||
&& grp->sg_state != SVCPOOL_CLOSING);
|
||||
|
||||
/*
|
||||
* Move this transport to the end of the active list to
|
||||
@ -1202,16 +1229,16 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
* If this was the last queued request, svc_getreq will end
|
||||
* up calling xprt_inactive to remove from the active list.
|
||||
*/
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
xprt->xp_thread = NULL;
|
||||
st->st_xprt = NULL;
|
||||
if (xprt->xp_active) {
|
||||
if (!svc_request_space_available(pool) ||
|
||||
!xprt_assignthread(xprt))
|
||||
TAILQ_INSERT_TAIL(&pool->sp_active,
|
||||
TAILQ_INSERT_TAIL(&grp->sg_active,
|
||||
xprt, xp_alink);
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
SVC_RELEASE(xprt);
|
||||
|
||||
/*
|
||||
@ -1228,7 +1255,7 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
}
|
||||
mtx_unlock(&st->st_lock);
|
||||
svc_change_space_used(pool, -sz);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
if (st->st_xprt) {
|
||||
@ -1236,46 +1263,43 @@ svc_run_internal(SVCPOOL *pool, bool_t ismaster)
|
||||
st->st_xprt = NULL;
|
||||
SVC_RELEASE(xprt);
|
||||
}
|
||||
|
||||
KASSERT(STAILQ_EMPTY(&st->st_reqs), ("stray reqs on exit"));
|
||||
LIST_REMOVE(st, st_link);
|
||||
pool->sp_threadcount--;
|
||||
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
|
||||
mtx_destroy(&st->st_lock);
|
||||
cv_destroy(&st->st_cond);
|
||||
mem_free(st, sizeof(*st));
|
||||
|
||||
grp->sg_threadcount--;
|
||||
if (!ismaster)
|
||||
wakeup(pool);
|
||||
wakeup(grp);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
svc_thread_start(void *arg)
|
||||
{
|
||||
|
||||
svc_run_internal((SVCPOOL *) arg, FALSE);
|
||||
svc_run_internal((SVCGROUP *) arg, FALSE);
|
||||
kthread_exit();
|
||||
}
|
||||
|
||||
static void
|
||||
svc_new_thread(SVCPOOL *pool)
|
||||
svc_new_thread(SVCGROUP *grp)
|
||||
{
|
||||
SVCPOOL *pool = grp->sg_pool;
|
||||
struct thread *td;
|
||||
|
||||
pool->sp_threadcount++;
|
||||
kthread_add(svc_thread_start, pool,
|
||||
pool->sp_proc, &td, 0, 0,
|
||||
grp->sg_threadcount++;
|
||||
kthread_add(svc_thread_start, grp, pool->sp_proc, &td, 0, 0,
|
||||
"%s: service", pool->sp_name);
|
||||
}
|
||||
|
||||
void
|
||||
svc_run(SVCPOOL *pool)
|
||||
{
|
||||
int i;
|
||||
int g, i;
|
||||
struct proc *p;
|
||||
struct thread *td;
|
||||
SVCGROUP *grp;
|
||||
|
||||
p = curproc;
|
||||
td = curthread;
|
||||
@ -1283,35 +1307,56 @@ svc_run(SVCPOOL *pool)
|
||||
"%s: master", pool->sp_name);
|
||||
pool->sp_state = SVCPOOL_ACTIVE;
|
||||
pool->sp_proc = p;
|
||||
pool->sp_lastcreatetime = time_uptime;
|
||||
pool->sp_threadcount = 1;
|
||||
|
||||
for (i = 1; i < pool->sp_minthreads; i++) {
|
||||
svc_new_thread(pool);
|
||||
/* Choose group count based on number of threads and CPUs. */
|
||||
pool->sp_groupcount = max(1, min(SVC_MAXGROUPS,
|
||||
min(pool->sp_maxthreads / 2, mp_ncpus) / 6));
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
grp->sg_minthreads = max(1,
|
||||
pool->sp_minthreads / pool->sp_groupcount);
|
||||
grp->sg_maxthreads = max(1,
|
||||
pool->sp_maxthreads / pool->sp_groupcount);
|
||||
grp->sg_lastcreatetime = time_uptime;
|
||||
}
|
||||
|
||||
svc_run_internal(pool, TRUE);
|
||||
/* Starting threads */
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
for (i = ((g == 0) ? 1 : 0); i < grp->sg_minthreads; i++)
|
||||
svc_new_thread(grp);
|
||||
}
|
||||
pool->sp_groups[0].sg_threadcount++;
|
||||
svc_run_internal(&pool->sp_groups[0], TRUE);
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
while (pool->sp_threadcount > 0)
|
||||
msleep(pool, &pool->sp_lock, 0, "svcexit", 0);
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
/* Waiting for threads to stop. */
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_lock(&grp->sg_lock);
|
||||
while (grp->sg_threadcount > 0)
|
||||
msleep(grp, &grp->sg_lock, 0, "svcexit", 0);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
svc_exit(SVCPOOL *pool)
|
||||
{
|
||||
SVCGROUP *grp;
|
||||
SVCTHREAD *st;
|
||||
int g;
|
||||
|
||||
mtx_lock(&pool->sp_lock);
|
||||
|
||||
if (pool->sp_state != SVCPOOL_CLOSING) {
|
||||
pool->sp_state = SVCPOOL_CLOSING;
|
||||
LIST_FOREACH(st, &pool->sp_idlethreads, st_ilink)
|
||||
cv_signal(&st->st_cond);
|
||||
pool->sp_state = SVCPOOL_CLOSING;
|
||||
for (g = 0; g < pool->sp_groupcount; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_lock(&grp->sg_lock);
|
||||
if (grp->sg_state != SVCPOOL_CLOSING) {
|
||||
grp->sg_state = SVCPOOL_CLOSING;
|
||||
LIST_FOREACH(st, &grp->sg_idlethreads, st_ilink)
|
||||
cv_signal(&st->st_cond);
|
||||
}
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
}
|
||||
|
||||
bool_t
|
||||
|
@ -137,6 +137,7 @@ struct xp_ops2 {
|
||||
|
||||
#ifdef _KERNEL
|
||||
struct __rpc_svcpool;
|
||||
struct __rpc_svcgroup;
|
||||
struct __rpc_svcthread;
|
||||
#endif
|
||||
|
||||
@ -150,6 +151,7 @@ typedef struct __rpc_svcxprt {
|
||||
volatile u_int xp_refs;
|
||||
struct sx xp_lock;
|
||||
struct __rpc_svcpool *xp_pool; /* owning pool (see below) */
|
||||
struct __rpc_svcgroup *xp_group; /* owning group (see below) */
|
||||
TAILQ_ENTRY(__rpc_svcxprt) xp_link;
|
||||
TAILQ_ENTRY(__rpc_svcxprt) xp_alink;
|
||||
bool_t xp_registered; /* xprt_register has been called */
|
||||
@ -245,8 +247,6 @@ struct svc_loss_callout {
|
||||
};
|
||||
TAILQ_HEAD(svc_loss_callout_list, svc_loss_callout);
|
||||
|
||||
struct __rpc_svcthread;
|
||||
|
||||
/*
|
||||
* Service request
|
||||
*/
|
||||
@ -296,7 +296,6 @@ typedef struct __rpc_svcthread {
|
||||
SVCXPRT *st_xprt; /* transport we are processing */
|
||||
struct svc_reqlist st_reqs; /* RPC requests to execute */
|
||||
struct cv st_cond; /* sleeping for work */
|
||||
LIST_ENTRY(__rpc_svcthread) st_link; /* all threads list */
|
||||
LIST_ENTRY(__rpc_svcthread) st_ilink; /* idle threads list */
|
||||
LIST_ENTRY(__rpc_svcthread) st_alink; /* application thread list */
|
||||
int st_p2; /* application workspace */
|
||||
@ -304,6 +303,36 @@ typedef struct __rpc_svcthread {
|
||||
} SVCTHREAD;
|
||||
LIST_HEAD(svcthread_list, __rpc_svcthread);
|
||||
|
||||
/*
|
||||
* A thread group contain all information needed to assign subset of
|
||||
* transports to subset of threads. On systems with many CPUs and many
|
||||
* threads that allows to reduce lock congestion and improve performance.
|
||||
* Hundreds of threads on dozens of CPUs sharing the single pool lock do
|
||||
* not scale well otherwise.
|
||||
*/
|
||||
TAILQ_HEAD(svcxprt_list, __rpc_svcxprt);
|
||||
enum svcpool_state {
|
||||
SVCPOOL_INIT, /* svc_run not called yet */
|
||||
SVCPOOL_ACTIVE, /* normal running state */
|
||||
SVCPOOL_THREADWANTED, /* new service thread requested */
|
||||
SVCPOOL_THREADSTARTING, /* new service thread started */
|
||||
SVCPOOL_CLOSING /* svc_exit called */
|
||||
};
|
||||
typedef struct __rpc_svcgroup {
|
||||
struct mtx_padalign sg_lock; /* protect the thread/req lists */
|
||||
struct __rpc_svcpool *sg_pool;
|
||||
enum svcpool_state sg_state; /* current pool state */
|
||||
struct svcxprt_list sg_xlist; /* all transports in the group */
|
||||
struct svcxprt_list sg_active; /* transports needing service */
|
||||
struct svcthread_list sg_idlethreads; /* idle service threads */
|
||||
|
||||
int sg_minthreads; /* minimum service thread count */
|
||||
int sg_maxthreads; /* maximum service thread count */
|
||||
int sg_threadcount; /* current service thread count */
|
||||
time_t sg_lastcreatetime; /* when we last started a thread */
|
||||
time_t sg_lastidlecheck; /* when we last checked idle transports */
|
||||
} SVCGROUP;
|
||||
|
||||
/*
|
||||
* In the kernel, we can't use global variables to store lists of
|
||||
* transports etc. since otherwise we could not have two unrelated RPC
|
||||
@ -316,32 +345,18 @@ LIST_HEAD(svcthread_list, __rpc_svcthread);
|
||||
* this to support something similar to the Solaris multi-threaded RPC
|
||||
* server.
|
||||
*/
|
||||
TAILQ_HEAD(svcxprt_list, __rpc_svcxprt);
|
||||
enum svcpool_state {
|
||||
SVCPOOL_INIT, /* svc_run not called yet */
|
||||
SVCPOOL_ACTIVE, /* normal running state */
|
||||
SVCPOOL_THREADWANTED, /* new service thread requested */
|
||||
SVCPOOL_THREADSTARTING, /* new service thread started */
|
||||
SVCPOOL_CLOSING /* svc_exit called */
|
||||
};
|
||||
typedef SVCTHREAD *pool_assign_fn(SVCTHREAD *, struct svc_req *);
|
||||
typedef void pool_done_fn(SVCTHREAD *, struct svc_req *);
|
||||
#define SVC_MAXGROUPS 16
|
||||
typedef struct __rpc_svcpool {
|
||||
struct mtx_padalign sp_lock; /* protect the transport lists */
|
||||
const char *sp_name; /* pool name (e.g. "nfsd", "NLM" */
|
||||
enum svcpool_state sp_state; /* current pool state */
|
||||
struct proc *sp_proc; /* process which is in svc_run */
|
||||
struct svcxprt_list sp_xlist; /* all transports in the pool */
|
||||
struct svcxprt_list sp_active; /* transports needing service */
|
||||
struct svc_callout_list sp_callouts; /* (prog,vers)->dispatch list */
|
||||
struct svc_loss_callout_list sp_lcallouts; /* loss->dispatch list */
|
||||
struct svcthread_list sp_threads; /* service threads */
|
||||
struct svcthread_list sp_idlethreads; /* idle service threads */
|
||||
int sp_minthreads; /* minimum service thread count */
|
||||
int sp_maxthreads; /* maximum service thread count */
|
||||
int sp_threadcount; /* current service thread count */
|
||||
time_t sp_lastcreatetime; /* when we last started a thread */
|
||||
time_t sp_lastidlecheck; /* when we last checked idle transports */
|
||||
|
||||
/*
|
||||
* Hooks to allow an application to control request to thread
|
||||
@ -364,6 +379,10 @@ typedef struct __rpc_svcpool {
|
||||
|
||||
struct replay_cache *sp_rcache; /* optional replay cache */
|
||||
struct sysctl_ctx_list sp_sysctl;
|
||||
|
||||
int sp_groupcount; /* Number of groups in the pool. */
|
||||
int sp_nextgroup; /* Next group to assign port. */
|
||||
SVCGROUP sp_groups[SVC_MAXGROUPS]; /* Thread/port groups. */
|
||||
} SVCPOOL;
|
||||
|
||||
#else
|
||||
|
@ -86,7 +86,8 @@ svc_create(
|
||||
rpcvers_t versnum, /* Version number */
|
||||
const char *nettype) /* Networktype token */
|
||||
{
|
||||
int num = 0;
|
||||
int g, num = 0;
|
||||
SVCGROUP *grp;
|
||||
SVCXPRT *xprt;
|
||||
struct netconfig *nconf;
|
||||
void *handle;
|
||||
@ -96,11 +97,14 @@ svc_create(
|
||||
return (0);
|
||||
}
|
||||
while ((nconf = __rpc_getconf(handle)) != NULL) {
|
||||
mtx_lock(&pool->sp_lock);
|
||||
TAILQ_FOREACH(xprt, &pool->sp_xlist, xp_link) {
|
||||
if (strcmp(xprt->xp_netid, nconf->nc_netid) == 0) {
|
||||
for (g = 0; g < SVC_MAXGROUPS; g++) {
|
||||
grp = &pool->sp_groups[g];
|
||||
mtx_lock(&grp->sg_lock);
|
||||
TAILQ_FOREACH(xprt, &grp->sg_xlist, xp_link) {
|
||||
if (strcmp(xprt->xp_netid, nconf->nc_netid))
|
||||
continue;
|
||||
/* Found an old one, use it */
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
(void) rpcb_unset(prognum, versnum, nconf);
|
||||
if (svc_reg(xprt, prognum, versnum,
|
||||
dispatch, nconf) == FALSE) {
|
||||
@ -108,15 +112,15 @@ svc_create(
|
||||
"svc_create: could not register prog %u vers %u on %s\n",
|
||||
(unsigned)prognum, (unsigned)versnum,
|
||||
nconf->nc_netid);
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
} else {
|
||||
num++;
|
||||
mtx_lock(&pool->sp_lock);
|
||||
mtx_lock(&grp->sg_lock);
|
||||
break;
|
||||
}
|
||||
}
|
||||
mtx_unlock(&grp->sg_lock);
|
||||
}
|
||||
mtx_unlock(&pool->sp_lock);
|
||||
if (xprt == NULL) {
|
||||
/* It was not found. Now create a new one */
|
||||
xprt = svc_tp_create(pool, dispatch, prognum, versnum,
|
||||
|
Loading…
Reference in New Issue
Block a user