freebsd-skq/sys/vm/vm_domain.c

375 lines
8.1 KiB
C
Raw Normal View History

Add an initial NUMA affinity/policy configuration for threads and processes. This is based on work done by jeff@ and jhb@, as well as the numa.diff patch that has been circulating when someone asks for first-touch NUMA on -10 or -11. * Introduce a simple set of VM policy and iterator types. * tie the policy types into the vm_phys path for now, mirroring how the initial first-touch allocation work was enabled. * add syscalls to control changing thread and process defaults. * add a global NUMA VM domain policy. * implement a simple cascade policy order - if a thread policy exists, use it; if a process policy exists, use it; use the default policy. * processes inherit policies from their parent processes, threads inherit policies from their parent threads. * add a simple tool (numactl) to query and modify default thread/process policities. * add documentation for the new syscalls, for numa and for numactl. * re-enable first touch NUMA again by default, as now policies can be set in a variety of methods. This is only relevant for very specific workloads. This doesn't pretend to be a final NUMA solution. The previous defaults in -HEAD (with MAXMEMDOM set) can be achieved by 'sysctl vm.default_policy=rr'. This is only relevant if MAXMEMDOM is set to something other than 1. Ie, if you're using GENERIC or a modified kernel with non-NUMA, then this is a glorified no-op for you. Thank you to Norse Corp for giving me access to rather large (for FreeBSD!) NUMA machines in order to develop and verify this. Thank you to Dell for providing me with dual socket sandybridge and westmere v3 hardware to do NUMA development with. Thank you to Scott Long at Netflix for providing me with access to the two-socket, four-domain haswell v3 hardware. Thank you to Peter Holm for running the stress testing suite against the NUMA branch during various stages of development! Tested: * MIPS (regression testing; non-NUMA) * i386 (regression testing; non-NUMA GENERIC) * amd64 (regression testing; non-NUMA GENERIC) * westmere, 2 socket (thankyou norse!) * sandy bridge, 2 socket (thankyou dell!) * ivy bridge, 2 socket (thankyou norse!) * westmere-EX, 4 socket / 1TB RAM (thankyou norse!) * haswell, 2 socket (thankyou norse!) * haswell v3, 2 socket (thankyou dell) * haswell v3, 2x18 core (thankyou scott long / netflix!) * Peter Holm ran a stress test suite on this work and found one issue, but has not been able to verify it (it doesn't look NUMA related, and he only saw it once over many testing runs.) * I've tested bhyve instances running in fixed NUMA domains and cpusets; all seems to work correctly. Verified: * intel-pcm - pcm-numa.x and pcm-memory.x, whilst selecting different NUMA policies for processes under test. Review: This was reviewed through phabricator (https://reviews.freebsd.org/D2559) as well as privately and via emails to freebsd-arch@. The git history with specific attributes is available at https://github.com/erikarn/freebsd/ in the NUMA branch (https://github.com/erikarn/freebsd/compare/local/adrian_numa_policy). This has been reviewed by a number of people (stas, rpaulo, kib, ngie, wblock) but not achieved a clear consensus. My hope is that with further exposure and testing more functionality can be implemented and evaluated. Notes: * The VM doesn't handle unbalanced domains very well, and if you have an overly unbalanced memory setup whilst under high memory pressure, VM page allocation may fail leading to a kernel panic. This was a problem in the past, but it's much more easily triggered now with these tools. * This work only controls the path through vm_phys; it doesn't yet strongly/predictably affect contigmalloc, KVA placement, UMA, etc. So, driver placement of memory isn't really guaranteed in any way. That's next on my plate. Sponsored by: Norse Corp, Inc.; Dell
2015-07-11 15:21:37 +00:00
/*-
* Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer,
* without modification.
* 2. Redistributions in binary form must reproduce at minimum a disclaimer
* similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
* redistribution must be conditioned upon including a substantially
* similar Disclaimer requirement for further binary redistribution.
*
* NO WARRANTY
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGES.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_vm.h"
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#if MAXMEMDOM > 1
#include <sys/proc.h>
#endif
#include <sys/queue.h>
#include <sys/rwlock.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/tree.h>
#include <sys/vmmeter.h>
#include <sys/seq.h>
#include <ddb/ddb.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_phys.h>
#include <vm/vm_domain.h>
static __inline int
vm_domain_rr_selectdomain(void)
{
#if MAXMEMDOM > 1
struct thread *td;
td = curthread;
td->td_dom_rr_idx++;
td->td_dom_rr_idx %= vm_ndomains;
return (td->td_dom_rr_idx);
#else
return (0);
#endif
}
/*
* This implements a very simple set of VM domain memory allocation
* policies and iterators.
*/
/*
* A VM domain policy represents a desired VM domain policy.
* Iterators implement searching through VM domains in a specific
* order.
*/
/*
* When setting a policy, the caller must establish their own
* exclusive write protection for the contents of the domain
* policy.
*/
int
vm_domain_policy_init(struct vm_domain_policy *vp)
{
bzero(vp, sizeof(*vp));
vp->p.policy = VM_POLICY_NONE;
vp->p.domain = -1;
return (0);
}
int
vm_domain_policy_set(struct vm_domain_policy *vp,
vm_domain_policy_type_t vt, int domain)
{
seq_write_begin(&vp->seq);
vp->p.policy = vt;
vp->p.domain = domain;
seq_write_end(&vp->seq);
return (0);
}
/*
* Take a local copy of a policy.
*
* The destination policy isn't write-barriered; this is used
* for doing local copies into something that isn't shared.
*/
void
vm_domain_policy_localcopy(struct vm_domain_policy *dst,
const struct vm_domain_policy *src)
{
seq_t seq;
for (;;) {
seq = seq_read(&src->seq);
*dst = *src;
if (seq_consistent(&src->seq, seq))
return;
cpu_spinwait();
}
}
/*
* Take a write-barrier copy of a policy.
*
* The destination policy is write -barriered; this is used
* for doing copies into policies that may be read by other
* threads.
*/
void
vm_domain_policy_copy(struct vm_domain_policy *dst,
const struct vm_domain_policy *src)
{
seq_t seq;
struct vm_domain_policy d;
for (;;) {
seq = seq_read(&src->seq);
d = *src;
if (seq_consistent(&src->seq, seq)) {
seq_write_begin(&dst->seq);
dst->p.domain = d.p.domain;
dst->p.policy = d.p.policy;
seq_write_end(&dst->seq);
return;
}
cpu_spinwait();
}
}
int
vm_domain_policy_validate(const struct vm_domain_policy *vp)
{
switch (vp->p.policy) {
case VM_POLICY_NONE:
case VM_POLICY_ROUND_ROBIN:
case VM_POLICY_FIRST_TOUCH:
case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
if (vp->p.domain == -1)
return (0);
return (-1);
case VM_POLICY_FIXED_DOMAIN:
case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains)
return (0);
return (-1);
default:
return (-1);
}
return (-1);
}
int
vm_domain_policy_cleanup(struct vm_domain_policy *vp)
{
/* For now, empty */
return (0);
}
int
vm_domain_iterator_init(struct vm_domain_iterator *vi)
{
/* Nothing to do for now */
return (0);
}
/*
* Manually setup an iterator with the given details.
*/
int
vm_domain_iterator_set(struct vm_domain_iterator *vi,
vm_domain_policy_type_t vt, int domain)
{
switch (vt) {
case VM_POLICY_FIXED_DOMAIN:
vi->policy = VM_POLICY_FIXED_DOMAIN;
vi->domain = domain;
vi->n = 1;
break;
case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN;
vi->domain = domain;
vi->n = vm_ndomains;
break;
case VM_POLICY_FIRST_TOUCH:
vi->policy = VM_POLICY_FIRST_TOUCH;
vi->domain = PCPU_GET(domain);
vi->n = 1;
break;
case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN;
vi->domain = PCPU_GET(domain);
vi->n = vm_ndomains;
break;
case VM_POLICY_ROUND_ROBIN:
default:
vi->policy = VM_POLICY_ROUND_ROBIN;
vi->domain = -1;
vi->n = vm_ndomains;
break;
}
return (0);
}
/*
* Setup an iterator based on the given policy.
*/
static inline void
_vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
const struct vm_domain_policy *vt)
{
/*
* Initialise the iterator.
*
* For first-touch, the initial domain is set
* via the current thread CPU domain.
*
* For fixed-domain, it's assumed that the
* caller has initialised the specific domain
* it is after.
*/
switch (vt->p.policy) {
case VM_POLICY_FIXED_DOMAIN:
vi->policy = vt->p.policy;
vi->domain = vt->p.domain;
vi->n = 1;
break;
case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
vi->policy = vt->p.policy;
vi->domain = vt->p.domain;
vi->n = vm_ndomains;
break;
case VM_POLICY_FIRST_TOUCH:
vi->policy = vt->p.policy;
vi->domain = PCPU_GET(domain);
vi->n = 1;
break;
case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
vi->policy = vt->p.policy;
vi->domain = PCPU_GET(domain);
vi->n = vm_ndomains;
break;
case VM_POLICY_ROUND_ROBIN:
default:
/*
* Default to round-robin policy.
*/
vi->policy = VM_POLICY_ROUND_ROBIN;
vi->domain = -1;
vi->n = vm_ndomains;
break;
}
}
void
vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
const struct vm_domain_policy *vt)
{
seq_t seq;
struct vm_domain_policy vt_lcl;
for (;;) {
seq = seq_read(&vt->seq);
vt_lcl = *vt;
if (seq_consistent(&vt->seq, seq)) {
_vm_domain_iterator_set_policy(vi, &vt_lcl);
return;
}
cpu_spinwait();
}
}
/*
* Return the next VM domain to use.
*
* Returns 0 w/ domain set to the next domain to use, or
* -1 to indicate no more domains are available.
*/
int
vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain)
{
/* General catch-all */
if (vi->n <= 0)
return (-1);
switch (vi->policy) {
case VM_POLICY_FIXED_DOMAIN:
case VM_POLICY_FIRST_TOUCH:
*domain = vi->domain;
vi->n--;
break;
case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
/*
* XXX TODO: skip over the rr'ed domain
* if it equals the one we started with.
*/
if (vi->n == vm_ndomains)
*domain = vi->domain;
else
*domain = vm_domain_rr_selectdomain();
vi->n--;
break;
case VM_POLICY_ROUND_ROBIN:
default:
*domain = vm_domain_rr_selectdomain();
vi->n--;
break;
}
return (0);
}
/*
* Returns 1 if the iteration is done, or 0 if it has not.
* This can only be called after at least one loop through
* the iterator. Ie, it's designed to be used as a tail
* check of a loop, not the head check of a loop.
*/
int
vm_domain_iterator_isdone(struct vm_domain_iterator *vi)
{
return (vi->n <= 0);
}
int
vm_domain_iterator_cleanup(struct vm_domain_iterator *vi)
{
return (0);
}