2018-01-12 22:57:57 +00:00
|
|
|
/*-
|
|
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
|
|
*
|
|
|
|
* Copyright (c) 2017, Jeffrey Roberson <jeff@freebsd.org>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice unmodified, this list of conditions, and the following
|
|
|
|
* disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
|
|
|
#include "opt_vm.h"
|
|
|
|
|
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
|
|
|
#include <sys/bitset.h>
|
|
|
|
#include <sys/domainset.h>
|
|
|
|
#include <sys/proc.h>
|
|
|
|
#include <sys/lock.h>
|
|
|
|
#include <sys/mutex.h>
|
|
|
|
#include <sys/malloc.h>
|
2018-10-23 16:35:58 +00:00
|
|
|
#include <sys/rwlock.h>
|
2018-01-12 22:57:57 +00:00
|
|
|
#include <sys/vmmeter.h>
|
|
|
|
|
|
|
|
#include <vm/vm.h>
|
|
|
|
#include <vm/vm_param.h>
|
|
|
|
#include <vm/vm_domainset.h>
|
|
|
|
#include <vm/vm_object.h>
|
|
|
|
#include <vm/vm_page.h>
|
|
|
|
#include <vm/vm_phys.h>
|
|
|
|
|
2018-01-14 03:36:03 +00:00
|
|
|
#ifdef NUMA
|
2018-01-12 22:57:57 +00:00
|
|
|
/*
|
|
|
|
* Iterators are written such that the first nowait pass has as short a
|
|
|
|
* codepath as possible to eliminate bloat from the allocator. It is
|
|
|
|
* assumed that most allocations are successful.
|
|
|
|
*/
|
|
|
|
|
2018-03-29 02:54:50 +00:00
|
|
|
static int vm_domainset_default_stride = 64;
|
|
|
|
|
2018-01-12 22:57:57 +00:00
|
|
|
/*
|
|
|
|
* Determine which policy is to be used for this allocation.
|
|
|
|
*/
|
|
|
|
static void
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,
|
|
|
|
int *iter, struct vm_object *obj, vm_pindex_t pindex)
|
2018-01-12 22:57:57 +00:00
|
|
|
{
|
|
|
|
|
2018-10-23 16:35:58 +00:00
|
|
|
di->di_domain = ds;
|
|
|
|
di->di_iter = iter;
|
|
|
|
di->di_policy = ds->ds_policy;
|
2018-03-29 02:54:50 +00:00
|
|
|
if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {
|
2018-05-19 23:49:13 +00:00
|
|
|
#if VM_NRESERVLEVEL > 0
|
2018-03-29 02:54:50 +00:00
|
|
|
if (vm_object_reserv(obj)) {
|
|
|
|
/*
|
|
|
|
* Color the pindex so we end up on the correct
|
|
|
|
* reservation boundary.
|
|
|
|
*/
|
|
|
|
pindex += obj->pg_color;
|
|
|
|
pindex >>= VM_LEVEL_0_ORDER;
|
2018-05-19 23:49:13 +00:00
|
|
|
} else
|
|
|
|
#endif
|
2018-03-29 02:54:50 +00:00
|
|
|
pindex /= vm_domainset_default_stride;
|
|
|
|
/*
|
|
|
|
* Offset pindex so the first page of each object does
|
|
|
|
* not end up in domain 0.
|
|
|
|
*/
|
|
|
|
if (obj != NULL)
|
|
|
|
pindex += (((uintptr_t)obj) / sizeof(*obj));
|
|
|
|
di->di_offset = pindex;
|
|
|
|
}
|
2018-09-06 20:02:19 +00:00
|
|
|
/* Skip domains below min on the first pass. */
|
2018-09-06 19:28:52 +00:00
|
|
|
di->di_minskip = true;
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)
|
|
|
|
{
|
|
|
|
|
2018-03-29 02:54:50 +00:00
|
|
|
*domain = di->di_domain->ds_order[
|
|
|
|
++(*di->di_iter) % di->di_domain->ds_cnt];
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)
|
|
|
|
{
|
|
|
|
int d;
|
|
|
|
|
|
|
|
do {
|
2018-03-29 02:54:50 +00:00
|
|
|
d = di->di_domain->ds_order[
|
|
|
|
++(*di->di_iter) % di->di_domain->ds_cnt];
|
|
|
|
} while (d == di->di_domain->ds_prefer);
|
|
|
|
*domain = d;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
vm_domainset_iter_interleave(struct vm_domainset_iter *di, int *domain)
|
|
|
|
{
|
|
|
|
int d;
|
|
|
|
|
|
|
|
d = di->di_offset % di->di_domain->ds_cnt;
|
|
|
|
*di->di_iter = d;
|
|
|
|
*domain = di->di_domain->ds_order[d];
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
|
|
|
|
{
|
|
|
|
|
|
|
|
KASSERT(di->di_n > 0,
|
|
|
|
("vm_domainset_iter_first: Invalid n %d", di->di_n));
|
2018-03-29 02:54:50 +00:00
|
|
|
switch (di->di_policy) {
|
2018-01-12 22:57:57 +00:00
|
|
|
case DOMAINSET_POLICY_FIRSTTOUCH:
|
|
|
|
/*
|
|
|
|
* To prevent impossible allocations we convert an invalid
|
|
|
|
* first-touch to round-robin.
|
|
|
|
*/
|
|
|
|
/* FALLTHROUGH */
|
2018-03-29 02:54:50 +00:00
|
|
|
case DOMAINSET_POLICY_INTERLEAVE:
|
|
|
|
/* FALLTHROUGH */
|
2018-01-12 22:57:57 +00:00
|
|
|
case DOMAINSET_POLICY_ROUNDROBIN:
|
|
|
|
vm_domainset_iter_rr(di, domain);
|
|
|
|
break;
|
|
|
|
case DOMAINSET_POLICY_PREFER:
|
|
|
|
vm_domainset_iter_prefer(di, domain);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
panic("vm_domainset_iter_first: Unknown policy %d",
|
2018-03-29 02:54:50 +00:00
|
|
|
di->di_policy);
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
KASSERT(*domain < vm_ndomains,
|
|
|
|
("vm_domainset_iter_next: Invalid domain %d", *domain));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
|
|
|
|
{
|
|
|
|
|
2018-03-29 02:54:50 +00:00
|
|
|
switch (di->di_policy) {
|
2018-01-12 22:57:57 +00:00
|
|
|
case DOMAINSET_POLICY_FIRSTTOUCH:
|
|
|
|
*domain = PCPU_GET(domain);
|
|
|
|
if (DOMAINSET_ISSET(*domain, &di->di_domain->ds_mask)) {
|
2018-03-29 02:54:50 +00:00
|
|
|
/*
|
|
|
|
* Add an extra iteration because we will visit the
|
|
|
|
* current domain a second time in the rr iterator.
|
|
|
|
*/
|
|
|
|
di->di_n = di->di_domain->ds_cnt + 1;
|
2018-01-12 22:57:57 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* To prevent impossible allocations we convert an invalid
|
|
|
|
* first-touch to round-robin.
|
|
|
|
*/
|
|
|
|
/* FALLTHROUGH */
|
|
|
|
case DOMAINSET_POLICY_ROUNDROBIN:
|
|
|
|
di->di_n = di->di_domain->ds_cnt;
|
|
|
|
vm_domainset_iter_rr(di, domain);
|
|
|
|
break;
|
|
|
|
case DOMAINSET_POLICY_PREFER:
|
|
|
|
*domain = di->di_domain->ds_prefer;
|
|
|
|
di->di_n = di->di_domain->ds_cnt;
|
|
|
|
break;
|
2018-03-29 02:54:50 +00:00
|
|
|
case DOMAINSET_POLICY_INTERLEAVE:
|
|
|
|
vm_domainset_iter_interleave(di, domain);
|
|
|
|
di->di_n = di->di_domain->ds_cnt;
|
|
|
|
break;
|
2018-01-12 22:57:57 +00:00
|
|
|
default:
|
|
|
|
panic("vm_domainset_iter_first: Unknown policy %d",
|
2018-03-29 02:54:50 +00:00
|
|
|
di->di_policy);
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
KASSERT(di->di_n > 0,
|
|
|
|
("vm_domainset_iter_first: Invalid n %d", di->di_n));
|
|
|
|
KASSERT(*domain < vm_ndomains,
|
|
|
|
("vm_domainset_iter_first: Invalid domain %d", *domain));
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
|
2018-03-29 02:54:50 +00:00
|
|
|
vm_pindex_t pindex, int *domain, int *req)
|
2018-01-12 22:57:57 +00:00
|
|
|
{
|
2018-10-23 16:35:58 +00:00
|
|
|
struct domainset_ref *dr;
|
2018-01-12 22:57:57 +00:00
|
|
|
|
2018-10-23 16:35:58 +00:00
|
|
|
/*
|
|
|
|
* Object policy takes precedence over thread policy. The policies
|
|
|
|
* are immutable and unsynchronized. Updates can race but pointer
|
|
|
|
* loads are assumed to be atomic.
|
|
|
|
*/
|
|
|
|
if (obj != NULL && obj->domain.dr_policy != NULL)
|
|
|
|
dr = &obj->domain;
|
|
|
|
else
|
|
|
|
dr = &curthread->td_domain;
|
|
|
|
vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex);
|
2018-01-12 22:57:57 +00:00
|
|
|
di->di_flags = *req;
|
|
|
|
*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
|
|
|
|
VM_ALLOC_NOWAIT;
|
|
|
|
vm_domainset_iter_first(di, domain);
|
2018-09-24 19:24:17 +00:00
|
|
|
if (vm_page_count_min_domain(*domain))
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_page(di, obj, domain);
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
|
|
|
|
int *domain)
|
2018-01-12 22:57:57 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
/* If there are more domains to visit we run the iterator. */
|
2018-09-06 19:28:52 +00:00
|
|
|
while (--di->di_n != 0) {
|
2018-01-12 22:57:57 +00:00
|
|
|
vm_domainset_iter_next(di, domain);
|
2018-09-24 19:24:17 +00:00
|
|
|
if (!di->di_minskip || !vm_page_count_min_domain(*domain))
|
2018-09-06 19:28:52 +00:00
|
|
|
return (0);
|
|
|
|
}
|
2018-10-23 16:35:58 +00:00
|
|
|
|
|
|
|
/* If we skipped domains below min restart the search. */
|
2018-09-06 19:28:52 +00:00
|
|
|
if (di->di_minskip) {
|
|
|
|
di->di_minskip = false;
|
|
|
|
vm_domainset_iter_first(di, domain);
|
2018-01-12 22:57:57 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If we visited all domains and this was a NOWAIT we return error. */
|
|
|
|
if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)
|
|
|
|
return (ENOMEM);
|
|
|
|
|
2018-10-23 16:35:58 +00:00
|
|
|
/* Wait for one of the domains to accumulate some free pages. */
|
|
|
|
if (obj != NULL)
|
|
|
|
VM_OBJECT_WUNLOCK(obj);
|
|
|
|
vm_wait_doms(&di->di_domain->ds_mask);
|
|
|
|
if (obj != NULL)
|
|
|
|
VM_OBJECT_WLOCK(obj);
|
|
|
|
if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0)
|
|
|
|
return (ENOMEM);
|
|
|
|
|
|
|
|
/* Restart the search. */
|
2018-01-12 22:57:57 +00:00
|
|
|
vm_domainset_iter_first(di, domain);
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2018-10-23 16:35:58 +00:00
|
|
|
static void
|
|
|
|
_vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain,
|
|
|
|
int *flags)
|
2018-01-12 22:57:57 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
di->di_flags = *flags;
|
|
|
|
*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
|
|
|
|
vm_domainset_iter_first(di, domain);
|
2018-09-24 19:24:17 +00:00
|
|
|
if (vm_page_count_min_domain(*domain))
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_policy(di, domain);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
|
|
|
|
struct domainset *ds, int *domain, int *flags)
|
|
|
|
{
|
|
|
|
|
|
|
|
vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0);
|
|
|
|
_vm_domainset_iter_policy_init(di, domain, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
|
|
|
|
struct domainset_ref *dr, int *domain, int *flags)
|
|
|
|
{
|
|
|
|
|
|
|
|
vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0);
|
|
|
|
_vm_domainset_iter_policy_init(di, domain, flags);
|
2018-01-12 22:57:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
|
2018-01-12 22:57:57 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
/* If there are more domains to visit we run the iterator. */
|
2018-09-06 19:28:52 +00:00
|
|
|
while (--di->di_n != 0) {
|
2018-01-12 22:57:57 +00:00
|
|
|
vm_domainset_iter_next(di, domain);
|
2018-09-24 19:24:17 +00:00
|
|
|
if (!di->di_minskip || !vm_page_count_min_domain(*domain))
|
2018-09-06 19:28:52 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2018-09-06 20:02:19 +00:00
|
|
|
/* If we skipped domains below min restart the search. */
|
2018-09-06 19:28:52 +00:00
|
|
|
if (di->di_minskip) {
|
|
|
|
di->di_minskip = false;
|
|
|
|
vm_domainset_iter_first(di, domain);
|
2018-01-12 22:57:57 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If we visited all domains and this was a NOWAIT we return error. */
|
|
|
|
if ((di->di_flags & M_WAITOK) == 0)
|
|
|
|
return (ENOMEM);
|
|
|
|
|
2018-10-23 16:35:58 +00:00
|
|
|
/* Wait for one of the domains to accumulate some free pages. */
|
|
|
|
vm_wait_doms(&di->di_domain->ds_mask);
|
|
|
|
|
|
|
|
/* Restart the search. */
|
2018-01-12 22:57:57 +00:00
|
|
|
vm_domainset_iter_first(di, domain);
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
2018-01-14 03:36:03 +00:00
|
|
|
|
|
|
|
#else /* !NUMA */
|
2018-10-23 16:35:58 +00:00
|
|
|
|
2018-01-14 03:36:03 +00:00
|
|
|
int
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
|
|
|
|
int *domain)
|
2018-01-14 03:36:03 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
return (EJUSTRETURN);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
|
|
|
|
vm_pindex_t pindex, int *domain, int *flags)
|
2018-01-14 03:36:03 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
*domain = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
|
2018-01-14 03:36:03 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
return (EJUSTRETURN);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2018-10-23 16:35:58 +00:00
|
|
|
vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
|
|
|
|
struct domainset *ds, int *domain, int *flags)
|
2018-01-14 03:36:03 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
*domain = 0;
|
|
|
|
}
|
|
|
|
|
2018-10-23 16:35:58 +00:00
|
|
|
#endif /* NUMA */
|