d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add malloc_domainset(9) and _domainset variants to other allocator KPIs.

Browse Source 
Remove malloc_domain(9) and most other _domain KPIs added in r327900.
The new functions allow the caller to specify a general NUMA domain
selection policy, rather than specifically requesting an allocation from
a specific domain.  The latter policy tends to interact poorly with
M_WAITOK, resulting in situations where a caller is blocked indefinitely
because the specified domain is depleted.  Most existing consumers of
the _domain KPIs are converted to instead use a DOMAINSET_PREF() policy,
in which we fall back to other domains to satisfy the allocation
request.

This change also defines a set of DOMAINSET_FIXED() policies, which
only permit allocations from the specified domain.

Discussed with:	gallatin, jeff
Reported and tested by:	pho (previous version)
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D17418
This commit is contained in:
Mark Johnston 2018-10-30 18:26:34 +00:00
parent 58b6812de1
commit 9978bd996b
20 changed files with 231 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ObsoleteFiles.inc
View File

@ -38,6 +38,8 @@
# xargs -n1 | sort | uniq -d; # xargs -n1 | sort | uniq -d;
# done # done
# 20181030: malloc_domain(9) KPI change
OLD_FILES+=share/man/man9/malloc_domain.9.gz
# 20181026: joy(4) removal # 20181026: joy(4) removal
OLD_FILES+=usr/share/man/man4/joy.4.gz OLD_FILES+=usr/share/man/man4/joy.4.gz
# 20181025: OpenSSL libraries version bump to avoid conflict with ports # 20181025: OpenSSL libraries version bump to avoid conflict with ports

5
share/man/man9/Makefile
View File

@ -812,7 +812,8 @@ MLINKS+=condvar.9 cv_broadcast.9 \
MLINKS+=config_intrhook.9 config_intrhook_disestablish.9 \ MLINKS+=config_intrhook.9 config_intrhook_disestablish.9 \
config_intrhook.9 config_intrhook_establish.9 \ config_intrhook.9 config_intrhook_establish.9 \
config_intrhook.9 config_intrhook_oneshot.9 config_intrhook.9 config_intrhook_oneshot.9
MLINKS+=contigmalloc.9 contigfree.9 MLINKS+=contigmalloc.9 contigmalloc_domainset.9 \
contigmalloc.9 contigfree.9
MLINKS+=casuword.9 casueword.9 \ MLINKS+=casuword.9 casueword.9 \
casuword.9 casueword32.9 \ casuword.9 casueword32.9 \
casuword.9 casuword32.9 casuword.9 casuword32.9
@ -1289,7 +1290,7 @@ MLINKS+=make_dev.9 destroy_dev.9 \
make_dev.9 make_dev_p.9 \ make_dev.9 make_dev_p.9 \
make_dev.9 make_dev_s.9 make_dev.9 make_dev_s.9
MLINKS+=malloc.9 free.9 \ MLINKS+=malloc.9 free.9 \
malloc.9 malloc_domain.9 \ malloc.9 malloc_domainset.9 \
malloc.9 free_domain.9 \ malloc.9 free_domain.9 \
malloc.9 mallocarray.9 \ malloc.9 mallocarray.9 \
malloc.9 MALLOC_DECLARE.9 \ malloc.9 MALLOC_DECLARE.9 \

28
share/man/man9/contigmalloc.9
View File

@ -25,7 +25,7 @@
.\" .\"
.\" $FreeBSD$ .\" $FreeBSD$
.\" .\"
.Dd January 29, 2015 .Dd October 30, 2018
.Dt CONTIGMALLOC 9 .Dt CONTIGMALLOC 9
.Os .Os
.Sh NAME .Sh NAME
@ -50,6 +50,19 @@
.Fa "unsigned long size" .Fa "unsigned long size"
.Fa "struct malloc_type *type" .Fa "struct malloc_type *type"
.Fc .Fc
.In sys/param.h
.In sys/domainset.h
.Ft "void *"
.Fo contigmalloc_domainset
.Fa "unsigned long size"
.Fa "struct malloc_type *type"
.Fa "struct domainset *ds"
.Fa "int flags"
.Fa "vm_paddr_t low"
.Fa "vm_paddr_t high"
.Fa "unsigned long alignment"
.Fa "vm_paddr_t boundary"
.Fc
.Sh DESCRIPTION .Sh DESCRIPTION
The The
.Fn contigmalloc .Fn contigmalloc
@ -70,6 +83,15 @@ address range of
bytes allocated from the kernel virtual address (KVA) map. bytes allocated from the kernel virtual address (KVA) map.
.Pp .Pp
The The
.Fn contigmalloc_domainset
variant allows the caller to additionally specify a
.Xr numa 4
domain selection policy.
See
.Xr domainset 9
for some example policies.
.Pp
The
.Fa flags .Fa flags
parameter modifies parameter modifies
.Fn contigmalloc Ns 's .Fn contigmalloc Ns 's
@ -90,7 +112,9 @@ Other flags (if present) are ignored.
The The
.Fn contigfree .Fn contigfree
function deallocates memory allocated by a previous call to function deallocates memory allocated by a previous call to
.Fn contigmalloc . .Fn contigmalloc
or
.Fn contigmalloc_domainset .
.Sh IMPLEMENTATION NOTES .Sh IMPLEMENTATION NOTES
The The
.Fn contigmalloc .Fn contigmalloc

23
share/man/man9/domainset.9
View File

@ -24,7 +24,7 @@
.\" .\"
.\" $FreeBSD$ .\" $FreeBSD$
.\" .\"
.Dd October 20, 2018 .Dd October 30, 2018
.Dt DOMAINSET 9 .Dt DOMAINSET 9
.Os .Os
.Sh NAME .Sh NAME
@ -43,7 +43,11 @@ struct domainset {
}; };
.Ed .Ed
.Pp .Pp
.Ft struct domainset *
.Fn DOMAINSET_FIXED domain
.Ft struct domainset *
.Fn DOMAINSET_RR .Fn DOMAINSET_RR
.Ft struct domainset *
.Fn DOMAINSET_PREF domain .Fn DOMAINSET_PREF domain
.Ft struct domainset * .Ft struct domainset *
.Fn domainset_create "const struct domainset *key" .Fn domainset_create "const struct domainset *key"
@ -98,11 +102,26 @@ efficiency higher and is preferential to round-robin for general use.
.El .El
.Pp .Pp
The The
.Fn DOMAINSET_FIXED ,
.Fn DOMAINSET_RR .Fn DOMAINSET_RR
and and
.Fn DOMAINSET_PREF .Fn DOMAINSET_PREF
provide pointers to global pre-defined policies for use when the macros provide pointers to global pre-defined policies for use when the
desired policy is known at compile time. desired policy is known at compile time.
.Fn DOMAINSET_FIXED
is a policy which only permits allocations from the specified domain.
.Fn DOMAINSET_RR
provides round-robin selection among all domains in the system.
The
.Fn DOMAINSET_PREF
policies attempt allocation from the specified domain, but unlike
.Fn DOMAINSET_FIXED
will fall back to other domains to satisfy the request.
These policies should be used in preference to
.Fn DOMAINSET_FIXED
to avoid blocking indefinitely on a
.Dv M_WAITOK
request.
The The
.Fn domainset_create .Fn domainset_create
function takes a partially filled in domainset as a key and returns a function takes a partially filled in domainset as a key and returns a

27
share/man/man9/malloc.9
View File

@ -29,7 +29,7 @@
.\" $NetBSD: malloc.9,v 1.3 1996/11/11 00:05:11 lukem Exp $ .\" $NetBSD: malloc.9,v 1.3 1996/11/11 00:05:11 lukem Exp $
.\" $FreeBSD$ .\" $FreeBSD$
.\" .\"
.Dd June 13, 2018 .Dd October 30, 2018
.Dt MALLOC 9 .Dt MALLOC 9
.Os .Os
.Sh NAME .Sh NAME
@ -46,13 +46,9 @@
.Ft void * .Ft void *
.Fn malloc "size_t size" "struct malloc_type *type" "int flags" .Fn malloc "size_t size" "struct malloc_type *type" "int flags"
.Ft void * .Ft void *
.Fn malloc_domain "size_t size" "struct malloc_type *type" "int domain" "int flags"
.Ft void *
.Fn mallocarray "size_t nmemb" "size_t size" "struct malloc_type *type" "int flags" .Fn mallocarray "size_t nmemb" "size_t size" "struct malloc_type *type" "int flags"
.Ft void .Ft void
.Fn free "void *addr" "struct malloc_type *type" .Fn free "void *addr" "struct malloc_type *type"
.Ft void
.Fn free_domain "void *addr" "struct malloc_type *type"
.Ft void * .Ft void *
.Fn realloc "void *addr" "size_t size" "struct malloc_type *type" "int flags" .Fn realloc "void *addr" "size_t size" "struct malloc_type *type" "int flags"
.Ft void * .Ft void *
@ -62,6 +58,12 @@
.In sys/malloc.h .In sys/malloc.h
.In sys/kernel.h .In sys/kernel.h
.Fn MALLOC_DEFINE type shortdesc longdesc .Fn MALLOC_DEFINE type shortdesc longdesc
.In sys/param.h
.In sys/domainset.h
.Ft void *
.Fn malloc_domainset "size_t size" "struct malloc_type *type" "struct domainset *ds" "int flags"
.Ft void
.Fn free_domain "void *addr" "struct malloc_type *type"
.Sh DESCRIPTION .Sh DESCRIPTION
The The
.Fn malloc .Fn malloc
@ -70,12 +72,15 @@ object whose size is specified by
.Fa size . .Fa size .
.Pp .Pp
The The
.Fn malloc_domain .Fn malloc_domainset
variant allocates the object from the specified memory domain. Memory allocated variant allocates memory from a specific
with this function should be returned with .Xr numa 4
.Fn free_domain . domain using the specified domain selection policy.
See See
.Xr numa 9 for more details. .Xr domainset 9
for some example policies.
Memory allocated with this function should be returned with
.Fn free_domain .
.Pp .Pp
The The
.Fn mallocarray .Fn mallocarray
@ -310,7 +315,9 @@ functions.
Failing consistency checks will cause a panic or a system console Failing consistency checks will cause a panic or a system console
message. message.
.Sh SEE ALSO .Sh SEE ALSO
.Xr numa 4 ,
.Xr vmstat 8 , .Xr vmstat 8 ,
.Xr contigmalloc 9 , .Xr contigmalloc 9 ,
.Xr domainset 9 ,
.Xr memguard 9 , .Xr memguard 9 ,
.Xr vnode 9 .Xr vnode 9

16
sys/dev/hwpmc/hwpmc_logging.c
View File

@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/capsicum.h> #include <sys/capsicum.h>
#include <sys/domainset.h>
#include <sys/file.h> #include <sys/file.h>
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/kthread.h> #include <sys/kthread.h>
@ -1231,8 +1232,8 @@ pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl)
void void
pmclog_initialize() pmclog_initialize()
{ {
int domain;
struct pmclog_buffer *plb; struct pmclog_buffer *plb;
int domain, ncpus, total;
if (pmclog_buffer_size <= 0 || pmclog_buffer_size > 16*1024) { if (pmclog_buffer_size <= 0 || pmclog_buffer_size > 16*1024) {
(void) printf("hwpmc: tunable logbuffersize=%d must be " (void) printf("hwpmc: tunable logbuffersize=%d must be "
@ -1253,16 +1254,17 @@ pmclog_initialize()
pmclog_buffer_size = PMC_LOG_BUFFER_SIZE; pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
} }
for (domain = 0; domain < vm_ndomains; domain++) { for (domain = 0; domain < vm_ndomains; domain++) {
int ncpus = pmc_dom_hdrs[domain]->pdbh_ncpus; ncpus = pmc_dom_hdrs[domain]->pdbh_ncpus;
int total = ncpus*pmc_nlogbuffers_pcpu; total = ncpus * pmc_nlogbuffers_pcpu;
plb = malloc_domain(sizeof(struct pmclog_buffer)*total, M_PMC, domain, M_WAITOK|M_ZERO); plb = malloc_domainset(sizeof(struct pmclog_buffer) * total,
M_PMC, DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
pmc_dom_hdrs[domain]->pdbh_plbs = plb; pmc_dom_hdrs[domain]->pdbh_plbs = plb;
for (int i = 0; i < total; i++, plb++) { for (; total > 0; total--, plb++) {
void *buf; void *buf;
buf = malloc_domain(1024 * pmclog_buffer_size, M_PMC, domain, buf = malloc_domainset(1024 * pmclog_buffer_size, M_PMC,
M_WAITOK|M_ZERO); DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
PMCLOG_INIT_BUFFER_DESCRIPTOR(plb, buf, domain); PMCLOG_INIT_BUFFER_DESCRIPTOR(plb, buf, domain);
pmc_plb_rele_unlocked(plb); pmc_plb_rele_unlocked(plb);
} }

54
sys/dev/hwpmc/hwpmc_mod.c
View File

@ -36,6 +36,7 @@
__FBSDID("$FreeBSD$"); __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/domainset.h>
#include <sys/eventhandler.h> #include <sys/eventhandler.h>
#include <sys/gtaskqueue.h> #include <sys/gtaskqueue.h>
#include <sys/jail.h> #include <sys/jail.h>
@ -78,14 +79,6 @@ __FBSDID("$FreeBSD$");
#include "hwpmc_soft.h" #include "hwpmc_soft.h"
#ifdef NUMA
#define NDOMAINS vm_ndomains
#else
#define NDOMAINS 1
#define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
#define free_domain(addr, type) free(addr, type)
#endif
#define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et) #define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et)
#define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et) #define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et)
@ -5643,15 +5636,16 @@ pmc_initialize(void)
continue; continue;
pc = pcpu_find(cpu); pc = pcpu_find(cpu);
domain = pc->pc_domain; domain = pc->pc_domain;
sb = malloc_domain(sizeof(struct pmc_samplebuffer) + sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain, pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
M_WAITOK|M_ZERO); DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
KASSERT(pmc_pcpu[cpu] != NULL, KASSERT(pmc_pcpu[cpu] != NULL,
("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples * sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO); pmc_nsamples * sizeof(uintptr_t), M_PMC,
DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains + ps->ps_pc = sb->ps_callchains +
@ -5659,35 +5653,27 @@ pmc_initialize(void)
pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb; pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
sb = malloc_domain(sizeof(struct pmc_samplebuffer) + sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain, pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
M_WAITOK|M_ZERO); DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
KASSERT(pmc_pcpu[cpu] != NULL,
("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
pmc_nsamples * sizeof(uintptr_t), M_PMC,
DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains + ps->ps_pc = sb->ps_callchains +
(n * pmc_callchaindepth); (n * pmc_callchaindepth);
pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb; pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
sb = malloc_domain(sizeof(struct pmc_samplebuffer) + sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain, pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
M_WAITOK|M_ZERO); DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
KASSERT(pmc_pcpu[cpu] != NULL, pmc_nsamples * sizeof(uintptr_t), M_PMC,
("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains + ps->ps_pc = sb->ps_callchains + n * pmc_callchaindepth;
(n * pmc_callchaindepth);
pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb; pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
} }

4
sys/i386/i386/pmap.c
View File

@ -864,8 +864,8 @@ pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
/* Inform UMA that this allocator uses kernel_map/object. */ /* Inform UMA that this allocator uses kernel_map/object. */
*flags = UMA_SLAB_KERNEL; *flags = UMA_SLAB_KERNEL;
return ((void *)kmem_alloc_contig_domain(domain, bytes, wait, 0x0ULL, return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT)); bytes, wait, 0x0ULL, 0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
} }
#endif #endif

7
sys/kern/kern_cpuset.c
View File

@ -119,6 +119,7 @@ __FBSDID("$FreeBSD$");
*/ */
LIST_HEAD(domainlist, domainset); LIST_HEAD(domainlist, domainset);
struct domainset __read_mostly domainset_fixed[MAXMEMDOM];
struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; struct domainset __read_mostly domainset_prefer[MAXMEMDOM];
struct domainset __read_mostly domainset_roundrobin; struct domainset __read_mostly domainset_roundrobin;
@ -1402,6 +1403,12 @@ domainset_init(void)
_domainset_create(dset, NULL); _domainset_create(dset, NULL);
for (i = 0; i < vm_ndomains; i++) { for (i = 0; i < vm_ndomains; i++) {
dset = &domainset_fixed[i];
DOMAINSET_ZERO(&dset->ds_mask);
DOMAINSET_SET(i, &dset->ds_mask);
dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
_domainset_create(dset, NULL);
dset = &domainset_prefer[i]; dset = &domainset_prefer[i];
DOMAINSET_COPY(&all_domains, &dset->ds_mask); DOMAINSET_COPY(&all_domains, &dset->ds_mask);
dset->ds_policy = DOMAINSET_POLICY_PREFER; dset->ds_policy = DOMAINSET_POLICY_PREFER;

30
sys/kern/kern_malloc.c
View File

@ -68,6 +68,7 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h> #include <vm/vm.h>
#include <vm/pmap.h> #include <vm/pmap.h>
#include <vm/vm_domainset.h>
#include <vm/vm_pageout.h> #include <vm/vm_pageout.h>
#include <vm/vm_param.h> #include <vm/vm_param.h>
#include <vm/vm_kern.h> #include <vm/vm_kern.h>
@ -453,13 +454,13 @@ contigmalloc(unsigned long size, struct malloc_type *type, int flags,
} }
void * void *
contigmalloc_domain(unsigned long size, struct malloc_type *type, contigmalloc_domainset(unsigned long size, struct malloc_type *type,
int domain, int flags, vm_paddr_t low, vm_paddr_t high, struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high,
unsigned long alignment, vm_paddr_t boundary) unsigned long alignment, vm_paddr_t boundary)
{ {
void *ret; void *ret;
ret = (void *)kmem_alloc_contig_domain(domain, size, flags, low, high, ret = (void *)kmem_alloc_contig_domainset(ds, size, flags, low, high,
alignment, boundary, VM_MEMATTR_DEFAULT); alignment, boundary, VM_MEMATTR_DEFAULT);
if (ret != NULL) if (ret != NULL)
malloc_type_allocated(type, round_page(size)); malloc_type_allocated(type, round_page(size));
@ -595,9 +596,8 @@ void *
return ((void *) va); return ((void *) va);
} }
void * static void *
malloc_domain(size_t size, struct malloc_type *mtp, int domain, malloc_domain(size_t size, struct malloc_type *mtp, int domain, int flags)
int flags)
{ {
int indx; int indx;
caddr_t va; caddr_t va;
@ -640,6 +640,24 @@ malloc_domain(size_t size, struct malloc_type *mtp, int domain,
return ((void *) va); return ((void *) va);
} }
void *
malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds,
int flags)
{
struct vm_domainset_iter di;
void *ret;
int domain;
vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do {
ret = malloc_domain(size, mtp, domain, flags);
if (ret != NULL)
break;
} while (vm_domainset_iter_policy(&di, &domain) == 0);
return (ret);
}
void * void *
mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags) mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags)
{ {

6
sys/kern/kern_mbuf.c
View File

@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/conf.h> #include <sys/conf.h>
#include <sys/domainset.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/mbuf.h> #include <sys/mbuf.h>
@ -590,8 +591,9 @@ mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
/* Inform UMA that this allocator uses kernel_map/object. */ /* Inform UMA that this allocator uses kernel_map/object. */
*flags = UMA_SLAB_KERNEL; *flags = UMA_SLAB_KERNEL;
return ((void *)kmem_alloc_contig_domain(domain, bytes, wait, return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
(vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); bytes, wait, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0,
VM_MEMATTR_DEFAULT));
} }
/* /*

29
sys/kern/kern_pmc.c
View File

@ -35,8 +35,9 @@ __FBSDID("$FreeBSD$");
#include "opt_hwpmc_hooks.h" #include "opt_hwpmc_hooks.h"
#include <sys/types.h> #include <sys/param.h>
#include <sys/ctype.h> #include <sys/ctype.h>
#include <sys/domainset.h>
#include <sys/param.h> #include <sys/param.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/kernel.h> #include <sys/kernel.h>
@ -330,22 +331,6 @@ pmc_soft_ev_release(struct pmc_soft *ps)
mtx_unlock_spin(&pmc_softs_mtx); mtx_unlock_spin(&pmc_softs_mtx);
} }
#ifdef NUMA
#define NDOMAINS vm_ndomains
static int
getdomain(int cpu)
{
struct pcpu *pc;
pc = pcpu_find(cpu);
return (pc->pc_domain);
}
#else
#define NDOMAINS 1
#define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
#define getdomain(cpu) 0
#endif
/* /*
* Initialise hwpmc. * Initialise hwpmc.
*/ */
@ -362,14 +347,16 @@ init_hwpmc(void *dummy __unused)
} }
pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS, pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS,
M_WAITOK | M_ZERO); M_WAITOK | M_ZERO);
for (domain = 0; domain < NDOMAINS; domain++) {
pmc_dom_hdrs[domain] = malloc_domain(sizeof(struct pmc_domain_buffer_header), M_PMC, domain, for (domain = 0; domain < vm_ndomains; domain++) {
M_WAITOK|M_ZERO); pmc_dom_hdrs[domain] = malloc_domainset(
sizeof(struct pmc_domain_buffer_header), M_PMC,
DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN); mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN);
TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head); TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head);
} }
CPU_FOREACH(cpu) { CPU_FOREACH(cpu) {
domain = getdomain(cpu); domain = pcpu_find(cpu)->pc_domain;
KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain)); KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain));
pmc_dom_hdrs[domain]->pdbh_ncpus++; pmc_dom_hdrs[domain]->pdbh_ncpus++;
} }

10
sys/kern/subr_busdma_bufalloc.c
View File

@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/bus.h> #include <sys/bus.h>
#include <sys/busdma_bufalloc.h> #include <sys/busdma_bufalloc.h>
#include <sys/domainset.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <vm/vm.h> #include <vm/vm.h>
@ -152,18 +153,15 @@ void *
busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_size_t size, int domain, busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_size_t size, int domain,
uint8_t *pflag, int wait) uint8_t *pflag, int wait)
{ {
#ifdef VM_MEMATTR_UNCACHEABLE
#ifdef VM_MEMATTR_UNCACHEABLE
/* Inform UMA that this allocator uses kernel_arena/object. */ /* Inform UMA that this allocator uses kernel_arena/object. */
*pflag = UMA_SLAB_KERNEL; *pflag = UMA_SLAB_KERNEL;
return ((void *)kmem_alloc_attr_domain(domain, size, wait, 0, return ((void *)kmem_alloc_attr_domainset(DOMAINSET_FIXED(domain), size,
BUS_SPACE_MAXADDR, VM_MEMATTR_UNCACHEABLE)); wait, 0, BUS_SPACE_MAXADDR, VM_MEMATTR_UNCACHEABLE));
#else #else
panic("VM_MEMATTR_UNCACHEABLE unavailable"); panic("VM_MEMATTR_UNCACHEABLE unavailable");
#endif /* VM_MEMATTR_UNCACHEABLE */ #endif /* VM_MEMATTR_UNCACHEABLE */
} }

3
sys/sys/domainset.h
View File

@ -96,7 +96,8 @@ struct domainset {
domainid_t ds_order[MAXMEMDOM]; /* nth domain table. */ domainid_t ds_order[MAXMEMDOM]; /* nth domain table. */
}; };
extern struct domainset domainset_prefer[MAXMEMDOM]; extern struct domainset domainset_fixed[MAXMEMDOM], domainset_prefer[MAXMEMDOM];
#define DOMAINSET_FIXED(domain) (&domainset_fixed[(domain)])
#define DOMAINSET_PREF(domain) (&domainset_prefer[(domain)]) #define DOMAINSET_PREF(domain) (&domainset_prefer[(domain)])
extern struct domainset domainset_roundrobin; extern struct domainset domainset_roundrobin;
#define DOMAINSET_RR() (&domainset_roundrobin) #define DOMAINSET_RR() (&domainset_roundrobin)

10
sys/sys/malloc.h
View File

@ -160,6 +160,7 @@ MALLOC_DECLARE(M_TEMP);
*/ */
MALLOC_DECLARE(M_IOV); MALLOC_DECLARE(M_IOV);
struct domainset;
extern struct mtx malloc_mtx; extern struct mtx malloc_mtx;
/* /*
@ -172,8 +173,8 @@ void *contigmalloc(unsigned long size, struct malloc_type *type, int flags,
vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t low, vm_paddr_t high, unsigned long alignment,
vm_paddr_t boundary) __malloc_like __result_use_check vm_paddr_t boundary) __malloc_like __result_use_check
__alloc_size(1) __alloc_align(6); __alloc_size(1) __alloc_align(6);
void *contigmalloc_domain(unsigned long size, struct malloc_type *type, void *contigmalloc_domainset(unsigned long size, struct malloc_type *type,
int domain, int flags, vm_paddr_t low, vm_paddr_t high, struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high,
unsigned long alignment, vm_paddr_t boundary) unsigned long alignment, vm_paddr_t boundary)
__malloc_like __result_use_check __alloc_size(1) __alloc_align(6); __malloc_like __result_use_check __alloc_size(1) __alloc_align(6);
void free(void *addr, struct malloc_type *type); void free(void *addr, struct malloc_type *type);
@ -230,8 +231,9 @@ void *malloc(size_t size, struct malloc_type *type, int flags) __malloc_like
_malloc_item; \ _malloc_item; \
}) })
void *malloc_domain(size_t size, struct malloc_type *type, int domain, void *malloc_domainset(size_t size, struct malloc_type *type,
int flags) __malloc_like __result_use_check __alloc_size(1); struct domainset *ds, int flags) __malloc_like __result_use_check
__alloc_size(1);
void *mallocarray(size_t nmemb, size_t size, struct malloc_type *type, void *mallocarray(size_t nmemb, size_t size, struct malloc_type *type,
int flags) __malloc_like __result_use_check int flags) __malloc_like __result_use_check
__alloc_size2(1, 2); __alloc_size2(1, 2);

10
sys/vm/uma_core.c
View File

@ -1172,7 +1172,7 @@ page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
void *p; /* Returned page */ void *p; /* Returned page */
*pflag = UMA_SLAB_KERNEL; *pflag = UMA_SLAB_KERNEL;
p = (void *) kmem_malloc_domain(domain, bytes, wait); p = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain), bytes, wait);
return (p); return (p);
} }
@ -3718,6 +3718,7 @@ uma_zone_exhausted_nolock(uma_zone_t zone)
void * void *
uma_large_malloc_domain(vm_size_t size, int domain, int wait) uma_large_malloc_domain(vm_size_t size, int domain, int wait)
{ {
struct domainset *policy;
vm_offset_t addr; vm_offset_t addr;
uma_slab_t slab; uma_slab_t slab;
@ -3729,10 +3730,9 @@ uma_large_malloc_domain(vm_size_t size, int domain, int wait)
slab = zone_alloc_item(slabzone, NULL, domain, wait); slab = zone_alloc_item(slabzone, NULL, domain, wait);
if (slab == NULL) if (slab == NULL)
return (NULL); return (NULL);
if (domain == UMA_ANYDOMAIN) policy = (domain == UMA_ANYDOMAIN) ? DOMAINSET_RR() :
addr = kmem_malloc(size, wait); DOMAINSET_FIXED(domain);
else addr = kmem_malloc_domainset(policy, size, wait);
addr = kmem_malloc_domain(domain, size, wait);
if (addr != 0) { if (addr != 0) {
vsetslab(addr, slab); vsetslab(addr, slab);
slab->us_data = (void *)addr; slab->us_data = (void *)addr;

14
sys/vm/vm_extern.h
View File

@ -44,6 +44,7 @@ struct vmem;
#ifdef _KERNEL #ifdef _KERNEL
struct cdev; struct cdev;
struct cdevsw; struct cdevsw;
struct domainset;
/* These operate on kernel virtual addresses only. */ /* These operate on kernel virtual addresses only. */
vm_offset_t kva_alloc(vm_size_t); vm_offset_t kva_alloc(vm_size_t);
@ -56,16 +57,17 @@ void kmap_free_wakeup(vm_map_t, vm_offset_t, vm_size_t);
/* These operate on virtual addresses backed by memory. */ /* These operate on virtual addresses backed by memory. */
vm_offset_t kmem_alloc_attr(vm_size_t size, int flags, vm_offset_t kmem_alloc_attr(vm_size_t size, int flags,
vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr); vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr);
vm_offset_t kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_offset_t kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size,
vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr); int flags, vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr);
vm_offset_t kmem_alloc_contig(vm_size_t size, int flags, vm_offset_t kmem_alloc_contig(vm_size_t size, int flags,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
vm_memattr_t memattr); vm_memattr_t memattr);
vm_offset_t kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_offset_t kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, int flags, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_memattr_t memattr); vm_paddr_t boundary, vm_memattr_t memattr);
vm_offset_t kmem_malloc(vm_size_t size, int flags); vm_offset_t kmem_malloc(vm_size_t size, int flags);
vm_offset_t kmem_malloc_domain(int domain, vm_size_t size, int flags); vm_offset_t kmem_malloc_domainset(struct domainset *ds, vm_size_t size,
int flags);
void kmem_free(vm_offset_t addr, vm_size_t size); void kmem_free(vm_offset_t addr, vm_size_t size);
/* This provides memory for previously allocated address space. */ /* This provides memory for previously allocated address space. */

40
sys/vm/vm_kern.c
View File

@ -175,7 +175,7 @@ kva_free(vm_offset_t addr, vm_size_t size)
* necessarily physically contiguous. If M_ZERO is specified through the * necessarily physically contiguous. If M_ZERO is specified through the
* given flags, then the pages are zeroed before they are mapped. * given flags, then the pages are zeroed before they are mapped.
*/ */
vm_offset_t static vm_offset_t
kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low, kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
vm_paddr_t high, vm_memattr_t memattr) vm_paddr_t high, vm_memattr_t memattr)
{ {
@ -231,11 +231,20 @@ vm_offset_t
kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high, kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high,
vm_memattr_t memattr) vm_memattr_t memattr)
{ {
return (kmem_alloc_attr_domainset(DOMAINSET_RR(), size, flags, low,
high, memattr));
}
vm_offset_t
kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size, int flags,
vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr)
{
struct vm_domainset_iter di; struct vm_domainset_iter di;
vm_offset_t addr; vm_offset_t addr;
int domain; int domain;
vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags); vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do { do {
addr = kmem_alloc_attr_domain(domain, size, flags, low, high, addr = kmem_alloc_attr_domain(domain, size, flags, low, high,
memattr); memattr);
@ -254,7 +263,7 @@ kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high,
* through the given flags, then the pages are zeroed before they are * through the given flags, then the pages are zeroed before they are
* mapped. * mapped.
*/ */
vm_offset_t static vm_offset_t
kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_paddr_t low, kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
vm_memattr_t memattr) vm_memattr_t memattr)
@ -315,11 +324,21 @@ vm_offset_t
kmem_alloc_contig(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high, kmem_alloc_contig(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high,
u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr) u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr)
{ {
return (kmem_alloc_contig_domainset(DOMAINSET_RR(), size, flags, low,
high, alignment, boundary, memattr));
}
vm_offset_t
kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size, int flags,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
vm_memattr_t memattr)
{
struct vm_domainset_iter di; struct vm_domainset_iter di;
vm_offset_t addr; vm_offset_t addr;
int domain; int domain;
vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags); vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do { do {
addr = kmem_alloc_contig_domain(domain, size, flags, low, high, addr = kmem_alloc_contig_domain(domain, size, flags, low, high,
alignment, boundary, memattr); alignment, boundary, memattr);
@ -368,11 +387,11 @@ kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max,
} }
/* /*
* kmem_malloc: * kmem_malloc_domain:
* *
* Allocate wired-down pages in the kernel's address space. * Allocate wired-down pages in the kernel's address space.
*/ */
vm_offset_t static vm_offset_t
kmem_malloc_domain(int domain, vm_size_t size, int flags) kmem_malloc_domain(int domain, vm_size_t size, int flags)
{ {
vmem_t *arena; vmem_t *arena;
@ -401,12 +420,19 @@ kmem_malloc_domain(int domain, vm_size_t size, int flags)
vm_offset_t vm_offset_t
kmem_malloc(vm_size_t size, int flags) kmem_malloc(vm_size_t size, int flags)
{
return (kmem_malloc_domainset(DOMAINSET_RR(), size, flags));
}
vm_offset_t
kmem_malloc_domainset(struct domainset *ds, vm_size_t size, int flags)
{ {
struct vm_domainset_iter di; struct vm_domainset_iter di;
vm_offset_t addr; vm_offset_t addr;
int domain; int domain;
vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags); vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do { do {
addr = kmem_malloc_domain(domain, size, flags); addr = kmem_malloc_domain(domain, size, flags);
if (addr != 0) if (addr != 0)

19
sys/x86/iommu/busdma_dmar.c
View File

@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/domainset.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/bus.h> #include <sys/bus.h>
#include <sys/conf.h> #include <sys/conf.h>
@ -373,16 +374,16 @@ dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__); WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__);
tag = (struct bus_dma_tag_dmar *)dmat; tag = (struct bus_dma_tag_dmar *)dmat;
map = malloc_domain(sizeof(*map), M_DMAR_DMAMAP, map = malloc_domainset(sizeof(*map), M_DMAR_DMAMAP,
tag->common.domain, M_NOWAIT | M_ZERO); DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO);
if (map == NULL) { if (map == NULL) {
*mapp = NULL; *mapp = NULL;
return (ENOMEM); return (ENOMEM);
} }
if (tag->segments == NULL) { if (tag->segments == NULL) {
tag->segments = malloc_domain(sizeof(bus_dma_segment_t) * tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
tag->common.nsegments, M_DMAR_DMAMAP, tag->common.nsegments, M_DMAR_DMAMAP,
tag->common.domain, M_NOWAIT); DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
if (tag->segments == NULL) { if (tag->segments == NULL) {
free_domain(map, M_DMAR_DMAMAP); free_domain(map, M_DMAR_DMAMAP);
*mapp = NULL; *mapp = NULL;
@ -447,13 +448,13 @@ dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
if (tag->common.maxsize < PAGE_SIZE && if (tag->common.maxsize < PAGE_SIZE &&
tag->common.alignment <= tag->common.maxsize && tag->common.alignment <= tag->common.maxsize &&
attr == VM_MEMATTR_DEFAULT) { attr == VM_MEMATTR_DEFAULT) {
*vaddr = malloc_domain(tag->common.maxsize, M_DEVBUF, *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
tag->common.domain, mflags); DOMAINSET_PREF(tag->common.domain), mflags);
map->flags |= BUS_DMAMAP_DMAR_MALLOC; map->flags |= BUS_DMAMAP_DMAR_MALLOC;
} else { } else {
*vaddr = (void *)kmem_alloc_attr_domain(tag->common.domain, *vaddr = (void *)kmem_alloc_attr_domainset(
tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR, DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
attr); mflags, 0ul, BUS_SPACE_MAXADDR, attr);
map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC; map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
} }
if (*vaddr == NULL) { if (*vaddr == NULL) {

38
sys/x86/x86/busdma_bounce.c
View File

@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/domainset.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/bus.h> #include <sys/bus.h>
#include <sys/interrupt.h> #include <sys/interrupt.h>
@ -294,9 +295,9 @@ bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
error = 0; error = 0;
if (dmat->segments == NULL) { if (dmat->segments == NULL) {
dmat->segments = (bus_dma_segment_t *)malloc_domain( dmat->segments = (bus_dma_segment_t *)malloc_domainset(
sizeof(bus_dma_segment_t) * dmat->common.nsegments, sizeof(bus_dma_segment_t) * dmat->common.nsegments,
M_DEVBUF, dmat->common.domain, M_NOWAIT); M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), M_NOWAIT);
if (dmat->segments == NULL) { if (dmat->segments == NULL) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d", CTR3(KTR_BUSDMA, "%s: tag %p error %d",
__func__, dmat, ENOMEM); __func__, dmat, ENOMEM);
@ -317,8 +318,8 @@ bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
} }
bz = dmat->bounce_zone; bz = dmat->bounce_zone;
*mapp = (bus_dmamap_t)malloc_domain(sizeof(**mapp), M_DEVBUF, *mapp = (bus_dmamap_t)malloc_domainset(sizeof(**mapp), M_DEVBUF,
dmat->common.domain, M_NOWAIT | M_ZERO); DOMAINSET_PREF(dmat->common.domain), M_NOWAIT | M_ZERO);
if (*mapp == NULL) { if (*mapp == NULL) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d", CTR3(KTR_BUSDMA, "%s: tag %p error %d",
__func__, dmat, ENOMEM); __func__, dmat, ENOMEM);
@ -411,9 +412,9 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
*mapp = NULL; *mapp = NULL;
if (dmat->segments == NULL) { if (dmat->segments == NULL) {
dmat->segments = (bus_dma_segment_t *)malloc_domain( dmat->segments = (bus_dma_segment_t *)malloc_domainset(
sizeof(bus_dma_segment_t) * dmat->common.nsegments, sizeof(bus_dma_segment_t) * dmat->common.nsegments,
M_DEVBUF, dmat->common.domain, mflags); M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), mflags);
if (dmat->segments == NULL) { if (dmat->segments == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->common.flags, ENOMEM); __func__, dmat, dmat->common.flags, ENOMEM);
@ -452,20 +453,21 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
(dmat->common.alignment <= dmat->common.maxsize) && (dmat->common.alignment <= dmat->common.maxsize) &&
dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
attr == VM_MEMATTR_DEFAULT) { attr == VM_MEMATTR_DEFAULT) {
*vaddr = malloc_domain(dmat->common.maxsize, M_DEVBUF, *vaddr = malloc_domainset(dmat->common.maxsize, M_DEVBUF,
dmat->common.domain, mflags); DOMAINSET_PREF(dmat->common.domain), mflags);
} else if (dmat->common.nsegments >= } else if (dmat->common.nsegments >=
howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) &&
dmat->common.alignment <= PAGE_SIZE && dmat->common.alignment <= PAGE_SIZE &&
(dmat->common.boundary % PAGE_SIZE) == 0) { (dmat->common.boundary % PAGE_SIZE) == 0) {
/* Page-based multi-segment allocations allowed */ /* Page-based multi-segment allocations allowed */
*vaddr = (void *)kmem_alloc_attr_domain(dmat->common.domain, *vaddr = (void *)kmem_alloc_attr_domainset(
dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, DOMAINSET_PREF(dmat->common.domain), dmat->common.maxsize,
attr); mflags, 0ul, dmat->common.lowaddr, attr);
dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC; dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC;
} else { } else {
*vaddr = (void *)kmem_alloc_contig_domain(dmat->common.domain, *vaddr = (void *)kmem_alloc_contig_domainset(
dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, DOMAINSET_PREF(dmat->common.domain), dmat->common.maxsize,
mflags, 0ul, dmat->common.lowaddr,
dmat->common.alignment != 0 ? dmat->common.alignment : 1ul, dmat->common.alignment != 0 ? dmat->common.alignment : 1ul,
dmat->common.boundary, attr); dmat->common.boundary, attr);
dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC; dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC;
@ -1149,14 +1151,14 @@ alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
while (numpages > 0) { while (numpages > 0) {
struct bounce_page *bpage; struct bounce_page *bpage;
bpage = (struct bounce_page *)malloc_domain(sizeof(*bpage), bpage = malloc_domainset(sizeof(*bpage), M_DEVBUF,
M_DEVBUF, dmat->common.domain, M_NOWAIT | M_ZERO); DOMAINSET_PREF(dmat->common.domain), M_NOWAIT | M_ZERO);
if (bpage == NULL) if (bpage == NULL)
break; break;
bpage->vaddr = (vm_offset_t)contigmalloc_domain(PAGE_SIZE, bpage->vaddr = (vm_offset_t)contigmalloc_domainset(PAGE_SIZE,
M_DEVBUF, dmat->common.domain, M_NOWAIT, 0ul, M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), M_NOWAIT,
bz->lowaddr, PAGE_SIZE, 0); 0ul, bz->lowaddr, PAGE_SIZE, 0);
if (bpage->vaddr == 0) { if (bpage->vaddr == 0) {
free_domain(bpage, M_DEVBUF); free_domain(bpage, M_DEVBUF);
break; break;