Fix some problems that manifest when NUMA domain 0 is empty.
- In uma_prealloc(), we need to check for an empty domain before the first allocation attempt, not after. Fix this by switching uma_prealloc() to use a vm_domainset iterator, which addresses the secondary issue of using a signed domain identifier in round-robin iteration. - Don't automatically create a page daemon for domain 0. - In domainset_empty_vm(), recompute ds_cnt and ds_order after excluding empty domains; otherwise we may frequently specify an empty domain when calling in to the page allocator, wasting CPU time. Convert DOMAINSET_PREF() policies for empty domains to round-robin. - When freeing bootstrap pages, don't count them towards the per-domain total page counts for now: some vm_phys segments are created before the SRAT is parsed and are thus always identified as being in domain 0 even when they are not. Then, when bootstrap pages are freed, they are added to a domain that we had previously thought was empty. Until this is corrected, we simply exclude them from the per-domain page count. Reported and tested by: Rajesh Kumar <rajfbsd@gmail.com> Reviewed by: gallatin MFC after: 2 weeks Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D17704
This commit is contained in:
parent
e35079db73
commit
920239efde
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=339925
@ -493,19 +493,28 @@ _domainset_create(struct domainset *domain, struct domainlist *freelist)
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* Are any of the domains in the mask empty? If so, silently
|
* Are any of the domains in the mask empty? If so, silently
|
||||||
* remove them. If only empty domains are present, we must
|
* remove them and update the domainset accordingly. If only empty
|
||||||
* return failure.
|
* domains are present, we must return failure.
|
||||||
*/
|
*/
|
||||||
static bool
|
static bool
|
||||||
domainset_empty_vm(struct domainset *domain)
|
domainset_empty_vm(struct domainset *domain)
|
||||||
{
|
{
|
||||||
int i, max;
|
int i, j, max;
|
||||||
|
|
||||||
max = DOMAINSET_FLS(&domain->ds_mask) + 1;
|
max = DOMAINSET_FLS(&domain->ds_mask) + 1;
|
||||||
for (i = 0; i < max; i++) {
|
for (i = 0; i < max; i++)
|
||||||
if (DOMAINSET_ISSET(i, &domain->ds_mask) &&
|
if (DOMAINSET_ISSET(i, &domain->ds_mask) && VM_DOMAIN_EMPTY(i))
|
||||||
VM_DOMAIN_EMPTY(i))
|
|
||||||
DOMAINSET_CLR(i, &domain->ds_mask);
|
DOMAINSET_CLR(i, &domain->ds_mask);
|
||||||
|
domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask);
|
||||||
|
max = DOMAINSET_FLS(&domain->ds_mask) + 1;
|
||||||
|
for (i = j = 0; i < max; i++) {
|
||||||
|
if (DOMAINSET_ISSET(i, &domain->ds_mask))
|
||||||
|
domain->ds_order[j++] = i;
|
||||||
|
else if (domain->ds_policy == DOMAINSET_POLICY_PREFER &&
|
||||||
|
domain->ds_prefer == i && domain->ds_cnt > 1) {
|
||||||
|
domain->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
|
||||||
|
domain->ds_prefer = -1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return (DOMAINSET_EMPTY(&domain->ds_mask));
|
return (DOMAINSET_EMPTY(&domain->ds_mask));
|
||||||
@ -1378,7 +1387,7 @@ cpuset_setithread(lwpid_t id, int cpu)
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* Initialize static domainsets after NUMA information is available. This is
|
* Initialize static domainsets after NUMA information is available. This is
|
||||||
* called very early during boot.
|
* called before memory allocators are initialized.
|
||||||
*/
|
*/
|
||||||
void
|
void
|
||||||
domainset_init(void)
|
domainset_init(void)
|
||||||
@ -1407,7 +1416,7 @@ domainset_init(void)
|
|||||||
void
|
void
|
||||||
domainset_zero(void)
|
domainset_zero(void)
|
||||||
{
|
{
|
||||||
struct domainset *dset;
|
struct domainset *dset, *tmp;
|
||||||
|
|
||||||
mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE);
|
mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE);
|
||||||
|
|
||||||
@ -1422,8 +1431,9 @@ domainset_zero(void)
|
|||||||
kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL);
|
kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL);
|
||||||
|
|
||||||
/* Remove empty domains from the global policies. */
|
/* Remove empty domains from the global policies. */
|
||||||
LIST_FOREACH(dset, &cpuset_domains, ds_link)
|
LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp)
|
||||||
(void)domainset_empty_vm(dset);
|
if (domainset_empty_vm(dset))
|
||||||
|
LIST_REMOVE(dset, ds_link);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -3608,29 +3608,30 @@ uma_zone_reserve_kva(uma_zone_t zone, int count)
|
|||||||
void
|
void
|
||||||
uma_prealloc(uma_zone_t zone, int items)
|
uma_prealloc(uma_zone_t zone, int items)
|
||||||
{
|
{
|
||||||
|
struct vm_domainset_iter di;
|
||||||
uma_domain_t dom;
|
uma_domain_t dom;
|
||||||
uma_slab_t slab;
|
uma_slab_t slab;
|
||||||
uma_keg_t keg;
|
uma_keg_t keg;
|
||||||
int domain, slabs;
|
int domain, flags, slabs;
|
||||||
|
|
||||||
keg = zone_first_keg(zone);
|
keg = zone_first_keg(zone);
|
||||||
if (keg == NULL)
|
if (keg == NULL)
|
||||||
return;
|
return;
|
||||||
KEG_LOCK(keg);
|
KEG_LOCK(keg);
|
||||||
slabs = items / keg->uk_ipers;
|
slabs = items / keg->uk_ipers;
|
||||||
domain = 0;
|
|
||||||
if (slabs * keg->uk_ipers < items)
|
if (slabs * keg->uk_ipers < items)
|
||||||
slabs++;
|
slabs++;
|
||||||
|
flags = M_WAITOK;
|
||||||
|
vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, &flags);
|
||||||
while (slabs-- > 0) {
|
while (slabs-- > 0) {
|
||||||
slab = keg_alloc_slab(keg, zone, domain, M_WAITOK);
|
slab = keg_alloc_slab(keg, zone, domain, flags);
|
||||||
if (slab == NULL)
|
if (slab == NULL)
|
||||||
return;
|
return;
|
||||||
MPASS(slab->us_keg == keg);
|
MPASS(slab->us_keg == keg);
|
||||||
dom = &keg->uk_domain[slab->us_domain];
|
dom = &keg->uk_domain[slab->us_domain];
|
||||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);
|
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);
|
||||||
do {
|
if (vm_domainset_iter_policy(&di, &domain) != 0)
|
||||||
domain = (domain + 1) % vm_ndomains;
|
break;
|
||||||
} while (VM_DOMAIN_EMPTY(domain));
|
|
||||||
}
|
}
|
||||||
KEG_UNLOCK(keg);
|
KEG_UNLOCK(keg);
|
||||||
}
|
}
|
||||||
|
@ -123,12 +123,18 @@ vm_mem_init(void *dummy)
|
|||||||
domainset_init();
|
domainset_init();
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Initializes resident memory structures. From here on, all physical
|
* Initialize resident memory structures. From here on, all physical
|
||||||
* memory is accounted for, and we use only virtual addresses.
|
* memory is accounted for, and we use only virtual addresses.
|
||||||
*/
|
*/
|
||||||
vm_set_page_size();
|
vm_set_page_size();
|
||||||
virtual_avail = vm_page_startup(virtual_avail);
|
virtual_avail = vm_page_startup(virtual_avail);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Set an initial domain policy for thread0 so that allocations
|
||||||
|
* can work.
|
||||||
|
*/
|
||||||
|
domainset_zero();
|
||||||
|
|
||||||
#ifdef UMA_MD_SMALL_ALLOC
|
#ifdef UMA_MD_SMALL_ALLOC
|
||||||
/* Announce page availability to UMA. */
|
/* Announce page availability to UMA. */
|
||||||
uma_startup1();
|
uma_startup1();
|
||||||
|
@ -800,7 +800,6 @@ kmem_bootstrap_free(vm_offset_t start, vm_size_t size)
|
|||||||
vmd = vm_pagequeue_domain(m);
|
vmd = vm_pagequeue_domain(m);
|
||||||
vm_domain_free_lock(vmd);
|
vm_domain_free_lock(vmd);
|
||||||
vm_phys_free_pages(m, 0);
|
vm_phys_free_pages(m, 0);
|
||||||
vmd->vmd_page_count++;
|
|
||||||
vm_domain_free_unlock(vmd);
|
vm_domain_free_unlock(vmd);
|
||||||
|
|
||||||
vm_domain_freecnt_inc(vmd, 1);
|
vm_domain_freecnt_inc(vmd, 1);
|
||||||
|
@ -855,11 +855,6 @@ vm_page_startup(vm_offset_t vaddr)
|
|||||||
*/
|
*/
|
||||||
vm_reserv_init();
|
vm_reserv_init();
|
||||||
#endif
|
#endif
|
||||||
/*
|
|
||||||
* Set an initial domain policy for thread0 so that allocations
|
|
||||||
* can work.
|
|
||||||
*/
|
|
||||||
domainset_zero();
|
|
||||||
|
|
||||||
return (vaddr);
|
return (vaddr);
|
||||||
}
|
}
|
||||||
|
@ -2072,41 +2072,41 @@ vm_pageout_init(void)
|
|||||||
static void
|
static void
|
||||||
vm_pageout(void)
|
vm_pageout(void)
|
||||||
{
|
{
|
||||||
int error;
|
struct proc *p;
|
||||||
int i;
|
struct thread *td;
|
||||||
|
int error, first, i;
|
||||||
|
|
||||||
|
p = curproc;
|
||||||
|
td = curthread;
|
||||||
|
|
||||||
swap_pager_swap_init();
|
swap_pager_swap_init();
|
||||||
snprintf(curthread->td_name, sizeof(curthread->td_name), "dom0");
|
for (first = -1, i = 0; i < vm_ndomains; i++) {
|
||||||
error = kthread_add(vm_pageout_laundry_worker, NULL, curproc, NULL,
|
|
||||||
0, 0, "laundry: dom0");
|
|
||||||
if (error != 0)
|
|
||||||
panic("starting laundry for domain 0, error %d", error);
|
|
||||||
for (i = 1; i < vm_ndomains; i++) {
|
|
||||||
if (VM_DOMAIN_EMPTY(i)) {
|
if (VM_DOMAIN_EMPTY(i)) {
|
||||||
if (bootverbose)
|
if (bootverbose)
|
||||||
printf("domain %d empty; skipping pageout\n",
|
printf("domain %d empty; skipping pageout\n",
|
||||||
i);
|
i);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (first == -1)
|
||||||
error = kthread_add(vm_pageout_worker, (void *)(uintptr_t)i,
|
first = i;
|
||||||
curproc, NULL, 0, 0, "dom%d", i);
|
else {
|
||||||
if (error != 0) {
|
error = kthread_add(vm_pageout_worker,
|
||||||
panic("starting pageout for domain %d, error %d\n",
|
(void *)(uintptr_t)i, p, NULL, 0, 0, "dom%d", i);
|
||||||
|
if (error != 0)
|
||||||
|
panic("starting pageout for domain %d: %d\n",
|
||||||
i, error);
|
i, error);
|
||||||
}
|
}
|
||||||
error = kthread_add(vm_pageout_laundry_worker,
|
error = kthread_add(vm_pageout_laundry_worker,
|
||||||
(void *)(uintptr_t)i, curproc, NULL, 0, 0,
|
(void *)(uintptr_t)i, p, NULL, 0, 0, "laundry: dom%d", i);
|
||||||
"laundry: dom%d", i);
|
|
||||||
if (error != 0)
|
if (error != 0)
|
||||||
panic("starting laundry for domain %d, error %d",
|
panic("starting laundry for domain %d: %d", i, error);
|
||||||
i, error);
|
|
||||||
}
|
}
|
||||||
error = kthread_add(uma_reclaim_worker, NULL, curproc, NULL,
|
error = kthread_add(uma_reclaim_worker, NULL, p, NULL, 0, 0, "uma");
|
||||||
0, 0, "uma");
|
|
||||||
if (error != 0)
|
if (error != 0)
|
||||||
panic("starting uma_reclaim helper, error %d\n", error);
|
panic("starting uma_reclaim helper, error %d\n", error);
|
||||||
vm_pageout_worker((void *)(uintptr_t)0);
|
|
||||||
|
snprintf(td->td_name, sizeof(td->td_name), "dom%d", first);
|
||||||
|
vm_pageout_worker((void *)(uintptr_t)first);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
Loading…
Reference in New Issue
Block a user