Use bogus_page to properly reduce number of I/Os in sendfile(2). The new

sendfile_swapin() loop works this way:

- Find first invalid page in the request.
- Do vm_pager_has_page() and get count of pages, that can be taken in
  single I/O.
- Trim valid pages from the end of the request.
- Cycle through the request and substitute to bogus_page all valid
  pages that are in the middle of the request.
- After I/O launched (pager copies array of pages into buf(9), it
  is important to restore proper page pointers with help vm_page_lookup().

Count bogus pages used and report them in sendfile stats.
This commit is contained in:
glebius 2016-11-17 21:02:55 +00:00
parent 89670e3aee
commit 8c35911278
3 changed files with 49 additions and 35 deletions

View File

@ -62,6 +62,8 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_object.h> #include <vm/vm_object.h>
#include <vm/vm_pager.h> #include <vm/vm_pager.h>
extern vm_page_t bogus_page;
/* /*
* Structure describing a single sendfile(2) I/O, which may consist of * Structure describing a single sendfile(2) I/O, which may consist of
* several underlying pager I/Os. * several underlying pager I/Os.
@ -258,7 +260,8 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, int error)
struct socket *so; struct socket *so;
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
vm_page_xunbusy(pg[i]); if (pg[i] != bogus_page)
vm_page_xunbusy(pg[i]);
if (error) if (error)
sfio->error = error; sfio->error = error;
@ -341,51 +344,53 @@ sendfile_swapin(vm_object_t obj, struct sf_io *sfio, off_t off, off_t len,
} }
/* /*
* Now 'i' points to first invalid page, iterate further * Next page is invalid. Check if it belongs to pager. It
* to make 'j' point at first valid after a bunch of * may not be there, which is a regular situation for shmem
* invalid ones. * pager. For vnode pager this happens only in case of
*/ * a sparse file.
for (j = i + 1; j < npages; j++)
if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
xfsize(j, npages, off, len))) {
SFSTAT_INC(sf_pages_valid);
break;
}
/*
* Now we got region of invalid pages between 'i' and 'j'.
* Check that they belong to pager. They may not be there,
* which is a regular situation for shmem pager. For vnode
* pager this happens only in case of sparse file.
* *
* Important feature of vm_pager_has_page() is the hint * Important feature of vm_pager_has_page() is the hint
* stored in 'a', about how many pages we can pagein after * stored in 'a', about how many pages we can pagein after
* this page in a single I/O. * this page in a single I/O.
*/ */
while (!vm_pager_has_page(obj, OFF_TO_IDX(vmoff(i, off)), if (!vm_pager_has_page(obj, OFF_TO_IDX(vmoff(i, off)), NULL,
NULL, &a) && i < j) { &a)) {
pmap_zero_page(pa[i]); pmap_zero_page(pa[i]);
pa[i]->valid = VM_PAGE_BITS_ALL; pa[i]->valid = VM_PAGE_BITS_ALL;
pa[i]->dirty = 0; pa[i]->dirty = 0;
vm_page_xunbusy(pa[i]); vm_page_xunbusy(pa[i]);
i++; i++;
}
if (i == j)
continue; continue;
}
/* /*
* We want to pagein as many pages as possible, limited only * We want to pagein as many pages as possible, limited only
* by the 'a' hint and actual request. * by the 'a' hint and actual request.
*
* We should not pagein into already valid page, thus if
* 'j' didn't reach last page, trim by that page.
*
* When the pagein fulfils the request, also specify readahead.
*/ */
if (j < npages)
a = min(a, j - i - 1);
count = min(a + 1, npages - i); count = min(a + 1, npages - i);
/*
* We should not pagein into a valid page, thus we first trim
* any valid pages off the end of request, and substitute
* to bogus_page those, that are in the middle.
*/
for (j = i + count - 1; j > i; j--) {
if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
xfsize(j, npages, off, len))) {
count--;
rhpages = 0;
} else
break;
}
for (j = i + 1; j < i + count - 1; j++)
if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
xfsize(j, npages, off, len))) {
vm_page_xunbusy(pa[j]);
SFSTAT_INC(sf_pages_valid);
SFSTAT_INC(sf_pages_bogus);
pa[j] = bogus_page;
}
refcount_acquire(&sfio->nios); refcount_acquire(&sfio->nios);
rv = vm_pager_get_pages_async(obj, pa + i, count, NULL, rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,
i + count == npages ? &rhpages : NULL, i + count == npages ? &rhpages : NULL,
@ -398,13 +403,18 @@ sendfile_swapin(vm_object_t obj, struct sf_io *sfio, off_t off, off_t len,
if (i + count == npages) if (i + count == npages)
SFSTAT_ADD(sf_rhpages_read, rhpages); SFSTAT_ADD(sf_rhpages_read, rhpages);
#ifdef INVARIANTS /*
for (j = i; j < i + count && j < npages; j++) * Restore the valid page pointers. They are already
KASSERT(pa[j] == vm_page_lookup(obj, * unbusied, but still wired.
OFF_TO_IDX(vmoff(j, off))), */
("pa[j] %p lookup %p\n", pa[j], for (j = i; j < i + count; j++)
vm_page_lookup(obj, OFF_TO_IDX(vmoff(j, off))))); if (pa[j] == bogus_page) {
#endif pa[j] = vm_page_lookup(obj,
OFF_TO_IDX(vmoff(j, off)));
KASSERT(pa[j], ("%s: page %p[%d] disappeared",
__func__, pa, j));
}
i += count; i += count;
nios++; nios++;
} }

View File

@ -41,6 +41,7 @@ struct sfstat { /* sendfile statistics */
uint64_t sf_busy; /* times aborted on a busy page */ uint64_t sf_busy; /* times aborted on a busy page */
uint64_t sf_allocfail; /* times sfbuf allocation failed */ uint64_t sf_allocfail; /* times sfbuf allocation failed */
uint64_t sf_allocwait; /* times sfbuf allocation had to wait */ uint64_t sf_allocwait; /* times sfbuf allocation had to wait */
uint64_t sf_pages_bogus; /* times bogus page was used */
}; };
#ifdef _KERNEL #ifdef _KERNEL

View File

@ -340,6 +340,9 @@ mbpr(void *kvmd, u_long mbaddr)
xo_emit("{:sendfile-pages-valid/%ju} " xo_emit("{:sendfile-pages-valid/%ju} "
"{N:pages were valid at time of a sendfile request}\n", "{N:pages were valid at time of a sendfile request}\n",
(uintmax_t)sfstat.sf_pages_valid); (uintmax_t)sfstat.sf_pages_valid);
xo_emit("{:sendfile-pages-bogus/%ju} "
"{N:pages were valid and substituted to bogus page}\n",
(uintmax_t)sfstat.sf_pages_bogus);
xo_emit("{:sendfile-requested-readahead/%ju} " xo_emit("{:sendfile-requested-readahead/%ju} "
"{N:pages were requested for read ahead by applications}\n", "{N:pages were requested for read ahead by applications}\n",
(uintmax_t)sfstat.sf_rhpages_requested); (uintmax_t)sfstat.sf_rhpages_requested);