Rewrite sendfile's header support so that headers are now sent in the first
packet along with data, instead of in their own packet. When serving files of size (packetsize - headersize) or smaller, this will result in one less packet crossing the network. Quick testing with thttpd and http_load has shown a noticeable performance improvement in this case (350 vs 330 fetches per second.) Included in this commit are two support routines, iov_to_uio, and m_uiotombuf; these routines are used by sendfile to construct the header mbuf chain that will be linked to the rest of the data in the socket buffer.
This commit is contained in:
parent
f2f51f8ab8
commit
beb699c7ba
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=125296
@ -483,3 +483,45 @@ copyinstrfrom(const void * __restrict src, void * __restrict dst, size_t len,
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
|
||||
int
|
||||
iov_to_uio(struct iovec *iovp, u_int iovcnt, struct uio *auio)
|
||||
{
|
||||
int error = 0, i;
|
||||
u_int iovlen;
|
||||
struct iovec *iov = NULL;
|
||||
|
||||
if (iovcnt < 0)
|
||||
panic("iovcnt < 0!\n");
|
||||
|
||||
/* note: can't use iovlen until iovcnt is validated */
|
||||
iovlen = iovcnt * sizeof (struct iovec);
|
||||
if (iovcnt > UIO_MAXIOV) {
|
||||
error = EINVAL;
|
||||
goto done;
|
||||
}
|
||||
MALLOC(iov, struct iovec *, iovlen, M_IOV, M_WAITOK);
|
||||
auio->uio_iov = iov;
|
||||
auio->uio_iovcnt = iovcnt;
|
||||
auio->uio_segflg = UIO_USERSPACE;
|
||||
auio->uio_offset = -1;
|
||||
if ((error = copyin(iovp, iov, iovlen)))
|
||||
goto done;
|
||||
auio->uio_resid = 0;
|
||||
for (i = 0; i < iovcnt; i++) {
|
||||
if (iov->iov_len > INT_MAX - auio->uio_resid) {
|
||||
error = EINVAL;
|
||||
goto done;
|
||||
}
|
||||
auio->uio_resid += iov->iov_len;
|
||||
iov++;
|
||||
}
|
||||
|
||||
done:
|
||||
if (error && auio->uio_iov) {
|
||||
FREE(auio->uio_iov, M_IOV);
|
||||
auio->uio_iov = NULL;
|
||||
}
|
||||
return (error);
|
||||
|
||||
}
|
||||
|
@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/limits.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/mac.h>
|
||||
#include <sys/malloc.h>
|
||||
@ -50,6 +51,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/domain.h>
|
||||
#include <sys/protosw.h>
|
||||
#include <sys/uio.h>
|
||||
|
||||
int max_linkhdr;
|
||||
int max_protohdr;
|
||||
@ -1028,3 +1030,57 @@ m_fragment(struct mbuf *m0, int how, int length)
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
struct mbuf *
|
||||
m_uiotombuf(struct uio *uio, int how, int len)
|
||||
{
|
||||
struct mbuf *m_new = NULL, *m_final = NULL;
|
||||
int progress = 0, error = 0, length, total;
|
||||
|
||||
if (len > 0)
|
||||
total = min(uio->uio_resid, len);
|
||||
else
|
||||
total = uio->uio_resid;
|
||||
|
||||
if (total > MHLEN)
|
||||
m_final = m_getcl(how, MT_DATA, M_PKTHDR);
|
||||
else
|
||||
m_final = m_gethdr(how, MT_DATA);
|
||||
|
||||
if (m_final == NULL)
|
||||
goto nospace;
|
||||
|
||||
m_new = m_final;
|
||||
|
||||
while (progress < total) {
|
||||
length = total - progress;
|
||||
if (length > MCLBYTES)
|
||||
length = MCLBYTES;
|
||||
|
||||
if (m_new == NULL) {
|
||||
if (length > MLEN)
|
||||
m_new = m_getcl(how, MT_DATA, 0);
|
||||
else
|
||||
m_new = m_get(how, MT_DATA);
|
||||
if (m_new == NULL)
|
||||
goto nospace;
|
||||
}
|
||||
|
||||
error = uiomove(mtod(m_new, void *), length, uio);
|
||||
if (error)
|
||||
goto nospace;
|
||||
progress += length;
|
||||
m_new->m_len = length;
|
||||
if (m_new != m_final)
|
||||
m_cat(m_final, m_new);
|
||||
m_new = NULL;
|
||||
}
|
||||
m_fixhdr(m_final);
|
||||
return (m_final);
|
||||
nospace:
|
||||
if (m_new)
|
||||
m_free(m_new);
|
||||
if (m_final)
|
||||
m_freem(m_final);
|
||||
return (NULL);
|
||||
}
|
||||
|
@ -1677,13 +1677,15 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
struct vnode *vp;
|
||||
struct vm_object *obj;
|
||||
struct socket *so = NULL;
|
||||
struct mbuf *m;
|
||||
struct mbuf *m, *m_header = NULL;
|
||||
struct sf_buf *sf;
|
||||
struct vm_page *pg;
|
||||
struct writev_args nuap;
|
||||
struct sf_hdtr hdtr;
|
||||
struct uio hdr_uio;
|
||||
off_t off, xfsize, hdtr_size, sbytes = 0;
|
||||
int error, s;
|
||||
int error, s, headersize = 0, headersent = 0;
|
||||
struct iovec *hdr_iov = NULL;
|
||||
|
||||
mtx_lock(&Giant);
|
||||
|
||||
@ -1731,19 +1733,25 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
if (error)
|
||||
goto done;
|
||||
/*
|
||||
* Send any headers. Wimp out and use writev(2).
|
||||
* Send any headers.
|
||||
*/
|
||||
if (hdtr.headers != NULL) {
|
||||
nuap.fd = uap->s;
|
||||
nuap.iovp = hdtr.headers;
|
||||
nuap.iovcnt = hdtr.hdr_cnt;
|
||||
error = writev(td, &nuap);
|
||||
hdr_uio.uio_td = td;
|
||||
hdr_uio.uio_rw = UIO_WRITE;
|
||||
error = iov_to_uio(hdtr.headers, hdtr.hdr_cnt,
|
||||
&hdr_uio);
|
||||
if (error)
|
||||
goto done;
|
||||
if (compat)
|
||||
sbytes += td->td_retval[0];
|
||||
else
|
||||
hdtr_size += td->td_retval[0];
|
||||
/* Cache hdr_iov, m_uiotombuf may change it. */
|
||||
hdr_iov = hdr_uio.uio_iov;
|
||||
if (hdr_uio.uio_resid > 0) {
|
||||
m_header = m_uiotombuf(&hdr_uio, M_DONTWAIT, 0);
|
||||
if (m_header == NULL)
|
||||
goto done;
|
||||
headersize = m_header->m_pkthdr.len;
|
||||
if (compat)
|
||||
sbytes += headersize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1901,7 +1909,10 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
/*
|
||||
* Get an mbuf header and set it up as having external storage.
|
||||
*/
|
||||
MGETHDR(m, M_TRYWAIT, MT_DATA);
|
||||
if (m_header)
|
||||
MGET(m, M_TRYWAIT, MT_DATA);
|
||||
else
|
||||
MGETHDR(m, M_TRYWAIT, MT_DATA);
|
||||
if (m == NULL) {
|
||||
error = ENOBUFS;
|
||||
sf_buf_free((void *)sf_buf_kva(sf), sf);
|
||||
@ -1915,6 +1926,14 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
EXT_SFBUF);
|
||||
m->m_data = (char *)sf_buf_kva(sf) + pgoff;
|
||||
m->m_pkthdr.len = m->m_len = xfsize;
|
||||
|
||||
if (m_header) {
|
||||
m_cat(m_header, m);
|
||||
m = m_header;
|
||||
m_header = NULL;
|
||||
m_fixhdr(m);
|
||||
}
|
||||
|
||||
/*
|
||||
* Add the buffer to the socket buffer chain.
|
||||
*/
|
||||
@ -1976,6 +1995,7 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
sbunlock(&so->so_snd);
|
||||
goto done;
|
||||
}
|
||||
headersent = 1;
|
||||
}
|
||||
sbunlock(&so->so_snd);
|
||||
|
||||
@ -1996,6 +2016,13 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
}
|
||||
|
||||
done:
|
||||
if (headersent) {
|
||||
if (!compat)
|
||||
hdtr_size += headersize;
|
||||
} else {
|
||||
if (compat)
|
||||
sbytes -= headersize;
|
||||
}
|
||||
/*
|
||||
* If there was no error we have to clear td->td_retval[0]
|
||||
* because it may have been set by writev.
|
||||
@ -2012,6 +2039,10 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
|
||||
vrele(vp);
|
||||
if (so)
|
||||
fputsock(so);
|
||||
if (hdr_iov)
|
||||
FREE(hdr_iov, M_IOV);
|
||||
if (m_header)
|
||||
m_freem(m_header);
|
||||
|
||||
mtx_unlock(&Giant);
|
||||
|
||||
|
@ -94,6 +94,9 @@ int uiomove(void *cp, int n, struct uio *uio);
|
||||
int uiomove_frombuf(void *buf, int buflen, struct uio *uio);
|
||||
int uiomoveco(void *cp, int n, struct uio *uio, struct vm_object *obj,
|
||||
int disposable);
|
||||
struct mbuf *
|
||||
m_uiotombuf(struct uio *uio, int how, int len);
|
||||
int iov_to_uio(struct iovec *iovp, u_int iovcnt, struct uio *auio);
|
||||
|
||||
#else /* !_KERNEL */
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user