d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
e4b16f2fb1
freebsd-dev/sys/kern/kern_sendfile.c
Mark Johnston 916c61a5ed Fix handling of errors from pru_send(PRUS_NOTREADY) 
PRUS_NOTREADY indicates that the caller has not yet populated the chain
with data, and so it is not ready for transmission.  This is used by
sendfile (for async I/O) and KTLS (for encryption).  In particular, if
pru_send returns an error, the caller is responsible for freeing the
chain since other implicit references to the data buffers exist.

For async sendfile, it happens that an error will only be returned if
the connection was dropped, in which case tcp_usr_ready() will handle
freeing the chain.  But since KTLS can be used in conjunction with the
regular socket I/O system calls, many more error cases - which do not
result in the connection being dropped - are reachable.  In these cases,
KTLS was effectively assuming success.

So:
- Change sosend_generic() to free the mbuf chain if
  pru_send(PRUS_NOTREADY) fails.  Nothing else owns a reference to the
  chain at that point.
- Similarly, in vn_sendfile() change the !async I/O && KTLS case to free
  the chain.
- If async I/O is still outstanding when pru_send fails in
  vn_sendfile(), set an error in the sfio structure so that the
  connection is aborted and the mbuf chain is freed.

Reviewed by:	gallatin, tuexen
Discussed with:	jhb
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D30349
2021-05-21 17:45:19 -04:00

1372 lines
34 KiB
C
Raw Blame History

/*-
* Copyright (c) 2013-2015 Gleb Smirnoff <glebius@FreeBSD.org>
* Copyright (c) 1998, David Greenman. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_kern_tls.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/ktls.h>
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vnode.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_pager.h>
static MALLOC_DEFINE(M_SENDFILE, "sendfile", "sendfile dynamic memory");
#define EXT_FLAG_SYNC EXT_FLAG_VENDOR1
#define EXT_FLAG_NOCACHE EXT_FLAG_VENDOR2
#define EXT_FLAG_CACHE_LAST EXT_FLAG_VENDOR3
/*
* Structure describing a single sendfile(2) I/O, which may consist of
* several underlying pager I/Os.
*
* The syscall context allocates the structure and initializes 'nios'
* to 1. As sendfile_swapin() runs through pages and starts asynchronous
* paging operations, it increments 'nios'.
*
* Every I/O completion calls sendfile_iodone(), which decrements the 'nios',
* and the syscall also calls sendfile_iodone() after allocating all mbufs,
* linking them and sending to socket. Whoever reaches zero 'nios' is
* responsible to * call pru_ready on the socket, to notify it of readyness
* of the data.
*/
struct sf_io {
volatile u_int nios;
u_int error;
int npages;
struct socket *so;
struct mbuf *m;
vm_object_t obj;
vm_pindex_t pindex0;
#ifdef KERN_TLS
struct ktls_session *tls;
#endif
vm_page_t pa[];
};
/*
* Structure used to track requests with SF_SYNC flag.
*/
struct sendfile_sync {
struct mtx mtx;
struct cv cv;
unsigned count;
bool waiting;
};
static void
sendfile_sync_destroy(struct sendfile_sync *sfs)
{
KASSERT(sfs->count == 0, ("sendfile sync %p still busy", sfs));
cv_destroy(&sfs->cv);
mtx_destroy(&sfs->mtx);
free(sfs, M_SENDFILE);
}
static void
sendfile_sync_signal(struct sendfile_sync *sfs)
{
mtx_lock(&sfs->mtx);
KASSERT(sfs->count > 0, ("sendfile sync %p not busy", sfs));
if (--sfs->count == 0) {
if (!sfs->waiting) {
/* The sendfile() waiter was interrupted by a signal. */
sendfile_sync_destroy(sfs);
return;
} else {
cv_signal(&sfs->cv);
}
}
mtx_unlock(&sfs->mtx);
}
counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
static void
sfstat_init(const void *unused)
{
COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t),
M_WAITOK);
}
SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL);
static int
sfstat_sysctl(SYSCTL_HANDLER_ARGS)
{
struct sfstat s;
COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t));
if (req->newptr)
COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t));
return (SYSCTL_OUT(req, &s, sizeof(s)));
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat,
CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
sfstat_sysctl, "I",
"sendfile statistics");
static void
sendfile_free_mext(struct mbuf *m)
{
struct sf_buf *sf;
vm_page_t pg;
int flags;
KASSERT(m->m_flags & M_EXT && m->m_ext.ext_type == EXT_SFBUF,
("%s: m %p !M_EXT or !EXT_SFBUF", __func__, m));
sf = m->m_ext.ext_arg1;
pg = sf_buf_page(sf);
flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0;
sf_buf_free(sf);
vm_page_release(pg, flags);
if (m->m_ext.ext_flags & EXT_FLAG_SYNC) {
struct sendfile_sync *sfs = m->m_ext.ext_arg2;
sendfile_sync_signal(sfs);
}
}
static void
sendfile_free_mext_pg(struct mbuf *m)
{
vm_page_t pg;
int flags, i;
bool cache_last;
M_ASSERTEXTPG(m);
cache_last = m->m_ext.ext_flags & EXT_FLAG_CACHE_LAST;
flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0;
for (i = 0; i < m->m_epg_npgs; i++) {
if (cache_last && i == m->m_epg_npgs - 1)
flags = 0;
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_release(pg, flags);
}
if (m->m_ext.ext_flags & EXT_FLAG_SYNC) {
struct sendfile_sync *sfs = m->m_ext.ext_arg1;
sendfile_sync_signal(sfs);
}
}
/*
* Helper function to calculate how much data to put into page i of n.
* Only first and last pages are special.
*/
static inline off_t
xfsize(int i, int n, off_t off, off_t len)
{
if (i == 0)
return (omin(PAGE_SIZE - (off & PAGE_MASK), len));
if (i == n - 1 && ((off + len) & PAGE_MASK) > 0)
return ((off + len) & PAGE_MASK);
return (PAGE_SIZE);
}
/*
* Helper function to get offset within object for i page.
*/
static inline vm_ooffset_t
vmoff(int i, off_t off)
{
if (i == 0)
return ((vm_ooffset_t)off);
return (trunc_page(off + i * PAGE_SIZE));
}
/*
* Helper function used when allocation of a page or sf_buf failed.
* Pretend as if we don't have enough space, subtract xfsize() of
* all pages that failed.
*/
static inline void
fixspace(int old, int new, off_t off, int *space)
{
KASSERT(old > new, ("%s: old %d new %d", __func__, old, new));
/* Subtract last one. */
*space -= xfsize(old - 1, old, off, *space);
old--;
if (new == old)
/* There was only one page. */
return;
/* Subtract first one. */
if (new == 0) {
*space -= xfsize(0, old, off, *space);
new++;
}
/* Rest of pages are full sized. */
*space -= (old - new) * PAGE_SIZE;
KASSERT(*space >= 0, ("%s: space went backwards", __func__));
}
/*
* Wait for all in-flight ios to complete, we must not unwire pages
* under them.
*/
static void
sendfile_iowait(struct sf_io *sfio, const char *wmesg)
{
while (atomic_load_int(&sfio->nios) != 1)
pause(wmesg, 1);
}
/*
* I/O completion callback.
*/
static void
sendfile_iodone(void *arg, vm_page_t *pa, int count, int error)
{
struct sf_io *sfio = arg;
struct socket *so;
int i;
if (error != 0)
sfio->error = error;
/*
* Restore the valid page pointers. They are already
* unbusied, but still wired.
*
* XXXKIB since pages are only wired, and we do not
* own the object lock, other users might have
* invalidated them in meantime. Similarly, after we
* unbusied the swapped-in pages, they can become
* invalid under us.
*/
MPASS(count == 0 || pa[0] != bogus_page);
for (i = 0; i < count; i++) {
if (pa[i] == bogus_page) {
sfio->pa[(pa[0]->pindex - sfio->pindex0) + i] =
pa[i] = vm_page_relookup(sfio->obj,
pa[0]->pindex + i);
KASSERT(pa[i] != NULL,
("%s: page %p[%d] disappeared",
__func__, pa, i));
} else {
vm_page_xunbusy_unchecked(pa[i]);
}
}
if (!refcount_release(&sfio->nios))
return;
#ifdef INVARIANTS
for (i = 1; i < sfio->npages; i++) {
if (sfio->pa[i] == NULL)
break;
KASSERT(vm_page_wired(sfio->pa[i]),
("sfio %p page %d %p not wired", sfio, i, sfio->pa[i]));
if (i == 0)
continue;
KASSERT(sfio->pa[0]->object == sfio->pa[i]->object,
("sfio %p page %d %p wrong owner %p %p", sfio, i,
sfio->pa[i], sfio->pa[0]->object, sfio->pa[i]->object));
KASSERT(sfio->pa[0]->pindex + i == sfio->pa[i]->pindex,
("sfio %p page %d %p wrong index %jx %jx", sfio, i,
sfio->pa[i], (uintmax_t)sfio->pa[0]->pindex,
(uintmax_t)sfio->pa[i]->pindex));
}
#endif
vm_object_pip_wakeup(sfio->obj);
if (sfio->m == NULL) {
/*
* Either I/O operation failed, or we failed to allocate
* buffers, or we bailed out on first busy page, or we
* succeeded filling the request without any I/Os. Anyway,
* pru_send hadn't been executed - nothing had been sent
* to the socket yet.
*/
MPASS((curthread->td_pflags & TDP_KTHREAD) == 0);
free(sfio, M_SENDFILE);
return;
}
#if defined(KERN_TLS) && defined(INVARIANTS)
if ((sfio->m->m_flags & M_EXTPG) != 0)
KASSERT(sfio->tls == sfio->m->m_epg_tls,
("TLS session mismatch"));
else
KASSERT(sfio->tls == NULL,
("non-ext_pgs mbuf with TLS session"));
#endif
so = sfio->so;
CURVNET_SET(so->so_vnet);
if (__predict_false(sfio->error)) {
/*
* I/O operation failed. The state of data in the socket
* is now inconsistent, and all what we can do is to tear
* it down. Protocol abort method would tear down protocol
* state, free all ready mbufs and detach not ready ones.
* We will free the mbufs corresponding to this I/O manually.
*
* The socket would be marked with EIO and made available
* for read, so that application receives EIO on next
* syscall and eventually closes the socket.
*/
so->so_proto->pr_usrreqs->pru_abort(so);
so->so_error = EIO;
mb_free_notready(sfio->m, sfio->npages);
#ifdef KERN_TLS
} else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) {
/*
* I/O operation is complete, but we still need to
* encrypt. We cannot do this in the interrupt thread
* of the disk controller, so forward the mbufs to a
* different thread.
*
* Donate the socket reference from sfio to rather
* than explicitly invoking soref().
*/
ktls_enqueue(sfio->m, so, sfio->npages);
goto out_with_ref;
#endif
} else
(void)(so->so_proto->pr_usrreqs->pru_ready)(so, sfio->m,
sfio->npages);
SOCK_LOCK(so);
sorele(so);
#ifdef KERN_TLS
out_with_ref:
#endif
CURVNET_RESTORE();
free(sfio, M_SENDFILE);
}
/*
* Iterate through pages vector and request paging for non-valid pages.
*/
static int
sendfile_swapin(vm_object_t obj, struct sf_io *sfio, int *nios, off_t off,
off_t len, int rhpages, int flags)
{
vm_page_t *pa;
int a, count, count1, grabbed, i, j, npages, rv;
pa = sfio->pa;
npages = sfio->npages;
*nios = 0;
flags = (flags & SF_NODISKIO) ? VM_ALLOC_NOWAIT : 0;
sfio->pindex0 = OFF_TO_IDX(off);
/*
* First grab all the pages and wire them. Note that we grab
* only required pages. Readahead pages are dealt with later.
*/
grabbed = vm_page_grab_pages_unlocked(obj, OFF_TO_IDX(off),
VM_ALLOC_NORMAL | VM_ALLOC_WIRED | flags, pa, npages);
if (grabbed < npages) {
for (int i = grabbed; i < npages; i++)
pa[i] = NULL;
npages = grabbed;
rhpages = 0;
}
for (i = 0; i < npages;) {
/* Skip valid pages. */
if (vm_page_is_valid(pa[i], vmoff(i, off) & PAGE_MASK,
xfsize(i, npages, off, len))) {
vm_page_xunbusy(pa[i]);
SFSTAT_INC(sf_pages_valid);
i++;
continue;
}
/*
* Next page is invalid. Check if it belongs to pager. It
* may not be there, which is a regular situation for shmem
* pager. For vnode pager this happens only in case of
* a sparse file.
*
* Important feature of vm_pager_has_page() is the hint
* stored in 'a', about how many pages we can pagein after
* this page in a single I/O.
*/
VM_OBJECT_RLOCK(obj);
if (!vm_pager_has_page(obj, OFF_TO_IDX(vmoff(i, off)), NULL,
&a)) {
VM_OBJECT_RUNLOCK(obj);
pmap_zero_page(pa[i]);
vm_page_valid(pa[i]);
MPASS(pa[i]->dirty == 0);
vm_page_xunbusy(pa[i]);
i++;
continue;
}
VM_OBJECT_RUNLOCK(obj);
/*
* We want to pagein as many pages as possible, limited only
* by the 'a' hint and actual request.
*/
count = min(a + 1, npages - i);
/*
* We should not pagein into a valid page because
* there might be still unfinished write tracked by
* e.g. a buffer, thus we substitute any valid pages
* with the bogus one.
*
* We must not leave around xbusy pages which are not
* part of the run passed to vm_pager_getpages(),
* otherwise pager might deadlock waiting for the busy
* status of the page, e.g. if it constitues the
* buffer needed to validate other page.
*
* First trim the end of the run consisting of the
* valid pages, then replace the rest of the valid
* with bogus.
*/
count1 = count;
for (j = i + count - 1; j > i; j--) {
if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
xfsize(j, npages, off, len))) {
vm_page_xunbusy(pa[j]);
SFSTAT_INC(sf_pages_valid);
count--;
} else {
break;
}
}
/*
* The last page in the run pa[i + count - 1] is
* guaranteed to be invalid by the trim above, so it
* is not replaced with bogus, thus -1 in the loop end
* condition.
*/
MPASS(pa[i + count - 1]->valid != VM_PAGE_BITS_ALL);
for (j = i + 1; j < i + count - 1; j++) {
if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
xfsize(j, npages, off, len))) {
vm_page_xunbusy(pa[j]);
SFSTAT_INC(sf_pages_valid);
SFSTAT_INC(sf_pages_bogus);
pa[j] = bogus_page;
}
}
refcount_acquire(&sfio->nios);
rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,
i + count == npages ? &rhpages : NULL,
&sendfile_iodone, sfio);
if (__predict_false(rv != VM_PAGER_OK)) {
sendfile_iowait(sfio, "sferrio");
/*
* Do remaining pages recovery before returning EIO.
* Pages from 0 to npages are wired.
* Pages from (i + count1) to npages are busied.
*/
for (j = 0; j < npages; j++) {
if (j >= i + count1)
vm_page_xunbusy(pa[j]);
KASSERT(pa[j] != NULL && pa[j] != bogus_page,
("%s: page %p[%d] I/O recovery failure",
__func__, pa, j));
vm_page_unwire(pa[j], PQ_INACTIVE);
pa[j] = NULL;
}
return (EIO);
}
SFSTAT_INC(sf_iocnt);
SFSTAT_ADD(sf_pages_read, count);
if (i + count == npages)
SFSTAT_ADD(sf_rhpages_read, rhpages);
i += count1;
(*nios)++;
}
if (*nios == 0 && npages != 0)
SFSTAT_INC(sf_noiocnt);
return (0);
}
static int
sendfile_getobj(struct thread *td, struct file *fp, vm_object_t *obj_res,
struct vnode **vp_res, struct shmfd **shmfd_res, off_t *obj_size,
int *bsize)
{
struct vattr va;
vm_object_t obj;
struct vnode *vp;
struct shmfd *shmfd;
int error;
error = 0;
vp = *vp_res = NULL;
obj = NULL;
shmfd = *shmfd_res = NULL;
*bsize = 0;
/*
* The file descriptor must be a regular file and have a
* backing VM object.
*/
if (fp->f_type == DTYPE_VNODE) {
vp = fp->f_vnode;
vn_lock(vp, LK_SHARED | LK_RETRY);
if (vp->v_type != VREG) {
error = EINVAL;
goto out;
}
*bsize = vp->v_mount->mnt_stat.f_iosize;
obj = vp->v_object;
if (obj == NULL) {
error = EINVAL;
goto out;
}
/*
* Use the pager size when available to simplify synchronization
* with filesystems, which otherwise must atomically update both
* the vnode pager size and file size.
*/
if (obj->type == OBJT_VNODE) {
VM_OBJECT_RLOCK(obj);
*obj_size = obj->un_pager.vnp.vnp_size;
} else {
error = VOP_GETATTR(vp, &va, td->td_ucred);
if (error != 0)
goto out;
*obj_size = va.va_size;
VM_OBJECT_RLOCK(obj);
}
} else if (fp->f_type == DTYPE_SHM) {
shmfd = fp->f_data;
obj = shmfd->shm_object;
VM_OBJECT_RLOCK(obj);
*obj_size = shmfd->shm_size;
} else {
error = EINVAL;
goto out;
}
if ((obj->flags & OBJ_DEAD) != 0) {
VM_OBJECT_RUNLOCK(obj);
error = EBADF;
goto out;
}
/*
* Temporarily increase the backing VM object's reference
* count so that a forced reclamation of its vnode does not
* immediately destroy it.
*/
vm_object_reference_locked(obj);
VM_OBJECT_RUNLOCK(obj);
*obj_res = obj;
*vp_res = vp;
*shmfd_res = shmfd;
out:
if (vp != NULL)
VOP_UNLOCK(vp);
return (error);
}
static int
sendfile_getsock(struct thread *td, int s, struct file **sock_fp,
struct socket **so)
{
int error;
*sock_fp = NULL;
*so = NULL;
/*
* The socket must be a stream socket and connected.
*/
error = getsock_cap(td, s, &cap_send_rights,
sock_fp, NULL, NULL);
if (error != 0)
return (error);
*so = (*sock_fp)->f_data;
if ((*so)->so_type != SOCK_STREAM)
return (EINVAL);
/*
* SCTP one-to-one style sockets currently don't work with
* sendfile(). So indicate EINVAL for now.
*/
if ((*so)->so_proto->pr_protocol == IPPROTO_SCTP)
return (EINVAL);
if (SOLISTENING(*so))
return (ENOTCONN);
return (0);
}
int
vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
struct thread *td)
{
struct file *sock_fp;
struct vnode *vp;
struct vm_object *obj;
vm_page_t pga;
struct socket *so;
#ifdef KERN_TLS
struct ktls_session *tls;
#endif
struct mbuf *m, *mh, *mhtail;
struct sf_buf *sf;
struct shmfd *shmfd;
struct sendfile_sync *sfs;
struct vattr va;
off_t off, sbytes, rem, obj_size, nobj_size;
int bsize, error, ext_pgs_idx, hdrlen, max_pgs, softerr;
#ifdef KERN_TLS
int tls_enq_cnt;
#endif
bool use_ext_pgs;
obj = NULL;
so = NULL;
m = mh = NULL;
sfs = NULL;
#ifdef KERN_TLS
tls = NULL;
#endif
hdrlen = sbytes = 0;
softerr = 0;
use_ext_pgs = false;
error = sendfile_getobj(td, fp, &obj, &vp, &shmfd, &obj_size, &bsize);
if (error != 0)
return (error);
error = sendfile_getsock(td, sockfd, &sock_fp, &so);
if (error != 0)
goto out;
#ifdef MAC
error = mac_socket_check_send(td->td_ucred, so);
if (error != 0)
goto out;
#endif
SFSTAT_INC(sf_syscalls);
SFSTAT_ADD(sf_rhpages_requested, SF_READAHEAD(flags));
if (flags & SF_SYNC) {
sfs = malloc(sizeof(*sfs), M_SENDFILE, M_WAITOK | M_ZERO);
mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF);
cv_init(&sfs->cv, "sendfile");
sfs->waiting = true;
}
rem = nbytes ? omin(nbytes, obj_size - offset) : obj_size - offset;
/*
* Protect against multiple writers to the socket.
*
* XXXRW: Historically this has assumed non-interruptibility, so now
* we implement that, but possibly shouldn't.
*/
(void)sblock(&so->so_snd, SBL_WAIT | SBL_NOINTR);
#ifdef KERN_TLS
tls = ktls_hold(so->so_snd.sb_tls_info);
#endif
/*
* Loop through the pages of the file, starting with the requested
* offset. Get a file page (do I/O if necessary), map the file page
* into an sf_buf, attach an mbuf header to the sf_buf, and queue
* it on the socket.
* This is done in two loops. The inner loop turns as many pages
* as it can, up to available socket buffer space, without blocking
* into mbufs to have it bulk delivered into the socket send buffer.
* The outer loop checks the state and available space of the socket
* and takes care of the overall progress.
*/
for (off = offset; rem > 0; ) {
struct sf_io *sfio;
vm_page_t *pa;
struct mbuf *m0, *mtail;
int nios, space, npages, rhpages;
mtail = NULL;
/*
* Check the socket state for ongoing connection,
* no errors and space in socket buffer.
* If space is low allow for the remainder of the
* file to be processed if it fits the socket buffer.
* Otherwise block in waiting for sufficient space
* to proceed, or if the socket is nonblocking, return
* to userland with EAGAIN while reporting how far
* we've come.
* We wait until the socket buffer has significant free
* space to do bulk sends. This makes good use of file
* system read ahead and allows packet segmentation
* offloading hardware to take over lots of work. If
* we were not careful here we would send off only one
* sfbuf at a time.
*/
SOCKBUF_LOCK(&so->so_snd);
if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2)
so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2;
retry_space:
if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
error = EPIPE;
SOCKBUF_UNLOCK(&so->so_snd);
goto done;
} else if (so->so_error) {
error = so->so_error;
so->so_error = 0;
SOCKBUF_UNLOCK(&so->so_snd);
goto done;
}
if ((so->so_state & SS_ISCONNECTED) == 0) {
SOCKBUF_UNLOCK(&so->so_snd);
error = ENOTCONN;
goto done;
}
space = sbspace(&so->so_snd);
if (space < rem &&
(space <= 0 ||
space < so->so_snd.sb_lowat)) {
if (so->so_state & SS_NBIO) {
SOCKBUF_UNLOCK(&so->so_snd);
error = EAGAIN;
goto done;
}
/*
* sbwait drops the lock while sleeping.
* When we loop back to retry_space the
* state may have changed and we retest
* for it.
*/
error = sbwait(&so->so_snd);
/*
* An error from sbwait usually indicates that we've
* been interrupted by a signal. If we've sent anything
* then return bytes sent, otherwise return the error.
*/
if (error != 0) {
SOCKBUF_UNLOCK(&so->so_snd);
goto done;
}
goto retry_space;
}
SOCKBUF_UNLOCK(&so->so_snd);
/*
* At the beginning of the first loop check if any headers
* are specified and copy them into mbufs. Reduce space in
* the socket buffer by the size of the header mbuf chain.
* Clear hdr_uio here and hdrlen at the end of the first loop.
*/
if (hdr_uio != NULL && hdr_uio->uio_resid > 0) {
hdr_uio->uio_td = td;
hdr_uio->uio_rw = UIO_WRITE;
#ifdef KERN_TLS
if (tls != NULL)
mh = m_uiotombuf(hdr_uio, M_WAITOK, space,
tls->params.max_frame_len, M_EXTPG);
else
#endif
mh = m_uiotombuf(hdr_uio, M_WAITOK,
space, 0, 0);
hdrlen = m_length(mh, &mhtail);
space -= hdrlen;
/*
* If header consumed all the socket buffer space,
* don't waste CPU cycles and jump to the end.
*/
if (space == 0) {
sfio = NULL;
nios = 0;
goto prepend_header;
}
hdr_uio = NULL;
}
if (vp != NULL) {
error = vn_lock(vp, LK_SHARED);
if (error != 0)
goto done;
/*
* Check to see if the file size has changed.
*/
if (obj->type == OBJT_VNODE) {
VM_OBJECT_RLOCK(obj);
nobj_size = obj->un_pager.vnp.vnp_size;
VM_OBJECT_RUNLOCK(obj);
} else {
error = VOP_GETATTR(vp, &va, td->td_ucred);
if (error != 0) {
VOP_UNLOCK(vp);
goto done;
}
nobj_size = va.va_size;
}
if (off >= nobj_size) {
VOP_UNLOCK(vp);
goto done;
}
if (nobj_size != obj_size) {
obj_size = nobj_size;
rem = nbytes ? omin(nbytes + offset, obj_size) :
obj_size;
rem -= off;
}
}
if (space > rem)
space = rem;
else if (space > PAGE_SIZE) {
/*
* Use page boundaries when possible for large
* requests.
*/
if (off & PAGE_MASK)
space -= (PAGE_SIZE - (off & PAGE_MASK));
space = trunc_page(space);
if (off & PAGE_MASK)
space += (PAGE_SIZE - (off & PAGE_MASK));
}
npages = howmany(space + (off & PAGE_MASK), PAGE_SIZE);
/*
* Calculate maximum allowed number of pages for readahead
* at this iteration. If SF_USER_READAHEAD was set, we don't
* do any heuristics and use exactly the value supplied by
* application. Otherwise, we allow readahead up to "rem".
* If application wants more, let it be, but there is no
* reason to go above maxphys. Also check against "obj_size",
* since vm_pager_has_page() can hint beyond EOF.
*/
if (flags & SF_USER_READAHEAD) {
rhpages = SF_READAHEAD(flags);
} else {
rhpages = howmany(rem + (off & PAGE_MASK), PAGE_SIZE) -
npages;
rhpages += SF_READAHEAD(flags);
}
rhpages = min(howmany(maxphys, PAGE_SIZE), rhpages);
rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) -
npages, rhpages);
sfio = malloc(sizeof(struct sf_io) +
npages * sizeof(vm_page_t), M_SENDFILE, M_WAITOK);
refcount_init(&sfio->nios, 1);
sfio->obj = obj;
sfio->error = 0;
sfio->m = NULL;
sfio->npages = npages;
#ifdef KERN_TLS
/*
* This doesn't use ktls_hold() because sfio->m will
* also have a reference on 'tls' that will be valid
* for all of sfio's lifetime.
*/
sfio->tls = tls;
#endif
vm_object_pip_add(obj, 1);
error = sendfile_swapin(obj, sfio, &nios, off, space, rhpages,
flags);
if (error != 0) {
if (vp != NULL)
VOP_UNLOCK(vp);
sendfile_iodone(sfio, NULL, 0, error);
goto done;
}
/*
* Loop and construct maximum sized mbuf chain to be bulk
* dumped into socket buffer.
*/
pa = sfio->pa;
/*
* Use unmapped mbufs if enabled for TCP. Unmapped
* bufs are restricted to TCP as that is what has been
* tested. In particular, unmapped mbufs have not
* been tested with UNIX-domain sockets.
*
* TLS frames always require unmapped mbufs.
*/
if ((mb_use_ext_pgs &&
so->so_proto->pr_protocol == IPPROTO_TCP)
#ifdef KERN_TLS
|| tls != NULL
#endif
) {
use_ext_pgs = true;
#ifdef KERN_TLS
if (tls != NULL)
max_pgs = num_pages(tls->params.max_frame_len);
else
#endif
max_pgs = MBUF_PEXT_MAX_PGS;
/* Start at last index, to wrap on first use. */
ext_pgs_idx = max_pgs - 1;
}
for (int i = 0; i < npages; i++) {
/*
* If a page wasn't grabbed successfully, then
* trim the array. Can happen only with SF_NODISKIO.
*/
if (pa[i] == NULL) {
SFSTAT_INC(sf_busy);
fixspace(npages, i, off, &space);
sfio->npages = i;
softerr = EBUSY;
break;
}
pga = pa[i];
if (pga == bogus_page)
pga = vm_page_relookup(obj, sfio->pindex0 + i);
if (use_ext_pgs) {
off_t xfs;
ext_pgs_idx++;
if (ext_pgs_idx == max_pgs) {
m0 = mb_alloc_ext_pgs(M_WAITOK,
sendfile_free_mext_pg);
if (flags & SF_NOCACHE) {
m0->m_ext.ext_flags |=
EXT_FLAG_NOCACHE;
/*
* See comment below regarding
* ignoring SF_NOCACHE for the
* last page.
*/
if ((npages - i <= max_pgs) &&
((off + space) & PAGE_MASK) &&
(rem > space || rhpages > 0))
m0->m_ext.ext_flags |=
EXT_FLAG_CACHE_LAST;
}
if (sfs != NULL) {
m0->m_ext.ext_flags |=
EXT_FLAG_SYNC;
m0->m_ext.ext_arg1 = sfs;
mtx_lock(&sfs->mtx);
sfs->count++;
mtx_unlock(&sfs->mtx);
}
ext_pgs_idx = 0;
/* Append to mbuf chain. */
if (mtail != NULL)
mtail->m_next = m0;
else
m = m0;
mtail = m0;
m0->m_epg_1st_off =
vmoff(i, off) & PAGE_MASK;
}
if (nios) {
mtail->m_flags |= M_NOTREADY;
m0->m_epg_nrdy++;
}
m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
m0->m_epg_npgs++;
xfs = xfsize(i, npages, off, space);
m0->m_epg_last_len = xfs;
MBUF_EXT_PGS_ASSERT_SANITY(m0);
mtail->m_len += xfs;
mtail->m_ext.ext_size += PAGE_SIZE;
continue;
}
/*
* Get a sendfile buf. When allocating the
* first buffer for mbuf chain, we usually
* wait as long as necessary, but this wait
* can be interrupted. For consequent
* buffers, do not sleep, since several
* threads might exhaust the buffers and then
* deadlock.
*/
sf = sf_buf_alloc(pga,
m != NULL ? SFB_NOWAIT : SFB_CATCH);
if (sf == NULL) {
SFSTAT_INC(sf_allocfail);
sendfile_iowait(sfio, "sfnosf");
for (int j = i; j < npages; j++) {
vm_page_unwire(pa[j], PQ_INACTIVE);
pa[j] = NULL;
}
if (m == NULL)
softerr = ENOBUFS;
fixspace(npages, i, off, &space);
sfio->npages = i;
break;
}
m0 = m_get(M_WAITOK, MT_DATA);
m0->m_ext.ext_buf = (char *)sf_buf_kva(sf);
m0->m_ext.ext_size = PAGE_SIZE;
m0->m_ext.ext_arg1 = sf;
m0->m_ext.ext_type = EXT_SFBUF;
m0->m_ext.ext_flags = EXT_FLAG_EMBREF;
m0->m_ext.ext_free = sendfile_free_mext;
/*
* SF_NOCACHE sets the page as being freed upon send.
* However, we ignore it for the last page in 'space',
* if the page is truncated, and we got more data to
* send (rem > space), or if we have readahead
* configured (rhpages > 0).
*/
if ((flags & SF_NOCACHE) &&
(i != npages - 1 ||
!((off + space) & PAGE_MASK) ||
!(rem > space || rhpages > 0)))
m0->m_ext.ext_flags |= EXT_FLAG_NOCACHE;
if (sfs != NULL) {
m0->m_ext.ext_flags |= EXT_FLAG_SYNC;
m0->m_ext.ext_arg2 = sfs;
mtx_lock(&sfs->mtx);
sfs->count++;
mtx_unlock(&sfs->mtx);
}
m0->m_ext.ext_count = 1;
m0->m_flags |= (M_EXT | M_RDONLY);
if (nios)
m0->m_flags |= M_NOTREADY;
m0->m_data = (char *)sf_buf_kva(sf) +
(vmoff(i, off) & PAGE_MASK);
m0->m_len = xfsize(i, npages, off, space);
/* Append to mbuf chain. */
if (mtail != NULL)
mtail->m_next = m0;
else
m = m0;
mtail = m0;
}
if (vp != NULL)
VOP_UNLOCK(vp);
/* Keep track of bytes processed. */
off += space;
rem -= space;
/*
* Prepend header, if any. Save pointer to first mbuf
* with a page.
*/
if (hdrlen) {
prepend_header:
m0 = mhtail->m_next = m;
m = mh;
mh = NULL;
} else
m0 = m;
if (m == NULL) {
KASSERT(softerr, ("%s: m NULL, no error", __func__));
error = softerr;
sendfile_iodone(sfio, NULL, 0, 0);
goto done;
}
/* Add the buffer chain to the socket buffer. */
KASSERT(m_length(m, NULL) == space + hdrlen,
("%s: mlen %u space %d hdrlen %d",
__func__, m_length(m, NULL), space, hdrlen));
CURVNET_SET(so->so_vnet);
#ifdef KERN_TLS
if (tls != NULL)
ktls_frame(m, tls, &tls_enq_cnt, TLS_RLTYPE_APP);
#endif
if (nios == 0) {
/*
* If sendfile_swapin() didn't initiate any I/Os,
* which happens if all data is cached in VM, or if
* the header consumed all socket buffer space and
* sfio is NULL, then we can send data right now
* without the PRUS_NOTREADY flag.
*/
if (sfio != NULL)
sendfile_iodone(sfio, NULL, 0, 0);
#ifdef KERN_TLS
if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td);
if (error != 0) {
m_freem(m);
} else {
soref(so);
ktls_enqueue(m, so, tls_enq_cnt);
}
} else
#endif
error = (*so->so_proto->pr_usrreqs->pru_send)
(so, 0, m, NULL, NULL, td);
} else {
sfio->so = so;
sfio->m = m0;
soref(so);
error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td);
sendfile_iodone(sfio, NULL, 0, error);
}
CURVNET_RESTORE();
m = NULL;
if (error)
goto done;
sbytes += space + hdrlen;
if (hdrlen)
hdrlen = 0;
if (softerr) {
error = softerr;
goto done;
}
}
/*
* Send trailers. Wimp out and use writev(2).
*/
if (trl_uio != NULL) {
sbunlock(&so->so_snd);
error = kern_writev(td, sockfd, trl_uio);
if (error == 0)
sbytes += td->td_retval[0];
goto out;
}
done:
sbunlock(&so->so_snd);
out:
/*
* If there was no error we have to clear td->td_retval[0]
* because it may have been set by writev.
*/
if (error == 0) {
td->td_retval[0] = 0;
}
if (sent != NULL) {
(*sent) = sbytes;
}
if (obj != NULL)
vm_object_deallocate(obj);
if (so)
fdrop(sock_fp, td);
if (m)
m_freem(m);
if (mh)
m_freem(mh);
if (sfs != NULL) {
mtx_lock(&sfs->mtx);
if (sfs->count != 0)
error = cv_wait_sig(&sfs->cv, &sfs->mtx);
if (sfs->count == 0) {
sendfile_sync_destroy(sfs);
} else {
sfs->waiting = false;
mtx_unlock(&sfs->mtx);
}
}
#ifdef KERN_TLS
if (tls != NULL)
ktls_free(tls);
#endif
if (error == ERESTART)
error = EINTR;
return (error);
}
static int
sendfile(struct thread *td, struct sendfile_args *uap, int compat)
{
struct sf_hdtr hdtr;
struct uio *hdr_uio, *trl_uio;
struct file *fp;
off_t sbytes;
int error;
/*
* File offset must be positive. If it goes beyond EOF
* we send only the header/trailer and no payload data.
*/
if (uap->offset < 0)
return (EINVAL);
sbytes = 0;
hdr_uio = trl_uio = NULL;
if (uap->hdtr != NULL) {
error = copyin(uap->hdtr, &hdtr, sizeof(hdtr));
if (error != 0)
goto out;
if (hdtr.headers != NULL) {
error = copyinuio(hdtr.headers, hdtr.hdr_cnt,
&hdr_uio);
if (error != 0)
goto out;
#ifdef COMPAT_FREEBSD4
/*
* In FreeBSD < 5.0 the nbytes to send also included
* the header. If compat is specified subtract the
* header size from nbytes.
*/
if (compat) {
if (uap->nbytes > hdr_uio->uio_resid)
uap->nbytes -= hdr_uio->uio_resid;
else
uap->nbytes = 0;
}
#endif
}
if (hdtr.trailers != NULL) {
error = copyinuio(hdtr.trailers, hdtr.trl_cnt,
&trl_uio);
if (error != 0)
goto out;
}
}
AUDIT_ARG_FD(uap->fd);
/*
* sendfile(2) can start at any offset within a file so we require
* CAP_READ+CAP_SEEK = CAP_PREAD.
*/
if ((error = fget_read(td, uap->fd, &cap_pread_rights, &fp)) != 0)
goto out;
error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, uap->offset,
uap->nbytes, &sbytes, uap->flags, td);
fdrop(fp, td);
if (uap->sbytes != NULL)
copyout(&sbytes, uap->sbytes, sizeof(off_t));
out:
free(hdr_uio, M_IOV);
free(trl_uio, M_IOV);
return (error);
}
/*
* sendfile(2)
*
* int sendfile(int fd, int s, off_t offset, size_t nbytes,
* struct sf_hdtr *hdtr, off_t *sbytes, int flags)
*
* Send a file specified by 'fd' and starting at 'offset' to a socket
* specified by 's'. Send only 'nbytes' of the file or until EOF if nbytes ==
* 0. Optionally add a header and/or trailer to the socket output. If
* specified, write the total number of bytes sent into *sbytes.
*/
int
sys_sendfile(struct thread *td, struct sendfile_args *uap)
{
return (sendfile(td, uap, 0));
}
#ifdef COMPAT_FREEBSD4
int
freebsd4_sendfile(struct thread *td, struct freebsd4_sendfile_args *uap)
{
struct sendfile_args args;
args.fd = uap->fd;
args.s = uap->s;
args.offset = uap->offset;
args.nbytes = uap->nbytes;
args.hdtr = uap->hdtr;
args.sbytes = uap->sbytes;
args.flags = uap->flags;
return (sendfile(td, &args, 1));
}
#endif /* COMPAT_FREEBSD4 */
Reference in New Issue View Git Blame Copy Permalink