1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 1982, 1986, 1988, 1990, 1993
|
|
|
|
* The Regents of the University of California. All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 4. Neither the name of the University nor the names of its contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
|
|
|
|
*/
|
|
|
|
|
2003-06-11 00:56:59 +00:00
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2000-11-20 01:35:25 +00:00
|
|
|
#include "opt_inet.h"
|
2002-07-31 03:03:22 +00:00
|
|
|
#include "opt_mac.h"
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#include "opt_zero.h"
|
2000-11-20 01:35:25 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
1997-03-23 03:37:54 +00:00
|
|
|
#include <sys/fcntl.h>
|
2003-04-29 13:36:06 +00:00
|
|
|
#include <sys/limits.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/lock.h>
|
2002-07-31 03:03:22 +00:00
|
|
|
#include <sys/mac.h>
|
2002-08-01 17:47:56 +00:00
|
|
|
#include <sys/malloc.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/mbuf.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/mutex.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/domain.h>
|
2000-04-16 18:53:38 +00:00
|
|
|
#include <sys/file.h> /* for struct knote */
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/kernel.h>
|
2000-04-16 18:53:38 +00:00
|
|
|
#include <sys/event.h>
|
1997-09-14 02:34:14 +00:00
|
|
|
#include <sys/poll.h>
|
1999-06-17 23:54:50 +00:00
|
|
|
#include <sys/proc.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/protosw.h>
|
|
|
|
#include <sys/socket.h>
|
|
|
|
#include <sys/socketvar.h>
|
|
|
|
#include <sys/resourcevar.h>
|
1994-10-02 17:35:40 +00:00
|
|
|
#include <sys/signalvar.h>
|
1995-11-03 18:33:46 +00:00
|
|
|
#include <sys/sysctl.h>
|
1998-03-28 10:33:27 +00:00
|
|
|
#include <sys/uio.h>
|
2000-06-04 04:28:31 +00:00
|
|
|
#include <sys/jail.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
|
2002-03-20 04:09:59 +00:00
|
|
|
#include <vm/uma.h>
|
1995-11-03 18:33:46 +00:00
|
|
|
|
1997-08-21 20:33:42 +00:00
|
|
|
|
2004-07-11 01:34:34 +00:00
|
|
|
static int soreceive_rcvoob(struct socket *so, struct uio *uio,
|
|
|
|
int flags);
|
|
|
|
|
2000-11-20 01:35:25 +00:00
|
|
|
#ifdef INET
|
2000-06-20 01:09:23 +00:00
|
|
|
static int do_setopt_accept_filter(struct socket *so, struct sockopt *sopt);
|
2000-11-20 01:35:25 +00:00
|
|
|
#endif
|
2000-06-20 01:09:23 +00:00
|
|
|
|
2003-03-02 15:56:49 +00:00
|
|
|
static void filt_sordetach(struct knote *kn);
|
|
|
|
static int filt_soread(struct knote *kn, long hint);
|
|
|
|
static void filt_sowdetach(struct knote *kn);
|
2000-04-16 18:53:38 +00:00
|
|
|
static int filt_sowrite(struct knote *kn, long hint);
|
|
|
|
static int filt_solisten(struct knote *kn, long hint);
|
|
|
|
|
2001-11-12 20:51:40 +00:00
|
|
|
static struct filterops solisten_filtops =
|
2001-02-15 16:34:11 +00:00
|
|
|
{ 1, NULL, filt_sordetach, filt_solisten };
|
|
|
|
static struct filterops soread_filtops =
|
|
|
|
{ 1, NULL, filt_sordetach, filt_soread };
|
2001-11-12 20:51:40 +00:00
|
|
|
static struct filterops sowrite_filtops =
|
2001-02-15 16:34:11 +00:00
|
|
|
{ 1, NULL, filt_sowdetach, filt_sowrite };
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2002-03-20 04:09:59 +00:00
|
|
|
uma_zone_t socket_zone;
|
1998-05-15 20:11:40 +00:00
|
|
|
so_gen_t so_gencnt; /* generation count for sockets */
|
|
|
|
|
1997-10-12 20:26:33 +00:00
|
|
|
MALLOC_DEFINE(M_SONAME, "soname", "socket name");
|
|
|
|
MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");
|
|
|
|
|
1999-02-16 10:49:55 +00:00
|
|
|
SYSCTL_DECL(_kern_ipc);
|
|
|
|
|
1995-11-03 18:33:46 +00:00
|
|
|
static int somaxconn = SOMAXCONN;
|
1999-11-22 02:45:11 +00:00
|
|
|
SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
|
1999-05-03 23:57:32 +00:00
|
|
|
&somaxconn, 0, "Maximum pending socket connection queue size");
|
2001-11-17 03:07:11 +00:00
|
|
|
static int numopensockets;
|
|
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
|
|
|
|
&numopensockets, 0, "Number of open sockets");
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
/* These aren't static because they're used in other files. */
|
|
|
|
int so_zero_copy_send = 1;
|
|
|
|
int so_zero_copy_receive = 1;
|
|
|
|
SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0,
|
|
|
|
"Zero copy controls");
|
|
|
|
SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW,
|
|
|
|
&so_zero_copy_receive, 0, "Enable zero copy receive");
|
|
|
|
SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW,
|
|
|
|
&so_zero_copy_send, 0, "Enable zero copy send");
|
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
2001-11-17 03:07:11 +00:00
|
|
|
|
2004-06-27 03:22:15 +00:00
|
|
|
/*
|
|
|
|
* accept_mtx locks down per-socket fields relating to accept queues. See
|
|
|
|
* socketvar.h for an annotation of the protected fields of struct socket.
|
|
|
|
*/
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
struct mtx accept_mtx;
|
|
|
|
MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF);
|
|
|
|
|
2004-06-27 03:22:15 +00:00
|
|
|
/*
|
|
|
|
* so_global_mtx protects so_gencnt, numopensockets, and the per-socket
|
|
|
|
* so_gencnt field.
|
|
|
|
*
|
|
|
|
* XXXRW: These variables might be better manipulated using atomic operations
|
|
|
|
* for improved efficiency.
|
|
|
|
*/
|
|
|
|
static struct mtx so_global_mtx;
|
|
|
|
MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF);
|
1994-05-25 09:21:21 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Socket operation routines.
|
|
|
|
* These routines are called by the routines in
|
|
|
|
* sys_socket.c or from a system process, and
|
|
|
|
* implement the semantics of socket operations by
|
|
|
|
* switching out to the protocol specific routines.
|
|
|
|
*/
|
1998-05-15 20:11:40 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get a socket structure from our zone, and initialize it.
|
|
|
|
* Note that it would probably be better to allocate socket
|
|
|
|
* and PCB at the same time, but I'm not convinced that all
|
|
|
|
* the protocols can be easily modified to do this.
|
2001-11-17 03:07:11 +00:00
|
|
|
*
|
|
|
|
* soalloc() returns a socket with a ref count of 0.
|
1998-05-15 20:11:40 +00:00
|
|
|
*/
|
|
|
|
struct socket *
|
2004-02-29 17:54:05 +00:00
|
|
|
soalloc(int mflags)
|
1998-05-15 20:11:40 +00:00
|
|
|
{
|
|
|
|
struct socket *so;
|
2002-10-05 21:23:47 +00:00
|
|
|
#ifdef MAC
|
|
|
|
int error;
|
|
|
|
#endif
|
1998-05-15 20:11:40 +00:00
|
|
|
|
2004-02-29 17:54:05 +00:00
|
|
|
so = uma_zalloc(socket_zone, mflags | M_ZERO);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (so != NULL) {
|
2002-10-05 21:23:47 +00:00
|
|
|
#ifdef MAC
|
2004-03-01 03:14:23 +00:00
|
|
|
error = mac_init_socket(so, mflags);
|
2002-10-05 21:23:47 +00:00
|
|
|
if (error != 0) {
|
|
|
|
uma_zfree(socket_zone, so);
|
|
|
|
so = NULL;
|
|
|
|
return so;
|
|
|
|
}
|
|
|
|
#endif
|
2004-06-12 16:08:41 +00:00
|
|
|
SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd");
|
|
|
|
SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv");
|
2002-05-31 11:52:35 +00:00
|
|
|
/* sx_init(&so->so_sxlock, "socket sxlock"); */
|
2000-01-14 02:53:29 +00:00
|
|
|
TAILQ_INIT(&so->so_aiojobq);
|
2004-06-27 03:22:15 +00:00
|
|
|
mtx_lock(&so_global_mtx);
|
|
|
|
so->so_gencnt = ++so_gencnt;
|
2001-11-17 03:07:11 +00:00
|
|
|
++numopensockets;
|
2004-06-27 03:22:15 +00:00
|
|
|
mtx_unlock(&so_global_mtx);
|
1998-05-15 20:11:40 +00:00
|
|
|
}
|
|
|
|
return so;
|
|
|
|
}
|
|
|
|
|
2001-11-17 03:07:11 +00:00
|
|
|
/*
|
|
|
|
* socreate returns a socket with a ref count of 1. The socket should be
|
|
|
|
* closed with soclose().
|
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
2001-12-31 17:45:16 +00:00
|
|
|
socreate(dom, aso, type, proto, cred, td)
|
1994-05-24 10:09:53 +00:00
|
|
|
int dom;
|
|
|
|
struct socket **aso;
|
2003-03-02 15:56:49 +00:00
|
|
|
int type;
|
1994-05-24 10:09:53 +00:00
|
|
|
int proto;
|
2001-12-31 17:45:16 +00:00
|
|
|
struct ucred *cred;
|
2001-09-12 08:38:13 +00:00
|
|
|
struct thread *td;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2003-03-02 15:56:49 +00:00
|
|
|
struct protosw *prp;
|
|
|
|
struct socket *so;
|
|
|
|
int error;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
if (proto)
|
|
|
|
prp = pffindproto(dom, proto, type);
|
|
|
|
else
|
|
|
|
prp = pffindtype(dom, type);
|
2000-06-04 04:28:31 +00:00
|
|
|
|
2004-04-09 13:23:51 +00:00
|
|
|
if (prp == NULL || prp->pr_usrreqs->pru_attach == NULL)
|
2000-06-13 15:44:04 +00:00
|
|
|
return (EPROTONOSUPPORT);
|
|
|
|
|
2002-08-12 16:49:03 +00:00
|
|
|
if (jailed(cred) && jail_socket_unixiproute_only &&
|
2000-06-04 04:28:31 +00:00
|
|
|
prp->pr_domain->dom_family != PF_LOCAL &&
|
|
|
|
prp->pr_domain->dom_family != PF_INET &&
|
|
|
|
prp->pr_domain->dom_family != PF_ROUTE) {
|
|
|
|
return (EPROTONOSUPPORT);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
if (prp->pr_type != type)
|
|
|
|
return (EPROTOTYPE);
|
2004-02-29 17:54:05 +00:00
|
|
|
so = soalloc(M_WAITOK);
|
2002-05-21 21:30:44 +00:00
|
|
|
if (so == NULL)
|
1998-05-15 20:11:40 +00:00
|
|
|
return (ENOBUFS);
|
|
|
|
|
1996-03-11 15:37:44 +00:00
|
|
|
TAILQ_INIT(&so->so_incomp);
|
|
|
|
TAILQ_INIT(&so->so_comp);
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_type = type;
|
2001-12-31 17:45:16 +00:00
|
|
|
so->so_cred = crhold(cred);
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_proto = prp;
|
2002-07-31 03:03:22 +00:00
|
|
|
#ifdef MAC
|
2002-08-12 16:49:03 +00:00
|
|
|
mac_create_socket(cred, so);
|
2002-07-31 03:03:22 +00:00
|
|
|
#endif
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_LOCK(so);
|
2001-11-17 03:07:11 +00:00
|
|
|
soref(so);
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_UNLOCK(so);
|
2001-09-12 08:38:13 +00:00
|
|
|
error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (error) {
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_LOCK(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_state |= SS_NOFDREF;
|
2001-11-17 03:07:11 +00:00
|
|
|
sorele(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
*aso = so;
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
2001-09-12 08:38:13 +00:00
|
|
|
sobind(so, nam, td)
|
1994-05-24 10:09:53 +00:00
|
|
|
struct socket *so;
|
1997-08-16 19:16:27 +00:00
|
|
|
struct sockaddr *nam;
|
2001-09-12 08:38:13 +00:00
|
|
|
struct thread *td;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
|
2004-06-22 03:49:22 +00:00
|
|
|
return ((*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td));
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
2002-11-02 05:14:31 +00:00
|
|
|
void
|
2001-11-17 03:07:11 +00:00
|
|
|
sodealloc(struct socket *so)
|
1998-05-15 20:11:40 +00:00
|
|
|
{
|
1999-09-19 02:17:02 +00:00
|
|
|
|
2001-11-17 03:07:11 +00:00
|
|
|
KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count));
|
2004-06-27 03:22:15 +00:00
|
|
|
mtx_lock(&so_global_mtx);
|
1998-05-15 20:11:40 +00:00
|
|
|
so->so_gencnt = ++so_gencnt;
|
2004-06-27 03:22:15 +00:00
|
|
|
mtx_unlock(&so_global_mtx);
|
1999-10-09 20:42:17 +00:00
|
|
|
if (so->so_rcv.sb_hiwat)
|
2000-09-05 22:11:13 +00:00
|
|
|
(void)chgsbsize(so->so_cred->cr_uidinfo,
|
2000-08-29 11:28:06 +00:00
|
|
|
&so->so_rcv.sb_hiwat, 0, RLIM_INFINITY);
|
1999-10-09 20:42:17 +00:00
|
|
|
if (so->so_snd.sb_hiwat)
|
2000-09-05 22:11:13 +00:00
|
|
|
(void)chgsbsize(so->so_cred->cr_uidinfo,
|
2000-08-29 11:28:06 +00:00
|
|
|
&so->so_snd.sb_hiwat, 0, RLIM_INFINITY);
|
2000-11-20 01:35:25 +00:00
|
|
|
#ifdef INET
|
2003-01-05 11:14:04 +00:00
|
|
|
/* remove acccept filter if one is present. */
|
|
|
|
if (so->so_accf != NULL)
|
|
|
|
do_setopt_accept_filter(so, NULL);
|
2002-07-31 03:03:22 +00:00
|
|
|
#endif
|
|
|
|
#ifdef MAC
|
|
|
|
mac_destroy_socket(so);
|
2000-11-20 01:35:25 +00:00
|
|
|
#endif
|
1999-09-19 02:17:02 +00:00
|
|
|
crfree(so->so_cred);
|
2004-06-12 16:08:41 +00:00
|
|
|
SOCKBUF_LOCK_DESTROY(&so->so_snd);
|
|
|
|
SOCKBUF_LOCK_DESTROY(&so->so_rcv);
|
2002-05-31 11:52:35 +00:00
|
|
|
/* sx_destroy(&so->so_sxlock); */
|
2002-04-08 03:04:22 +00:00
|
|
|
uma_zfree(socket_zone, so);
|
2004-06-27 03:22:15 +00:00
|
|
|
/*
|
|
|
|
* XXXRW: Seems like a shame to grab the mutex again down here, but
|
|
|
|
* we don't want to decrement the socket count until after we free
|
|
|
|
* the socket, and we can't increment the gencnt on the socket after
|
|
|
|
* we free, it so...
|
|
|
|
*/
|
|
|
|
mtx_lock(&so_global_mtx);
|
2001-11-17 03:07:11 +00:00
|
|
|
--numopensockets;
|
2004-06-27 03:22:15 +00:00
|
|
|
mtx_unlock(&so_global_mtx);
|
1998-05-15 20:11:40 +00:00
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
2001-09-12 08:38:13 +00:00
|
|
|
solisten(so, backlog, td)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1994-05-24 10:09:53 +00:00
|
|
|
int backlog;
|
2001-09-12 08:38:13 +00:00
|
|
|
struct thread *td;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2004-06-22 03:49:22 +00:00
|
|
|
int error;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2004-06-20 17:38:19 +00:00
|
|
|
/*
|
|
|
|
* XXXRW: Ordering issue here -- perhaps we need to set
|
|
|
|
* SO_ACCEPTCONN before the call to pru_listen()?
|
|
|
|
* XXXRW: General atomic test-and-set concerns here also.
|
|
|
|
*/
|
2003-10-21 18:28:36 +00:00
|
|
|
if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
|
2004-06-22 03:49:22 +00:00
|
|
|
SS_ISDISCONNECTING))
|
2003-01-17 19:20:00 +00:00
|
|
|
return (EINVAL);
|
2001-09-12 08:38:13 +00:00
|
|
|
error = (*so->so_proto->pr_usrreqs->pru_listen)(so, td);
|
2004-06-22 03:49:22 +00:00
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_LOCK();
|
2004-06-17 22:48:11 +00:00
|
|
|
if (TAILQ_EMPTY(&so->so_comp)) {
|
|
|
|
SOCK_LOCK(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_options |= SO_ACCEPTCONN;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_UNLOCK(so);
|
|
|
|
}
|
1995-11-03 18:33:46 +00:00
|
|
|
if (backlog < 0 || backlog > somaxconn)
|
|
|
|
backlog = somaxconn;
|
|
|
|
so->so_qlimit = backlog;
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_UNLOCK();
|
1994-05-24 10:09:53 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
void
|
1994-05-24 10:09:53 +00:00
|
|
|
sofree(so)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2004-03-31 03:48:35 +00:00
|
|
|
struct socket *head;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2001-11-17 03:07:11 +00:00
|
|
|
KASSERT(so->so_count == 0, ("socket %p so_count not 0", so));
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_LOCK_ASSERT(so);
|
2001-11-17 03:07:11 +00:00
|
|
|
|
2004-06-12 20:47:32 +00:00
|
|
|
if (so->so_pcb != NULL || (so->so_state & SS_NOFDREF) == 0) {
|
|
|
|
SOCK_UNLOCK(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
return;
|
2004-06-12 20:47:32 +00:00
|
|
|
}
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_UNLOCK(so);
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_LOCK();
|
|
|
|
head = so->so_head;
|
|
|
|
if (head != NULL) {
|
|
|
|
KASSERT((so->so_qstate & SQ_COMP) != 0 ||
|
|
|
|
(so->so_qstate & SQ_INCOMP) != 0,
|
|
|
|
("sofree: so_head != NULL, but neither SQ_COMP nor "
|
|
|
|
"SQ_INCOMP"));
|
|
|
|
KASSERT((so->so_qstate & SQ_COMP) == 0 ||
|
|
|
|
(so->so_qstate & SQ_INCOMP) == 0,
|
|
|
|
("sofree: so->so_qstate is SQ_COMP and also SQ_INCOMP"));
|
|
|
|
/*
|
|
|
|
* accept(2) is responsible draining the completed
|
|
|
|
* connection queue and freeing those sockets, so
|
|
|
|
* we just return here if this socket is currently
|
|
|
|
* on the completed connection queue. Otherwise,
|
|
|
|
* accept(2) may hang after select(2) has indicating
|
|
|
|
* that a listening socket was ready. If it's an
|
|
|
|
* incomplete connection, we remove it from the queue
|
|
|
|
* and free it; otherwise, it won't be released until
|
|
|
|
* the listening socket is closed.
|
|
|
|
*/
|
|
|
|
if ((so->so_qstate & SQ_COMP) != 0) {
|
|
|
|
ACCEPT_UNLOCK();
|
1999-02-02 07:23:28 +00:00
|
|
|
return;
|
1996-03-11 15:37:44 +00:00
|
|
|
}
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
TAILQ_REMOVE(&head->so_incomp, so, so_list);
|
|
|
|
head->so_incqlen--;
|
2004-06-01 02:42:56 +00:00
|
|
|
so->so_qstate &= ~SQ_INCOMP;
|
1996-03-11 15:37:44 +00:00
|
|
|
so->so_head = NULL;
|
2002-05-31 11:52:35 +00:00
|
|
|
}
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
KASSERT((so->so_qstate & SQ_COMP) == 0 &&
|
|
|
|
(so->so_qstate & SQ_INCOMP) == 0,
|
|
|
|
("sofree: so_head == NULL, but still SQ_COMP(%d) or SQ_INCOMP(%d)",
|
|
|
|
so->so_qstate & SQ_COMP, so->so_qstate & SQ_INCOMP));
|
|
|
|
ACCEPT_UNLOCK();
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
2004-02-12 01:48:40 +00:00
|
|
|
so->so_snd.sb_flags |= SB_NOINTR;
|
|
|
|
(void)sblock(&so->so_snd, M_WAITOK);
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
/*
|
|
|
|
* socantsendmore_locked() drops the socket buffer mutex so that it
|
|
|
|
* can safely perform wakeups. Re-acquire the mutex before
|
|
|
|
* continuing.
|
|
|
|
*/
|
|
|
|
socantsendmore_locked(so);
|
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
2004-02-12 01:48:40 +00:00
|
|
|
sbunlock(&so->so_snd);
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
sbrelease_locked(&so->so_snd, so);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
sorflush(so);
|
1998-05-15 20:11:40 +00:00
|
|
|
sodealloc(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Close a socket on last file table reference removal.
|
|
|
|
* Initiate disconnect if connected.
|
|
|
|
* Free socket when disconnect complete.
|
2001-11-17 03:07:11 +00:00
|
|
|
*
|
|
|
|
* This function will sorele() the socket. Note that soclose() may be
|
|
|
|
* called prior to the ref count reaching zero. The actual socket
|
|
|
|
* structure will not be freed until the ref count reaches zero.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1994-05-24 10:09:53 +00:00
|
|
|
soclose(so)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
int error = 0;
|
|
|
|
|
2002-05-06 19:31:28 +00:00
|
|
|
funsetown(&so->so_sigio);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (so->so_options & SO_ACCEPTCONN) {
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
struct socket *sp;
|
|
|
|
ACCEPT_LOCK();
|
|
|
|
while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) {
|
|
|
|
TAILQ_REMOVE(&so->so_incomp, sp, so_list);
|
|
|
|
so->so_incqlen--;
|
|
|
|
sp->so_qstate &= ~SQ_INCOMP;
|
|
|
|
sp->so_head = NULL;
|
|
|
|
ACCEPT_UNLOCK();
|
1996-03-11 15:37:44 +00:00
|
|
|
(void) soabort(sp);
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_LOCK();
|
1996-04-16 03:50:08 +00:00
|
|
|
}
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) {
|
1999-01-25 16:58:56 +00:00
|
|
|
TAILQ_REMOVE(&so->so_comp, sp, so_list);
|
1999-02-02 07:23:28 +00:00
|
|
|
so->so_qlen--;
|
2004-06-01 02:42:56 +00:00
|
|
|
sp->so_qstate &= ~SQ_COMP;
|
1999-02-02 07:23:28 +00:00
|
|
|
sp->so_head = NULL;
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_UNLOCK();
|
1996-03-11 15:37:44 +00:00
|
|
|
(void) soabort(sp);
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_LOCK();
|
1996-04-16 03:50:08 +00:00
|
|
|
}
|
Integrate accept locking from rwatson_netperf, introducing a new
global mutex, accept_mtx, which serializes access to the following
fields across all sockets:
so_qlen so_incqlen so_qstate
so_comp so_incomp so_list
so_head
While providing only coarse granularity, this approach avoids lock
order issues between sockets by avoiding ownership of the fields
by a specific socket and its per-socket mutexes.
While here, rewrite soclose(), sofree(), soaccept(), and
sonewconn() to add assertions, close additional races and address
lock order concerns. In particular:
- Reorganize the optimistic concurrency behavior in accept1() to
always allocate a file descriptor with falloc() so that if we do
find a socket, we don't have to encounter the "Oh, there wasn't
a socket" race that can occur if falloc() sleeps in the current
code, which broke inbound accept() ordering, not to mention
requiring backing out socket state changes in a way that raced
with the protocol level. We may want to add a lockless read of
the queue state if polling of empty queues proves to be important
to optimize.
- In accept1(), soref() the socket while holding the accept lock
so that the socket cannot be free'd in a race with the protocol
layer. Likewise in netgraph equivilents of the accept1() code.
- In sonewconn(), loop waiting for the queue to be small enough to
insert our new socket once we've committed to inserting it, or
races can occur that cause the incomplete socket queue to
overfill. In the previously implementation, it was sufficient
to simply tested once since calling soabort() didn't release
synchronization permitting another thread to insert a socket as
we discard a previous one.
- In soclose()/sofree()/et al, it is the responsibility of the
caller to remove a socket from the incomplete connection queue
before calling soabort(), which prevents soabort() from having
to walk into the accept socket to release the socket from its
queue, and avoids races when releasing the accept mutex to enter
soabort(), permitting soabort() to avoid lock ordering issues
with the caller.
- Generally cluster accept queue related operations together
throughout these functions in order to facilitate locking.
Annotate new locking in socketvar.h.
2004-06-02 04:15:39 +00:00
|
|
|
ACCEPT_UNLOCK();
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
if (so->so_pcb == NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto discard;
|
|
|
|
if (so->so_state & SS_ISCONNECTED) {
|
|
|
|
if ((so->so_state & SS_ISDISCONNECTING) == 0) {
|
|
|
|
error = sodisconnect(so);
|
2002-05-31 11:52:35 +00:00
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
if (so->so_options & SO_LINGER) {
|
|
|
|
if ((so->so_state & SS_ISDISCONNECTING) &&
|
2002-05-31 11:52:35 +00:00
|
|
|
(so->so_state & SS_NBIO))
|
1994-05-24 10:09:53 +00:00
|
|
|
goto drop;
|
1994-10-02 17:35:40 +00:00
|
|
|
while (so->so_state & SS_ISCONNECTED) {
|
2002-06-29 00:29:12 +00:00
|
|
|
error = tsleep(&so->so_timeo,
|
1999-04-24 18:22:34 +00:00
|
|
|
PSOCK | PCATCH, "soclos", so->so_linger * hz);
|
1994-10-02 17:35:40 +00:00
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
1994-10-02 17:35:40 +00:00
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
drop:
|
2004-04-09 13:23:51 +00:00
|
|
|
if (so->so_pcb != NULL) {
|
1996-07-11 16:32:50 +00:00
|
|
|
int error2 = (*so->so_proto->pr_usrreqs->pru_detach)(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (error == 0)
|
|
|
|
error = error2;
|
|
|
|
}
|
|
|
|
discard:
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_LOCK(so);
|
2004-06-20 17:47:51 +00:00
|
|
|
KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF"));
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_state |= SS_NOFDREF;
|
2001-11-17 03:07:11 +00:00
|
|
|
sorele(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2004-06-26 17:12:29 +00:00
|
|
|
* soabort() must not be called with any socket locks held, as it calls
|
|
|
|
* into the protocol, which will call back into the socket code causing
|
|
|
|
* it to acquire additional socket locks that may cause recursion or lock
|
|
|
|
* order reversals.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1994-05-24 10:09:53 +00:00
|
|
|
soabort(so)
|
|
|
|
struct socket *so;
|
|
|
|
{
|
2000-03-18 08:56:56 +00:00
|
|
|
int error;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2000-03-18 08:56:56 +00:00
|
|
|
error = (*so->so_proto->pr_usrreqs->pru_abort)(so);
|
|
|
|
if (error) {
|
2004-06-12 20:47:32 +00:00
|
|
|
SOCK_LOCK(so);
|
2001-11-17 03:07:11 +00:00
|
|
|
sotryfree(so); /* note: does not decrement the ref count */
|
2000-03-18 08:56:56 +00:00
|
|
|
return error;
|
|
|
|
}
|
|
|
|
return (0);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1994-05-24 10:09:53 +00:00
|
|
|
soaccept(so, nam)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1997-08-16 19:16:27 +00:00
|
|
|
struct sockaddr **nam;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
int error;
|
|
|
|
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_LOCK(so);
|
|
|
|
KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF"));
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_state &= ~SS_NOFDREF;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_UNLOCK(so);
|
2001-03-09 08:16:40 +00:00
|
|
|
error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam);
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
2001-09-12 08:38:13 +00:00
|
|
|
soconnect(so, nam, td)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1997-08-16 19:16:27 +00:00
|
|
|
struct sockaddr *nam;
|
2001-09-12 08:38:13 +00:00
|
|
|
struct thread *td;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
int error;
|
|
|
|
|
2002-05-31 11:52:35 +00:00
|
|
|
if (so->so_options & SO_ACCEPTCONN)
|
1994-05-24 10:09:53 +00:00
|
|
|
return (EOPNOTSUPP);
|
|
|
|
/*
|
|
|
|
* If protocol is connection-based, can only connect once.
|
|
|
|
* Otherwise, if connected, try to disconnect first.
|
|
|
|
* This allows user to disconnect by connecting to, e.g.,
|
|
|
|
* a null address.
|
|
|
|
*/
|
2002-05-31 11:52:35 +00:00
|
|
|
if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
|
|
|
|
((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
|
|
|
|
(error = sodisconnect(so))))
|
|
|
|
error = EISCONN;
|
|
|
|
else
|
|
|
|
error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td);
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1994-05-24 10:09:53 +00:00
|
|
|
soconnect2(so1, so2)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so1;
|
1994-05-24 10:09:53 +00:00
|
|
|
struct socket *so2;
|
|
|
|
{
|
|
|
|
|
2004-06-22 03:49:22 +00:00
|
|
|
return ((*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2));
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1994-05-24 10:09:53 +00:00
|
|
|
sodisconnect(so)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
int error;
|
|
|
|
|
2004-06-18 02:57:55 +00:00
|
|
|
if ((so->so_state & SS_ISCONNECTED) == 0)
|
|
|
|
return (ENOTCONN);
|
|
|
|
if (so->so_state & SS_ISDISCONNECTING)
|
|
|
|
return (EALREADY);
|
1996-07-11 16:32:50 +00:00
|
|
|
error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
2003-02-19 05:47:46 +00:00
|
|
|
#define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Send on a socket.
|
|
|
|
* If send must go all at once and message is larger than
|
|
|
|
* send buffering, then hard error.
|
|
|
|
* Lock against other senders.
|
|
|
|
* If must go all at once and not enough room now, then
|
|
|
|
* inform user that this would block and do nothing.
|
|
|
|
* Otherwise, if nonblocking, send as much as possible.
|
|
|
|
* The data to be sent is described by "uio" if nonzero,
|
|
|
|
* otherwise by the mbuf chain "top" (which must be null
|
|
|
|
* if uio is not). Data provided in mbuf chain must be small
|
|
|
|
* enough to send all at once.
|
|
|
|
*
|
|
|
|
* Returns nonzero on error, timeout or signal; callers
|
|
|
|
* must check for short counts if EINTR/ERESTART are returned.
|
|
|
|
* Data and control buffers are freed on return.
|
|
|
|
*/
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
|
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
struct so_zerocopy_stats{
|
|
|
|
int size_ok;
|
|
|
|
int align_ok;
|
|
|
|
int found_ifp;
|
|
|
|
};
|
|
|
|
struct so_zerocopy_stats so_zerocp_stats = {0,0,0};
|
|
|
|
#include <netinet/in.h>
|
|
|
|
#include <net/route.h>
|
|
|
|
#include <netinet/in_pcb.h>
|
|
|
|
#include <vm/vm.h>
|
|
|
|
#include <vm/vm_page.h>
|
|
|
|
#include <vm/vm_object.h>
|
|
|
|
#endif /*ZERO_COPY_SOCKETS*/
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
2001-09-12 08:38:13 +00:00
|
|
|
sosend(so, addr, uio, top, control, flags, td)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1997-08-16 19:16:27 +00:00
|
|
|
struct sockaddr *addr;
|
1994-05-24 10:09:53 +00:00
|
|
|
struct uio *uio;
|
|
|
|
struct mbuf *top;
|
|
|
|
struct mbuf *control;
|
|
|
|
int flags;
|
2001-09-12 08:38:13 +00:00
|
|
|
struct thread *td;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
struct mbuf **mp;
|
2003-03-02 15:56:49 +00:00
|
|
|
struct mbuf *m;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
long space, len = 0, resid;
|
2004-06-22 03:49:22 +00:00
|
|
|
int clen = 0, error, dontroute;
|
1994-05-24 10:09:53 +00:00
|
|
|
int atomic = sosendallatonce(so) || top;
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
int cow_send;
|
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2004-04-09 13:23:51 +00:00
|
|
|
if (uio != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
resid = uio->uio_resid;
|
|
|
|
else
|
|
|
|
resid = top->m_pkthdr.len;
|
|
|
|
/*
|
|
|
|
* In theory resid should be unsigned.
|
|
|
|
* However, space must be signed, as it might be less than 0
|
|
|
|
* if we over-committed, and we must use a signed comparison
|
|
|
|
* of space and resid. On the other hand, a negative resid
|
|
|
|
* causes us to loop sending 0-length segments to the protocol.
|
1997-11-09 05:07:40 +00:00
|
|
|
*
|
|
|
|
* Also check to make sure that MSG_EOR isn't used on SOCK_STREAM
|
|
|
|
* type sockets since that's an error.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
1999-01-27 21:50:00 +00:00
|
|
|
if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
|
1997-11-09 05:07:40 +00:00
|
|
|
error = EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
dontroute =
|
|
|
|
(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
|
|
|
|
(so->so_proto->pr_flags & PR_ATOMIC);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (td != NULL)
|
2001-09-12 08:38:13 +00:00
|
|
|
td->td_proc->p_stats->p_ru.ru_msgsnd++;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (control != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
clen = control->m_len;
|
2004-06-22 03:49:22 +00:00
|
|
|
#define snderr(errno) { error = (errno); goto release; }
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
restart:
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_snd);
|
1994-10-02 17:35:40 +00:00
|
|
|
error = sblock(&so->so_snd, SBLOCKWAIT(flags));
|
|
|
|
if (error)
|
2004-06-19 03:23:14 +00:00
|
|
|
goto out_locked;
|
1994-05-24 10:09:53 +00:00
|
|
|
do {
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_snd);
|
2004-06-14 18:16:22 +00:00
|
|
|
if (so->so_snd.sb_state & SBS_CANTSENDMORE)
|
1994-05-24 10:09:53 +00:00
|
|
|
snderr(EPIPE);
|
1998-02-19 19:38:20 +00:00
|
|
|
if (so->so_error) {
|
|
|
|
error = so->so_error;
|
|
|
|
so->so_error = 0;
|
|
|
|
goto release;
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
if ((so->so_state & SS_ISCONNECTED) == 0) {
|
1995-02-07 02:01:16 +00:00
|
|
|
/*
|
|
|
|
* `sendto' and `sendmsg' is allowed on a connection-
|
|
|
|
* based socket if it supports implied connect.
|
|
|
|
* Return ENOTCONN if not connected and no address is
|
|
|
|
* supplied.
|
|
|
|
*/
|
|
|
|
if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
|
|
|
|
(so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
|
1994-05-24 10:09:53 +00:00
|
|
|
if ((so->so_state & SS_ISCONFIRMING) == 0 &&
|
|
|
|
!(resid == 0 && clen != 0))
|
|
|
|
snderr(ENOTCONN);
|
2004-04-09 13:23:51 +00:00
|
|
|
} else if (addr == NULL)
|
1995-02-07 02:01:16 +00:00
|
|
|
snderr(so->so_proto->pr_flags & PR_CONNREQUIRED ?
|
|
|
|
ENOTCONN : EDESTADDRREQ);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
space = sbspace(&so->so_snd);
|
|
|
|
if (flags & MSG_OOB)
|
|
|
|
space += 1024;
|
1994-10-02 17:35:40 +00:00
|
|
|
if ((atomic && resid > so->so_snd.sb_hiwat) ||
|
2002-05-31 11:52:35 +00:00
|
|
|
clen > so->so_snd.sb_hiwat)
|
1994-05-24 10:09:53 +00:00
|
|
|
snderr(EMSGSIZE);
|
2002-02-28 11:22:40 +00:00
|
|
|
if (space < resid + clen &&
|
1994-05-24 10:09:53 +00:00
|
|
|
(atomic || space < so->so_snd.sb_lowat || space < clen)) {
|
2004-06-01 01:18:51 +00:00
|
|
|
if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO))
|
1994-05-24 10:09:53 +00:00
|
|
|
snderr(EWOULDBLOCK);
|
|
|
|
sbunlock(&so->so_snd);
|
|
|
|
error = sbwait(&so->so_snd);
|
|
|
|
if (error)
|
2004-06-19 03:23:14 +00:00
|
|
|
goto out_locked;
|
1994-05-24 10:09:53 +00:00
|
|
|
goto restart;
|
|
|
|
}
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
mp = ⊤
|
|
|
|
space -= clen;
|
|
|
|
do {
|
|
|
|
if (uio == NULL) {
|
|
|
|
/*
|
|
|
|
* Data is prepackaged in "top".
|
|
|
|
*/
|
|
|
|
resid = 0;
|
|
|
|
if (flags & MSG_EOR)
|
|
|
|
top->m_flags |= M_EOR;
|
|
|
|
} else do {
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
cow_send = 0;
|
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
1994-05-29 07:48:17 +00:00
|
|
|
if (resid >= MINCLSIZE) {
|
2003-03-02 15:56:49 +00:00
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (top == NULL) {
|
|
|
|
MGETHDR(m, M_TRYWAIT, MT_DATA);
|
|
|
|
if (m == NULL) {
|
|
|
|
error = ENOBUFS;
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
goto release;
|
|
|
|
}
|
|
|
|
m->m_pkthdr.len = 0;
|
|
|
|
m->m_pkthdr.rcvif = (struct ifnet *)0;
|
|
|
|
} else {
|
|
|
|
MGET(m, M_TRYWAIT, MT_DATA);
|
|
|
|
if (m == NULL) {
|
|
|
|
error = ENOBUFS;
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
goto release;
|
|
|
|
}
|
|
|
|
}
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
if (so_zero_copy_send &&
|
2003-03-02 15:56:49 +00:00
|
|
|
resid>=PAGE_SIZE &&
|
|
|
|
space>=PAGE_SIZE &&
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
uio->uio_iov->iov_len>=PAGE_SIZE) {
|
|
|
|
so_zerocp_stats.size_ok++;
|
|
|
|
if (!((vm_offset_t)
|
|
|
|
uio->uio_iov->iov_base & PAGE_MASK)){
|
|
|
|
so_zerocp_stats.align_ok++;
|
|
|
|
cow_send = socow_setup(m, uio);
|
|
|
|
}
|
2003-03-02 15:56:49 +00:00
|
|
|
}
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (!cow_send) {
|
|
|
|
MCLGET(m, M_TRYWAIT);
|
|
|
|
if ((m->m_flags & M_EXT) == 0) {
|
|
|
|
m_free(m);
|
|
|
|
m = NULL;
|
|
|
|
} else {
|
|
|
|
len = min(min(MCLBYTES, resid), space);
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
len = PAGE_SIZE;
|
|
|
|
#else /* ZERO_COPY_SOCKETS */
|
|
|
|
if (top == NULL) {
|
|
|
|
m = m_getcl(M_TRYWAIT, MT_DATA, M_PKTHDR);
|
|
|
|
m->m_pkthdr.len = 0;
|
|
|
|
m->m_pkthdr.rcvif = (struct ifnet *)0;
|
|
|
|
} else
|
|
|
|
m = m_getcl(M_TRYWAIT, MT_DATA, 0);
|
|
|
|
len = min(min(MCLBYTES, resid), space);
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
1994-05-24 10:09:53 +00:00
|
|
|
} else {
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (top == NULL) {
|
|
|
|
m = m_gethdr(M_TRYWAIT, MT_DATA);
|
|
|
|
m->m_pkthdr.len = 0;
|
|
|
|
m->m_pkthdr.rcvif = (struct ifnet *)0;
|
|
|
|
|
|
|
|
len = min(min(MHLEN, resid), space);
|
|
|
|
/*
|
|
|
|
* For datagram protocols, leave room
|
|
|
|
* for protocol headers in first mbuf.
|
|
|
|
*/
|
|
|
|
if (atomic && m && len < MHLEN)
|
|
|
|
MH_ALIGN(m, len);
|
|
|
|
} else {
|
|
|
|
m = m_get(M_TRYWAIT, MT_DATA);
|
|
|
|
len = min(min(MLEN, resid), space);
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (m == NULL) {
|
|
|
|
error = ENOBUFS;
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
goto release;
|
|
|
|
}
|
|
|
|
|
1994-05-29 07:48:17 +00:00
|
|
|
space -= len;
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
if (cow_send)
|
|
|
|
error = 0;
|
|
|
|
else
|
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
2003-03-02 15:50:23 +00:00
|
|
|
error = uiomove(mtod(m, void *), (int)len, uio);
|
1994-05-24 10:09:53 +00:00
|
|
|
resid = uio->uio_resid;
|
|
|
|
m->m_len = len;
|
|
|
|
*mp = m;
|
|
|
|
top->m_pkthdr.len += len;
|
2004-06-19 03:23:14 +00:00
|
|
|
if (error) {
|
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
goto release;
|
2004-06-19 03:23:14 +00:00
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
mp = &m->m_next;
|
|
|
|
if (resid <= 0) {
|
|
|
|
if (flags & MSG_EOR)
|
|
|
|
top->m_flags |= M_EOR;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} while (space > 0 && atomic);
|
2004-06-17 22:48:11 +00:00
|
|
|
if (dontroute) {
|
|
|
|
SOCK_LOCK(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_options |= SO_DONTROUTE;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_UNLOCK(so);
|
|
|
|
}
|
1999-06-04 02:27:06 +00:00
|
|
|
/*
|
2004-06-14 18:16:22 +00:00
|
|
|
* XXX all the SBS_CANTSENDMORE checks previously
|
1999-06-04 02:27:06 +00:00
|
|
|
* done could be out of date. We could have recieved
|
|
|
|
* a reset packet in an interrupt or maybe we slept
|
|
|
|
* while doing page faults in uiomove() etc. We could
|
|
|
|
* probably recheck again inside the splnet() protection
|
|
|
|
* here, but there are probably other places that this
|
|
|
|
* also happens. We must rethink this.
|
|
|
|
*/
|
1996-07-11 16:32:50 +00:00
|
|
|
error = (*so->so_proto->pr_usrreqs->pru_send)(so,
|
|
|
|
(flags & MSG_OOB) ? PRUS_OOB :
|
1995-02-07 02:01:16 +00:00
|
|
|
/*
|
|
|
|
* If the user set MSG_EOF, the protocol
|
|
|
|
* understands this flag and nothing left to
|
|
|
|
* send then use PRU_SEND_EOF instead of PRU_SEND.
|
|
|
|
*/
|
|
|
|
((flags & MSG_EOF) &&
|
|
|
|
(so->so_proto->pr_flags & PR_IMPLOPCL) &&
|
|
|
|
(resid <= 0)) ?
|
1999-01-20 17:32:01 +00:00
|
|
|
PRUS_EOF :
|
|
|
|
/* If there is more to send set PRUS_MORETOCOME */
|
1999-01-20 17:45:22 +00:00
|
|
|
(resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
|
2001-09-12 08:38:13 +00:00
|
|
|
top, addr, control, td);
|
2004-06-18 04:02:56 +00:00
|
|
|
if (dontroute) {
|
|
|
|
SOCK_LOCK(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_options &= ~SO_DONTROUTE;
|
2004-06-18 04:02:56 +00:00
|
|
|
SOCK_UNLOCK(so);
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
clen = 0;
|
2004-04-09 13:23:51 +00:00
|
|
|
control = NULL;
|
|
|
|
top = NULL;
|
1994-05-24 10:09:53 +00:00
|
|
|
mp = ⊤
|
2004-06-19 03:23:14 +00:00
|
|
|
if (error) {
|
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
goto release;
|
2004-06-19 03:23:14 +00:00
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
} while (resid && space > 0);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
} while (resid);
|
|
|
|
|
|
|
|
release:
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
sbunlock(&so->so_snd);
|
2004-06-19 03:23:14 +00:00
|
|
|
out_locked:
|
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_snd);
|
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
out:
|
2004-04-09 13:23:51 +00:00
|
|
|
if (top != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
m_freem(top);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (control != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
m_freem(control);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
2004-07-11 01:34:34 +00:00
|
|
|
/*
|
|
|
|
* The part of soreceive() that implements reading non-inline out-of-band
|
|
|
|
* data from a socket. For more complete comments, see soreceive(), from
|
|
|
|
* which this code originated.
|
|
|
|
*
|
|
|
|
* XXXRW: Note that soreceive_rcvoob(), unlike the remainder of soreceve(),
|
|
|
|
* is unable to return an mbuf chain to the caller.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
soreceive_rcvoob(so, uio, flags)
|
|
|
|
struct socket *so;
|
|
|
|
struct uio *uio;
|
|
|
|
int flags;
|
|
|
|
{
|
|
|
|
struct protosw *pr = so->so_proto;
|
|
|
|
struct mbuf *m;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
|
|
|
|
|
|
|
|
m = m_get(M_TRYWAIT, MT_DATA);
|
|
|
|
if (m == NULL)
|
|
|
|
return (ENOBUFS);
|
|
|
|
error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
|
|
|
|
if (error)
|
|
|
|
goto bad;
|
|
|
|
do {
|
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
if (so_zero_copy_receive) {
|
|
|
|
vm_page_t pg;
|
|
|
|
int disposable;
|
|
|
|
|
|
|
|
if ((m->m_flags & M_EXT)
|
|
|
|
&& (m->m_ext.ext_type == EXT_DISPOSABLE))
|
|
|
|
disposable = 1;
|
|
|
|
else
|
|
|
|
disposable = 0;
|
|
|
|
|
|
|
|
pg = PHYS_TO_VM_PAGE(vtophys(mtod(m, caddr_t)));
|
|
|
|
if (uio->uio_offset == -1)
|
|
|
|
uio->uio_offset =IDX_TO_OFF(pg->pindex);
|
|
|
|
|
|
|
|
error = uiomoveco(mtod(m, void *),
|
|
|
|
min(uio->uio_resid, m->m_len),
|
|
|
|
uio, pg->object,
|
|
|
|
disposable);
|
|
|
|
} else
|
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
|
|
|
error = uiomove(mtod(m, void *),
|
|
|
|
(int) min(uio->uio_resid, m->m_len), uio);
|
|
|
|
m = m_free(m);
|
|
|
|
} while (uio->uio_resid && error == 0 && m);
|
|
|
|
bad:
|
|
|
|
if (m != NULL)
|
|
|
|
m_freem(m);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Implement receive operations on a socket.
|
|
|
|
* We depend on the way that records are added to the sockbuf
|
|
|
|
* by sbappend*. In particular, each record (mbufs linked through m_next)
|
|
|
|
* must begin with an address if the protocol so specifies,
|
|
|
|
* followed by an optional mbuf or mbufs containing ancillary data,
|
|
|
|
* and then zero or more mbufs of data.
|
|
|
|
* In order to avoid blocking network interrupts for the entire time here,
|
|
|
|
* we splx() while doing the actual copy to user space.
|
|
|
|
* Although the sockbuf is locked, new data may still be appended,
|
|
|
|
* and thus we must maintain consistency of the sockbuf during that time.
|
|
|
|
*
|
|
|
|
* The caller may receive the data as a single mbuf chain by supplying
|
|
|
|
* an mbuf **mp0 for use in returning the chain. The uio is then used
|
|
|
|
* only for the count in uio_resid.
|
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1997-08-16 19:16:27 +00:00
|
|
|
soreceive(so, psa, uio, mp0, controlp, flagsp)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1997-08-16 19:16:27 +00:00
|
|
|
struct sockaddr **psa;
|
1994-05-24 10:09:53 +00:00
|
|
|
struct uio *uio;
|
|
|
|
struct mbuf **mp0;
|
|
|
|
struct mbuf **controlp;
|
|
|
|
int *flagsp;
|
|
|
|
{
|
2001-10-04 13:11:48 +00:00
|
|
|
struct mbuf *m, **mp;
|
2004-06-22 03:49:22 +00:00
|
|
|
int flags, len, error, offset;
|
1994-05-24 10:09:53 +00:00
|
|
|
struct protosw *pr = so->so_proto;
|
|
|
|
struct mbuf *nextrecord;
|
1994-05-25 09:21:21 +00:00
|
|
|
int moff, type = 0;
|
1994-05-24 10:09:53 +00:00
|
|
|
int orig_resid = uio->uio_resid;
|
|
|
|
|
|
|
|
mp = mp0;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (psa != NULL)
|
2004-07-11 01:22:40 +00:00
|
|
|
*psa = NULL;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (controlp != NULL)
|
2004-07-11 01:22:40 +00:00
|
|
|
*controlp = NULL;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (flagsp != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
flags = *flagsp &~ MSG_EOR;
|
|
|
|
else
|
|
|
|
flags = 0;
|
2004-07-11 01:34:34 +00:00
|
|
|
if (flags & MSG_OOB)
|
|
|
|
return (soreceive_rcvoob(so, uio, flags));
|
2004-04-09 13:23:51 +00:00
|
|
|
if (mp != NULL)
|
|
|
|
*mp = NULL;
|
2002-05-31 11:52:35 +00:00
|
|
|
if (so->so_state & SS_ISCONFIRMING && uio->uio_resid)
|
1996-07-11 16:32:50 +00:00
|
|
|
(*pr->pr_usrreqs->pru_rcvd)(so, 0);
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
restart:
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
1994-10-02 17:35:40 +00:00
|
|
|
error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
|
|
|
|
if (error)
|
2004-06-19 03:23:14 +00:00
|
|
|
goto out;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
m = so->so_rcv.sb_mb;
|
|
|
|
/*
|
|
|
|
* If we have less data than requested, block awaiting more
|
|
|
|
* (subject to any timeout) if:
|
|
|
|
* 1. the current count is less than the low water mark, or
|
|
|
|
* 2. MSG_WAITALL is set, and it is possible to do the entire
|
|
|
|
* receive operation at once if we block (resid <= hiwat).
|
|
|
|
* 3. MSG_DONTWAIT is not set
|
|
|
|
* If MSG_WAITALL is set but resid is larger than the receive buffer,
|
|
|
|
* we have to do the receive in sections, and thus risk returning
|
|
|
|
* a short count if a timeout or signal occurs after we start.
|
|
|
|
*/
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_rcv.sb_cc < uio->uio_resid) &&
|
|
|
|
(so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
|
|
|
|
((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
|
2004-04-09 13:23:51 +00:00
|
|
|
m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
|
|
|
|
KASSERT(m != NULL || !so->so_rcv.sb_cc,
|
2002-07-24 14:21:41 +00:00
|
|
|
("receive: m == %p so->so_rcv.sb_cc == %u",
|
2001-04-27 13:42:50 +00:00
|
|
|
m, so->so_rcv.sb_cc));
|
1994-05-24 10:09:53 +00:00
|
|
|
if (so->so_error) {
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto dontblock;
|
|
|
|
error = so->so_error;
|
|
|
|
if ((flags & MSG_PEEK) == 0)
|
|
|
|
so->so_error = 0;
|
|
|
|
goto release;
|
|
|
|
}
|
2004-07-10 03:47:15 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2004-06-14 18:16:22 +00:00
|
|
|
if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
|
1994-05-24 10:09:53 +00:00
|
|
|
if (m)
|
|
|
|
goto dontblock;
|
|
|
|
else
|
|
|
|
goto release;
|
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
for (; m != NULL; m = m->m_next)
|
1994-05-24 10:09:53 +00:00
|
|
|
if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
|
|
|
|
m = so->so_rcv.sb_mb;
|
|
|
|
goto dontblock;
|
|
|
|
}
|
|
|
|
if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
|
|
|
|
(so->so_proto->pr_flags & PR_CONNREQUIRED)) {
|
|
|
|
error = ENOTCONN;
|
|
|
|
goto release;
|
|
|
|
}
|
2002-05-31 11:52:35 +00:00
|
|
|
if (uio->uio_resid == 0)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto release;
|
2004-06-01 01:18:51 +00:00
|
|
|
if ((so->so_state & SS_NBIO) ||
|
|
|
|
(flags & (MSG_DONTWAIT|MSG_NBIO))) {
|
1994-05-24 10:09:53 +00:00
|
|
|
error = EWOULDBLOCK;
|
|
|
|
goto release;
|
|
|
|
}
|
2003-10-28 05:47:40 +00:00
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
sbunlock(&so->so_rcv);
|
|
|
|
error = sbwait(&so->so_rcv);
|
|
|
|
if (error)
|
2004-06-19 03:23:14 +00:00
|
|
|
goto out;
|
1994-05-24 10:09:53 +00:00
|
|
|
goto restart;
|
|
|
|
}
|
|
|
|
dontblock:
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2001-09-12 08:38:13 +00:00
|
|
|
if (uio->uio_td)
|
|
|
|
uio->uio_td->td_proc->p_stats->p_ru.ru_msgrcv++;
|
2003-10-28 05:47:40 +00:00
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
nextrecord = m->m_nextpkt;
|
|
|
|
if (pr->pr_flags & PR_ADDR) {
|
2001-11-12 20:50:06 +00:00
|
|
|
KASSERT(m->m_type == MT_SONAME,
|
|
|
|
("m->m_type == %d", m->m_type));
|
1994-05-24 10:09:53 +00:00
|
|
|
orig_resid = 0;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (psa != NULL)
|
2004-03-01 03:14:23 +00:00
|
|
|
*psa = sodupsockaddr(mtod(m, struct sockaddr *),
|
2004-06-19 03:23:14 +00:00
|
|
|
M_NOWAIT);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (flags & MSG_PEEK) {
|
|
|
|
m = m->m_next;
|
|
|
|
} else {
|
|
|
|
sbfree(&so->so_rcv, m);
|
2002-02-05 02:00:56 +00:00
|
|
|
so->so_rcv.sb_mb = m_free(m);
|
1997-08-16 19:16:27 +00:00
|
|
|
m = so->so_rcv.sb_mb;
|
2004-07-10 21:43:35 +00:00
|
|
|
if (m != NULL)
|
|
|
|
m->m_nextpkt = nextrecord;
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
while (m != NULL && m->m_type == MT_CONTROL && error == 0) {
|
1994-05-24 10:09:53 +00:00
|
|
|
if (flags & MSG_PEEK) {
|
2004-04-09 13:23:51 +00:00
|
|
|
if (controlp != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
*controlp = m_copy(m, 0, m->m_len);
|
|
|
|
m = m->m_next;
|
|
|
|
} else {
|
|
|
|
sbfree(&so->so_rcv, m);
|
2001-10-04 13:11:48 +00:00
|
|
|
so->so_rcv.sb_mb = m->m_next;
|
|
|
|
m->m_next = NULL;
|
2004-06-19 03:23:14 +00:00
|
|
|
if (pr->pr_domain->dom_externalize) {
|
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
|
|
|
error = (*pr->pr_domain->dom_externalize)
|
|
|
|
(m, controlp);
|
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
|
|
|
} else if (controlp != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
*controlp = m;
|
2001-10-04 13:11:48 +00:00
|
|
|
else
|
|
|
|
m_freem(m);
|
|
|
|
m = so->so_rcv.sb_mb;
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
if (controlp != NULL) {
|
1994-05-24 10:09:53 +00:00
|
|
|
orig_resid = 0;
|
2003-04-14 14:44:36 +00:00
|
|
|
while (*controlp != NULL)
|
2001-10-04 13:11:48 +00:00
|
|
|
controlp = &(*controlp)->m_next;
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m != NULL) {
|
2003-10-28 05:47:40 +00:00
|
|
|
if ((flags & MSG_PEEK) == 0) {
|
1994-05-24 10:09:53 +00:00
|
|
|
m->m_nextpkt = nextrecord;
|
2003-10-28 05:47:40 +00:00
|
|
|
/*
|
|
|
|
* If nextrecord == NULL (this is a single chain),
|
|
|
|
* then sb_lastrecord may not be valid here if m
|
|
|
|
* was changed earlier.
|
|
|
|
*/
|
|
|
|
if (nextrecord == NULL) {
|
|
|
|
KASSERT(so->so_rcv.sb_mb == m,
|
|
|
|
("receive tailq 1"));
|
|
|
|
so->so_rcv.sb_lastrecord = m;
|
|
|
|
}
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
type = m->m_type;
|
|
|
|
if (type == MT_OOBDATA)
|
|
|
|
flags |= MSG_OOB;
|
2003-10-28 05:47:40 +00:00
|
|
|
} else {
|
|
|
|
if ((flags & MSG_PEEK) == 0) {
|
|
|
|
KASSERT(so->so_rcv.sb_mb == m,("receive tailq 2"));
|
|
|
|
so->so_rcv.sb_mb = nextrecord;
|
|
|
|
SB_EMPTY_FIXUP(&so->so_rcv);
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2003-10-28 05:47:40 +00:00
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
moff = 0;
|
|
|
|
offset = 0;
|
2004-04-09 13:23:51 +00:00
|
|
|
while (m != NULL && uio->uio_resid > 0 && error == 0) {
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (m->m_type == MT_OOBDATA) {
|
|
|
|
if (type != MT_OOBDATA)
|
|
|
|
break;
|
|
|
|
} else if (type == MT_OOBDATA)
|
|
|
|
break;
|
1999-01-08 17:31:30 +00:00
|
|
|
else
|
|
|
|
KASSERT(m->m_type == MT_DATA || m->m_type == MT_HEADER,
|
2001-11-12 20:50:06 +00:00
|
|
|
("m->m_type == %d", m->m_type));
|
2004-06-14 18:16:22 +00:00
|
|
|
so->so_rcv.sb_state &= ~SBS_RCVATMARK;
|
1994-05-24 10:09:53 +00:00
|
|
|
len = uio->uio_resid;
|
|
|
|
if (so->so_oobmark && len > so->so_oobmark - offset)
|
|
|
|
len = so->so_oobmark - offset;
|
|
|
|
if (len > m->m_len - moff)
|
|
|
|
len = m->m_len - moff;
|
|
|
|
/*
|
|
|
|
* If mp is set, just pass back the mbufs.
|
|
|
|
* Otherwise copy them out via the uio, then free.
|
|
|
|
* Sockbuf must be consistent here (points to current mbuf,
|
|
|
|
* it points to next record) when we drop priority;
|
|
|
|
* we must note any additions to the sockbuf when we
|
|
|
|
* block interrupts again.
|
|
|
|
*/
|
2004-04-09 13:23:51 +00:00
|
|
|
if (mp == NULL) {
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2003-10-28 05:47:40 +00:00
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
#ifdef ZERO_COPY_SOCKETS
|
|
|
|
if (so_zero_copy_receive) {
|
|
|
|
vm_page_t pg;
|
|
|
|
int disposable;
|
|
|
|
|
|
|
|
if ((m->m_flags & M_EXT)
|
|
|
|
&& (m->m_ext.ext_type == EXT_DISPOSABLE))
|
|
|
|
disposable = 1;
|
|
|
|
else
|
|
|
|
disposable = 0;
|
2003-03-02 15:56:49 +00:00
|
|
|
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
pg = PHYS_TO_VM_PAGE(vtophys(mtod(m, caddr_t) +
|
|
|
|
moff));
|
|
|
|
|
|
|
|
if (uio->uio_offset == -1)
|
|
|
|
uio->uio_offset =IDX_TO_OFF(pg->pindex);
|
|
|
|
|
2003-03-02 15:50:23 +00:00
|
|
|
error = uiomoveco(mtod(m, char *) + moff,
|
At long last, commit the zero copy sockets code.
MAKEDEV: Add MAKEDEV glue for the ti(4) device nodes.
ti.4: Update the ti(4) man page to include information on the
TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS kernel options,
and also include information about the new character
device interface and the associated ioctls.
man9/Makefile: Add jumbo.9 and zero_copy.9 man pages and associated
links.
jumbo.9: New man page describing the jumbo buffer allocator
interface and operation.
zero_copy.9: New man page describing the general characteristics of
the zero copy send and receive code, and what an
application author should do to take advantage of the
zero copy functionality.
NOTES: Add entries for ZERO_COPY_SOCKETS, TI_PRIVATE_JUMBOS,
TI_JUMBO_HDRSPLIT, MSIZE, and MCLSHIFT.
conf/files: Add uipc_jumbo.c and uipc_cow.c.
conf/options: Add the 5 options mentioned above.
kern_subr.c: Receive side zero copy implementation. This takes
"disposable" pages attached to an mbuf, gives them to
a user process, and then recycles the user's page.
This is only active when ZERO_COPY_SOCKETS is turned on
and the kern.ipc.zero_copy.receive sysctl variable is
set to 1.
uipc_cow.c: Send side zero copy functions. Takes a page written
by the user and maps it copy on write and assigns it
kernel virtual address space. Removes copy on write
mapping once the buffer has been freed by the network
stack.
uipc_jumbo.c: Jumbo disposable page allocator code. This allocates
(optionally) disposable pages for network drivers that
want to give the user the option of doing zero copy
receive.
uipc_socket.c: Add kern.ipc.zero_copy.{send,receive} sysctls that are
enabled if ZERO_COPY_SOCKETS is turned on.
Add zero copy send support to sosend() -- pages get
mapped into the kernel instead of getting copied if
they meet size and alignment restrictions.
uipc_syscalls.c:Un-staticize some of the sf* functions so that they
can be used elsewhere. (uipc_cow.c)
if_media.c: In the SIOCGIFMEDIA ioctl in ifmedia_ioctl(), avoid
calling malloc() with M_WAITOK. Return an error if
the M_NOWAIT malloc fails.
The ti(4) driver and the wi(4) driver, at least, call
this with a mutex held. This causes witness warnings
for 'ifconfig -a' with a wi(4) or ti(4) board in the
system. (I've only verified for ti(4)).
ip_output.c: Fragment large datagrams so that each segment contains
a multiple of PAGE_SIZE amount of data plus headers.
This allows the receiver to potentially do page
flipping on receives.
if_ti.c: Add zero copy receive support to the ti(4) driver. If
TI_PRIVATE_JUMBOS is not defined, it now uses the
jumbo(9) buffer allocator for jumbo receive buffers.
Add a new character device interface for the ti(4)
driver for the new debugging interface. This allows
(a patched version of) gdb to talk to the Tigon board
and debug the firmware. There are also a few additional
debugging ioctls available through this interface.
Add header splitting support to the ti(4) driver.
Tweak some of the default interrupt coalescing
parameters to more useful defaults.
Add hooks for supporting transmit flow control, but
leave it turned off with a comment describing why it
is turned off.
if_tireg.h: Change the firmware rev to 12.4.11, since we're really
at 12.4.11 plus fixes from 12.4.13.
Add defines needed for debugging.
Remove the ti_stats structure, it is now defined in
sys/tiio.h.
ti_fw.h: 12.4.11 firmware.
ti_fw2.h: 12.4.11 firmware, plus selected fixes from 12.4.13,
and my header splitting patches. Revision 12.4.13
doesn't handle 10/100 negotiation properly. (This
firmware is the same as what was in the tree previously,
with the addition of header splitting support.)
sys/jumbo.h: Jumbo buffer allocator interface.
sys/mbuf.h: Add a new external mbuf type, EXT_DISPOSABLE, to
indicate that the payload buffer can be thrown away /
flipped to a userland process.
socketvar.h: Add prototype for socow_setup.
tiio.h: ioctl interface to the character portion of the ti(4)
driver, plus associated structure/type definitions.
uio.h: Change prototype for uiomoveco() so that we'll know
whether the source page is disposable.
ufs_readwrite.c:Update for new prototype of uiomoveco().
vm_fault.c: In vm_fault(), check to see whether we need to do a page
based copy on write fault.
vm_object.c: Add a new function, vm_object_allocate_wait(). This
does the same thing that vm_object allocate does, except
that it gives the caller the opportunity to specify whether
it should wait on the uma_zalloc() of the object structre.
This allows vm objects to be allocated while holding a
mutex. (Without generating WITNESS warnings.)
vm_object_allocate() is implemented as a call to
vm_object_allocate_wait() with the malloc flag set to
M_WAITOK.
vm_object.h: Add prototype for vm_object_allocate_wait().
vm_page.c: Add page-based copy on write setup, clear and fault
routines.
vm_page.h: Add page based COW function prototypes and variable in
the vm_page structure.
Many thanks to Drew Gallatin, who wrote the zero copy send and receive
code, and to all the other folks who have tested and reviewed this code
over the years.
2002-06-26 03:37:47 +00:00
|
|
|
(int)len, uio,pg->object,
|
|
|
|
disposable);
|
|
|
|
} else
|
|
|
|
#endif /* ZERO_COPY_SOCKETS */
|
2003-03-02 15:50:23 +00:00
|
|
|
error = uiomove(mtod(m, char *) + moff, (int)len, uio);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
1996-11-29 19:03:42 +00:00
|
|
|
if (error)
|
|
|
|
goto release;
|
1994-05-24 10:09:53 +00:00
|
|
|
} else
|
|
|
|
uio->uio_resid -= len;
|
2004-07-10 03:47:15 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (len == m->m_len - moff) {
|
|
|
|
if (m->m_flags & M_EOR)
|
|
|
|
flags |= MSG_EOR;
|
|
|
|
if (flags & MSG_PEEK) {
|
|
|
|
m = m->m_next;
|
|
|
|
moff = 0;
|
|
|
|
} else {
|
|
|
|
nextrecord = m->m_nextpkt;
|
|
|
|
sbfree(&so->so_rcv, m);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (mp != NULL) {
|
1994-05-24 10:09:53 +00:00
|
|
|
*mp = m;
|
|
|
|
mp = &m->m_next;
|
|
|
|
so->so_rcv.sb_mb = m = m->m_next;
|
2004-04-09 13:23:51 +00:00
|
|
|
*mp = NULL;
|
1994-05-24 10:09:53 +00:00
|
|
|
} else {
|
2002-02-05 02:00:56 +00:00
|
|
|
so->so_rcv.sb_mb = m_free(m);
|
1994-05-24 10:09:53 +00:00
|
|
|
m = so->so_rcv.sb_mb;
|
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m != NULL) {
|
1994-05-24 10:09:53 +00:00
|
|
|
m->m_nextpkt = nextrecord;
|
2003-10-28 05:47:40 +00:00
|
|
|
if (nextrecord == NULL)
|
|
|
|
so->so_rcv.sb_lastrecord = m;
|
|
|
|
} else {
|
|
|
|
so->so_rcv.sb_mb = nextrecord;
|
|
|
|
SB_EMPTY_FIXUP(&so->so_rcv);
|
|
|
|
}
|
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (flags & MSG_PEEK)
|
|
|
|
moff += len;
|
|
|
|
else {
|
2004-07-05 19:29:33 +00:00
|
|
|
if (mp != NULL) {
|
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
2003-02-19 05:47:46 +00:00
|
|
|
*mp = m_copym(m, 0, len, M_TRYWAIT);
|
2004-07-05 19:29:33 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
m->m_data += len;
|
|
|
|
m->m_len -= len;
|
|
|
|
so->so_rcv.sb_cc -= len;
|
|
|
|
}
|
|
|
|
}
|
2004-07-10 03:47:15 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (so->so_oobmark) {
|
|
|
|
if ((flags & MSG_PEEK) == 0) {
|
|
|
|
so->so_oobmark -= len;
|
|
|
|
if (so->so_oobmark == 0) {
|
2004-06-14 18:16:22 +00:00
|
|
|
so->so_rcv.sb_state |= SBS_RCVATMARK;
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
offset += len;
|
|
|
|
if (offset == so->so_oobmark)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (flags & MSG_EOR)
|
|
|
|
break;
|
|
|
|
/*
|
|
|
|
* If the MSG_WAITALL flag is set (for non-atomic socket),
|
|
|
|
* we must not quit until "uio->uio_resid == 0" or an error
|
|
|
|
* termination. If a signal/timeout occurs, return
|
|
|
|
* with a short count but without error.
|
|
|
|
* Keep sockbuf locked against other readers.
|
|
|
|
*/
|
2004-04-09 13:23:51 +00:00
|
|
|
while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
|
|
|
|
!sosendallatonce(so) && nextrecord == NULL) {
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2004-06-14 18:16:22 +00:00
|
|
|
if (so->so_error || so->so_rcv.sb_state & SBS_CANTRCVMORE)
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
2001-03-16 22:37:06 +00:00
|
|
|
/*
|
|
|
|
* Notify the protocol that some data has been
|
|
|
|
* drained before blocking.
|
|
|
|
*/
|
2004-06-19 03:23:14 +00:00
|
|
|
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb != NULL) {
|
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
2001-03-16 22:37:06 +00:00
|
|
|
(*pr->pr_usrreqs->pru_rcvd)(so, flags);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
|
|
|
}
|
2003-10-28 05:47:40 +00:00
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
error = sbwait(&so->so_rcv);
|
2004-06-19 03:23:14 +00:00
|
|
|
if (error)
|
|
|
|
goto release;
|
1994-10-02 17:35:40 +00:00
|
|
|
m = so->so_rcv.sb_mb;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
nextrecord = m->m_nextpkt;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2004-07-10 03:47:15 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m != NULL && pr->pr_flags & PR_ATOMIC) {
|
1994-05-24 10:09:53 +00:00
|
|
|
flags |= MSG_TRUNC;
|
2004-07-10 04:38:06 +00:00
|
|
|
if ((flags & MSG_PEEK) == 0)
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
(void) sbdroprecord_locked(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
if ((flags & MSG_PEEK) == 0) {
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m == NULL) {
|
2003-10-28 05:47:40 +00:00
|
|
|
/*
|
|
|
|
* First part is an inline SB_EMPTY_FIXUP(). Second
|
|
|
|
* part makes sure sb_lastrecord is up-to-date if
|
|
|
|
* there is still data in the socket buffer.
|
|
|
|
*/
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_rcv.sb_mb = nextrecord;
|
2003-10-28 05:47:40 +00:00
|
|
|
if (so->so_rcv.sb_mb == NULL) {
|
|
|
|
so->so_rcv.sb_mbtail = NULL;
|
|
|
|
so->so_rcv.sb_lastrecord = NULL;
|
|
|
|
} else if (nextrecord->m_nextpkt == NULL)
|
|
|
|
so->so_rcv.sb_lastrecord = nextrecord;
|
|
|
|
}
|
|
|
|
SBLASTRECORDCHK(&so->so_rcv);
|
|
|
|
SBLASTMBUFCHK(&so->so_rcv);
|
2004-06-19 03:23:14 +00:00
|
|
|
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) {
|
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
1996-07-11 16:32:50 +00:00
|
|
|
(*pr->pr_usrreqs->pru_rcvd)(so, flags);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (orig_resid == uio->uio_resid && orig_resid &&
|
2004-06-14 18:16:22 +00:00
|
|
|
(flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
|
1994-05-24 10:09:53 +00:00
|
|
|
sbunlock(&so->so_rcv);
|
|
|
|
goto restart;
|
|
|
|
}
|
1995-05-30 08:16:23 +00:00
|
|
|
|
2004-04-09 13:23:51 +00:00
|
|
|
if (flagsp != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
*flagsp |= flags;
|
|
|
|
release:
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
sbunlock(&so->so_rcv);
|
2004-06-19 03:23:14 +00:00
|
|
|
out:
|
2004-07-10 03:47:15 +00:00
|
|
|
SOCKBUF_LOCK_ASSERT(&so->so_rcv);
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1994-05-24 10:09:53 +00:00
|
|
|
soshutdown(so, how)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
|
|
|
int how;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2003-03-02 15:56:49 +00:00
|
|
|
struct protosw *pr = so->so_proto;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2001-02-27 13:48:07 +00:00
|
|
|
if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
|
|
|
|
return (EINVAL);
|
|
|
|
|
|
|
|
if (how != SHUT_WR)
|
1994-05-24 10:09:53 +00:00
|
|
|
sorflush(so);
|
2001-02-27 13:48:07 +00:00
|
|
|
if (how != SHUT_RD)
|
1996-07-11 16:32:50 +00:00
|
|
|
return ((*pr->pr_usrreqs->pru_shutdown)(so));
|
1994-05-24 10:09:53 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
void
|
1994-05-24 10:09:53 +00:00
|
|
|
sorflush(so)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2003-03-02 15:56:49 +00:00
|
|
|
struct sockbuf *sb = &so->so_rcv;
|
|
|
|
struct protosw *pr = so->so_proto;
|
1994-05-24 10:09:53 +00:00
|
|
|
struct sockbuf asb;
|
|
|
|
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
/*
|
|
|
|
* XXXRW: This is quite ugly. The existing code made a copy of the
|
|
|
|
* socket buffer, then zero'd the original to clear the buffer
|
|
|
|
* fields. However, with mutexes in the socket buffer, this causes
|
|
|
|
* problems. We only clear the zeroable bits of the original;
|
|
|
|
* however, we have to initialize and destroy the mutex in the copy
|
|
|
|
* so that dom_dispose() and sbrelease() can lock t as needed.
|
|
|
|
*/
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_LOCK(sb);
|
1994-05-24 10:09:53 +00:00
|
|
|
sb->sb_flags |= SB_NOINTR;
|
2003-02-19 05:47:46 +00:00
|
|
|
(void) sblock(sb, M_WAITOK);
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
/*
|
|
|
|
* socantrcvmore_locked() drops the socket buffer mutex so that it
|
|
|
|
* can safely perform wakeups. Re-acquire the mutex before
|
|
|
|
* continuing.
|
|
|
|
*/
|
|
|
|
socantrcvmore_locked(so);
|
|
|
|
SOCKBUF_LOCK(sb);
|
1994-05-24 10:09:53 +00:00
|
|
|
sbunlock(sb);
|
2003-07-17 23:49:10 +00:00
|
|
|
/*
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
* Invalidate/clear most of the sockbuf structure, but leave
|
|
|
|
* selinfo and mutex data unchanged.
|
2003-07-17 23:49:10 +00:00
|
|
|
*/
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
bzero(&asb, offsetof(struct sockbuf, sb_startzero));
|
|
|
|
bcopy(&sb->sb_startzero, &asb.sb_startzero,
|
|
|
|
sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
|
2003-07-17 23:49:10 +00:00
|
|
|
bzero(&sb->sb_startzero,
|
|
|
|
sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
|
2004-06-19 03:23:14 +00:00
|
|
|
SOCKBUF_UNLOCK(sb);
|
2003-07-17 23:49:10 +00:00
|
|
|
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
SOCKBUF_LOCK_INIT(&asb, "so_rcv");
|
2004-04-09 13:23:51 +00:00
|
|
|
if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
(*pr->pr_domain->dom_dispose)(asb.sb_mb);
|
1999-10-09 20:42:17 +00:00
|
|
|
sbrelease(&asb, so);
|
Merge next step in socket buffer locking:
- sowakeup() now asserts the socket buffer lock on entry. Move
the call to KNOTE higher in sowakeup() so that it is made with
the socket buffer lock held for consistency with other calls.
Release the socket buffer lock prior to calling into pgsigio(),
so_upcall(), or aio_swake(). Locking for this event management
will need revisiting in the future, but this model avoids lock
order reversals when upcalls into other subsystems result in
socket/socket buffer operations. Assert that the socket buffer
lock is not held at the end of the function.
- Wrapper macros for sowakeup(), sorwakeup() and sowwakeup(), now
have _locked versions which assert the socket buffer lock on
entry. If a wakeup is required by sb_notify(), invoke
sowakeup(); otherwise, unconditionally release the socket buffer
lock. This results in the socket buffer lock being released
whether a wakeup is required or not.
- Break out socantsendmore() into socantsendmore_locked() that
asserts the socket buffer lock. socantsendmore()
unconditionally locks the socket buffer before calling
socantsendmore_locked(). Note that both functions return with
the socket buffer unlocked as socantsendmore_locked() calls
sowwakeup_locked() which has the same properties. Assert that
the socket buffer is unlocked on return.
- Break out socantrcvmore() into socantrcvmore_locked() that
asserts the socket buffer lock. socantrcvmore() unconditionally
locks the socket buffer before calling socantrcvmore_locked().
Note that both functions return with the socket buffer unlocked
as socantrcvmore_locked() calls sorwakeup_locked() which has
similar properties. Assert that the socket buffer is unlocked
on return.
- Break out sbrelease() into a sbrelease_locked() that asserts the
socket buffer lock. sbrelease() unconditionally locks the
socket buffer before calling sbrelease_locked().
sbrelease_locked() now invokes sbflush_locked() instead of
sbflush().
- Assert the socket buffer lock in socket buffer sanity check
functions sblastrecordchk(), sblastmbufchk().
- Assert the socket buffer lock in SBLINKRECORD().
- Break out various sbappend() functions into sbappend_locked()
(and variations on that name) that assert the socket buffer
lock. The !_locked() variations unconditionally lock the socket
buffer before calling their _locked counterparts. Internally,
make sure to call _locked() support routines, etc, if already
holding the socket buffer lock.
- Break out sbinsertoob() into sbinsertoob_locked() that asserts
the socket buffer lock. sbinsertoob() unconditionally locks the
socket buffer before calling sbinsertoob_locked().
- Break out sbflush() into sbflush_locked() that asserts the
socket buffer lock. sbflush() unconditionally locks the socket
buffer before calling sbflush_locked(). Update panic strings
for new function names.
- Break out sbdrop() into sbdrop_locked() that asserts the socket
buffer lock. sbdrop() unconditionally locks the socket buffer
before calling sbdrop_locked().
- Break out sbdroprecord() into sbdroprecord_locked() that asserts
the socket buffer lock. sbdroprecord() unconditionally locks
the socket buffer before calling sbdroprecord_locked().
- sofree() now calls socantsendmore_locked() and re-acquires the
socket buffer lock on return. It also now calls
sbrelease_locked().
- sorflush() now calls socantrcvmore_locked() and re-acquires the
socket buffer lock on return. Clean up/mess up other behavior
in sorflush() relating to the temporary stack copy of the socket
buffer used with dom_dispose by more properly initializing the
temporary copy, and selectively bzeroing/copying more carefully
to prevent WITNESS from getting confused by improperly
initialized mutexes. Annotate why that's necessary, or at
least, needed.
- soisconnected() now calls sbdrop_locked() before unlocking the
socket buffer to avoid locking overhead.
Some parts of this change were:
Submitted by: sam
Sponsored by: FreeBSD Foundation
Obtained from: BSD/OS
2004-06-21 00:20:43 +00:00
|
|
|
SOCKBUF_LOCK_DESTROY(&asb);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
2000-11-20 01:35:25 +00:00
|
|
|
#ifdef INET
|
2000-06-20 01:09:23 +00:00
|
|
|
static int
|
|
|
|
do_setopt_accept_filter(so, sopt)
|
|
|
|
struct socket *so;
|
|
|
|
struct sockopt *sopt;
|
|
|
|
{
|
|
|
|
struct accept_filter_arg *afap = NULL;
|
|
|
|
struct accept_filter *afp;
|
|
|
|
struct so_accf *af = so->so_accf;
|
|
|
|
int error = 0;
|
|
|
|
|
2000-07-20 12:17:17 +00:00
|
|
|
/* do not set/remove accept filters on non listen sockets */
|
|
|
|
if ((so->so_options & SO_ACCEPTCONN) == 0) {
|
|
|
|
error = EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2000-06-20 01:09:23 +00:00
|
|
|
/* removing the filter */
|
|
|
|
if (sopt == NULL) {
|
|
|
|
if (af != NULL) {
|
2001-11-12 20:51:40 +00:00
|
|
|
if (af->so_accept_filter != NULL &&
|
2000-06-20 01:09:23 +00:00
|
|
|
af->so_accept_filter->accf_destroy != NULL) {
|
|
|
|
af->so_accept_filter->accf_destroy(so);
|
|
|
|
}
|
|
|
|
if (af->so_accept_filter_str != NULL) {
|
|
|
|
FREE(af->so_accept_filter_str, M_ACCF);
|
|
|
|
}
|
|
|
|
FREE(af, M_ACCF);
|
|
|
|
so->so_accf = NULL;
|
|
|
|
}
|
|
|
|
so->so_options &= ~SO_ACCEPTFILTER;
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
/* adding a filter */
|
|
|
|
/* must remove previous filter first */
|
|
|
|
if (af != NULL) {
|
|
|
|
error = EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/* don't put large objects on the kernel stack */
|
2003-02-19 05:47:46 +00:00
|
|
|
MALLOC(afap, struct accept_filter_arg *, sizeof(*afap), M_TEMP, M_WAITOK);
|
2000-06-20 01:09:23 +00:00
|
|
|
error = sooptcopyin(sopt, afap, sizeof *afap, sizeof *afap);
|
|
|
|
afap->af_name[sizeof(afap->af_name)-1] = '\0';
|
|
|
|
afap->af_arg[sizeof(afap->af_arg)-1] = '\0';
|
|
|
|
if (error)
|
|
|
|
goto out;
|
|
|
|
afp = accept_filt_get(afap->af_name);
|
|
|
|
if (afp == NULL) {
|
|
|
|
error = ENOENT;
|
|
|
|
goto out;
|
|
|
|
}
|
2003-02-19 05:47:46 +00:00
|
|
|
MALLOC(af, struct so_accf *, sizeof(*af), M_ACCF, M_WAITOK | M_ZERO);
|
2000-06-20 01:09:23 +00:00
|
|
|
if (afp->accf_create != NULL) {
|
|
|
|
if (afap->af_name[0] != '\0') {
|
|
|
|
int len = strlen(afap->af_name) + 1;
|
|
|
|
|
2003-02-19 05:47:46 +00:00
|
|
|
MALLOC(af->so_accept_filter_str, char *, len, M_ACCF, M_WAITOK);
|
2000-06-20 01:09:23 +00:00
|
|
|
strcpy(af->so_accept_filter_str, afap->af_name);
|
|
|
|
}
|
|
|
|
af->so_accept_filter_arg = afp->accf_create(so, afap->af_arg);
|
|
|
|
if (af->so_accept_filter_arg == NULL) {
|
|
|
|
FREE(af->so_accept_filter_str, M_ACCF);
|
|
|
|
FREE(af, M_ACCF);
|
|
|
|
so->so_accf = NULL;
|
|
|
|
error = EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
af->so_accept_filter = afp;
|
|
|
|
so->so_accf = af;
|
|
|
|
so->so_options |= SO_ACCEPTFILTER;
|
|
|
|
out:
|
|
|
|
if (afap != NULL)
|
|
|
|
FREE(afap, M_TEMP);
|
|
|
|
return (error);
|
|
|
|
}
|
2000-11-20 01:35:25 +00:00
|
|
|
#endif /* INET */
|
2000-06-20 01:09:23 +00:00
|
|
|
|
1998-08-23 03:07:17 +00:00
|
|
|
/*
|
|
|
|
* Perhaps this routine, and sooptcopyout(), below, ought to come in
|
|
|
|
* an additional variant to handle the case where the option value needs
|
|
|
|
* to be some kind of integer, but not a specific size.
|
|
|
|
* In addition to their use here, these functions are also called by the
|
|
|
|
* protocol-level pr_ctloutput() routines.
|
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
1998-08-23 03:07:17 +00:00
|
|
|
sooptcopyin(sopt, buf, len, minlen)
|
|
|
|
struct sockopt *sopt;
|
|
|
|
void *buf;
|
|
|
|
size_t len;
|
|
|
|
size_t minlen;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
1998-08-23 03:07:17 +00:00
|
|
|
size_t valsize;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the user gives us more than we wanted, we ignore it,
|
|
|
|
* but if we don't get the minimum length the caller
|
|
|
|
* wants, we return EINVAL. On success, sopt->sopt_valsize
|
|
|
|
* is set to however much we actually retrieved.
|
|
|
|
*/
|
|
|
|
if ((valsize = sopt->sopt_valsize) < minlen)
|
|
|
|
return EINVAL;
|
|
|
|
if (valsize > len)
|
|
|
|
sopt->sopt_valsize = valsize = len;
|
|
|
|
|
2004-04-09 13:23:51 +00:00
|
|
|
if (sopt->sopt_td != NULL)
|
1998-08-23 03:07:17 +00:00
|
|
|
return (copyin(sopt->sopt_val, buf, valsize));
|
1994-05-24 10:09:53 +00:00
|
|
|
|
1998-08-23 03:07:17 +00:00
|
|
|
bcopy(sopt->sopt_val, buf, valsize);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
sosetopt(so, sopt)
|
|
|
|
struct socket *so;
|
|
|
|
struct sockopt *sopt;
|
|
|
|
{
|
|
|
|
int error, optval;
|
|
|
|
struct linger l;
|
|
|
|
struct timeval tv;
|
1999-05-21 15:54:40 +00:00
|
|
|
u_long val;
|
2002-08-01 03:45:40 +00:00
|
|
|
#ifdef MAC
|
|
|
|
struct mac extmac;
|
2002-10-28 21:17:53 +00:00
|
|
|
#endif
|
1998-08-23 03:07:17 +00:00
|
|
|
|
|
|
|
error = 0;
|
|
|
|
if (sopt->sopt_level != SOL_SOCKET) {
|
1994-05-24 10:09:53 +00:00
|
|
|
if (so->so_proto && so->so_proto->pr_ctloutput)
|
|
|
|
return ((*so->so_proto->pr_ctloutput)
|
1998-08-23 03:07:17 +00:00
|
|
|
(so, sopt));
|
1994-05-24 10:09:53 +00:00
|
|
|
error = ENOPROTOOPT;
|
|
|
|
} else {
|
1998-08-23 03:07:17 +00:00
|
|
|
switch (sopt->sopt_name) {
|
2000-11-20 01:35:25 +00:00
|
|
|
#ifdef INET
|
|
|
|
case SO_ACCEPTFILTER:
|
|
|
|
error = do_setopt_accept_filter(so, sopt);
|
|
|
|
if (error)
|
|
|
|
goto bad;
|
|
|
|
break;
|
|
|
|
#endif
|
1994-05-24 10:09:53 +00:00
|
|
|
case SO_LINGER:
|
1998-08-23 03:07:17 +00:00
|
|
|
error = sooptcopyin(sopt, &l, sizeof l, sizeof l);
|
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto bad;
|
1998-08-23 03:07:17 +00:00
|
|
|
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_LOCK(so);
|
1998-08-23 03:07:17 +00:00
|
|
|
so->so_linger = l.l_linger;
|
|
|
|
if (l.l_onoff)
|
|
|
|
so->so_options |= SO_LINGER;
|
|
|
|
else
|
|
|
|
so->so_options &= ~SO_LINGER;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_UNLOCK(so);
|
1998-08-23 03:07:17 +00:00
|
|
|
break;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_DEBUG:
|
|
|
|
case SO_KEEPALIVE:
|
|
|
|
case SO_DONTROUTE:
|
|
|
|
case SO_USELOOPBACK:
|
|
|
|
case SO_BROADCAST:
|
|
|
|
case SO_REUSEADDR:
|
|
|
|
case SO_REUSEPORT:
|
|
|
|
case SO_OOBINLINE:
|
1996-05-09 20:15:26 +00:00
|
|
|
case SO_TIMESTAMP:
|
2004-01-31 10:40:25 +00:00
|
|
|
case SO_BINTIME:
|
2002-06-20 18:52:54 +00:00
|
|
|
case SO_NOSIGPIPE:
|
1998-08-23 03:07:17 +00:00
|
|
|
error = sooptcopyin(sopt, &optval, sizeof optval,
|
|
|
|
sizeof optval);
|
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto bad;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_LOCK(so);
|
1998-08-23 03:07:17 +00:00
|
|
|
if (optval)
|
|
|
|
so->so_options |= sopt->sopt_name;
|
1994-05-24 10:09:53 +00:00
|
|
|
else
|
1998-08-23 03:07:17 +00:00
|
|
|
so->so_options &= ~sopt->sopt_name;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCK_UNLOCK(so);
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case SO_SNDBUF:
|
|
|
|
case SO_RCVBUF:
|
|
|
|
case SO_SNDLOWAT:
|
|
|
|
case SO_RCVLOWAT:
|
1998-08-23 03:07:17 +00:00
|
|
|
error = sooptcopyin(sopt, &optval, sizeof optval,
|
|
|
|
sizeof optval);
|
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto bad;
|
1997-06-27 15:28:54 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Values < 1 make no sense for any of these
|
|
|
|
* options, so disallow them.
|
|
|
|
*/
|
|
|
|
if (optval < 1) {
|
|
|
|
error = EINVAL;
|
|
|
|
goto bad;
|
|
|
|
}
|
|
|
|
|
1998-08-23 03:07:17 +00:00
|
|
|
switch (sopt->sopt_name) {
|
1994-05-24 10:09:53 +00:00
|
|
|
case SO_SNDBUF:
|
|
|
|
case SO_RCVBUF:
|
1998-08-23 03:07:17 +00:00
|
|
|
if (sbreserve(sopt->sopt_name == SO_SNDBUF ?
|
1999-10-09 20:42:17 +00:00
|
|
|
&so->so_snd : &so->so_rcv, (u_long)optval,
|
2001-09-12 08:38:13 +00:00
|
|
|
so, curthread) == 0) {
|
1994-05-24 10:09:53 +00:00
|
|
|
error = ENOBUFS;
|
|
|
|
goto bad;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
1997-06-27 15:28:54 +00:00
|
|
|
/*
|
|
|
|
* Make sure the low-water is never greater than
|
|
|
|
* the high-water.
|
|
|
|
*/
|
1994-05-24 10:09:53 +00:00
|
|
|
case SO_SNDLOWAT:
|
2004-06-24 04:28:30 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
1997-06-27 15:28:54 +00:00
|
|
|
so->so_snd.sb_lowat =
|
|
|
|
(optval > so->so_snd.sb_hiwat) ?
|
|
|
|
so->so_snd.sb_hiwat : optval;
|
2004-06-24 04:28:30 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
case SO_RCVLOWAT:
|
2004-06-24 04:28:30 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
1997-06-27 15:28:54 +00:00
|
|
|
so->so_rcv.sb_lowat =
|
|
|
|
(optval > so->so_rcv.sb_hiwat) ?
|
|
|
|
so->so_rcv.sb_hiwat : optval;
|
2004-06-24 04:28:30 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case SO_SNDTIMEO:
|
|
|
|
case SO_RCVTIMEO:
|
1998-08-23 03:07:17 +00:00
|
|
|
error = sooptcopyin(sopt, &tv, sizeof tv,
|
|
|
|
sizeof tv);
|
|
|
|
if (error)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto bad;
|
1998-08-23 03:07:17 +00:00
|
|
|
|
1999-05-21 15:54:40 +00:00
|
|
|
/* assert(hz > 0); */
|
|
|
|
if (tv.tv_sec < 0 || tv.tv_sec > SHRT_MAX / hz ||
|
|
|
|
tv.tv_usec < 0 || tv.tv_usec >= 1000000) {
|
|
|
|
error = EDOM;
|
|
|
|
goto bad;
|
|
|
|
}
|
|
|
|
/* assert(tick > 0); */
|
|
|
|
/* assert(ULONG_MAX - SHRT_MAX >= 1000000); */
|
|
|
|
val = (u_long)(tv.tv_sec * hz) + tv.tv_usec / tick;
|
|
|
|
if (val > SHRT_MAX) {
|
1994-05-24 10:09:53 +00:00
|
|
|
error = EDOM;
|
|
|
|
goto bad;
|
|
|
|
}
|
2002-11-27 13:34:04 +00:00
|
|
|
if (val == 0 && tv.tv_usec != 0)
|
|
|
|
val = 1;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
1998-08-23 03:07:17 +00:00
|
|
|
switch (sopt->sopt_name) {
|
1994-05-24 10:09:53 +00:00
|
|
|
case SO_SNDTIMEO:
|
|
|
|
so->so_snd.sb_timeo = val;
|
|
|
|
break;
|
|
|
|
case SO_RCVTIMEO:
|
|
|
|
so->so_rcv.sb_timeo = val;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
2002-08-01 03:45:40 +00:00
|
|
|
case SO_LABEL:
|
|
|
|
#ifdef MAC
|
|
|
|
error = sooptcopyin(sopt, &extmac, sizeof extmac,
|
|
|
|
sizeof extmac);
|
|
|
|
if (error)
|
|
|
|
goto bad;
|
2003-11-16 18:25:20 +00:00
|
|
|
error = mac_setsockopt_label(sopt->sopt_td->td_ucred,
|
|
|
|
so, &extmac);
|
2002-10-28 21:17:53 +00:00
|
|
|
#else
|
2002-08-01 03:45:40 +00:00
|
|
|
error = EOPNOTSUPP;
|
2002-10-28 21:17:53 +00:00
|
|
|
#endif
|
2002-08-01 03:45:40 +00:00
|
|
|
break;
|
1994-05-24 10:09:53 +00:00
|
|
|
default:
|
|
|
|
error = ENOPROTOOPT;
|
|
|
|
break;
|
|
|
|
}
|
2004-04-09 13:23:51 +00:00
|
|
|
if (error == 0 && so->so_proto != NULL &&
|
|
|
|
so->so_proto->pr_ctloutput != NULL) {
|
1994-05-24 10:09:53 +00:00
|
|
|
(void) ((*so->so_proto->pr_ctloutput)
|
1998-08-23 03:07:17 +00:00
|
|
|
(so, sopt));
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
bad:
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
1998-08-23 03:07:17 +00:00
|
|
|
/* Helper routine for getsockopt */
|
1994-05-25 09:21:21 +00:00
|
|
|
int
|
2003-08-05 00:27:54 +00:00
|
|
|
sooptcopyout(struct sockopt *sopt, const void *buf, size_t len)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
1998-08-23 03:07:17 +00:00
|
|
|
int error;
|
|
|
|
size_t valsize;
|
|
|
|
|
|
|
|
error = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Documented get behavior is that we always return a value,
|
|
|
|
* possibly truncated to fit in the user's buffer.
|
1998-08-31 18:07:23 +00:00
|
|
|
* Traditional behavior is that we always tell the user
|
|
|
|
* precisely how much we copied, rather than something useful
|
|
|
|
* like the total amount we had available for her.
|
|
|
|
* Note that this interface is not idempotent; the entire answer must
|
1998-08-23 03:07:17 +00:00
|
|
|
* generated ahead of time.
|
|
|
|
*/
|
1998-08-31 15:34:55 +00:00
|
|
|
valsize = min(len, sopt->sopt_valsize);
|
1998-08-31 18:07:23 +00:00
|
|
|
sopt->sopt_valsize = valsize;
|
2004-04-09 13:23:51 +00:00
|
|
|
if (sopt->sopt_val != NULL) {
|
|
|
|
if (sopt->sopt_td != NULL)
|
1998-08-23 03:07:17 +00:00
|
|
|
error = copyout(buf, sopt->sopt_val, valsize);
|
|
|
|
else
|
|
|
|
bcopy(buf, sopt->sopt_val, valsize);
|
|
|
|
}
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
sogetopt(so, sopt)
|
|
|
|
struct socket *so;
|
|
|
|
struct sockopt *sopt;
|
|
|
|
{
|
|
|
|
int error, optval;
|
|
|
|
struct linger l;
|
|
|
|
struct timeval tv;
|
2000-11-20 01:35:25 +00:00
|
|
|
#ifdef INET
|
2000-06-20 01:09:23 +00:00
|
|
|
struct accept_filter_arg *afap;
|
2000-11-20 01:35:25 +00:00
|
|
|
#endif
|
2002-08-01 03:45:40 +00:00
|
|
|
#ifdef MAC
|
|
|
|
struct mac extmac;
|
2002-10-28 21:17:53 +00:00
|
|
|
#endif
|
1994-05-24 10:09:53 +00:00
|
|
|
|
1998-08-23 03:07:17 +00:00
|
|
|
error = 0;
|
|
|
|
if (sopt->sopt_level != SOL_SOCKET) {
|
1994-05-24 10:09:53 +00:00
|
|
|
if (so->so_proto && so->so_proto->pr_ctloutput) {
|
|
|
|
return ((*so->so_proto->pr_ctloutput)
|
1998-08-23 03:07:17 +00:00
|
|
|
(so, sopt));
|
1994-05-24 10:09:53 +00:00
|
|
|
} else
|
|
|
|
return (ENOPROTOOPT);
|
|
|
|
} else {
|
1998-08-23 03:07:17 +00:00
|
|
|
switch (sopt->sopt_name) {
|
2000-11-20 01:35:25 +00:00
|
|
|
#ifdef INET
|
2000-06-20 01:09:23 +00:00
|
|
|
case SO_ACCEPTFILTER:
|
2002-05-31 11:52:35 +00:00
|
|
|
if ((so->so_options & SO_ACCEPTCONN) == 0)
|
|
|
|
return (EINVAL);
|
2000-06-20 01:09:23 +00:00
|
|
|
MALLOC(afap, struct accept_filter_arg *, sizeof(*afap),
|
2003-02-19 05:47:46 +00:00
|
|
|
M_TEMP, M_WAITOK | M_ZERO);
|
2000-06-20 01:09:23 +00:00
|
|
|
if ((so->so_options & SO_ACCEPTFILTER) != 0) {
|
|
|
|
strcpy(afap->af_name, so->so_accf->so_accept_filter->accf_name);
|
|
|
|
if (so->so_accf->so_accept_filter_str != NULL)
|
|
|
|
strcpy(afap->af_arg, so->so_accf->so_accept_filter_str);
|
|
|
|
}
|
|
|
|
error = sooptcopyout(sopt, afap, sizeof(*afap));
|
|
|
|
FREE(afap, M_TEMP);
|
|
|
|
break;
|
2000-11-20 01:35:25 +00:00
|
|
|
#endif
|
2001-11-12 20:51:40 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
case SO_LINGER:
|
2004-06-20 17:50:42 +00:00
|
|
|
/*
|
|
|
|
* XXXRW: We grab the lock here to get a consistent
|
|
|
|
* snapshot of both fields. This may not really
|
|
|
|
* be necessary.
|
|
|
|
*/
|
|
|
|
SOCK_LOCK(so);
|
1998-08-23 03:07:17 +00:00
|
|
|
l.l_onoff = so->so_options & SO_LINGER;
|
|
|
|
l.l_linger = so->so_linger;
|
2004-06-20 17:50:42 +00:00
|
|
|
SOCK_UNLOCK(so);
|
1998-08-23 03:07:17 +00:00
|
|
|
error = sooptcopyout(sopt, &l, sizeof l);
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case SO_USELOOPBACK:
|
|
|
|
case SO_DONTROUTE:
|
|
|
|
case SO_DEBUG:
|
|
|
|
case SO_KEEPALIVE:
|
|
|
|
case SO_REUSEADDR:
|
|
|
|
case SO_REUSEPORT:
|
|
|
|
case SO_BROADCAST:
|
|
|
|
case SO_OOBINLINE:
|
1996-05-09 20:15:26 +00:00
|
|
|
case SO_TIMESTAMP:
|
2004-01-31 10:40:25 +00:00
|
|
|
case SO_BINTIME:
|
2002-06-20 18:52:54 +00:00
|
|
|
case SO_NOSIGPIPE:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_options & sopt->sopt_name;
|
|
|
|
integer:
|
|
|
|
error = sooptcopyout(sopt, &optval, sizeof optval);
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case SO_TYPE:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_type;
|
|
|
|
goto integer;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_ERROR:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_error;
|
1994-05-24 10:09:53 +00:00
|
|
|
so->so_error = 0;
|
1998-08-23 03:07:17 +00:00
|
|
|
goto integer;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_SNDBUF:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_snd.sb_hiwat;
|
|
|
|
goto integer;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_RCVBUF:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_rcv.sb_hiwat;
|
|
|
|
goto integer;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_SNDLOWAT:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_snd.sb_lowat;
|
|
|
|
goto integer;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_RCVLOWAT:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = so->so_rcv.sb_lowat;
|
|
|
|
goto integer;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
case SO_SNDTIMEO:
|
|
|
|
case SO_RCVTIMEO:
|
1998-08-23 03:07:17 +00:00
|
|
|
optval = (sopt->sopt_name == SO_SNDTIMEO ?
|
|
|
|
so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
|
|
|
|
|
|
|
|
tv.tv_sec = optval / hz;
|
|
|
|
tv.tv_usec = (optval % hz) * tick;
|
|
|
|
error = sooptcopyout(sopt, &tv, sizeof tv);
|
2001-11-12 20:51:40 +00:00
|
|
|
break;
|
2002-08-01 03:45:40 +00:00
|
|
|
case SO_LABEL:
|
|
|
|
#ifdef MAC
|
2003-11-16 03:53:36 +00:00
|
|
|
error = sooptcopyin(sopt, &extmac, sizeof(extmac),
|
|
|
|
sizeof(extmac));
|
|
|
|
if (error)
|
|
|
|
return (error);
|
2003-11-16 18:25:20 +00:00
|
|
|
error = mac_getsockopt_label(sopt->sopt_td->td_ucred,
|
|
|
|
so, &extmac);
|
2002-08-01 03:45:40 +00:00
|
|
|
if (error)
|
|
|
|
return (error);
|
|
|
|
error = sooptcopyout(sopt, &extmac, sizeof extmac);
|
2002-10-28 21:17:53 +00:00
|
|
|
#else
|
2002-08-01 03:45:40 +00:00
|
|
|
error = EOPNOTSUPP;
|
2002-10-28 21:17:53 +00:00
|
|
|
#endif
|
2002-08-01 03:45:40 +00:00
|
|
|
break;
|
|
|
|
case SO_PEERLABEL:
|
|
|
|
#ifdef MAC
|
2003-11-16 03:53:36 +00:00
|
|
|
error = sooptcopyin(sopt, &extmac, sizeof(extmac),
|
|
|
|
sizeof(extmac));
|
|
|
|
if (error)
|
|
|
|
return (error);
|
2003-11-16 18:25:20 +00:00
|
|
|
error = mac_getsockopt_peerlabel(
|
2002-08-01 03:45:40 +00:00
|
|
|
sopt->sopt_td->td_ucred, so, &extmac);
|
|
|
|
if (error)
|
|
|
|
return (error);
|
|
|
|
error = sooptcopyout(sopt, &extmac, sizeof extmac);
|
2002-10-28 21:17:53 +00:00
|
|
|
#else
|
2002-08-01 03:45:40 +00:00
|
|
|
error = EOPNOTSUPP;
|
2002-10-28 21:17:53 +00:00
|
|
|
#endif
|
2002-08-01 03:45:40 +00:00
|
|
|
break;
|
1994-05-24 10:09:53 +00:00
|
|
|
default:
|
1998-08-23 03:07:17 +00:00
|
|
|
error = ENOPROTOOPT;
|
|
|
|
break;
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
1998-08-23 03:07:17 +00:00
|
|
|
return (error);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
1999-11-22 02:45:11 +00:00
|
|
|
/* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */
|
|
|
|
int
|
|
|
|
soopt_getm(struct sockopt *sopt, struct mbuf **mp)
|
|
|
|
{
|
|
|
|
struct mbuf *m, *m_prev;
|
|
|
|
int sopt_size = sopt->sopt_valsize;
|
|
|
|
|
2003-02-19 05:47:46 +00:00
|
|
|
MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m == NULL)
|
1999-11-22 02:45:11 +00:00
|
|
|
return ENOBUFS;
|
|
|
|
if (sopt_size > MLEN) {
|
2003-02-19 05:47:46 +00:00
|
|
|
MCLGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT);
|
1999-11-22 02:45:11 +00:00
|
|
|
if ((m->m_flags & M_EXT) == 0) {
|
|
|
|
m_free(m);
|
|
|
|
return ENOBUFS;
|
|
|
|
}
|
|
|
|
m->m_len = min(MCLBYTES, sopt_size);
|
|
|
|
} else {
|
|
|
|
m->m_len = min(MLEN, sopt_size);
|
|
|
|
}
|
|
|
|
sopt_size -= m->m_len;
|
|
|
|
*mp = m;
|
|
|
|
m_prev = m;
|
|
|
|
|
|
|
|
while (sopt_size) {
|
2003-02-19 05:47:46 +00:00
|
|
|
MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA);
|
2004-04-09 13:23:51 +00:00
|
|
|
if (m == NULL) {
|
1999-11-22 02:45:11 +00:00
|
|
|
m_freem(*mp);
|
|
|
|
return ENOBUFS;
|
|
|
|
}
|
|
|
|
if (sopt_size > MLEN) {
|
2004-04-09 13:23:51 +00:00
|
|
|
MCLGET(m, sopt->sopt_td != NULL ? M_TRYWAIT :
|
|
|
|
M_DONTWAIT);
|
1999-11-22 02:45:11 +00:00
|
|
|
if ((m->m_flags & M_EXT) == 0) {
|
2004-01-18 14:02:53 +00:00
|
|
|
m_freem(m);
|
1999-11-22 02:45:11 +00:00
|
|
|
m_freem(*mp);
|
|
|
|
return ENOBUFS;
|
|
|
|
}
|
|
|
|
m->m_len = min(MCLBYTES, sopt_size);
|
|
|
|
} else {
|
|
|
|
m->m_len = min(MLEN, sopt_size);
|
|
|
|
}
|
|
|
|
sopt_size -= m->m_len;
|
|
|
|
m_prev->m_next = m;
|
|
|
|
m_prev = m;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* XXX; copyin sopt data into mbuf chain for (__FreeBSD__ < 3) routines. */
|
|
|
|
int
|
|
|
|
soopt_mcopyin(struct sockopt *sopt, struct mbuf *m)
|
|
|
|
{
|
|
|
|
struct mbuf *m0 = m;
|
|
|
|
|
|
|
|
if (sopt->sopt_val == NULL)
|
|
|
|
return 0;
|
|
|
|
while (m != NULL && sopt->sopt_valsize >= m->m_len) {
|
2001-09-12 08:38:13 +00:00
|
|
|
if (sopt->sopt_td != NULL) {
|
1999-11-22 02:45:11 +00:00
|
|
|
int error;
|
|
|
|
|
|
|
|
error = copyin(sopt->sopt_val, mtod(m, char *),
|
|
|
|
m->m_len);
|
|
|
|
if (error != 0) {
|
|
|
|
m_freem(m0);
|
|
|
|
return(error);
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
bcopy(sopt->sopt_val, mtod(m, char *), m->m_len);
|
|
|
|
sopt->sopt_valsize -= m->m_len;
|
2004-06-08 13:08:19 +00:00
|
|
|
sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
|
1999-11-22 02:45:11 +00:00
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */
|
|
|
|
panic("ip6_sooptmcopyin");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* XXX; copyout mbuf chain data into soopt for (__FreeBSD__ < 3) routines. */
|
|
|
|
int
|
|
|
|
soopt_mcopyout(struct sockopt *sopt, struct mbuf *m)
|
|
|
|
{
|
|
|
|
struct mbuf *m0 = m;
|
|
|
|
size_t valsize = 0;
|
|
|
|
|
|
|
|
if (sopt->sopt_val == NULL)
|
|
|
|
return 0;
|
|
|
|
while (m != NULL && sopt->sopt_valsize >= m->m_len) {
|
2001-09-12 08:38:13 +00:00
|
|
|
if (sopt->sopt_td != NULL) {
|
1999-11-22 02:45:11 +00:00
|
|
|
int error;
|
|
|
|
|
|
|
|
error = copyout(mtod(m, char *), sopt->sopt_val,
|
|
|
|
m->m_len);
|
|
|
|
if (error != 0) {
|
|
|
|
m_freem(m0);
|
|
|
|
return(error);
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
bcopy(mtod(m, char *), sopt->sopt_val, m->m_len);
|
|
|
|
sopt->sopt_valsize -= m->m_len;
|
2004-06-08 13:08:19 +00:00
|
|
|
sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
|
1999-11-22 02:45:11 +00:00
|
|
|
valsize += m->m_len;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
if (m != NULL) {
|
|
|
|
/* enough soopt buffer should be given from user-land */
|
|
|
|
m_freem(m0);
|
|
|
|
return(EINVAL);
|
|
|
|
}
|
|
|
|
sopt->sopt_valsize = valsize;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
void
|
1994-05-24 10:09:53 +00:00
|
|
|
sohasoutofband(so)
|
2003-03-02 15:56:49 +00:00
|
|
|
struct socket *so;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
Installed the second patch attached to kern/7899 with some changes suggested
by bde, a few other tweaks to get the patch to apply cleanly again and
some improvements to the comments.
This change closes some fairly minor security holes associated with
F_SETOWN, fixes a few bugs, and removes some limitations that F_SETOWN
had on tty devices. For more details, see the description on the PR.
Because this patch increases the size of the proc and pgrp structures,
it is necessary to re-install the includes and recompile libkvm,
the vinum lkm, fstat, gcore, gdb, ipfilter, ps, top, and w.
PR: kern/7899
Reviewed by: bde, elvind
1998-11-11 10:04:13 +00:00
|
|
|
if (so->so_sigio != NULL)
|
2002-05-01 20:44:46 +00:00
|
|
|
pgsigio(&so->so_sigio, SIGURG, 0);
|
2003-11-09 09:17:26 +00:00
|
|
|
selwakeuppri(&so->so_rcv.sb_sel, PSOCK);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
1997-04-27 20:01:29 +00:00
|
|
|
|
|
|
|
int
|
Make similar changes to fo_stat() and fo_poll() as made earlier to
fo_read() and fo_write(): explicitly use the cred argument to fo_poll()
as "active_cred" using the passed file descriptor's f_cred reference
to provide access to the file credential. Add an active_cred
argument to fo_stat() so that implementers have access to the active
credential as well as the file credential. Generally modify callers
of fo_stat() to pass in td->td_ucred rather than fp->f_cred, which
was redundantly provided via the fp argument. This set of modifications
also permits threads to perform these operations on behalf of another
thread without modifying their credential.
Trickle this change down into fo_stat/poll() implementations:
- badfo_poll(), badfo_stat(): modify/add arguments.
- kqueue_poll(), kqueue_stat(): modify arguments.
- pipe_poll(), pipe_stat(): modify/add arguments, pass active_cred to
MAC checks rather than td->td_ucred.
- soo_poll(), soo_stat(): modify/add arguments, pass fp->f_cred rather
than cred to pru_sopoll() to maintain current semantics.
- sopoll(): moidfy arguments.
- vn_poll(), vn_statfile(): modify/add arguments, pass new arguments
to vn_stat(). Pass active_cred to MAC and fp->f_cred to VOP_POLL()
to maintian current semantics.
- vn_close(): rename cred to file_cred to reflect reality while I'm here.
- vn_stat(): Add active_cred and file_cred arguments to vn_stat()
and consumers so that this distinction is maintained at the VFS
as well as 'struct file' layer. Pass active_cred instead of
td->td_ucred to MAC and to VOP_GETATTR() to maintain current semantics.
- fifofs: modify the creation of a "filetemp" so that the file
credential is properly initialized and can be used in the socket
code if desired. Pass ap->a_td->td_ucred as the active
credential to soo_poll(). If we teach the vnop interface about
the distinction between file and active credentials, we would use
the active credential here.
Note that current inconsistent passing of active_cred vs. file_cred to
VOP's is maintained. It's not clear why GETATTR would be authorized
using active_cred while POLL would be authorized using file_cred at
the file system level.
Obtained from: TrustedBSD Project
Sponsored by: DARPA, NAI Labs
2002-08-16 12:52:03 +00:00
|
|
|
sopoll(struct socket *so, int events, struct ucred *active_cred,
|
|
|
|
struct thread *td)
|
1997-04-27 20:01:29 +00:00
|
|
|
{
|
1997-09-14 02:34:14 +00:00
|
|
|
int revents = 0;
|
1997-04-27 20:01:29 +00:00
|
|
|
|
2002-05-31 11:52:35 +00:00
|
|
|
if (events & (POLLIN | POLLRDNORM))
|
1997-09-14 02:34:14 +00:00
|
|
|
if (soreadable(so))
|
|
|
|
revents |= events & (POLLIN | POLLRDNORM);
|
1997-04-27 20:01:29 +00:00
|
|
|
|
2002-01-14 22:03:48 +00:00
|
|
|
if (events & POLLINIGNEOF)
|
|
|
|
if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat ||
|
|
|
|
!TAILQ_EMPTY(&so->so_comp) || so->so_error)
|
|
|
|
revents |= POLLINIGNEOF;
|
|
|
|
|
2002-05-31 11:52:35 +00:00
|
|
|
if (events & (POLLOUT | POLLWRNORM))
|
1997-09-14 02:34:14 +00:00
|
|
|
if (sowriteable(so))
|
|
|
|
revents |= events & (POLLOUT | POLLWRNORM);
|
|
|
|
|
2002-05-31 11:52:35 +00:00
|
|
|
if (events & (POLLPRI | POLLRDBAND))
|
2004-06-14 18:16:22 +00:00
|
|
|
if (so->so_oobmark || (so->so_rcv.sb_state & SBS_RCVATMARK))
|
1997-09-14 02:34:14 +00:00
|
|
|
revents |= events & (POLLPRI | POLLRDBAND);
|
|
|
|
|
|
|
|
if (revents == 0) {
|
2002-01-14 22:03:48 +00:00
|
|
|
if (events &
|
|
|
|
(POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM |
|
|
|
|
POLLRDBAND)) {
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
2004-06-24 00:54:26 +00:00
|
|
|
selrecord(td, &so->so_rcv.sb_sel);
|
1997-09-14 02:34:14 +00:00
|
|
|
so->so_rcv.sb_flags |= SB_SEL;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
1997-04-27 20:01:29 +00:00
|
|
|
}
|
|
|
|
|
1997-09-14 02:34:14 +00:00
|
|
|
if (events & (POLLOUT | POLLWRNORM)) {
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
2004-06-24 00:54:26 +00:00
|
|
|
selrecord(td, &so->so_snd.sb_sel);
|
1997-09-14 02:34:14 +00:00
|
|
|
so->so_snd.sb_flags |= SB_SEL;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
1997-04-27 20:01:29 +00:00
|
|
|
}
|
|
|
|
}
|
1997-09-14 02:34:14 +00:00
|
|
|
|
|
|
|
return (revents);
|
1997-04-27 20:01:29 +00:00
|
|
|
}
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2001-02-15 16:34:11 +00:00
|
|
|
int
|
2002-12-23 21:37:28 +00:00
|
|
|
soo_kqfilter(struct file *fp, struct knote *kn)
|
2000-04-16 18:53:38 +00:00
|
|
|
{
|
2003-01-13 00:33:17 +00:00
|
|
|
struct socket *so = kn->kn_fp->f_data;
|
2001-02-15 16:34:11 +00:00
|
|
|
struct sockbuf *sb;
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2001-02-15 16:34:11 +00:00
|
|
|
switch (kn->kn_filter) {
|
|
|
|
case EVFILT_READ:
|
|
|
|
if (so->so_options & SO_ACCEPTCONN)
|
|
|
|
kn->kn_fop = &solisten_filtops;
|
|
|
|
else
|
|
|
|
kn->kn_fop = &soread_filtops;
|
|
|
|
sb = &so->so_rcv;
|
|
|
|
break;
|
|
|
|
case EVFILT_WRITE:
|
|
|
|
kn->kn_fop = &sowrite_filtops;
|
|
|
|
sb = &so->so_snd;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return (1);
|
|
|
|
}
|
|
|
|
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_LOCK(sb);
|
2001-02-15 16:34:11 +00:00
|
|
|
SLIST_INSERT_HEAD(&sb->sb_sel.si_note, kn, kn_selnext);
|
|
|
|
sb->sb_flags |= SB_KNOTE;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_UNLOCK(sb);
|
2000-04-16 18:53:38 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
filt_sordetach(struct knote *kn)
|
|
|
|
{
|
2003-01-13 00:33:17 +00:00
|
|
|
struct socket *so = kn->kn_fp->f_data;
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
2000-05-26 02:09:24 +00:00
|
|
|
SLIST_REMOVE(&so->so_rcv.sb_sel.si_note, kn, knote, kn_selnext);
|
2000-04-16 18:53:38 +00:00
|
|
|
if (SLIST_EMPTY(&so->so_rcv.sb_sel.si_note))
|
|
|
|
so->so_rcv.sb_flags &= ~SB_KNOTE;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
2000-04-16 18:53:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*ARGSUSED*/
|
|
|
|
static int
|
|
|
|
filt_soread(struct knote *kn, long hint)
|
|
|
|
{
|
2003-01-13 00:33:17 +00:00
|
|
|
struct socket *so = kn->kn_fp->f_data;
|
2004-06-18 02:57:55 +00:00
|
|
|
int need_lock, result;
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2004-06-18 02:57:55 +00:00
|
|
|
/*
|
|
|
|
* XXXRW: Conditional locking because filt_soread() can be called
|
|
|
|
* either from KNOTE() in the socket context where the socket buffer
|
|
|
|
* lock is already held, or from kqueue() itself.
|
|
|
|
*/
|
|
|
|
need_lock = !SOCKBUF_OWNED(&so->so_rcv);
|
|
|
|
if (need_lock)
|
|
|
|
SOCKBUF_LOCK(&so->so_rcv);
|
2002-11-01 21:27:59 +00:00
|
|
|
kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
|
2004-06-14 18:16:22 +00:00
|
|
|
if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
|
2001-11-12 20:51:40 +00:00
|
|
|
kn->kn_flags |= EV_EOF;
|
2001-02-24 01:33:12 +00:00
|
|
|
kn->kn_fflags = so->so_error;
|
2004-03-29 18:06:15 +00:00
|
|
|
result = 1;
|
|
|
|
} else if (so->so_error) /* temporary udp error */
|
|
|
|
result = 1;
|
|
|
|
else if (kn->kn_sfflags & NOTE_LOWAT)
|
|
|
|
result = (kn->kn_data >= kn->kn_sdata);
|
|
|
|
else
|
|
|
|
result = (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat);
|
2004-06-18 02:57:55 +00:00
|
|
|
if (need_lock)
|
|
|
|
SOCKBUF_UNLOCK(&so->so_rcv);
|
2004-03-29 18:06:15 +00:00
|
|
|
return (result);
|
2000-04-16 18:53:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
filt_sowdetach(struct knote *kn)
|
|
|
|
{
|
2003-01-13 00:33:17 +00:00
|
|
|
struct socket *so = kn->kn_fp->f_data;
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
2000-05-26 02:09:24 +00:00
|
|
|
SLIST_REMOVE(&so->so_snd.sb_sel.si_note, kn, knote, kn_selnext);
|
2000-04-16 18:53:38 +00:00
|
|
|
if (SLIST_EMPTY(&so->so_snd.sb_sel.si_note))
|
|
|
|
so->so_snd.sb_flags &= ~SB_KNOTE;
|
2004-06-17 22:48:11 +00:00
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
2000-04-16 18:53:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*ARGSUSED*/
|
|
|
|
static int
|
|
|
|
filt_sowrite(struct knote *kn, long hint)
|
|
|
|
{
|
2003-01-13 00:33:17 +00:00
|
|
|
struct socket *so = kn->kn_fp->f_data;
|
2004-06-18 02:57:55 +00:00
|
|
|
int need_lock, result;
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2004-06-18 02:57:55 +00:00
|
|
|
/*
|
|
|
|
* XXXRW: Conditional locking because filt_soread() can be called
|
|
|
|
* either from KNOTE() in the socket context where the socket buffer
|
|
|
|
* lock is already held, or from kqueue() itself.
|
|
|
|
*/
|
|
|
|
need_lock = !SOCKBUF_OWNED(&so->so_snd);
|
|
|
|
if (need_lock)
|
|
|
|
SOCKBUF_LOCK(&so->so_snd);
|
2000-04-16 18:53:38 +00:00
|
|
|
kn->kn_data = sbspace(&so->so_snd);
|
2004-06-14 18:16:22 +00:00
|
|
|
if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
|
2001-11-12 20:51:40 +00:00
|
|
|
kn->kn_flags |= EV_EOF;
|
2001-02-24 01:33:12 +00:00
|
|
|
kn->kn_fflags = so->so_error;
|
2004-03-29 18:06:15 +00:00
|
|
|
result = 1;
|
|
|
|
} else if (so->so_error) /* temporary udp error */
|
|
|
|
result = 1;
|
|
|
|
else if (((so->so_state & SS_ISCONNECTED) == 0) &&
|
2002-05-31 11:52:35 +00:00
|
|
|
(so->so_proto->pr_flags & PR_CONNREQUIRED))
|
2004-03-29 18:06:15 +00:00
|
|
|
result = 0;
|
|
|
|
else if (kn->kn_sfflags & NOTE_LOWAT)
|
|
|
|
result = (kn->kn_data >= kn->kn_sdata);
|
|
|
|
else
|
|
|
|
result = (kn->kn_data >= so->so_snd.sb_lowat);
|
2004-06-18 02:57:55 +00:00
|
|
|
if (need_lock)
|
|
|
|
SOCKBUF_UNLOCK(&so->so_snd);
|
2004-03-29 18:06:15 +00:00
|
|
|
return (result);
|
2000-04-16 18:53:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*ARGSUSED*/
|
|
|
|
static int
|
|
|
|
filt_solisten(struct knote *kn, long hint)
|
|
|
|
{
|
2003-01-13 00:33:17 +00:00
|
|
|
struct socket *so = kn->kn_fp->f_data;
|
2000-04-16 18:53:38 +00:00
|
|
|
|
2002-04-26 02:07:46 +00:00
|
|
|
kn->kn_data = so->so_qlen;
|
2000-04-16 18:53:38 +00:00
|
|
|
return (! TAILQ_EMPTY(&so->so_comp));
|
|
|
|
}
|
2001-10-05 07:06:32 +00:00
|
|
|
|
|
|
|
int
|
|
|
|
socheckuid(struct socket *so, uid_t uid)
|
|
|
|
{
|
|
|
|
|
|
|
|
if (so == NULL)
|
|
|
|
return (EPERM);
|
|
|
|
if (so->so_cred->cr_uid == uid)
|
|
|
|
return (0);
|
|
|
|
return (EPERM);
|
|
|
|
}
|