2005-01-06 23:35:40 +00:00
|
|
|
/*-
|
1994-05-24 10:09:53 +00:00
|
|
|
* Copyright (c) 1982, 1986, 1988, 1991, 1993
|
|
|
|
* The Regents of the University of California. All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 4. Neither the name of the University nor the names of its contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
|
|
|
|
*/
|
|
|
|
|
2003-06-11 00:56:59 +00:00
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2002-07-31 01:51:34 +00:00
|
|
|
#include "opt_mac.h"
|
2001-06-01 09:51:14 +00:00
|
|
|
#include "opt_param.h"
|
2003-04-15 02:14:43 +00:00
|
|
|
#include "opt_mbuf_stress_test.h"
|
2002-07-31 01:51:34 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/kernel.h>
|
2004-02-01 07:56:44 +00:00
|
|
|
#include <sys/limits.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/lock.h>
|
2002-07-31 01:51:34 +00:00
|
|
|
#include <sys/mac.h>
|
2002-08-01 17:47:56 +00:00
|
|
|
#include <sys/malloc.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/mbuf.h>
|
1997-02-24 20:30:58 +00:00
|
|
|
#include <sys/sysctl.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/domain.h>
|
|
|
|
#include <sys/protosw.h>
|
2004-02-01 07:56:44 +00:00
|
|
|
#include <sys/uio.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
|
1995-07-29 11:44:31 +00:00
|
|
|
int max_linkhdr;
|
|
|
|
int max_protohdr;
|
|
|
|
int max_hdr;
|
|
|
|
int max_datalen;
|
2003-06-17 02:34:40 +00:00
|
|
|
#ifdef MBUF_STRESS_TEST
|
2003-03-29 05:48:36 +00:00
|
|
|
int m_defragpackets;
|
|
|
|
int m_defragbytes;
|
|
|
|
int m_defraguseless;
|
|
|
|
int m_defragfailure;
|
2003-04-15 02:14:43 +00:00
|
|
|
int m_defragrandomfailures;
|
|
|
|
#endif
|
Big mbuf subsystem diff #1: incorporate mutexes and fix things up somewhat
to accomodate the changes.
Here's a list of things that have changed (I may have left out a few); for a
relatively complete list, see http://people.freebsd.org/~bmilekic/mtx_journal
* Remove old (once useful) mcluster code for MCLBYTES > PAGE_SIZE which
nobody uses anymore. It was great while it lasted, but now we're moving
onto bigger and better things (Approved by: wollman).
* Practically re-wrote the allocation macros in sys/sys/mbuf.h to accomodate
new allocations which grab the necessary lock.
* Make sure that necessary mbstat variables are manipulated with
corresponding atomic() routines.
* Changed the "wait" routines, cleaned it up, made one routine that does
the job.
* Generalized MWAKEUP() macro. Got rid of m_retry and m_retryhdr, as they
are now included in the generalized "wait" routines.
* Sleep routines now use msleep().
* Free lists have locks.
* etc... probably other stuff I'm missing...
Things to look out for and work on later:
* find a better way to (dynamically) adjust EXT_COUNTERS
* move necessity to recurse on a lock from drain routines by providing
lock-free lower-level version of MFREE() (and possibly m_free()?).
* checkout include of mutex.h in sys/sys/mbuf.h - probably violating
general philosophy here.
The code has been reviewed quite a bit, but problems may arise... please,
don't panic! Send me Emails: bmilekic@freebsd.org
Reviewed by: jlemon, cp, alfred, others?
2000-09-30 06:30:39 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* sysctl(8) exported objects
|
|
|
|
*/
|
1999-02-16 10:49:55 +00:00
|
|
|
SYSCTL_DECL(_kern_ipc);
|
1997-02-24 20:30:58 +00:00
|
|
|
SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW,
|
|
|
|
&max_linkhdr, 0, "");
|
|
|
|
SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW,
|
|
|
|
&max_protohdr, 0, "");
|
|
|
|
SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, "");
|
|
|
|
SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW,
|
|
|
|
&max_datalen, 0, "");
|
2003-06-17 02:34:40 +00:00
|
|
|
#ifdef MBUF_STRESS_TEST
|
2003-03-29 05:48:36 +00:00
|
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
|
|
|
|
&m_defragpackets, 0, "");
|
|
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
|
|
|
|
&m_defragbytes, 0, "");
|
|
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
|
|
|
|
&m_defraguseless, 0, "");
|
|
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
|
|
|
|
&m_defragfailure, 0, "");
|
2003-04-15 02:14:43 +00:00
|
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
|
|
|
|
&m_defragrandomfailures, 0, "");
|
|
|
|
#endif
|
1994-05-24 10:09:53 +00:00
|
|
|
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
/*
|
|
|
|
* Malloc-type for external ext_buf ref counts.
|
|
|
|
*/
|
2005-02-10 12:02:37 +00:00
|
|
|
static MALLOC_DEFINE(M_MBUF, "mbextcnt", "mbuf external ref counts");
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate a given length worth of mbufs and/or clusters (whatever fits
|
|
|
|
* best) and return a pointer to the top of the allocated chain. If an
|
|
|
|
* existing mbuf chain is provided, then we will append the new chain
|
|
|
|
* to the existing one but still return the top of the newly allocated
|
|
|
|
* chain.
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
|
|
|
m_getm(struct mbuf *m, int len, int how, short type)
|
|
|
|
{
|
|
|
|
struct mbuf *mb, *top, *cur, *mtail;
|
|
|
|
int num, rem;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
KASSERT(len >= 0, ("m_getm(): len is < 0"));
|
|
|
|
|
|
|
|
/* If m != NULL, we will append to the end of that chain. */
|
|
|
|
if (m != NULL)
|
|
|
|
for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
|
|
|
|
else
|
|
|
|
mtail = NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate how many mbufs+clusters ("packets") we need and how much
|
|
|
|
* leftover there is after that and allocate the first mbuf+cluster
|
|
|
|
* if required.
|
|
|
|
*/
|
|
|
|
num = len / MCLBYTES;
|
|
|
|
rem = len % MCLBYTES;
|
|
|
|
top = cur = NULL;
|
|
|
|
if (num > 0) {
|
|
|
|
if ((top = cur = m_getcl(how, type, 0)) == NULL)
|
|
|
|
goto failed;
|
2004-06-09 14:53:35 +00:00
|
|
|
top->m_len = 0;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
}
|
|
|
|
num--;
|
|
|
|
|
|
|
|
for (i = 0; i < num; i++) {
|
|
|
|
mb = m_getcl(how, type, 0);
|
|
|
|
if (mb == NULL)
|
|
|
|
goto failed;
|
|
|
|
mb->m_len = 0;
|
|
|
|
cur = (cur->m_next = mb);
|
|
|
|
}
|
|
|
|
if (rem > 0) {
|
|
|
|
mb = (rem > MINCLSIZE) ?
|
|
|
|
m_getcl(how, type, 0) : m_get(how, type);
|
|
|
|
if (mb == NULL)
|
|
|
|
goto failed;
|
|
|
|
mb->m_len = 0;
|
|
|
|
if (cur == NULL)
|
|
|
|
top = mb;
|
|
|
|
else
|
|
|
|
cur->m_next = mb;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (mtail != NULL)
|
|
|
|
mtail->m_next = top;
|
|
|
|
return top;
|
|
|
|
failed:
|
|
|
|
if (top != NULL)
|
|
|
|
m_freem(top);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Free an entire chain of mbufs and associated external buffers, if
|
|
|
|
* applicable.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
m_freem(struct mbuf *mb)
|
|
|
|
{
|
|
|
|
|
|
|
|
while (mb != NULL)
|
|
|
|
mb = m_free(mb);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*-
|
|
|
|
* Configure a provided mbuf to refer to the provided external storage
|
|
|
|
* buffer and setup a reference count for said buffer. If the setting
|
|
|
|
* up of the reference count fails, the M_EXT bit will not be set. If
|
|
|
|
* successfull, the M_EXT bit is set in the mbuf's flags.
|
|
|
|
*
|
|
|
|
* Arguments:
|
|
|
|
* mb The existing mbuf to which to attach the provided buffer.
|
|
|
|
* buf The address of the provided external storage buffer.
|
|
|
|
* size The size of the provided buffer.
|
|
|
|
* freef A pointer to a routine that is responsible for freeing the
|
|
|
|
* provided external storage buffer.
|
|
|
|
* args A pointer to an argument structure (of any type) to be passed
|
|
|
|
* to the provided freef routine (may be NULL).
|
|
|
|
* flags Any other flags to be passed to the provided mbuf.
|
|
|
|
* type The type that the external storage buffer should be
|
|
|
|
* labeled with.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* Nothing.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
|
|
|
|
void (*freef)(void *, void *), void *args, int flags, int type)
|
|
|
|
{
|
|
|
|
u_int *ref_cnt = NULL;
|
|
|
|
|
|
|
|
/* XXX Shouldn't be adding EXT_CLUSTER with this API */
|
|
|
|
if (type == EXT_CLUSTER)
|
|
|
|
ref_cnt = (u_int *)uma_find_refcnt(zone_clust,
|
|
|
|
mb->m_ext.ext_buf);
|
|
|
|
else if (type == EXT_EXTREF)
|
2005-03-06 20:09:00 +00:00
|
|
|
ref_cnt = __DEVOLATILE(u_int *, mb->m_ext.ref_cnt);
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
mb->m_ext.ref_cnt = (ref_cnt == NULL) ?
|
|
|
|
malloc(sizeof(u_int), M_MBUF, M_NOWAIT) : (u_int *)ref_cnt;
|
|
|
|
if (mb->m_ext.ref_cnt != NULL) {
|
|
|
|
*(mb->m_ext.ref_cnt) = 1;
|
|
|
|
mb->m_flags |= (M_EXT | flags);
|
|
|
|
mb->m_ext.ext_buf = buf;
|
|
|
|
mb->m_data = mb->m_ext.ext_buf;
|
|
|
|
mb->m_ext.ext_size = size;
|
|
|
|
mb->m_ext.ext_free = freef;
|
|
|
|
mb->m_ext.ext_args = args;
|
|
|
|
mb->m_ext.ext_type = type;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Non-directly-exported function to clean up after mbufs with M_EXT
|
|
|
|
* storage attached to them if the reference count hits 0.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
mb_free_ext(struct mbuf *m)
|
|
|
|
{
|
2004-06-10 00:04:27 +00:00
|
|
|
u_int cnt;
|
2005-02-10 22:23:02 +00:00
|
|
|
int dofree;
|
|
|
|
|
|
|
|
/* Account for lazy ref count assign. */
|
|
|
|
if (m->m_ext.ref_cnt == NULL)
|
|
|
|
dofree = 1;
|
|
|
|
else
|
|
|
|
dofree = 0;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
|
2004-06-10 00:04:27 +00:00
|
|
|
/*
|
|
|
|
* This is tricky. We need to make sure to decrement the
|
|
|
|
* refcount in a safe way but to also clean up if we're the
|
|
|
|
* last reference. This method seems to do it without race.
|
|
|
|
*/
|
2005-02-10 22:23:02 +00:00
|
|
|
while (dofree == 0) {
|
2005-03-06 20:09:00 +00:00
|
|
|
cnt = *(m->m_ext.ref_cnt);
|
2004-06-10 00:04:27 +00:00
|
|
|
if (atomic_cmpset_int(m->m_ext.ref_cnt, cnt, cnt - 1)) {
|
2005-02-10 22:23:02 +00:00
|
|
|
if (cnt == 1)
|
|
|
|
dofree = 1;
|
2004-06-10 00:04:27 +00:00
|
|
|
break;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
}
|
2005-02-10 22:23:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (dofree) {
|
|
|
|
/*
|
|
|
|
* Do the free, should be safe.
|
|
|
|
*/
|
|
|
|
if (m->m_ext.ext_type == EXT_PACKET) {
|
|
|
|
uma_zfree(zone_pack, m);
|
|
|
|
return;
|
|
|
|
} else if (m->m_ext.ext_type == EXT_CLUSTER) {
|
|
|
|
uma_zfree(zone_clust, m->m_ext.ext_buf);
|
|
|
|
m->m_ext.ext_buf = NULL;
|
|
|
|
} else {
|
|
|
|
(*(m->m_ext.ext_free))(m->m_ext.ext_buf,
|
|
|
|
m->m_ext.ext_args);
|
|
|
|
if (m->m_ext.ext_type != EXT_EXTREF) {
|
|
|
|
if (m->m_ext.ref_cnt != NULL)
|
2005-03-06 20:09:00 +00:00
|
|
|
free(__DEVOLATILE(u_int *,
|
|
|
|
m->m_ext.ref_cnt), M_MBUF);
|
2005-02-10 22:23:02 +00:00
|
|
|
m->m_ext.ref_cnt = NULL;
|
|
|
|
}
|
|
|
|
m->m_ext.ext_buf = NULL;
|
|
|
|
}
|
|
|
|
}
|
2004-06-11 18:17:25 +00:00
|
|
|
uma_zfree(zone_mbuf, m);
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
}
|
|
|
|
|
2005-08-29 19:45:39 +00:00
|
|
|
/*
|
|
|
|
* Clean up mbuf (chain) from any tags and packet headers.
|
|
|
|
*/
|
|
|
|
void
|
2005-08-30 20:07:49 +00:00
|
|
|
m_demote(struct mbuf *m0)
|
2005-08-29 19:45:39 +00:00
|
|
|
{
|
|
|
|
struct mbuf *m;
|
|
|
|
|
|
|
|
for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
|
|
|
|
if (m->m_flags & M_PKTHDR) {
|
|
|
|
m_tag_delete_chain(m, NULL);
|
|
|
|
m->m_flags &= ~M_PKTHDR;
|
|
|
|
bzero(&m->m_pkthdr, sizeof(struct pkthdr));
|
|
|
|
}
|
2005-08-30 20:07:49 +00:00
|
|
|
if (m->m_type == MT_HEADER)
|
2005-08-29 19:45:39 +00:00
|
|
|
m->m_type = MT_DATA;
|
2005-08-30 20:07:49 +00:00
|
|
|
if (m != m0 && m->m_nextpkt != NULL)
|
2005-08-29 19:45:39 +00:00
|
|
|
m->m_nextpkt = NULL;
|
|
|
|
m->m_flags = m->m_flags & (M_EXT|M_EOR|M_RDONLY|M_FREELIST);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-08-29 19:58:56 +00:00
|
|
|
/*
|
|
|
|
* Sanity checks on mbuf (chain).
|
|
|
|
* Returns 0 bad, 1 good, panic worse.
|
|
|
|
* sanitize, 0 run M_SANITY_ACTION, 1 garble things so they blow up later.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
m_sanity(struct mbuf *m0, int sanitize)
|
|
|
|
{
|
|
|
|
struct mbuf *m;
|
|
|
|
caddr_t a, b;
|
|
|
|
int pktlen = 0;
|
|
|
|
|
|
|
|
#define M_SANITY_ACTION(s) return (0)
|
|
|
|
/* #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) */
|
|
|
|
|
|
|
|
m = m0;
|
|
|
|
while (m) {
|
|
|
|
/*
|
|
|
|
* Basic pointer checks. If any of these fails then some
|
|
|
|
* unrelated kernel memory before or after us is trashed.
|
|
|
|
* No way to recover from that.
|
|
|
|
*/
|
|
|
|
a = (m->m_flags & M_EXT ? m->m_ext.ext_buf :
|
|
|
|
(m->m_flags & M_PKTHDR ? (caddr_t)(&m->m_pktdat) :
|
|
|
|
(caddr_t)(&m->m_dat)) );
|
|
|
|
b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
|
|
|
|
(m->m_flags & M_PKTHDR ? MHLEN : MLEN)));
|
|
|
|
if ((caddr_t)m->m_data < a)
|
|
|
|
M_SANITY_ACTION("m_data outside mbuf data range left");
|
|
|
|
if ((caddr_t)m->m_data > b)
|
|
|
|
M_SANITY_ACTION("m_data outside mbuf data range right");
|
|
|
|
if ((caddr_t)m->m_data + m->m_len > b)
|
|
|
|
M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
|
|
|
|
if (m->m_flags & M_PKTHDR && m->m_pkthdr.header) {
|
|
|
|
if ((caddr_t)m->m_pkthdr.header < a ||
|
|
|
|
(caddr_t)m->m_pkthdr.header > b)
|
|
|
|
M_SANITY_ACTION("m_pkthdr.header outside mbuf data range");
|
|
|
|
}
|
|
|
|
|
|
|
|
/* m->m_nextpkt may only be set on first mbuf in chain. */
|
|
|
|
if (m != m0 && m->m_nextpkt) {
|
|
|
|
if (sanitize) {
|
|
|
|
m_freem(m->m_nextpkt);
|
|
|
|
m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
|
|
|
|
} else
|
|
|
|
M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
|
|
|
|
}
|
|
|
|
|
|
|
|
/* correct type correlations. */
|
|
|
|
if (m->m_type == MT_HEADER && !(m->m_flags & M_PKTHDR)) {
|
|
|
|
if (sanitize)
|
|
|
|
m->m_type = MT_DATA;
|
|
|
|
else
|
|
|
|
M_SANITY_ACTION("MT_HEADER set but not M_PKTHDR");
|
|
|
|
}
|
|
|
|
|
|
|
|
/* packet length (not mbuf length!) calculation */
|
|
|
|
if (m0->m_flags & M_PKTHDR)
|
|
|
|
pktlen += m->m_len;
|
|
|
|
|
|
|
|
/* m_tags may only be attached to first mbuf in chain. */
|
|
|
|
if (m != m0 && m->m_flags & M_PKTHDR &&
|
|
|
|
!SLIST_EMPTY(&m->m_pkthdr.tags)) {
|
|
|
|
if (sanitize) {
|
|
|
|
m_tag_delete_chain(m, NULL);
|
|
|
|
/* put in 0xDEADC0DE perhaps? */
|
|
|
|
}
|
|
|
|
else
|
|
|
|
M_SANITY_ACTION("m_tags on in-chain mbuf");
|
|
|
|
}
|
|
|
|
|
|
|
|
/* M_PKTHDR may only be set on first mbuf in chain */
|
|
|
|
if (m != m0 && m->m_flags & M_PKTHDR) {
|
|
|
|
if (sanitize) {
|
|
|
|
bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
|
|
|
|
m->m_flags &= ~M_PKTHDR;
|
|
|
|
/* put in 0xDEADCODE and leave hdr flag in */
|
|
|
|
} else
|
|
|
|
M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
|
|
|
|
}
|
|
|
|
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
if (pktlen && pktlen != m0->m_pkthdr.len) {
|
|
|
|
if (sanitize)
|
|
|
|
m0->m_pkthdr.len = 0;
|
|
|
|
else
|
|
|
|
M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
|
|
|
|
}
|
|
|
|
#undef M_SANITY_ACTION
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2002-07-30 18:28:58 +00:00
|
|
|
/*
|
2002-12-30 20:22:40 +00:00
|
|
|
* "Move" mbuf pkthdr from "from" to "to".
|
2002-07-30 18:28:58 +00:00
|
|
|
* "from" must have M_PKTHDR set, and "to" must be empty.
|
|
|
|
*/
|
|
|
|
void
|
2002-12-30 20:22:40 +00:00
|
|
|
m_move_pkthdr(struct mbuf *to, struct mbuf *from)
|
2002-07-30 18:28:58 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
#if 0
|
2002-12-30 20:22:40 +00:00
|
|
|
/* see below for why these are not enabled */
|
2003-04-08 14:25:47 +00:00
|
|
|
M_ASSERTPKTHDR(to);
|
Move MAC label storage for mbufs into m_tags from the m_pkthdr structure,
returning some additional room in the first mbuf in a chain, and
avoiding feature-specific contents in the mbuf header. To do this:
- Modify mbuf_to_label() to extract the tag, returning NULL if not
found.
- Introduce mac_init_mbuf_tag() which does most of the work
mac_init_mbuf() used to do, except on an m_tag rather than an
mbuf.
- Scale back mac_init_mbuf() to perform m_tag allocation and invoke
mac_init_mbuf_tag().
- Replace mac_destroy_mbuf() with mac_destroy_mbuf_tag(), since
m_tag's are now GC'd deep in the m_tag/mbuf code rather than
at a higher level when mbufs are directly free()'d.
- Add mac_copy_mbuf_tag() to support m_copy_pkthdr() and related
notions.
- Generally change all references to mbuf labels so that they use
mbuf_to_label() rather than &mbuf->m_pkthdr.label. This
required no changes in the MAC policies (yay!).
- Tweak mbuf release routines to not call mac_destroy_mbuf(),
tag destruction takes care of it for us now.
- Remove MAC magic from m_copy_pkthdr() and m_move_pkthdr() --
the existing m_tag support does all this for us. Note that
we can no longer just zero the m_tag list on the target mbuf,
rather, we have to delete the chain because m_tag's will
already be hung off freshly allocated mbuf's.
- Tweak m_tag copying routines so that if we're copying a MAC
m_tag, we don't do a binary copy, rather, we initialize the
new storage and do a deep copy of the label.
- Remove use of MAC_FLAG_INITIALIZED in a few bizarre places
having to do with mbuf header copies previously.
- When an mbuf is copied in ip_input(), we no longer need to
explicitly copy the label because it will get handled by the
m_tag code now.
- No longer any weird handling of MAC labels in if_loop.c during
header copies.
- Add MPC_LOADTIME_FLAG_LABELMBUFS flag to Biba, MLS, mac_test.
In mac_test, handle the label==NULL case, since it can be
dynamically loaded.
In order to improve performance with this change, introduce the notion
of "lazy MAC label allocation" -- only allocate m_tag storage for MAC
labels if we're running with a policy that uses MAC labels on mbufs.
Policies declare this intent by setting the MPC_LOADTIME_FLAG_LABELMBUFS
flag in their load-time flags field during declaration. Note: this
opens up the possibility of post-boot policy modules getting back NULL
slot entries even though they have policy invariants of non-NULL slot
entries, as the policy might have been loaded after the mbuf was
allocated, leaving the mbuf without label storage. Policies that cannot
handle this case must be declared as NOTLATE, or must be modified.
- mac_labelmbufs holds the current cumulative status as to whether
any policies require mbuf labeling or not. This is updated whenever
the active policy set changes by the function mac_policy_updateflags().
The function iterates the list and checks whether any have the
flag set. Write access to this variable is protected by the policy
list; read access is currently not protected for performance reasons.
This might change if it causes problems.
- Add MAC_POLICY_LIST_ASSERT_EXCLUSIVE() to permit the flags update
function to assert appropriate locks.
- This makes allocation in mac_init_mbuf() conditional on the flag.
Reviewed by: sam
Obtained from: TrustedBSD Project
Sponsored by: DARPA, Network Associates Laboratories
2003-04-14 20:39:06 +00:00
|
|
|
/* Note: with MAC, this may not be a good assertion. */
|
2002-12-30 20:22:40 +00:00
|
|
|
KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
|
|
|
|
("m_move_pkthdr: to has tags"));
|
2002-07-31 01:51:34 +00:00
|
|
|
#endif
|
|
|
|
#ifdef MAC
|
Move MAC label storage for mbufs into m_tags from the m_pkthdr structure,
returning some additional room in the first mbuf in a chain, and
avoiding feature-specific contents in the mbuf header. To do this:
- Modify mbuf_to_label() to extract the tag, returning NULL if not
found.
- Introduce mac_init_mbuf_tag() which does most of the work
mac_init_mbuf() used to do, except on an m_tag rather than an
mbuf.
- Scale back mac_init_mbuf() to perform m_tag allocation and invoke
mac_init_mbuf_tag().
- Replace mac_destroy_mbuf() with mac_destroy_mbuf_tag(), since
m_tag's are now GC'd deep in the m_tag/mbuf code rather than
at a higher level when mbufs are directly free()'d.
- Add mac_copy_mbuf_tag() to support m_copy_pkthdr() and related
notions.
- Generally change all references to mbuf labels so that they use
mbuf_to_label() rather than &mbuf->m_pkthdr.label. This
required no changes in the MAC policies (yay!).
- Tweak mbuf release routines to not call mac_destroy_mbuf(),
tag destruction takes care of it for us now.
- Remove MAC magic from m_copy_pkthdr() and m_move_pkthdr() --
the existing m_tag support does all this for us. Note that
we can no longer just zero the m_tag list on the target mbuf,
rather, we have to delete the chain because m_tag's will
already be hung off freshly allocated mbuf's.
- Tweak m_tag copying routines so that if we're copying a MAC
m_tag, we don't do a binary copy, rather, we initialize the
new storage and do a deep copy of the label.
- Remove use of MAC_FLAG_INITIALIZED in a few bizarre places
having to do with mbuf header copies previously.
- When an mbuf is copied in ip_input(), we no longer need to
explicitly copy the label because it will get handled by the
m_tag code now.
- No longer any weird handling of MAC labels in if_loop.c during
header copies.
- Add MPC_LOADTIME_FLAG_LABELMBUFS flag to Biba, MLS, mac_test.
In mac_test, handle the label==NULL case, since it can be
dynamically loaded.
In order to improve performance with this change, introduce the notion
of "lazy MAC label allocation" -- only allocate m_tag storage for MAC
labels if we're running with a policy that uses MAC labels on mbufs.
Policies declare this intent by setting the MPC_LOADTIME_FLAG_LABELMBUFS
flag in their load-time flags field during declaration. Note: this
opens up the possibility of post-boot policy modules getting back NULL
slot entries even though they have policy invariants of non-NULL slot
entries, as the policy might have been loaded after the mbuf was
allocated, leaving the mbuf without label storage. Policies that cannot
handle this case must be declared as NOTLATE, or must be modified.
- mac_labelmbufs holds the current cumulative status as to whether
any policies require mbuf labeling or not. This is updated whenever
the active policy set changes by the function mac_policy_updateflags().
The function iterates the list and checks whether any have the
flag set. Write access to this variable is protected by the policy
list; read access is currently not protected for performance reasons.
This might change if it causes problems.
- Add MAC_POLICY_LIST_ASSERT_EXCLUSIVE() to permit the flags update
function to assert appropriate locks.
- This makes allocation in mac_init_mbuf() conditional on the flag.
Reviewed by: sam
Obtained from: TrustedBSD Project
Sponsored by: DARPA, Network Associates Laboratories
2003-04-14 20:39:06 +00:00
|
|
|
/*
|
|
|
|
* XXXMAC: It could be this should also occur for non-MAC?
|
|
|
|
*/
|
2002-07-31 01:51:34 +00:00
|
|
|
if (to->m_flags & M_PKTHDR)
|
Move MAC label storage for mbufs into m_tags from the m_pkthdr structure,
returning some additional room in the first mbuf in a chain, and
avoiding feature-specific contents in the mbuf header. To do this:
- Modify mbuf_to_label() to extract the tag, returning NULL if not
found.
- Introduce mac_init_mbuf_tag() which does most of the work
mac_init_mbuf() used to do, except on an m_tag rather than an
mbuf.
- Scale back mac_init_mbuf() to perform m_tag allocation and invoke
mac_init_mbuf_tag().
- Replace mac_destroy_mbuf() with mac_destroy_mbuf_tag(), since
m_tag's are now GC'd deep in the m_tag/mbuf code rather than
at a higher level when mbufs are directly free()'d.
- Add mac_copy_mbuf_tag() to support m_copy_pkthdr() and related
notions.
- Generally change all references to mbuf labels so that they use
mbuf_to_label() rather than &mbuf->m_pkthdr.label. This
required no changes in the MAC policies (yay!).
- Tweak mbuf release routines to not call mac_destroy_mbuf(),
tag destruction takes care of it for us now.
- Remove MAC magic from m_copy_pkthdr() and m_move_pkthdr() --
the existing m_tag support does all this for us. Note that
we can no longer just zero the m_tag list on the target mbuf,
rather, we have to delete the chain because m_tag's will
already be hung off freshly allocated mbuf's.
- Tweak m_tag copying routines so that if we're copying a MAC
m_tag, we don't do a binary copy, rather, we initialize the
new storage and do a deep copy of the label.
- Remove use of MAC_FLAG_INITIALIZED in a few bizarre places
having to do with mbuf header copies previously.
- When an mbuf is copied in ip_input(), we no longer need to
explicitly copy the label because it will get handled by the
m_tag code now.
- No longer any weird handling of MAC labels in if_loop.c during
header copies.
- Add MPC_LOADTIME_FLAG_LABELMBUFS flag to Biba, MLS, mac_test.
In mac_test, handle the label==NULL case, since it can be
dynamically loaded.
In order to improve performance with this change, introduce the notion
of "lazy MAC label allocation" -- only allocate m_tag storage for MAC
labels if we're running with a policy that uses MAC labels on mbufs.
Policies declare this intent by setting the MPC_LOADTIME_FLAG_LABELMBUFS
flag in their load-time flags field during declaration. Note: this
opens up the possibility of post-boot policy modules getting back NULL
slot entries even though they have policy invariants of non-NULL slot
entries, as the policy might have been loaded after the mbuf was
allocated, leaving the mbuf without label storage. Policies that cannot
handle this case must be declared as NOTLATE, or must be modified.
- mac_labelmbufs holds the current cumulative status as to whether
any policies require mbuf labeling or not. This is updated whenever
the active policy set changes by the function mac_policy_updateflags().
The function iterates the list and checks whether any have the
flag set. Write access to this variable is protected by the policy
list; read access is currently not protected for performance reasons.
This might change if it causes problems.
- Add MAC_POLICY_LIST_ASSERT_EXCLUSIVE() to permit the flags update
function to assert appropriate locks.
- This makes allocation in mac_init_mbuf() conditional on the flag.
Reviewed by: sam
Obtained from: TrustedBSD Project
Sponsored by: DARPA, Network Associates Laboratories
2003-04-14 20:39:06 +00:00
|
|
|
m_tag_delete_chain(to, NULL);
|
2002-07-30 18:28:58 +00:00
|
|
|
#endif
|
2005-03-08 17:52:01 +00:00
|
|
|
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
|
|
|
|
if ((to->m_flags & M_EXT) == 0)
|
|
|
|
to->m_data = to->m_pktdat;
|
2002-12-30 20:22:40 +00:00
|
|
|
to->m_pkthdr = from->m_pkthdr; /* especially tags */
|
|
|
|
SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
|
|
|
|
from->m_flags &= ~M_PKTHDR;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Duplicate "from"'s mbuf pkthdr in "to".
|
|
|
|
* "from" must have M_PKTHDR set, and "to" must be empty.
|
|
|
|
* In particular, this does a deep copy of the packet tags.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
|
|
|
|
{
|
|
|
|
|
|
|
|
#if 0
|
|
|
|
/*
|
|
|
|
* The mbuf allocator only initializes the pkthdr
|
|
|
|
* when the mbuf is allocated with MGETHDR. Many users
|
|
|
|
* (e.g. m_copy*, m_prepend) use MGET and then
|
|
|
|
* smash the pkthdr as needed causing these
|
|
|
|
* assertions to trip. For now just disable them.
|
|
|
|
*/
|
2003-04-08 14:25:47 +00:00
|
|
|
M_ASSERTPKTHDR(to);
|
Move MAC label storage for mbufs into m_tags from the m_pkthdr structure,
returning some additional room in the first mbuf in a chain, and
avoiding feature-specific contents in the mbuf header. To do this:
- Modify mbuf_to_label() to extract the tag, returning NULL if not
found.
- Introduce mac_init_mbuf_tag() which does most of the work
mac_init_mbuf() used to do, except on an m_tag rather than an
mbuf.
- Scale back mac_init_mbuf() to perform m_tag allocation and invoke
mac_init_mbuf_tag().
- Replace mac_destroy_mbuf() with mac_destroy_mbuf_tag(), since
m_tag's are now GC'd deep in the m_tag/mbuf code rather than
at a higher level when mbufs are directly free()'d.
- Add mac_copy_mbuf_tag() to support m_copy_pkthdr() and related
notions.
- Generally change all references to mbuf labels so that they use
mbuf_to_label() rather than &mbuf->m_pkthdr.label. This
required no changes in the MAC policies (yay!).
- Tweak mbuf release routines to not call mac_destroy_mbuf(),
tag destruction takes care of it for us now.
- Remove MAC magic from m_copy_pkthdr() and m_move_pkthdr() --
the existing m_tag support does all this for us. Note that
we can no longer just zero the m_tag list on the target mbuf,
rather, we have to delete the chain because m_tag's will
already be hung off freshly allocated mbuf's.
- Tweak m_tag copying routines so that if we're copying a MAC
m_tag, we don't do a binary copy, rather, we initialize the
new storage and do a deep copy of the label.
- Remove use of MAC_FLAG_INITIALIZED in a few bizarre places
having to do with mbuf header copies previously.
- When an mbuf is copied in ip_input(), we no longer need to
explicitly copy the label because it will get handled by the
m_tag code now.
- No longer any weird handling of MAC labels in if_loop.c during
header copies.
- Add MPC_LOADTIME_FLAG_LABELMBUFS flag to Biba, MLS, mac_test.
In mac_test, handle the label==NULL case, since it can be
dynamically loaded.
In order to improve performance with this change, introduce the notion
of "lazy MAC label allocation" -- only allocate m_tag storage for MAC
labels if we're running with a policy that uses MAC labels on mbufs.
Policies declare this intent by setting the MPC_LOADTIME_FLAG_LABELMBUFS
flag in their load-time flags field during declaration. Note: this
opens up the possibility of post-boot policy modules getting back NULL
slot entries even though they have policy invariants of non-NULL slot
entries, as the policy might have been loaded after the mbuf was
allocated, leaving the mbuf without label storage. Policies that cannot
handle this case must be declared as NOTLATE, or must be modified.
- mac_labelmbufs holds the current cumulative status as to whether
any policies require mbuf labeling or not. This is updated whenever
the active policy set changes by the function mac_policy_updateflags().
The function iterates the list and checks whether any have the
flag set. Write access to this variable is protected by the policy
list; read access is currently not protected for performance reasons.
This might change if it causes problems.
- Add MAC_POLICY_LIST_ASSERT_EXCLUSIVE() to permit the flags update
function to assert appropriate locks.
- This makes allocation in mac_init_mbuf() conditional on the flag.
Reviewed by: sam
Obtained from: TrustedBSD Project
Sponsored by: DARPA, Network Associates Laboratories
2003-04-14 20:39:06 +00:00
|
|
|
/* Note: with MAC, this may not be a good assertion. */
|
2002-12-30 20:22:40 +00:00
|
|
|
KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
|
|
|
|
#endif
|
2004-07-21 07:12:24 +00:00
|
|
|
MBUF_CHECKSLEEP(how);
|
2002-12-30 20:22:40 +00:00
|
|
|
#ifdef MAC
|
|
|
|
if (to->m_flags & M_PKTHDR)
|
Move MAC label storage for mbufs into m_tags from the m_pkthdr structure,
returning some additional room in the first mbuf in a chain, and
avoiding feature-specific contents in the mbuf header. To do this:
- Modify mbuf_to_label() to extract the tag, returning NULL if not
found.
- Introduce mac_init_mbuf_tag() which does most of the work
mac_init_mbuf() used to do, except on an m_tag rather than an
mbuf.
- Scale back mac_init_mbuf() to perform m_tag allocation and invoke
mac_init_mbuf_tag().
- Replace mac_destroy_mbuf() with mac_destroy_mbuf_tag(), since
m_tag's are now GC'd deep in the m_tag/mbuf code rather than
at a higher level when mbufs are directly free()'d.
- Add mac_copy_mbuf_tag() to support m_copy_pkthdr() and related
notions.
- Generally change all references to mbuf labels so that they use
mbuf_to_label() rather than &mbuf->m_pkthdr.label. This
required no changes in the MAC policies (yay!).
- Tweak mbuf release routines to not call mac_destroy_mbuf(),
tag destruction takes care of it for us now.
- Remove MAC magic from m_copy_pkthdr() and m_move_pkthdr() --
the existing m_tag support does all this for us. Note that
we can no longer just zero the m_tag list on the target mbuf,
rather, we have to delete the chain because m_tag's will
already be hung off freshly allocated mbuf's.
- Tweak m_tag copying routines so that if we're copying a MAC
m_tag, we don't do a binary copy, rather, we initialize the
new storage and do a deep copy of the label.
- Remove use of MAC_FLAG_INITIALIZED in a few bizarre places
having to do with mbuf header copies previously.
- When an mbuf is copied in ip_input(), we no longer need to
explicitly copy the label because it will get handled by the
m_tag code now.
- No longer any weird handling of MAC labels in if_loop.c during
header copies.
- Add MPC_LOADTIME_FLAG_LABELMBUFS flag to Biba, MLS, mac_test.
In mac_test, handle the label==NULL case, since it can be
dynamically loaded.
In order to improve performance with this change, introduce the notion
of "lazy MAC label allocation" -- only allocate m_tag storage for MAC
labels if we're running with a policy that uses MAC labels on mbufs.
Policies declare this intent by setting the MPC_LOADTIME_FLAG_LABELMBUFS
flag in their load-time flags field during declaration. Note: this
opens up the possibility of post-boot policy modules getting back NULL
slot entries even though they have policy invariants of non-NULL slot
entries, as the policy might have been loaded after the mbuf was
allocated, leaving the mbuf without label storage. Policies that cannot
handle this case must be declared as NOTLATE, or must be modified.
- mac_labelmbufs holds the current cumulative status as to whether
any policies require mbuf labeling or not. This is updated whenever
the active policy set changes by the function mac_policy_updateflags().
The function iterates the list and checks whether any have the
flag set. Write access to this variable is protected by the policy
list; read access is currently not protected for performance reasons.
This might change if it causes problems.
- Add MAC_POLICY_LIST_ASSERT_EXCLUSIVE() to permit the flags update
function to assert appropriate locks.
- This makes allocation in mac_init_mbuf() conditional on the flag.
Reviewed by: sam
Obtained from: TrustedBSD Project
Sponsored by: DARPA, Network Associates Laboratories
2003-04-14 20:39:06 +00:00
|
|
|
m_tag_delete_chain(to, NULL);
|
2002-12-30 20:22:40 +00:00
|
|
|
#endif
|
2003-03-28 05:57:48 +00:00
|
|
|
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
|
|
|
|
if ((to->m_flags & M_EXT) == 0)
|
|
|
|
to->m_data = to->m_pktdat;
|
2002-07-30 18:28:58 +00:00
|
|
|
to->m_pkthdr = from->m_pkthdr;
|
2002-12-30 20:22:40 +00:00
|
|
|
SLIST_INIT(&to->m_pkthdr.tags);
|
2003-04-14 16:04:10 +00:00
|
|
|
return (m_tag_copy_chain(to, from, MBTOM(how)));
|
2002-07-30 18:28:58 +00:00
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Lesser-used path for M_PREPEND:
|
|
|
|
* allocate new mbuf to prepend to chain,
|
|
|
|
* copy junk along.
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-02-11 05:02:06 +00:00
|
|
|
m_prepend(struct mbuf *m, int len, int how)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
struct mbuf *mn;
|
|
|
|
|
2003-07-19 06:03:48 +00:00
|
|
|
if (m->m_flags & M_PKTHDR)
|
|
|
|
MGETHDR(mn, how, m->m_type);
|
|
|
|
else
|
|
|
|
MGET(mn, how, m->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (mn == NULL) {
|
1994-05-24 10:09:53 +00:00
|
|
|
m_freem(m);
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
Move MAC label storage for mbufs into m_tags from the m_pkthdr structure,
returning some additional room in the first mbuf in a chain, and
avoiding feature-specific contents in the mbuf header. To do this:
- Modify mbuf_to_label() to extract the tag, returning NULL if not
found.
- Introduce mac_init_mbuf_tag() which does most of the work
mac_init_mbuf() used to do, except on an m_tag rather than an
mbuf.
- Scale back mac_init_mbuf() to perform m_tag allocation and invoke
mac_init_mbuf_tag().
- Replace mac_destroy_mbuf() with mac_destroy_mbuf_tag(), since
m_tag's are now GC'd deep in the m_tag/mbuf code rather than
at a higher level when mbufs are directly free()'d.
- Add mac_copy_mbuf_tag() to support m_copy_pkthdr() and related
notions.
- Generally change all references to mbuf labels so that they use
mbuf_to_label() rather than &mbuf->m_pkthdr.label. This
required no changes in the MAC policies (yay!).
- Tweak mbuf release routines to not call mac_destroy_mbuf(),
tag destruction takes care of it for us now.
- Remove MAC magic from m_copy_pkthdr() and m_move_pkthdr() --
the existing m_tag support does all this for us. Note that
we can no longer just zero the m_tag list on the target mbuf,
rather, we have to delete the chain because m_tag's will
already be hung off freshly allocated mbuf's.
- Tweak m_tag copying routines so that if we're copying a MAC
m_tag, we don't do a binary copy, rather, we initialize the
new storage and do a deep copy of the label.
- Remove use of MAC_FLAG_INITIALIZED in a few bizarre places
having to do with mbuf header copies previously.
- When an mbuf is copied in ip_input(), we no longer need to
explicitly copy the label because it will get handled by the
m_tag code now.
- No longer any weird handling of MAC labels in if_loop.c during
header copies.
- Add MPC_LOADTIME_FLAG_LABELMBUFS flag to Biba, MLS, mac_test.
In mac_test, handle the label==NULL case, since it can be
dynamically loaded.
In order to improve performance with this change, introduce the notion
of "lazy MAC label allocation" -- only allocate m_tag storage for MAC
labels if we're running with a policy that uses MAC labels on mbufs.
Policies declare this intent by setting the MPC_LOADTIME_FLAG_LABELMBUFS
flag in their load-time flags field during declaration. Note: this
opens up the possibility of post-boot policy modules getting back NULL
slot entries even though they have policy invariants of non-NULL slot
entries, as the policy might have been loaded after the mbuf was
allocated, leaving the mbuf without label storage. Policies that cannot
handle this case must be declared as NOTLATE, or must be modified.
- mac_labelmbufs holds the current cumulative status as to whether
any policies require mbuf labeling or not. This is updated whenever
the active policy set changes by the function mac_policy_updateflags().
The function iterates the list and checks whether any have the
flag set. Write access to this variable is protected by the policy
list; read access is currently not protected for performance reasons.
This might change if it causes problems.
- Add MAC_POLICY_LIST_ASSERT_EXCLUSIVE() to permit the flags update
function to assert appropriate locks.
- This makes allocation in mac_init_mbuf() conditional on the flag.
Reviewed by: sam
Obtained from: TrustedBSD Project
Sponsored by: DARPA, Network Associates Laboratories
2003-04-14 20:39:06 +00:00
|
|
|
if (m->m_flags & M_PKTHDR)
|
2002-12-30 20:22:40 +00:00
|
|
|
M_MOVE_PKTHDR(mn, m);
|
1994-05-24 10:09:53 +00:00
|
|
|
mn->m_next = m;
|
|
|
|
m = mn;
|
|
|
|
if (len < MHLEN)
|
|
|
|
MH_ALIGN(m, len);
|
|
|
|
m->m_len = len;
|
|
|
|
return (m);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make a copy of an mbuf chain starting "off0" bytes from the beginning,
|
|
|
|
* continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
|
2003-02-19 05:47:46 +00:00
|
|
|
* The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller.
|
1999-12-01 22:31:32 +00:00
|
|
|
* Note that the copy is read-only, because clusters are not copied,
|
|
|
|
* only their reference counts are incremented.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-02-11 05:02:06 +00:00
|
|
|
m_copym(struct mbuf *m, int off0, int len, int wait)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2001-02-11 05:02:06 +00:00
|
|
|
struct mbuf *n, **np;
|
|
|
|
int off = off0;
|
1994-05-24 10:09:53 +00:00
|
|
|
struct mbuf *top;
|
|
|
|
int copyhdr = 0;
|
|
|
|
|
1999-10-13 09:55:42 +00:00
|
|
|
KASSERT(off >= 0, ("m_copym, negative off %d", off));
|
|
|
|
KASSERT(len >= 0, ("m_copym, negative len %d", len));
|
2004-07-21 07:12:24 +00:00
|
|
|
MBUF_CHECKSLEEP(wait);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (off == 0 && m->m_flags & M_PKTHDR)
|
|
|
|
copyhdr = 1;
|
|
|
|
while (off > 0) {
|
1999-10-13 09:55:42 +00:00
|
|
|
KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
|
1994-05-24 10:09:53 +00:00
|
|
|
if (off < m->m_len)
|
|
|
|
break;
|
|
|
|
off -= m->m_len;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
np = ⊤
|
|
|
|
top = 0;
|
|
|
|
while (len > 0) {
|
2001-02-11 05:02:06 +00:00
|
|
|
if (m == NULL) {
|
1999-10-13 09:55:42 +00:00
|
|
|
KASSERT(len == M_COPYALL,
|
|
|
|
("m_copym, length > size of mbuf chain"));
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
}
|
2003-07-19 06:03:48 +00:00
|
|
|
if (copyhdr)
|
|
|
|
MGETHDR(n, wait, m->m_type);
|
|
|
|
else
|
|
|
|
MGET(n, wait, m->m_type);
|
1994-05-24 10:09:53 +00:00
|
|
|
*np = n;
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n == NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto nospace;
|
|
|
|
if (copyhdr) {
|
2002-12-30 20:22:40 +00:00
|
|
|
if (!m_dup_pkthdr(n, m, wait))
|
|
|
|
goto nospace;
|
1994-05-24 10:09:53 +00:00
|
|
|
if (len == M_COPYALL)
|
|
|
|
n->m_pkthdr.len -= off0;
|
|
|
|
else
|
|
|
|
n->m_pkthdr.len = len;
|
|
|
|
copyhdr = 0;
|
|
|
|
}
|
|
|
|
n->m_len = min(len, m->m_len - off);
|
|
|
|
if (m->m_flags & M_EXT) {
|
|
|
|
n->m_data = m->m_data + off;
|
|
|
|
n->m_ext = m->m_ext;
|
|
|
|
n->m_flags |= M_EXT;
|
2000-08-19 08:32:59 +00:00
|
|
|
MEXT_ADD_REF(m);
|
2005-02-10 22:23:02 +00:00
|
|
|
n->m_ext.ref_cnt = m->m_ext.ref_cnt;
|
1994-05-24 10:09:53 +00:00
|
|
|
} else
|
|
|
|
bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
|
2002-09-18 22:33:52 +00:00
|
|
|
(u_int)n->m_len);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (len != M_COPYALL)
|
|
|
|
len -= n->m_len;
|
|
|
|
off = 0;
|
|
|
|
m = m->m_next;
|
|
|
|
np = &n->m_next;
|
|
|
|
}
|
2001-06-22 06:35:32 +00:00
|
|
|
if (top == NULL)
|
|
|
|
mbstat.m_mcfail++; /* XXX: No consistency. */
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
return (top);
|
|
|
|
nospace:
|
|
|
|
m_freem(top);
|
2001-06-22 06:35:32 +00:00
|
|
|
mbstat.m_mcfail++; /* XXX: No consistency. */
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
2005-08-29 20:15:33 +00:00
|
|
|
/*
|
|
|
|
* Returns mbuf chain with new head for the prepending case.
|
|
|
|
* Copies from mbuf (chain) n from off for len to mbuf (chain) m
|
|
|
|
* either prepending or appending the data.
|
|
|
|
* The resulting mbuf (chain) m is fully writeable.
|
|
|
|
* m is destination (is made writeable)
|
|
|
|
* n is source, off is offset in source, len is len from offset
|
|
|
|
* dir, 0 append, 1 prepend
|
|
|
|
* how, wait or nowait
|
|
|
|
*/
|
|
|
|
|
|
|
|
static int
|
|
|
|
m_bcopyxxx(void *s, void *t, u_int len)
|
|
|
|
{
|
|
|
|
bcopy(s, t, (size_t)len);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct mbuf *
|
|
|
|
m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
|
|
|
|
int prep, int how)
|
|
|
|
{
|
|
|
|
struct mbuf *mm, *x, *z;
|
|
|
|
caddr_t p;
|
|
|
|
int i, mlen, nlen = 0;
|
|
|
|
caddr_t buf[MLEN];
|
|
|
|
|
|
|
|
KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
|
|
|
|
KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
|
|
|
|
KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
|
|
|
|
KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
|
|
|
|
|
|
|
|
/* Make sure environment is sane. */
|
|
|
|
for (z = m; z != NULL; z = z->m_next) {
|
|
|
|
mlen += z->m_len;
|
|
|
|
if (!M_WRITABLE(z)) {
|
|
|
|
/* Make clusters writeable. */
|
|
|
|
if (z->m_flags & M_RDONLY)
|
|
|
|
return NULL; /* Can't handle ext ref. */
|
|
|
|
x = m_getcl(how, MT_DATA, 0);
|
|
|
|
if (!x)
|
|
|
|
return NULL;
|
|
|
|
bcopy(z->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
|
|
|
|
p = x->m_ext.ext_buf + (z->m_data - z->m_ext.ext_buf);
|
|
|
|
MEXT_REM_REF(z); /* XXX */
|
|
|
|
z->m_data = p;
|
|
|
|
x->m_flags &= ~M_EXT;
|
|
|
|
(void)m_free(x);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
mm = prep ? m : z;
|
|
|
|
for (z = n; z != NULL; z = z->m_next)
|
|
|
|
nlen += z->m_len;
|
|
|
|
if (len == M_COPYALL)
|
|
|
|
len = nlen - off;
|
|
|
|
if (off + len > nlen || len < 1)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Append/prepend the data. Allocating mbufs as necessary.
|
|
|
|
*/
|
|
|
|
/* Shortcut if enough free space in first/last mbuf. */
|
|
|
|
if (!prep && M_TRAILINGSPACE(mm) >= len) {
|
|
|
|
m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
|
|
|
|
mm->m_len);
|
|
|
|
mm->m_len += len;
|
|
|
|
mm->m_pkthdr.len += len;
|
|
|
|
return m;
|
|
|
|
}
|
|
|
|
if (prep && M_LEADINGSPACE(mm) >= len) {
|
|
|
|
mm->m_data = mtod(mm, caddr_t) - len;
|
|
|
|
m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
|
|
|
|
mm->m_len += len;
|
|
|
|
mm->m_pkthdr.len += len;
|
|
|
|
return mm;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Expand first/last mbuf to cluster if possible. */
|
|
|
|
if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
|
|
|
|
bcopy(mm->m_data, &buf, mm->m_len);
|
|
|
|
m_clget(mm, how);
|
|
|
|
if (!(mm->m_flags & M_EXT))
|
|
|
|
return NULL;
|
|
|
|
bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
|
|
|
|
mm->m_data = mm->m_ext.ext_buf;
|
|
|
|
mm->m_pkthdr.header = NULL;
|
|
|
|
}
|
|
|
|
if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
|
|
|
|
bcopy(mm->m_data, &buf, mm->m_len);
|
|
|
|
m_clget(mm, how);
|
|
|
|
if (!(mm->m_flags & M_EXT))
|
|
|
|
return NULL;
|
|
|
|
bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
|
|
|
|
mm->m_ext.ext_size - mm->m_len, mm->m_len);
|
|
|
|
mm->m_data = (caddr_t)mm->m_ext.ext_buf +
|
|
|
|
mm->m_ext.ext_size - mm->m_len;
|
|
|
|
mm->m_pkthdr.header = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Append/prepend as many mbuf (clusters) as necessary to fit len. */
|
|
|
|
if (!prep && len > M_TRAILINGSPACE(mm)) {
|
|
|
|
if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
if (prep && len > M_LEADINGSPACE(mm)) {
|
|
|
|
if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
|
|
|
|
return NULL;
|
|
|
|
i = 0;
|
|
|
|
for (x = z; x != NULL; x = x->m_next) {
|
|
|
|
i += x->m_flags & M_EXT ? x->m_ext.ext_size :
|
|
|
|
(x->m_flags & M_PKTHDR ? MHLEN : MLEN);
|
|
|
|
if (!x->m_next)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
z->m_data += i - len;
|
|
|
|
m_move_pkthdr(mm, z);
|
|
|
|
x->m_next = mm;
|
|
|
|
mm = z;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Seek to start position in source mbuf. Optimization for long chains. */
|
|
|
|
while (off > 0) {
|
|
|
|
if (off < n->m_len)
|
|
|
|
break;
|
|
|
|
off -= n->m_len;
|
|
|
|
n = n->m_next;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Copy data into target mbuf. */
|
|
|
|
z = mm;
|
|
|
|
while (len > 0) {
|
|
|
|
KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
|
|
|
|
i = M_TRAILINGSPACE(z);
|
|
|
|
m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
|
|
|
|
z->m_len += i;
|
|
|
|
/* fixup pkthdr.len if necessary */
|
|
|
|
if ((prep ? mm : m)->m_flags & M_PKTHDR)
|
|
|
|
(prep ? mm : m)->m_pkthdr.len += i;
|
|
|
|
off += i;
|
|
|
|
len -= i;
|
|
|
|
z = z->m_next;
|
|
|
|
}
|
|
|
|
return (prep ? mm : m);
|
|
|
|
}
|
|
|
|
|
1996-05-08 19:38:27 +00:00
|
|
|
/*
|
|
|
|
* Copy an entire packet, including header (which must be present).
|
|
|
|
* An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
|
1999-12-01 22:31:32 +00:00
|
|
|
* Note that the copy is read-only, because clusters are not copied,
|
|
|
|
* only their reference counts are incremented.
|
2001-02-20 08:23:41 +00:00
|
|
|
* Preserve alignment of the first mbuf so if the creator has left
|
|
|
|
* some room at the beginning (e.g. for inserting protocol headers)
|
|
|
|
* the copies still have the room available.
|
1996-05-08 19:38:27 +00:00
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-02-11 05:02:06 +00:00
|
|
|
m_copypacket(struct mbuf *m, int how)
|
1996-05-08 19:38:27 +00:00
|
|
|
{
|
|
|
|
struct mbuf *top, *n, *o;
|
|
|
|
|
2004-07-21 07:12:24 +00:00
|
|
|
MBUF_CHECKSLEEP(how);
|
1996-05-08 19:38:27 +00:00
|
|
|
MGET(n, how, m->m_type);
|
|
|
|
top = n;
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n == NULL)
|
1996-05-08 19:38:27 +00:00
|
|
|
goto nospace;
|
|
|
|
|
2002-12-30 20:22:40 +00:00
|
|
|
if (!m_dup_pkthdr(n, m, how))
|
|
|
|
goto nospace;
|
1996-05-08 19:38:27 +00:00
|
|
|
n->m_len = m->m_len;
|
|
|
|
if (m->m_flags & M_EXT) {
|
|
|
|
n->m_data = m->m_data;
|
|
|
|
n->m_ext = m->m_ext;
|
|
|
|
n->m_flags |= M_EXT;
|
2000-08-19 08:32:59 +00:00
|
|
|
MEXT_ADD_REF(m);
|
2005-02-10 22:23:02 +00:00
|
|
|
n->m_ext.ref_cnt = m->m_ext.ref_cnt;
|
1996-05-08 19:38:27 +00:00
|
|
|
} else {
|
2001-02-20 08:23:41 +00:00
|
|
|
n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
|
1996-05-08 19:38:27 +00:00
|
|
|
bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
|
|
|
|
}
|
|
|
|
|
|
|
|
m = m->m_next;
|
|
|
|
while (m) {
|
|
|
|
MGET(o, how, m->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (o == NULL)
|
1996-05-08 19:38:27 +00:00
|
|
|
goto nospace;
|
|
|
|
|
|
|
|
n->m_next = o;
|
|
|
|
n = n->m_next;
|
|
|
|
|
|
|
|
n->m_len = m->m_len;
|
|
|
|
if (m->m_flags & M_EXT) {
|
|
|
|
n->m_data = m->m_data;
|
|
|
|
n->m_ext = m->m_ext;
|
|
|
|
n->m_flags |= M_EXT;
|
2000-08-19 08:32:59 +00:00
|
|
|
MEXT_ADD_REF(m);
|
2005-02-10 22:23:02 +00:00
|
|
|
n->m_ext.ref_cnt = m->m_ext.ref_cnt;
|
1996-05-08 19:38:27 +00:00
|
|
|
} else {
|
|
|
|
bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
|
|
|
|
}
|
|
|
|
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
return top;
|
|
|
|
nospace:
|
|
|
|
m_freem(top);
|
2001-06-22 06:35:32 +00:00
|
|
|
mbstat.m_mcfail++; /* XXX: No consistency. */
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
1996-05-08 19:38:27 +00:00
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Copy data from an mbuf chain starting "off" bytes from the beginning,
|
|
|
|
* continuing for "len" bytes, into the indicated buffer.
|
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
void
|
2001-08-19 04:30:13 +00:00
|
|
|
m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-09-18 22:33:52 +00:00
|
|
|
u_int count;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
1999-10-13 09:55:42 +00:00
|
|
|
KASSERT(off >= 0, ("m_copydata, negative off %d", off));
|
|
|
|
KASSERT(len >= 0, ("m_copydata, negative len %d", len));
|
1994-05-24 10:09:53 +00:00
|
|
|
while (off > 0) {
|
1999-10-13 09:55:42 +00:00
|
|
|
KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
|
1994-05-24 10:09:53 +00:00
|
|
|
if (off < m->m_len)
|
|
|
|
break;
|
|
|
|
off -= m->m_len;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
while (len > 0) {
|
1999-10-13 09:55:42 +00:00
|
|
|
KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
|
1994-05-24 10:09:53 +00:00
|
|
|
count = min(m->m_len - off, len);
|
|
|
|
bcopy(mtod(m, caddr_t) + off, cp, count);
|
|
|
|
len -= count;
|
|
|
|
cp += count;
|
|
|
|
off = 0;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
1999-12-01 22:31:32 +00:00
|
|
|
/*
|
|
|
|
* Copy a packet header mbuf chain into a completely new chain, including
|
|
|
|
* copying any mbuf clusters. Use this instead of m_copypacket() when
|
|
|
|
* you need a writable copy of an mbuf chain.
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-02-11 05:02:06 +00:00
|
|
|
m_dup(struct mbuf *m, int how)
|
1999-12-01 22:31:32 +00:00
|
|
|
{
|
|
|
|
struct mbuf **p, *top = NULL;
|
|
|
|
int remain, moff, nsize;
|
|
|
|
|
2004-07-21 07:12:24 +00:00
|
|
|
MBUF_CHECKSLEEP(how);
|
1999-12-01 22:31:32 +00:00
|
|
|
/* Sanity check */
|
|
|
|
if (m == NULL)
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
2003-04-08 14:25:47 +00:00
|
|
|
M_ASSERTPKTHDR(m);
|
1999-12-01 22:31:32 +00:00
|
|
|
|
|
|
|
/* While there's more data, get a new mbuf, tack it on, and fill it */
|
|
|
|
remain = m->m_pkthdr.len;
|
|
|
|
moff = 0;
|
|
|
|
p = ⊤
|
|
|
|
while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */
|
|
|
|
struct mbuf *n;
|
|
|
|
|
|
|
|
/* Get the next new mbuf */
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (remain >= MINCLSIZE) {
|
|
|
|
n = m_getcl(how, m->m_type, 0);
|
|
|
|
nsize = MCLBYTES;
|
|
|
|
} else {
|
|
|
|
n = m_get(how, m->m_type);
|
|
|
|
nsize = MLEN;
|
|
|
|
}
|
1999-12-01 22:31:32 +00:00
|
|
|
if (n == NULL)
|
|
|
|
goto nospace;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
|
|
|
|
if (top == NULL) { /* First one, must be PKTHDR */
|
|
|
|
if (!m_dup_pkthdr(n, m, how)) {
|
|
|
|
m_free(n);
|
1999-12-01 22:31:32 +00:00
|
|
|
goto nospace;
|
|
|
|
}
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
nsize = MHLEN;
|
1999-12-01 22:31:32 +00:00
|
|
|
}
|
|
|
|
n->m_len = 0;
|
|
|
|
|
|
|
|
/* Link it into the new chain */
|
|
|
|
*p = n;
|
|
|
|
p = &n->m_next;
|
|
|
|
|
|
|
|
/* Copy data from original mbuf(s) into new mbuf */
|
|
|
|
while (n->m_len < nsize && m != NULL) {
|
|
|
|
int chunk = min(nsize - n->m_len, m->m_len - moff);
|
|
|
|
|
|
|
|
bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
|
|
|
|
moff += chunk;
|
|
|
|
n->m_len += chunk;
|
|
|
|
remain -= chunk;
|
|
|
|
if (moff == m->m_len) {
|
|
|
|
m = m->m_next;
|
|
|
|
moff = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Check correct total mbuf length */
|
|
|
|
KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
|
2001-12-10 05:51:45 +00:00
|
|
|
("%s: bogus m_pkthdr.len", __func__));
|
1999-12-01 22:31:32 +00:00
|
|
|
}
|
|
|
|
return (top);
|
|
|
|
|
|
|
|
nospace:
|
|
|
|
m_freem(top);
|
2001-06-22 06:35:32 +00:00
|
|
|
mbstat.m_mcfail++; /* XXX: No consistency. */
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
1999-12-01 22:31:32 +00:00
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Concatenate mbuf chain n to m.
|
|
|
|
* Both chains must be of the same type (e.g. MT_DATA).
|
|
|
|
* Any m_pkthdr is not updated.
|
|
|
|
*/
|
1994-05-25 09:21:21 +00:00
|
|
|
void
|
2001-02-11 05:02:06 +00:00
|
|
|
m_cat(struct mbuf *m, struct mbuf *n)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
|
|
|
while (m->m_next)
|
|
|
|
m = m->m_next;
|
|
|
|
while (n) {
|
|
|
|
if (m->m_flags & M_EXT ||
|
|
|
|
m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
|
|
|
|
/* just join the two chains */
|
|
|
|
m->m_next = n;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
/* splat the data from one into the other */
|
|
|
|
bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
|
|
|
|
(u_int)n->m_len);
|
|
|
|
m->m_len += n->m_len;
|
|
|
|
n = m_free(n);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
1994-05-25 09:21:21 +00:00
|
|
|
void
|
2001-02-11 05:02:06 +00:00
|
|
|
m_adj(struct mbuf *mp, int req_len)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2001-02-11 05:02:06 +00:00
|
|
|
int len = req_len;
|
|
|
|
struct mbuf *m;
|
|
|
|
int count;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
|
|
|
if ((m = mp) == NULL)
|
|
|
|
return;
|
|
|
|
if (len >= 0) {
|
|
|
|
/*
|
|
|
|
* Trim from head.
|
|
|
|
*/
|
|
|
|
while (m != NULL && len > 0) {
|
|
|
|
if (m->m_len <= len) {
|
|
|
|
len -= m->m_len;
|
|
|
|
m->m_len = 0;
|
|
|
|
m = m->m_next;
|
|
|
|
} else {
|
|
|
|
m->m_len -= len;
|
|
|
|
m->m_data += len;
|
|
|
|
len = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
m = mp;
|
|
|
|
if (mp->m_flags & M_PKTHDR)
|
|
|
|
m->m_pkthdr.len -= (req_len - len);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Trim from tail. Scan the mbuf chain,
|
|
|
|
* calculating its length and finding the last mbuf.
|
|
|
|
* If the adjustment only affects this mbuf, then just
|
|
|
|
* adjust and return. Otherwise, rescan and truncate
|
|
|
|
* after the remaining size.
|
|
|
|
*/
|
|
|
|
len = -len;
|
|
|
|
count = 0;
|
|
|
|
for (;;) {
|
|
|
|
count += m->m_len;
|
|
|
|
if (m->m_next == (struct mbuf *)0)
|
|
|
|
break;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
if (m->m_len >= len) {
|
|
|
|
m->m_len -= len;
|
|
|
|
if (mp->m_flags & M_PKTHDR)
|
|
|
|
mp->m_pkthdr.len -= len;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
count -= len;
|
|
|
|
if (count < 0)
|
|
|
|
count = 0;
|
|
|
|
/*
|
|
|
|
* Correct length for chain is "count".
|
|
|
|
* Find the mbuf with last data, adjust its length,
|
|
|
|
* and toss data from remaining mbufs on chain.
|
|
|
|
*/
|
|
|
|
m = mp;
|
|
|
|
if (m->m_flags & M_PKTHDR)
|
|
|
|
m->m_pkthdr.len = count;
|
|
|
|
for (; m; m = m->m_next) {
|
|
|
|
if (m->m_len >= count) {
|
|
|
|
m->m_len = count;
|
2005-02-24 00:40:33 +00:00
|
|
|
if (m->m_next != NULL) {
|
|
|
|
m_freem(m->m_next);
|
|
|
|
m->m_next = NULL;
|
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
count -= m->m_len;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Rearange an mbuf chain so that len bytes are contiguous
|
|
|
|
* and in the data area of an mbuf (so that mtod and dtom
|
|
|
|
* will work for a structure of size len). Returns the resulting
|
|
|
|
* mbuf chain on success, frees it and returns null on failure.
|
|
|
|
* If there is room, it will add up to max_protohdr-len extra bytes to the
|
|
|
|
* contiguous region in an attempt to avoid being called next time.
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-02-11 05:02:06 +00:00
|
|
|
m_pullup(struct mbuf *n, int len)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2001-02-11 05:02:06 +00:00
|
|
|
struct mbuf *m;
|
|
|
|
int count;
|
1994-05-24 10:09:53 +00:00
|
|
|
int space;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If first mbuf has no cluster, and has room for len bytes
|
|
|
|
* without shifting current data, pullup into it,
|
|
|
|
* otherwise allocate a new mbuf to prepend to the chain.
|
|
|
|
*/
|
|
|
|
if ((n->m_flags & M_EXT) == 0 &&
|
|
|
|
n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
|
|
|
|
if (n->m_len >= len)
|
|
|
|
return (n);
|
|
|
|
m = n;
|
|
|
|
n = n->m_next;
|
|
|
|
len -= m->m_len;
|
|
|
|
} else {
|
|
|
|
if (len > MHLEN)
|
|
|
|
goto bad;
|
2003-02-19 05:47:46 +00:00
|
|
|
MGET(m, M_DONTWAIT, n->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (m == NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto bad;
|
|
|
|
m->m_len = 0;
|
2002-12-30 20:22:40 +00:00
|
|
|
if (n->m_flags & M_PKTHDR)
|
|
|
|
M_MOVE_PKTHDR(m, n);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
|
|
|
|
do {
|
|
|
|
count = min(min(max(len, max_protohdr), space), n->m_len);
|
|
|
|
bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
|
2002-09-18 22:33:52 +00:00
|
|
|
(u_int)count);
|
1994-05-24 10:09:53 +00:00
|
|
|
len -= count;
|
|
|
|
m->m_len += count;
|
|
|
|
n->m_len -= count;
|
|
|
|
space -= count;
|
|
|
|
if (n->m_len)
|
|
|
|
n->m_data += count;
|
|
|
|
else
|
|
|
|
n = m_free(n);
|
|
|
|
} while (len > 0 && n);
|
|
|
|
if (len > 0) {
|
|
|
|
(void) m_free(m);
|
|
|
|
goto bad;
|
|
|
|
}
|
|
|
|
m->m_next = n;
|
|
|
|
return (m);
|
|
|
|
bad:
|
|
|
|
m_freem(n);
|
2001-06-22 06:35:32 +00:00
|
|
|
mbstat.m_mpfail++; /* XXX: No consistency. */
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
2005-03-17 19:34:57 +00:00
|
|
|
/*
|
|
|
|
* Like m_pullup(), except a new mbuf is always allocated, and we allow
|
|
|
|
* the amount of empty space before the data in the new mbuf to be specified
|
|
|
|
* (in the event that the caller expects to prepend later).
|
|
|
|
*/
|
|
|
|
int MSFail;
|
|
|
|
|
|
|
|
struct mbuf *
|
|
|
|
m_copyup(struct mbuf *n, int len, int dstoff)
|
|
|
|
{
|
|
|
|
struct mbuf *m;
|
|
|
|
int count, space;
|
|
|
|
|
|
|
|
if (len > (MHLEN - dstoff))
|
|
|
|
goto bad;
|
|
|
|
MGET(m, M_DONTWAIT, n->m_type);
|
|
|
|
if (m == NULL)
|
|
|
|
goto bad;
|
|
|
|
m->m_len = 0;
|
|
|
|
if (n->m_flags & M_PKTHDR)
|
|
|
|
M_MOVE_PKTHDR(m, n);
|
|
|
|
m->m_data += dstoff;
|
|
|
|
space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
|
|
|
|
do {
|
|
|
|
count = min(min(max(len, max_protohdr), space), n->m_len);
|
|
|
|
memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
|
|
|
|
(unsigned)count);
|
|
|
|
len -= count;
|
|
|
|
m->m_len += count;
|
|
|
|
n->m_len -= count;
|
|
|
|
space -= count;
|
|
|
|
if (n->m_len)
|
|
|
|
n->m_data += count;
|
|
|
|
else
|
|
|
|
n = m_free(n);
|
|
|
|
} while (len > 0 && n);
|
|
|
|
if (len > 0) {
|
|
|
|
(void) m_free(m);
|
|
|
|
goto bad;
|
|
|
|
}
|
|
|
|
m->m_next = n;
|
|
|
|
return (m);
|
|
|
|
bad:
|
|
|
|
m_freem(n);
|
|
|
|
MSFail++;
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Partition an mbuf chain in two pieces, returning the tail --
|
|
|
|
* all but the first len0 bytes. In case of failure, it returns NULL and
|
|
|
|
* attempts to restore the chain to its original state.
|
2002-05-31 22:09:57 +00:00
|
|
|
*
|
|
|
|
* Note that the resulting mbufs might be read-only, because the new
|
|
|
|
* mbuf can end up sharing an mbuf cluster with the original mbuf if
|
|
|
|
* the "breaking point" happens to lie within a cluster mbuf. Use the
|
|
|
|
* M_WRITABLE() macro to check for this case.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-02-11 05:02:06 +00:00
|
|
|
m_split(struct mbuf *m0, int len0, int wait)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2001-02-11 05:02:06 +00:00
|
|
|
struct mbuf *m, *n;
|
2002-09-18 22:33:52 +00:00
|
|
|
u_int len = len0, remain;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2004-07-21 07:12:24 +00:00
|
|
|
MBUF_CHECKSLEEP(wait);
|
1994-05-24 10:09:53 +00:00
|
|
|
for (m = m0; m && len > m->m_len; m = m->m_next)
|
|
|
|
len -= m->m_len;
|
2001-02-11 05:02:06 +00:00
|
|
|
if (m == NULL)
|
|
|
|
return (NULL);
|
1994-05-24 10:09:53 +00:00
|
|
|
remain = m->m_len - len;
|
|
|
|
if (m0->m_flags & M_PKTHDR) {
|
|
|
|
MGETHDR(n, wait, m0->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n == NULL)
|
|
|
|
return (NULL);
|
1994-05-24 10:09:53 +00:00
|
|
|
n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
|
|
|
|
n->m_pkthdr.len = m0->m_pkthdr.len - len0;
|
|
|
|
m0->m_pkthdr.len = len0;
|
|
|
|
if (m->m_flags & M_EXT)
|
|
|
|
goto extpacket;
|
|
|
|
if (remain > MHLEN) {
|
|
|
|
/* m can't be the lead packet */
|
|
|
|
MH_ALIGN(n, 0);
|
|
|
|
n->m_next = m_split(m, len, wait);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n->m_next == NULL) {
|
1994-05-24 10:09:53 +00:00
|
|
|
(void) m_free(n);
|
2001-02-11 05:02:06 +00:00
|
|
|
return (NULL);
|
2002-04-12 00:01:50 +00:00
|
|
|
} else {
|
|
|
|
n->m_len = 0;
|
1994-05-24 10:09:53 +00:00
|
|
|
return (n);
|
2002-04-12 00:01:50 +00:00
|
|
|
}
|
1994-05-24 10:09:53 +00:00
|
|
|
} else
|
|
|
|
MH_ALIGN(n, remain);
|
|
|
|
} else if (remain == 0) {
|
|
|
|
n = m->m_next;
|
2001-02-11 05:02:06 +00:00
|
|
|
m->m_next = NULL;
|
1994-05-24 10:09:53 +00:00
|
|
|
return (n);
|
|
|
|
} else {
|
|
|
|
MGET(n, wait, m->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n == NULL)
|
|
|
|
return (NULL);
|
1994-05-24 10:09:53 +00:00
|
|
|
M_ALIGN(n, remain);
|
|
|
|
}
|
|
|
|
extpacket:
|
|
|
|
if (m->m_flags & M_EXT) {
|
|
|
|
n->m_flags |= M_EXT;
|
|
|
|
n->m_ext = m->m_ext;
|
2000-08-19 08:32:59 +00:00
|
|
|
MEXT_ADD_REF(m);
|
2005-02-10 22:23:02 +00:00
|
|
|
n->m_ext.ref_cnt = m->m_ext.ref_cnt;
|
1994-05-24 10:09:53 +00:00
|
|
|
n->m_data = m->m_data + len;
|
|
|
|
} else {
|
|
|
|
bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
|
|
|
|
}
|
|
|
|
n->m_len = remain;
|
|
|
|
m->m_len = len;
|
|
|
|
n->m_next = m->m_next;
|
2001-02-11 05:02:06 +00:00
|
|
|
m->m_next = NULL;
|
1994-05-24 10:09:53 +00:00
|
|
|
return (n);
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Routine to copy from device local memory into mbufs.
|
2001-06-20 19:48:35 +00:00
|
|
|
* Note that `off' argument is offset into first mbuf of target chain from
|
|
|
|
* which to begin copying the data to.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
struct mbuf *
|
2001-06-20 19:48:35 +00:00
|
|
|
m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
|
2001-02-11 05:02:06 +00:00
|
|
|
void (*copy)(char *from, caddr_t to, u_int len))
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2001-02-11 05:02:06 +00:00
|
|
|
struct mbuf *m;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
struct mbuf *top = NULL, **mp = ⊤
|
2001-06-20 19:48:35 +00:00
|
|
|
int len;
|
|
|
|
|
|
|
|
if (off < 0 || off > MHLEN)
|
|
|
|
return (NULL);
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
while (totlen > 0) {
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (top == NULL) { /* First one, must be PKTHDR */
|
|
|
|
if (totlen + off >= MINCLSIZE) {
|
|
|
|
m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
|
2001-06-20 19:48:35 +00:00
|
|
|
len = MCLBYTES;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
} else {
|
|
|
|
m = m_gethdr(M_DONTWAIT, MT_DATA);
|
|
|
|
len = MHLEN;
|
|
|
|
|
|
|
|
/* Place initial small packet/header at end of mbuf */
|
|
|
|
if (m && totlen + off + max_linkhdr <= MLEN) {
|
|
|
|
m->m_data += max_linkhdr;
|
|
|
|
len -= max_linkhdr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (m == NULL)
|
|
|
|
return NULL;
|
|
|
|
m->m_pkthdr.rcvif = ifp;
|
|
|
|
m->m_pkthdr.len = totlen;
|
1994-05-24 10:09:53 +00:00
|
|
|
} else {
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (totlen + off >= MINCLSIZE) {
|
|
|
|
m = m_getcl(M_DONTWAIT, MT_DATA, 0);
|
|
|
|
len = MCLBYTES;
|
|
|
|
} else {
|
|
|
|
m = m_get(M_DONTWAIT, MT_DATA);
|
|
|
|
len = MLEN;
|
|
|
|
}
|
|
|
|
if (m == NULL) {
|
|
|
|
m_freem(top);
|
|
|
|
return NULL;
|
2001-06-20 19:48:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (off) {
|
|
|
|
m->m_data += off;
|
|
|
|
len -= off;
|
|
|
|
off = 0;
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2001-06-20 19:48:35 +00:00
|
|
|
m->m_len = len = min(totlen, len);
|
1994-05-24 10:09:53 +00:00
|
|
|
if (copy)
|
2002-09-18 22:33:52 +00:00
|
|
|
copy(buf, mtod(m, caddr_t), (u_int)len);
|
1994-05-24 10:09:53 +00:00
|
|
|
else
|
2002-09-18 22:33:52 +00:00
|
|
|
bcopy(buf, mtod(m, caddr_t), (u_int)len);
|
2001-06-20 19:48:35 +00:00
|
|
|
buf += len;
|
1994-05-24 10:09:53 +00:00
|
|
|
*mp = m;
|
|
|
|
mp = &m->m_next;
|
|
|
|
totlen -= len;
|
|
|
|
}
|
|
|
|
return (top);
|
|
|
|
}
|
1994-10-04 06:50:01 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Copy data from a buffer back into the indicated mbuf chain,
|
|
|
|
* starting "off" bytes from the beginning, extending the mbuf
|
|
|
|
* chain if necessary.
|
|
|
|
*/
|
|
|
|
void
|
2004-04-18 13:01:28 +00:00
|
|
|
m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
|
1994-10-04 06:50:01 +00:00
|
|
|
{
|
2001-02-11 05:02:06 +00:00
|
|
|
int mlen;
|
|
|
|
struct mbuf *m = m0, *n;
|
1994-10-04 06:50:01 +00:00
|
|
|
int totlen = 0;
|
|
|
|
|
2001-02-11 05:02:06 +00:00
|
|
|
if (m0 == NULL)
|
1994-10-04 06:50:01 +00:00
|
|
|
return;
|
|
|
|
while (off > (mlen = m->m_len)) {
|
|
|
|
off -= mlen;
|
|
|
|
totlen += mlen;
|
2001-02-11 05:02:06 +00:00
|
|
|
if (m->m_next == NULL) {
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
n = m_get(M_DONTWAIT, m->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n == NULL)
|
1994-10-04 06:50:01 +00:00
|
|
|
goto out;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
bzero(mtod(n, caddr_t), MLEN);
|
1994-10-04 06:50:01 +00:00
|
|
|
n->m_len = min(MLEN, len + off);
|
|
|
|
m->m_next = n;
|
|
|
|
}
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
while (len > 0) {
|
|
|
|
mlen = min (m->m_len - off, len);
|
2002-09-18 22:33:52 +00:00
|
|
|
bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
|
1994-10-04 06:50:01 +00:00
|
|
|
cp += mlen;
|
|
|
|
len -= mlen;
|
|
|
|
mlen += off;
|
|
|
|
off = 0;
|
|
|
|
totlen += mlen;
|
|
|
|
if (len == 0)
|
|
|
|
break;
|
2001-02-11 05:02:06 +00:00
|
|
|
if (m->m_next == NULL) {
|
2003-02-19 05:47:46 +00:00
|
|
|
n = m_get(M_DONTWAIT, m->m_type);
|
2001-02-11 05:02:06 +00:00
|
|
|
if (n == NULL)
|
1994-10-04 06:50:01 +00:00
|
|
|
break;
|
|
|
|
n->m_len = min(MLEN, len);
|
|
|
|
m->m_next = n;
|
|
|
|
}
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
|
|
|
|
m->m_pkthdr.len = totlen;
|
|
|
|
}
|
1999-11-01 15:03:20 +00:00
|
|
|
|
2004-12-08 05:42:02 +00:00
|
|
|
/*
|
|
|
|
* Append the specified data to the indicated mbuf chain,
|
|
|
|
* Extend the mbuf chain if the new data does not fit in
|
|
|
|
* existing space.
|
|
|
|
*
|
|
|
|
* Return 1 if able to complete the job; otherwise 0.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
m_append(struct mbuf *m0, int len, c_caddr_t cp)
|
|
|
|
{
|
|
|
|
struct mbuf *m, *n;
|
|
|
|
int remainder, space;
|
|
|
|
|
|
|
|
for (m = m0; m->m_next != NULL; m = m->m_next)
|
|
|
|
;
|
|
|
|
remainder = len;
|
|
|
|
space = M_TRAILINGSPACE(m);
|
|
|
|
if (space > 0) {
|
|
|
|
/*
|
|
|
|
* Copy into available space.
|
|
|
|
*/
|
|
|
|
if (space > remainder)
|
|
|
|
space = remainder;
|
|
|
|
bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
|
|
|
|
m->m_len += space;
|
|
|
|
cp += space, remainder -= space;
|
|
|
|
}
|
|
|
|
while (remainder > 0) {
|
|
|
|
/*
|
|
|
|
* Allocate a new mbuf; could check space
|
|
|
|
* and allocate a cluster instead.
|
|
|
|
*/
|
|
|
|
n = m_get(M_DONTWAIT, m->m_type);
|
|
|
|
if (n == NULL)
|
|
|
|
break;
|
|
|
|
n->m_len = min(MLEN, remainder);
|
2004-12-15 19:04:07 +00:00
|
|
|
bcopy(cp, mtod(n, caddr_t), n->m_len);
|
|
|
|
cp += n->m_len, remainder -= n->m_len;
|
2004-12-08 05:42:02 +00:00
|
|
|
m->m_next = n;
|
|
|
|
m = n;
|
|
|
|
}
|
|
|
|
if (m0->m_flags & M_PKTHDR)
|
|
|
|
m0->m_pkthdr.len += len - remainder;
|
|
|
|
return (remainder == 0);
|
|
|
|
}
|
|
|
|
|
2003-12-15 21:49:41 +00:00
|
|
|
/*
|
|
|
|
* Apply function f to the data in an mbuf chain starting "off" bytes from
|
|
|
|
* the beginning, continuing for "len" bytes.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
m_apply(struct mbuf *m, int off, int len,
|
2003-12-16 14:13:47 +00:00
|
|
|
int (*f)(void *, void *, u_int), void *arg)
|
2003-12-15 21:49:41 +00:00
|
|
|
{
|
2003-12-16 14:13:47 +00:00
|
|
|
u_int count;
|
2003-12-15 21:49:41 +00:00
|
|
|
int rval;
|
|
|
|
|
|
|
|
KASSERT(off >= 0, ("m_apply, negative off %d", off));
|
|
|
|
KASSERT(len >= 0, ("m_apply, negative len %d", len));
|
|
|
|
while (off > 0) {
|
|
|
|
KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
|
|
|
|
if (off < m->m_len)
|
|
|
|
break;
|
|
|
|
off -= m->m_len;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
while (len > 0) {
|
|
|
|
KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
|
|
|
|
count = min(m->m_len - off, len);
|
|
|
|
rval = (*f)(arg, mtod(m, caddr_t) + off, count);
|
|
|
|
if (rval)
|
|
|
|
return (rval);
|
|
|
|
len -= count;
|
|
|
|
off = 0;
|
|
|
|
m = m->m_next;
|
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return a pointer to mbuf/offset of location in mbuf chain.
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
|
|
|
m_getptr(struct mbuf *m, int loc, int *off)
|
|
|
|
{
|
|
|
|
|
|
|
|
while (loc >= 0) {
|
2003-12-16 14:13:47 +00:00
|
|
|
/* Normal end of search. */
|
2003-12-15 21:49:41 +00:00
|
|
|
if (m->m_len > loc) {
|
|
|
|
*off = loc;
|
|
|
|
return (m);
|
|
|
|
} else {
|
|
|
|
loc -= m->m_len;
|
|
|
|
if (m->m_next == NULL) {
|
|
|
|
if (loc == 0) {
|
2003-12-16 14:13:47 +00:00
|
|
|
/* Point at the end of valid data. */
|
2003-12-15 21:49:41 +00:00
|
|
|
*off = m->m_len;
|
|
|
|
return (m);
|
2003-12-16 14:13:47 +00:00
|
|
|
}
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
m = m->m_next;
|
2003-12-15 21:49:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
1999-11-01 15:03:20 +00:00
|
|
|
void
|
2004-09-28 18:40:18 +00:00
|
|
|
m_print(const struct mbuf *m, int maxlen)
|
1999-11-01 15:03:20 +00:00
|
|
|
{
|
|
|
|
int len;
|
2004-09-28 18:40:18 +00:00
|
|
|
int pdata;
|
1999-12-20 18:10:00 +00:00
|
|
|
const struct mbuf *m2;
|
1999-11-01 15:03:20 +00:00
|
|
|
|
2004-09-28 18:40:18 +00:00
|
|
|
if (m->m_flags & M_PKTHDR)
|
|
|
|
len = m->m_pkthdr.len;
|
|
|
|
else
|
|
|
|
len = -1;
|
1999-11-01 15:03:20 +00:00
|
|
|
m2 = m;
|
2004-09-28 18:40:18 +00:00
|
|
|
while (m2 != NULL && (len == -1 || len)) {
|
|
|
|
pdata = m2->m_len;
|
|
|
|
if (maxlen != -1 && pdata > maxlen)
|
|
|
|
pdata = maxlen;
|
|
|
|
printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
|
|
|
|
m2->m_next, m2->m_flags, "\20\20freelist\17skipfw"
|
|
|
|
"\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
|
|
|
|
"\3eor\2pkthdr\1ext", pdata ? "" : "\n");
|
|
|
|
if (pdata)
|
|
|
|
printf(", %*D\n", m2->m_len, (u_char *)m2->m_data, "-");
|
|
|
|
if (len != -1)
|
|
|
|
len -= m2->m_len;
|
1999-11-01 15:03:20 +00:00
|
|
|
m2 = m2->m_next;
|
|
|
|
}
|
2004-09-28 18:40:18 +00:00
|
|
|
if (len > 0)
|
|
|
|
printf("%d bytes unaccounted for.\n", len);
|
1999-11-01 15:03:20 +00:00
|
|
|
return;
|
|
|
|
}
|
2002-09-18 13:41:37 +00:00
|
|
|
|
2002-09-18 22:33:52 +00:00
|
|
|
u_int
|
2002-09-18 13:41:37 +00:00
|
|
|
m_fixhdr(struct mbuf *m0)
|
|
|
|
{
|
2002-09-18 22:33:52 +00:00
|
|
|
u_int len;
|
2002-09-18 13:41:37 +00:00
|
|
|
|
2002-09-18 14:57:35 +00:00
|
|
|
len = m_length(m0, NULL);
|
|
|
|
m0->m_pkthdr.len = len;
|
|
|
|
return (len);
|
|
|
|
}
|
|
|
|
|
2002-09-18 22:33:52 +00:00
|
|
|
u_int
|
2002-09-18 14:57:35 +00:00
|
|
|
m_length(struct mbuf *m0, struct mbuf **last)
|
|
|
|
{
|
|
|
|
struct mbuf *m;
|
2002-09-18 22:33:52 +00:00
|
|
|
u_int len;
|
2002-09-18 14:57:35 +00:00
|
|
|
|
|
|
|
len = 0;
|
|
|
|
for (m = m0; m != NULL; m = m->m_next) {
|
2002-09-18 13:41:37 +00:00
|
|
|
len += m->m_len;
|
2002-09-18 14:57:35 +00:00
|
|
|
if (m->m_next == NULL)
|
|
|
|
break;
|
2002-09-18 13:41:37 +00:00
|
|
|
}
|
2002-09-18 14:57:35 +00:00
|
|
|
if (last != NULL)
|
|
|
|
*last = m;
|
|
|
|
return (len);
|
2002-09-18 13:41:37 +00:00
|
|
|
}
|
2003-03-29 05:48:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Defragment a mbuf chain, returning the shortest possible
|
|
|
|
* chain of mbufs and clusters. If allocation fails and
|
|
|
|
* this cannot be completed, NULL will be returned, but
|
|
|
|
* the passed in chain will be unchanged. Upon success,
|
|
|
|
* the original chain will be freed, and the new chain
|
|
|
|
* will be returned.
|
|
|
|
*
|
|
|
|
* If a non-packet header is passed in, the original
|
|
|
|
* mbuf (chain?) will be returned unharmed.
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
|
|
|
m_defrag(struct mbuf *m0, int how)
|
|
|
|
{
|
2004-02-05 08:29:27 +00:00
|
|
|
struct mbuf *m_new = NULL, *m_final = NULL;
|
|
|
|
int progress = 0, length;
|
2003-03-29 05:48:36 +00:00
|
|
|
|
2004-07-21 07:12:24 +00:00
|
|
|
MBUF_CHECKSLEEP(how);
|
2003-03-29 05:48:36 +00:00
|
|
|
if (!(m0->m_flags & M_PKTHDR))
|
|
|
|
return (m0);
|
|
|
|
|
2003-07-19 06:03:48 +00:00
|
|
|
m_fixhdr(m0); /* Needed sanity check */
|
|
|
|
|
2003-04-15 02:14:43 +00:00
|
|
|
#ifdef MBUF_STRESS_TEST
|
|
|
|
if (m_defragrandomfailures) {
|
|
|
|
int temp = arc4random() & 0xff;
|
|
|
|
if (temp == 0xba)
|
|
|
|
goto nospace;
|
|
|
|
}
|
|
|
|
#endif
|
2003-03-29 05:48:36 +00:00
|
|
|
|
|
|
|
if (m0->m_pkthdr.len > MHLEN)
|
|
|
|
m_final = m_getcl(how, MT_DATA, M_PKTHDR);
|
|
|
|
else
|
|
|
|
m_final = m_gethdr(how, MT_DATA);
|
|
|
|
|
|
|
|
if (m_final == NULL)
|
|
|
|
goto nospace;
|
|
|
|
|
2003-12-23 02:36:43 +00:00
|
|
|
if (m_dup_pkthdr(m_final, m0, how) == 0)
|
2003-03-29 05:48:36 +00:00
|
|
|
goto nospace;
|
|
|
|
|
|
|
|
m_new = m_final;
|
|
|
|
|
|
|
|
while (progress < m0->m_pkthdr.len) {
|
|
|
|
length = m0->m_pkthdr.len - progress;
|
|
|
|
if (length > MCLBYTES)
|
|
|
|
length = MCLBYTES;
|
|
|
|
|
|
|
|
if (m_new == NULL) {
|
|
|
|
if (length > MLEN)
|
|
|
|
m_new = m_getcl(how, MT_DATA, 0);
|
|
|
|
else
|
|
|
|
m_new = m_get(how, MT_DATA);
|
|
|
|
if (m_new == NULL)
|
|
|
|
goto nospace;
|
|
|
|
}
|
|
|
|
|
|
|
|
m_copydata(m0, progress, length, mtod(m_new, caddr_t));
|
|
|
|
progress += length;
|
|
|
|
m_new->m_len = length;
|
|
|
|
if (m_new != m_final)
|
|
|
|
m_cat(m_final, m_new);
|
|
|
|
m_new = NULL;
|
|
|
|
}
|
2003-06-17 02:34:40 +00:00
|
|
|
#ifdef MBUF_STRESS_TEST
|
2003-03-29 05:48:36 +00:00
|
|
|
if (m0->m_next == NULL)
|
|
|
|
m_defraguseless++;
|
2003-06-17 02:34:40 +00:00
|
|
|
#endif
|
2003-03-29 05:48:36 +00:00
|
|
|
m_freem(m0);
|
|
|
|
m0 = m_final;
|
2003-06-17 02:34:40 +00:00
|
|
|
#ifdef MBUF_STRESS_TEST
|
2003-03-29 05:48:36 +00:00
|
|
|
m_defragpackets++;
|
|
|
|
m_defragbytes += m0->m_pkthdr.len;
|
2003-06-17 02:34:40 +00:00
|
|
|
#endif
|
2003-03-29 05:48:36 +00:00
|
|
|
return (m0);
|
|
|
|
nospace:
|
2003-06-17 02:34:40 +00:00
|
|
|
#ifdef MBUF_STRESS_TEST
|
2003-03-29 05:48:36 +00:00
|
|
|
m_defragfailure++;
|
2003-06-17 02:34:40 +00:00
|
|
|
#endif
|
2003-03-29 05:48:36 +00:00
|
|
|
if (m_final)
|
|
|
|
m_freem(m_final);
|
|
|
|
return (NULL);
|
|
|
|
}
|
2003-09-01 05:55:37 +00:00
|
|
|
|
|
|
|
#ifdef MBUF_STRESS_TEST
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fragment an mbuf chain. There's no reason you'd ever want to do
|
|
|
|
* this in normal usage, but it's great for stress testing various
|
|
|
|
* mbuf consumers.
|
|
|
|
*
|
|
|
|
* If fragmentation is not possible, the original chain will be
|
|
|
|
* returned.
|
|
|
|
*
|
|
|
|
* Possible length values:
|
|
|
|
* 0 no fragmentation will occur
|
|
|
|
* > 0 each fragment will be of the specified length
|
|
|
|
* -1 each fragment will be the same random value in length
|
|
|
|
* -2 each fragment's length will be entirely random
|
|
|
|
* (Random values range from 1 to 256)
|
|
|
|
*/
|
|
|
|
struct mbuf *
|
|
|
|
m_fragment(struct mbuf *m0, int how, int length)
|
|
|
|
{
|
2004-02-05 08:29:27 +00:00
|
|
|
struct mbuf *m_new = NULL, *m_final = NULL;
|
|
|
|
int progress = 0;
|
2003-09-01 05:55:37 +00:00
|
|
|
|
|
|
|
if (!(m0->m_flags & M_PKTHDR))
|
|
|
|
return (m0);
|
|
|
|
|
|
|
|
if ((length == 0) || (length < -2))
|
|
|
|
return (m0);
|
|
|
|
|
|
|
|
m_fixhdr(m0); /* Needed sanity check */
|
|
|
|
|
|
|
|
m_final = m_getcl(how, MT_DATA, M_PKTHDR);
|
|
|
|
|
|
|
|
if (m_final == NULL)
|
|
|
|
goto nospace;
|
|
|
|
|
2003-12-25 01:17:27 +00:00
|
|
|
if (m_dup_pkthdr(m_final, m0, how) == 0)
|
2003-09-01 05:55:37 +00:00
|
|
|
goto nospace;
|
|
|
|
|
|
|
|
m_new = m_final;
|
|
|
|
|
|
|
|
if (length == -1)
|
|
|
|
length = 1 + (arc4random() & 255);
|
|
|
|
|
|
|
|
while (progress < m0->m_pkthdr.len) {
|
|
|
|
int fraglen;
|
|
|
|
|
|
|
|
if (length > 0)
|
|
|
|
fraglen = length;
|
|
|
|
else
|
|
|
|
fraglen = 1 + (arc4random() & 255);
|
|
|
|
if (fraglen > m0->m_pkthdr.len - progress)
|
|
|
|
fraglen = m0->m_pkthdr.len - progress;
|
|
|
|
|
|
|
|
if (fraglen > MCLBYTES)
|
|
|
|
fraglen = MCLBYTES;
|
|
|
|
|
|
|
|
if (m_new == NULL) {
|
|
|
|
m_new = m_getcl(how, MT_DATA, 0);
|
|
|
|
if (m_new == NULL)
|
|
|
|
goto nospace;
|
|
|
|
}
|
|
|
|
|
|
|
|
m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
|
|
|
|
progress += fraglen;
|
|
|
|
m_new->m_len = fraglen;
|
|
|
|
if (m_new != m_final)
|
|
|
|
m_cat(m_final, m_new);
|
|
|
|
m_new = NULL;
|
|
|
|
}
|
|
|
|
m_freem(m0);
|
|
|
|
m0 = m_final;
|
|
|
|
return (m0);
|
|
|
|
nospace:
|
|
|
|
if (m_final)
|
|
|
|
m_freem(m_final);
|
|
|
|
/* Return the original chain on failure */
|
|
|
|
return (m0);
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
2004-02-01 07:56:44 +00:00
|
|
|
|
|
|
|
struct mbuf *
|
2005-05-04 18:55:03 +00:00
|
|
|
m_uiotombuf(struct uio *uio, int how, int len, int align)
|
2004-02-01 07:56:44 +00:00
|
|
|
{
|
2004-02-05 08:29:27 +00:00
|
|
|
struct mbuf *m_new = NULL, *m_final = NULL;
|
|
|
|
int progress = 0, error = 0, length, total;
|
2004-02-01 07:56:44 +00:00
|
|
|
|
|
|
|
if (len > 0)
|
|
|
|
total = min(uio->uio_resid, len);
|
|
|
|
else
|
|
|
|
total = uio->uio_resid;
|
2005-05-04 18:55:03 +00:00
|
|
|
if (align >= MHLEN)
|
|
|
|
goto nospace;
|
|
|
|
if (total + align > MHLEN)
|
2004-02-01 07:56:44 +00:00
|
|
|
m_final = m_getcl(how, MT_DATA, M_PKTHDR);
|
|
|
|
else
|
|
|
|
m_final = m_gethdr(how, MT_DATA);
|
|
|
|
if (m_final == NULL)
|
|
|
|
goto nospace;
|
2005-05-04 18:55:03 +00:00
|
|
|
m_final->m_data += align;
|
2004-02-01 07:56:44 +00:00
|
|
|
m_new = m_final;
|
|
|
|
while (progress < total) {
|
|
|
|
length = total - progress;
|
|
|
|
if (length > MCLBYTES)
|
|
|
|
length = MCLBYTES;
|
|
|
|
if (m_new == NULL) {
|
|
|
|
if (length > MLEN)
|
|
|
|
m_new = m_getcl(how, MT_DATA, 0);
|
|
|
|
else
|
|
|
|
m_new = m_get(how, MT_DATA);
|
|
|
|
if (m_new == NULL)
|
|
|
|
goto nospace;
|
|
|
|
}
|
|
|
|
error = uiomove(mtod(m_new, void *), length, uio);
|
|
|
|
if (error)
|
|
|
|
goto nospace;
|
|
|
|
progress += length;
|
|
|
|
m_new->m_len = length;
|
|
|
|
if (m_new != m_final)
|
|
|
|
m_cat(m_final, m_new);
|
|
|
|
m_new = NULL;
|
|
|
|
}
|
|
|
|
m_fixhdr(m_final);
|
|
|
|
return (m_final);
|
|
|
|
nospace:
|
|
|
|
if (m_new)
|
|
|
|
m_free(m_new);
|
|
|
|
if (m_final)
|
|
|
|
m_freem(m_final);
|
|
|
|
return (NULL);
|
|
|
|
}
|
2005-07-30 01:32:16 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the m_data pointer of a newly-allocated mbuf
|
|
|
|
* to place an object of the specified size at the
|
|
|
|
* end of the mbuf, longword aligned.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
m_align(struct mbuf *m, int len)
|
|
|
|
{
|
|
|
|
int adjust;
|
|
|
|
|
|
|
|
if (m->m_flags & M_EXT)
|
|
|
|
adjust = m->m_ext.ext_size - len;
|
|
|
|
else if (m->m_flags & M_PKTHDR)
|
|
|
|
adjust = MHLEN - len;
|
|
|
|
else
|
|
|
|
adjust = MLEN - len;
|
|
|
|
m->m_data += adjust &~ (sizeof(long)-1);
|
|
|
|
}
|