Reimplement the netisr framework in order to support parallel netisr
threads:
- Support up to one netisr thread per CPU, each processings its own
workstream, or set of per-protocol queues. Threads may be bound
to specific CPUs, or allowed to migrate, based on a global policy.
In the future it would be desirable to support topology-centric
policies, such as "one netisr per package".
- Allow each protocol to advertise an ordering policy, which can
currently be one of:
NETISR_POLICY_SOURCE: packets must maintain ordering with respect to
an implicit or explicit source (such as an interface or socket).
NETISR_POLICY_FLOW: make use of mbuf flow identifiers to place work,
as well as allowing protocols to provide a flow generation function
for mbufs without flow identifers (m2flow). Falls back on
NETISR_POLICY_SOURCE if now flow ID is available.
NETISR_POLICY_CPU: allow protocols to inspect and assign a CPU for
each packet handled by netisr (m2cpuid).
- Provide utility functions for querying the number of workstreams
being used, as well as a mapping function from workstream to CPU ID,
which protocols may use in work placement decisions.
- Add explicit interfaces to get and set per-protocol queue limits, and
get and clear drop counters, which query data or apply changes across
all workstreams.
- Add a more extensible netisr registration interface, in which
protocols declare 'struct netisr_handler' structures for each
registered NETISR_ type. These include name, handler function,
optional mbuf to flow ID function, optional mbuf to CPU ID function,
queue limit, and ordering policy. Padding is present to allow these
to be expanded in the future. If no queue limit is declared, then
a default is used.
- Queue limits are now per-workstream, and raised from the previous
IFQ_MAXLEN default of 50 to 256.
- All protocols are updated to use the new registration interface, and
with the exception of netnatm, default queue limits. Most protocols
register as NETISR_POLICY_SOURCE, except IPv4 and IPv6, which use
NETISR_POLICY_FLOW, and will therefore take advantage of driver-
generated flow IDs if present.
- Formalize a non-packet based interface between interface polling and
the netisr, rather than having polling pretend to be two protocols.
Provide two explicit hooks in the netisr worker for start and end
events for runs: netisr_poll() and netisr_pollmore(), as well as a
function, netisr_sched_poll(), to allow the polling code to schedule
netisr execution. DEVICE_POLLING still embeds single-netisr
assumptions in its implementation, so for now if it is compiled into
the kernel, a single and un-bound netisr thread is enforced
regardless of tunable configuration.
In the default configuration, the new netisr implementation maintains
the same basic assumptions as the previous implementation: a single,
un-bound worker thread processes all deferred work, and direct dispatch
is enabled by default wherever possible.
Performance measurement shows a marginal performance improvement over
the old implementation due to the use of batched dequeue.
An rmlock is used to synchronize use and registration/unregistration
using the framework; currently, synchronized use is disabled
(replicating current netisr policy) due to a measurable 3%-6% hit in
ping-pong micro-benchmarking. It will be enabled once further rmlock
optimization has taken place. However, in practice, netisrs are
rarely registered or unregistered at runtime.
A new man page for netisr will follow, but since one doesn't currently
exist, it hasn't been updated.
This change is not appropriate for MFC, although the polling shutdown
handler should be merged to 7-STABLE.
Bump __FreeBSD_version.
Reviewed by: bz
This commit is contained in:
Robert Watson
parent
79d6b3f34a
commit
d4b5cae49b
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=193219
@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/kthread.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/eventhandler.h>
|
||||
#include <sys/resourcevar.h>
|
||||
#include <sys/socket.h> /* needed by net/if.h */
|
||||
#include <sys/sockio.h>
|
||||
@ -48,8 +49,6 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/route.h>
|
||||
#include <net/vnet.h>
|
||||
|
||||
static void netisr_poll(void); /* the two netisr handlers */
|
||||
static void netisr_pollmore(void);
|
||||
static int poll_switch(SYSCTL_HANDLER_ARGS);
|
||||
|
||||
void hardclock_device_poll(void); /* hook from hardclock */
|
||||
@ -110,6 +109,10 @@ SYSCTL_NODE(_kern, OID_AUTO, polling, CTLFLAG_RW, 0,
|
||||
SYSCTL_UINT(_kern_polling, OID_AUTO, burst, CTLFLAG_RD,
|
||||
&poll_burst, 0, "Current polling burst size");
|
||||
|
||||
static int netisr_poll_scheduled;
|
||||
static int netisr_pollmore_scheduled;
|
||||
static int poll_shutting_down;
|
||||
|
||||
static int poll_burst_max_sysctl(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
uint32_t val = poll_burst_max;
|
||||
@ -259,13 +262,20 @@ struct pollrec {
|
||||
|
||||
static struct pollrec pr[POLL_LIST_LEN];
|
||||
|
||||
static void
|
||||
poll_shutdown(void *arg, int howto)
|
||||
{
|
||||
|
||||
poll_shutting_down = 1;
|
||||
}
|
||||
|
||||
static void
|
||||
init_device_poll(void)
|
||||
{
|
||||
|
||||
mtx_init(&poll_mtx, "polling", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_POLL, (netisr_t *)netisr_poll, NULL, 0);
|
||||
netisr_register(NETISR_POLLMORE, (netisr_t *)netisr_pollmore, NULL, 0);
|
||||
EVENTHANDLER_REGISTER(shutdown_post_sync, poll_shutdown, NULL,
|
||||
SHUTDOWN_PRI_LAST);
|
||||
}
|
||||
SYSINIT(device_poll, SI_SUB_CLOCKS, SI_ORDER_MIDDLE, init_device_poll, NULL);
|
||||
|
||||
@ -289,7 +299,7 @@ hardclock_device_poll(void)
|
||||
static struct timeval prev_t, t;
|
||||
int delta;
|
||||
|
||||
if (poll_handlers == 0)
|
||||
if (poll_handlers == 0 || poll_shutting_down)
|
||||
return;
|
||||
|
||||
microuptime(&t);
|
||||
@ -314,7 +324,9 @@ hardclock_device_poll(void)
|
||||
if (phase != 0)
|
||||
suspect++;
|
||||
phase = 1;
|
||||
schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE);
|
||||
netisr_poll_scheduled = 1;
|
||||
netisr_pollmore_scheduled = 1;
|
||||
netisr_sched_poll();
|
||||
phase = 2;
|
||||
}
|
||||
if (pending_polls++ > 0)
|
||||
@ -365,9 +377,16 @@ netisr_pollmore()
|
||||
int kern_load;
|
||||
|
||||
mtx_lock(&poll_mtx);
|
||||
if (!netisr_pollmore_scheduled) {
|
||||
mtx_unlock(&poll_mtx);
|
||||
return;
|
||||
}
|
||||
netisr_pollmore_scheduled = 0;
|
||||
phase = 5;
|
||||
if (residual_burst > 0) {
|
||||
schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE);
|
||||
netisr_poll_scheduled = 1;
|
||||
netisr_pollmore_scheduled = 1;
|
||||
netisr_sched_poll();
|
||||
mtx_unlock(&poll_mtx);
|
||||
/* will run immediately on return, followed by netisrs */
|
||||
return;
|
||||
@ -397,23 +416,29 @@ netisr_pollmore()
|
||||
poll_burst -= (poll_burst / 8);
|
||||
if (poll_burst < 1)
|
||||
poll_burst = 1;
|
||||
schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE);
|
||||
netisr_poll_scheduled = 1;
|
||||
netisr_pollmore_scheduled = 1;
|
||||
netisr_sched_poll();
|
||||
phase = 6;
|
||||
}
|
||||
mtx_unlock(&poll_mtx);
|
||||
}
|
||||
|
||||
/*
|
||||
* netisr_poll is scheduled by schednetisr when appropriate, typically once
|
||||
* per tick.
|
||||
* netisr_poll is typically scheduled once per tick.
|
||||
*/
|
||||
static void
|
||||
void
|
||||
netisr_poll(void)
|
||||
{
|
||||
int i, cycles;
|
||||
enum poll_cmd arg = POLL_ONLY;
|
||||
|
||||
mtx_lock(&poll_mtx);
|
||||
if (!netisr_poll_scheduled) {
|
||||
mtx_unlock(&poll_mtx);
|
||||
return;
|
||||
}
|
||||
netisr_poll_scheduled = 0;
|
||||
phase = 3;
|
||||
if (residual_burst == 0) { /* first call in this tick */
|
||||
microuptime(&poll_start_t);
|
||||
|
||||
1228
sys/net/netisr.c
1228
sys/net/netisr.c
File diff suppressed because it is too large
Load Diff
142
sys/net/netisr.h
142
sys/net/netisr.h
@ -1,6 +1,6 @@
|
||||
/*-
|
||||
* Copyright (c) 1980, 1986, 1989, 1993
|
||||
* The Regents of the University of California. All rights reserved.
|
||||
* Copyright (c) 2007-2009 Robert N. M. Watson
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
@ -10,14 +10,11 @@
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* 4. Neither the name of the University nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
@ -26,20 +23,18 @@
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*
|
||||
* @(#)netisr.h 8.1 (Berkeley) 6/10/93
|
||||
* $FreeBSD$
|
||||
*/
|
||||
|
||||
#ifndef _NET_NETISR_H_
|
||||
#define _NET_NETISR_H_
|
||||
#ifdef _KERNEL
|
||||
|
||||
/*
|
||||
* The netisr (network interrupt service routine) provides a deferred
|
||||
* execution evironment in which (generally inbound) network processing can
|
||||
* take place. Protocols register handlers and, optionally, packet queues;
|
||||
* when packets are delivered to the queue, the protocol handler will be
|
||||
* executed directly, or via deferred dispatch depending on the
|
||||
* circumstances.
|
||||
* take place. Protocols register handlers which will be executed directly,
|
||||
* or via deferred dispatch, depending on the circumstances.
|
||||
*
|
||||
* Historically, this was implemented by the BSD software ISR facility; it is
|
||||
* now implemented via a software ithread (SWI).
|
||||
@ -53,37 +48,108 @@
|
||||
#define NETISR_ATALK1 17 /* Appletalk phase 1 */
|
||||
#define NETISR_ARP 18 /* same as AF_LINK */
|
||||
#define NETISR_IPX 23 /* same as AF_IPX */
|
||||
#define NETISR_ETHER 24 /* ethernet input */
|
||||
#define NETISR_IPV6 27
|
||||
#define NETISR_NATM 28
|
||||
#define NETISR_POLLMORE 31 /* polling callback, must be last */
|
||||
|
||||
#ifndef LOCORE
|
||||
#ifdef _KERNEL
|
||||
|
||||
void legacy_setsoftnet(void);
|
||||
|
||||
extern volatile unsigned int netisr; /* scheduling bits for network */
|
||||
#define schednetisr(anisr) do { \
|
||||
atomic_set_rel_int(&netisr, 1 << (anisr)); \
|
||||
legacy_setsoftnet(); \
|
||||
} while (0)
|
||||
/* used to atomically schedule multiple netisrs */
|
||||
#define schednetisrbits(isrbits) do { \
|
||||
atomic_set_rel_int(&netisr, isrbits); \
|
||||
legacy_setsoftnet(); \
|
||||
} while (0)
|
||||
|
||||
struct ifqueue;
|
||||
/*-
|
||||
* Protocols express ordering constraints and affinity preferences by
|
||||
* implementing one or neither of nh_m2flow and nh_m2cpuid, which are used by
|
||||
* netisr to determine which per-CPU workstream to assign mbufs to.
|
||||
*
|
||||
* The following policies may be used by protocols:
|
||||
*
|
||||
* NETISR_POLICY_SOURCE - netisr should maintain source ordering without
|
||||
* advice from the protocol. netisr will ignore any
|
||||
* flow IDs present on the mbuf for the purposes of
|
||||
* work placement.
|
||||
*
|
||||
* NETISR_POLICY_FLOW - netisr should maintain flow ordering as defined by
|
||||
* the mbuf header flow ID field. If the protocol
|
||||
* implements nh_m2flow, then netisr will query the
|
||||
* protocol in the event that the mbuf doesn't have a
|
||||
* flow ID, falling back on source ordering.
|
||||
*
|
||||
* NETISR_POLICY_CPU - netisr will delegate all work placement decisions to
|
||||
* the protocol, querying nh_m2cpuid for each packet.
|
||||
*
|
||||
* Protocols might make decisions about work placement based on an existing
|
||||
* calculated flow ID on the mbuf, such as one provided in hardware, the
|
||||
* receive interface pointed to by the mbuf (if any), the optional source
|
||||
* identifier passed at some dispatch points, or even parse packet headers to
|
||||
* calculate a flow. Both protocol handlers may return a new mbuf pointer
|
||||
* for the chain, or NULL if the packet proves invalid or m_pullup() fails.
|
||||
*
|
||||
* XXXRW: If we eventually support dynamic reconfiguration, there should be
|
||||
* protocol handlers to notify them of CPU configuration changes so that they
|
||||
* can rebalance work.
|
||||
*/
|
||||
struct mbuf;
|
||||
typedef void netisr_handler_t (struct mbuf *m);
|
||||
typedef struct mbuf *netisr_m2cpuid_t(struct mbuf *m, uintptr_t source,
|
||||
u_int *cpuid);
|
||||
typedef struct mbuf *netisr_m2flow_t(struct mbuf *m, uintptr_t source);
|
||||
|
||||
typedef void netisr_t (struct mbuf *);
|
||||
|
||||
void netisr_dispatch(int, struct mbuf *);
|
||||
int netisr_queue(int, struct mbuf *);
|
||||
void netisr_register(int, netisr_t *, struct ifqueue *, int);
|
||||
void netisr_unregister(int);
|
||||
#define NETISR_POLICY_SOURCE 1 /* Maintain source ordering. */
|
||||
#define NETISR_POLICY_FLOW 2 /* Maintain flow ordering. */
|
||||
#define NETISR_POLICY_CPU 3 /* Protocol determines CPU placement. */
|
||||
|
||||
#endif
|
||||
#endif
|
||||
/*
|
||||
* Data structure describing a protocol handler.
|
||||
*/
|
||||
struct netisr_handler {
|
||||
const char *nh_name; /* Character string protocol name. */
|
||||
netisr_handler_t *nh_handler; /* Protocol handler. */
|
||||
netisr_m2flow_t *nh_m2flow; /* Query flow for untagged packet. */
|
||||
netisr_m2cpuid_t *nh_m2cpuid; /* Query CPU to process mbuf on. */
|
||||
u_int nh_proto; /* Integer protocol ID. */
|
||||
u_int nh_qlimit; /* Maximum per-CPU queue depth. */
|
||||
u_int nh_policy; /* Work placement policy. */
|
||||
u_int nh_ispare[5]; /* For future use. */
|
||||
void *nh_pspare[4]; /* For future use. */
|
||||
};
|
||||
|
||||
#endif
|
||||
/*
|
||||
* Register, unregister, and other netisr handler management functions.
|
||||
*/
|
||||
void netisr_clearqdrops(const struct netisr_handler *nhp);
|
||||
void netisr_getqdrops(const struct netisr_handler *nhp,
|
||||
u_int64_t *qdropsp);
|
||||
void netisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp);
|
||||
void netisr_register(const struct netisr_handler *nhp);
|
||||
int netisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit);
|
||||
void netisr_unregister(const struct netisr_handler *nhp);
|
||||
|
||||
/*
|
||||
* Process a packet destined for a protocol, and attempt direct dispatch.
|
||||
* Supplemental source ordering information can be passed using the _src
|
||||
* variant.
|
||||
*/
|
||||
int netisr_dispatch(u_int proto, struct mbuf *m);
|
||||
int netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m);
|
||||
int netisr_queue(u_int proto, struct mbuf *m);
|
||||
int netisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m);
|
||||
|
||||
/*
|
||||
* Provide a default implementation of "map an ID to a CPU ID".
|
||||
*/
|
||||
u_int netisr_default_flow2cpu(u_int flowid);
|
||||
|
||||
/*
|
||||
* Utility routines to return the number of CPUs participting in netisr, and
|
||||
* to return a mapping from a number to a CPU ID that can be used with the
|
||||
* scheduler.
|
||||
*/
|
||||
u_int netisr_get_cpucount(void);
|
||||
u_int netisr_get_cpuid(u_int cpunumber);
|
||||
|
||||
/*
|
||||
* Interfaces between DEVICE_POLLING and netisr.
|
||||
*/
|
||||
void netisr_sched_poll(void);
|
||||
void netisr_poll(void);
|
||||
void netisr_pollmore(void);
|
||||
|
||||
#endif /* !_KERNEL */
|
||||
#endif /* !_NET_NETISR_H_ */
|
||||
|
||||
@ -90,11 +90,7 @@ MTX_SYSINIT(rtsock, &rtsock_mtx, "rtsock route_cb lock", MTX_DEF);
|
||||
#define RTSOCK_UNLOCK() mtx_unlock(&rtsock_mtx)
|
||||
#define RTSOCK_LOCK_ASSERT() mtx_assert(&rtsock_mtx, MA_OWNED)
|
||||
|
||||
static struct ifqueue rtsintrq;
|
||||
|
||||
SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RD, 0, "");
|
||||
SYSCTL_INT(_net_route, OID_AUTO, netisr_maxqlen, CTLFLAG_RW,
|
||||
&rtsintrq.ifq_maxlen, 0, "maximum routing socket dispatch queue length");
|
||||
|
||||
struct walkarg {
|
||||
int w_tmemsize;
|
||||
@ -119,16 +115,38 @@ static void rt_getmetrics(const struct rt_metrics_lite *in,
|
||||
struct rt_metrics *out);
|
||||
static void rt_dispatch(struct mbuf *, const struct sockaddr *);
|
||||
|
||||
static struct netisr_handler rtsock_nh = {
|
||||
.nh_name = "rtsock",
|
||||
.nh_handler = rts_input,
|
||||
.nh_proto = NETISR_ROUTE,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
static int
|
||||
sysctl_route_netisr_maxqlen(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
int error, qlimit;
|
||||
|
||||
netisr_getqlimit(&rtsock_nh, &qlimit);
|
||||
error = sysctl_handle_int(oidp, &qlimit, 0, req);
|
||||
if (error || !req->newptr)
|
||||
return (error);
|
||||
if (qlimit < 1)
|
||||
return (EINVAL);
|
||||
return (netisr_setqlimit(&rtsock_nh, qlimit));
|
||||
}
|
||||
SYSCTL_PROC(_net_route, OID_AUTO, netisr_maxqlen, CTLTYPE_INT|CTLFLAG_RW,
|
||||
0, 0, sysctl_route_netisr_maxqlen, "I",
|
||||
"maximum routing socket dispatch queue length");
|
||||
|
||||
static void
|
||||
rts_init(void)
|
||||
{
|
||||
int tmp;
|
||||
|
||||
rtsintrq.ifq_maxlen = 256;
|
||||
if (TUNABLE_INT_FETCH("net.route.netisr_maxqlen", &tmp))
|
||||
rtsintrq.ifq_maxlen = tmp;
|
||||
mtx_init(&rtsintrq.ifq_mtx, "rts_inq", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_ROUTE, rts_input, &rtsintrq, 0);
|
||||
rtsock_nh.nh_qlimit = tmp;
|
||||
netisr_register(&rtsock_nh);
|
||||
}
|
||||
SYSINIT(rtsock, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, rts_init, 0);
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*-
|
||||
* Copyright (c) 2004-2005 Robert N. M. Watson
|
||||
* Copyright (c) 2004-2009 Robert N. M. Watson
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -70,7 +70,26 @@
|
||||
static u_long ddp_sendspace = DDP_MAXSZ; /* Max ddp size + 1 (ddp_type) */
|
||||
static u_long ddp_recvspace = 10 * (587 + sizeof(struct sockaddr_at));
|
||||
|
||||
static struct ifqueue atintrq1, atintrq2, aarpintrq;
|
||||
static const struct netisr_handler atalk1_nh = {
|
||||
.nh_name = "atalk1",
|
||||
.nh_handler = at1intr,
|
||||
.nh_proto = NETISR_ATALK1,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
static const struct netisr_handler atalk2_nh = {
|
||||
.nh_name = "atalk2",
|
||||
.nh_handler = at2intr,
|
||||
.nh_proto = NETISR_ATALK2,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
static const struct netisr_handler aarp_nh = {
|
||||
.nh_name = "aarp",
|
||||
.nh_handler = aarpintr,
|
||||
.nh_proto = NETISR_AARP,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
static int
|
||||
ddp_attach(struct socket *so, int proto, struct thread *td)
|
||||
@ -256,16 +275,10 @@ void
|
||||
ddp_init(void)
|
||||
{
|
||||
|
||||
atintrq1.ifq_maxlen = IFQ_MAXLEN;
|
||||
atintrq2.ifq_maxlen = IFQ_MAXLEN;
|
||||
aarpintrq.ifq_maxlen = IFQ_MAXLEN;
|
||||
mtx_init(&atintrq1.ifq_mtx, "at1_inq", NULL, MTX_DEF);
|
||||
mtx_init(&atintrq2.ifq_mtx, "at2_inq", NULL, MTX_DEF);
|
||||
mtx_init(&aarpintrq.ifq_mtx, "aarp_inq", NULL, MTX_DEF);
|
||||
DDP_LIST_LOCK_INIT();
|
||||
netisr_register(NETISR_ATALK1, at1intr, &atintrq1, 0);
|
||||
netisr_register(NETISR_ATALK2, at2intr, &atintrq2, 0);
|
||||
netisr_register(NETISR_AARP, aarpintr, &aarpintrq, 0);
|
||||
netisr_register(&atalk1_nh);
|
||||
netisr_register(&atalk2_nh);
|
||||
netisr_register(&aarp_nh);
|
||||
}
|
||||
|
||||
#if 0
|
||||
|
||||
@ -96,8 +96,6 @@ static int arp_proxyall;
|
||||
SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, max_age,
|
||||
CTLFLAG_RW, arpt_keep, 0, "ARP entry lifetime in seconds");
|
||||
|
||||
static struct ifqueue arpintrq;
|
||||
|
||||
SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, maxtries,
|
||||
CTLFLAG_RW, arp_maxtries, 0,
|
||||
"ARP resolution attempts before returning error");
|
||||
@ -118,6 +116,13 @@ static void arptimer(void *);
|
||||
static void in_arpinput(struct mbuf *);
|
||||
#endif
|
||||
|
||||
static const struct netisr_handler arp_nh = {
|
||||
.nh_name = "arp",
|
||||
.nh_handler = arpintr,
|
||||
.nh_proto = NETISR_ARP,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
#ifndef VIMAGE_GLOBALS
|
||||
static const vnet_modinfo_t vnet_arp_modinfo = {
|
||||
.vmi_id = VNET_MOD_ARP,
|
||||
@ -823,8 +828,6 @@ arp_init(void)
|
||||
arp_iattach(NULL);
|
||||
#endif
|
||||
|
||||
arpintrq.ifq_maxlen = 50;
|
||||
mtx_init(&arpintrq.ifq_mtx, "arp_inq", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_ARP, arpintr, &arpintrq, 0);
|
||||
netisr_register(&arp_nh);
|
||||
}
|
||||
SYSINIT(arp, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, arp_init, 0);
|
||||
|
||||
@ -144,6 +144,13 @@ static int sysctl_igmp_ifinfo(SYSCTL_HANDLER_ARGS);
|
||||
static vnet_attach_fn vnet_igmp_iattach;
|
||||
static vnet_detach_fn vnet_igmp_idetach;
|
||||
|
||||
static const struct netisr_handler igmp_nh = {
|
||||
.nh_name = "igmp",
|
||||
.nh_handler = igmp_intr,
|
||||
.nh_proto = NETISR_IGMP,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
/*
|
||||
* System-wide globals.
|
||||
*
|
||||
@ -189,11 +196,6 @@ struct mtx igmp_mtx;
|
||||
struct mbuf *m_raopt; /* Router Alert option */
|
||||
MALLOC_DEFINE(M_IGMP, "igmp", "igmp state");
|
||||
|
||||
/*
|
||||
* Global netisr output queue.
|
||||
*/
|
||||
struct ifqueue igmpoq;
|
||||
|
||||
/*
|
||||
* VIMAGE-wide globals.
|
||||
*
|
||||
@ -3537,12 +3539,9 @@ igmp_sysinit(void)
|
||||
|
||||
IGMP_LOCK_INIT();
|
||||
|
||||
mtx_init(&igmpoq.ifq_mtx, "igmpoq_mtx", NULL, MTX_DEF);
|
||||
IFQ_SET_MAXLEN(&igmpoq, IFQ_MAXLEN);
|
||||
|
||||
m_raopt = igmp_ra_alloc();
|
||||
|
||||
netisr_register(NETISR_IGMP, igmp_intr, &igmpoq, 0);
|
||||
netisr_register(&igmp_nh);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -3551,8 +3550,7 @@ igmp_sysuninit(void)
|
||||
|
||||
CTR1(KTR_IGMPV3, "%s: tearing down", __func__);
|
||||
|
||||
netisr_unregister(NETISR_IGMP);
|
||||
mtx_destroy(&igmpoq.ifq_mtx);
|
||||
netisr_unregister(&igmp_nh);
|
||||
|
||||
m_free(m_raopt);
|
||||
m_raopt = NULL;
|
||||
|
||||
@ -472,7 +472,7 @@ div_output(struct socket *so, struct mbuf *m, struct sockaddr_in *sin,
|
||||
SOCK_UNLOCK(so);
|
||||
#endif
|
||||
/* Send packet to input processing via netisr */
|
||||
netisr_queue(NETISR_IP, m);
|
||||
netisr_queue_src(NETISR_IP, (uintptr_t)so, m);
|
||||
}
|
||||
|
||||
return error;
|
||||
|
||||
@ -164,18 +164,17 @@ SYSCTL_V_INT(V_NET, vnet_inet, _net_inet_ip, OID_AUTO,
|
||||
|
||||
struct pfil_head inet_pfil_hook; /* Packet filter hooks */
|
||||
|
||||
static struct ifqueue ipintrq;
|
||||
static int ipqmaxlen = IFQ_MAXLEN;
|
||||
static struct netisr_handler ip_nh = {
|
||||
.nh_name = "ip",
|
||||
.nh_handler = ip_input,
|
||||
.nh_proto = NETISR_IP,
|
||||
.nh_policy = NETISR_POLICY_FLOW,
|
||||
};
|
||||
|
||||
extern struct domain inetdomain;
|
||||
extern struct protosw inetsw[];
|
||||
u_char ip_protox[IPPROTO_MAX];
|
||||
|
||||
SYSCTL_INT(_net_inet_ip, IPCTL_INTRQMAXLEN, intr_queue_maxlen, CTLFLAG_RW,
|
||||
&ipintrq.ifq_maxlen, 0, "Maximum size of the IP input queue");
|
||||
SYSCTL_INT(_net_inet_ip, IPCTL_INTRQDROPS, intr_queue_drops, CTLFLAG_RD,
|
||||
&ipintrq.ifq_drops, 0,
|
||||
"Number of packets dropped from the IP input queue");
|
||||
|
||||
SYSCTL_V_STRUCT(V_NET, vnet_inet, _net_inet_ip, IPCTL_STATS, stats, CTLFLAG_RW,
|
||||
ipstat, ipstat, "IP statistics (struct ipstat, netinet/ip_var.h)");
|
||||
@ -249,6 +248,44 @@ static void vnet_inet_register()
|
||||
SYSINIT(inet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, vnet_inet_register, 0);
|
||||
#endif
|
||||
|
||||
static int
|
||||
sysctl_netinet_intr_queue_maxlen(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
int error, qlimit;
|
||||
|
||||
netisr_getqlimit(&ip_nh, &qlimit);
|
||||
error = sysctl_handle_int(oidp, &qlimit, 0, req);
|
||||
if (error || !req->newptr)
|
||||
return (error);
|
||||
if (qlimit < 1)
|
||||
return (EINVAL);
|
||||
return (netisr_setqlimit(&ip_nh, qlimit));
|
||||
}
|
||||
SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQMAXLEN, intr_queue_maxlen,
|
||||
CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_netinet_intr_queue_maxlen, "I",
|
||||
"Maximum size of the IP input queue");
|
||||
|
||||
static int
|
||||
sysctl_netinet_intr_queue_drops(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
u_int64_t qdrops_long;
|
||||
int error, qdrops;
|
||||
|
||||
netisr_getqdrops(&ip_nh, &qdrops_long);
|
||||
qdrops = qdrops_long;
|
||||
error = sysctl_handle_int(oidp, &qdrops, 0, req);
|
||||
if (error || !req->newptr)
|
||||
return (error);
|
||||
if (qdrops != 0)
|
||||
return (EINVAL);
|
||||
netisr_clearqdrops(&ip_nh);
|
||||
return (0);
|
||||
}
|
||||
|
||||
SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQDROPS, intr_queue_drops,
|
||||
CTLTYPE_INT|CTLFLAG_RD, 0, 0, sysctl_netinet_intr_queue_drops, "I",
|
||||
"Number of packets dropped from the IP input queue");
|
||||
|
||||
/*
|
||||
* IP initialization: fill in IP protocol switch table.
|
||||
* All protocols not implemented in kernel go to raw IP protocol handler.
|
||||
@ -347,10 +384,7 @@ ip_init(void)
|
||||
|
||||
/* Initialize various other remaining things. */
|
||||
IPQ_LOCK_INIT();
|
||||
ipintrq.ifq_maxlen = ipqmaxlen;
|
||||
mtx_init(&ipintrq.ifq_mtx, "ip_inq", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_IP, ip_input, &ipintrq, 0);
|
||||
|
||||
netisr_register(&ip_nh);
|
||||
ip_ft = flowtable_alloc(ip_output_flowtable_size, FL_PCPU);
|
||||
}
|
||||
|
||||
|
||||
@ -120,7 +120,13 @@ __FBSDID("$FreeBSD$");
|
||||
extern struct domain inet6domain;
|
||||
|
||||
u_char ip6_protox[IPPROTO_MAX];
|
||||
static struct ifqueue ip6intrq;
|
||||
|
||||
static struct netisr_handler ip6_nh = {
|
||||
.nh_name = "ip6",
|
||||
.nh_handler = ip6_input,
|
||||
.nh_proto = NETISR_IPV6,
|
||||
.nh_policy = NETISR_POLICY_FLOW,
|
||||
};
|
||||
|
||||
#ifndef VIMAGE
|
||||
#ifndef VIMAGE_GLOBALS
|
||||
@ -129,7 +135,6 @@ struct vnet_inet6 vnet_inet6_0;
|
||||
#endif
|
||||
|
||||
#ifdef VIMAGE_GLOBALS
|
||||
static int ip6qmaxlen;
|
||||
struct in6_ifaddr *in6_ifaddr;
|
||||
struct ip6stat ip6stat;
|
||||
|
||||
@ -186,7 +191,6 @@ ip6_init(void)
|
||||
struct ip6protosw *pr;
|
||||
int i;
|
||||
|
||||
V_ip6qmaxlen = IFQ_MAXLEN;
|
||||
V_in6_maxmtu = 0;
|
||||
#ifdef IP6_AUTO_LINKLOCAL
|
||||
V_ip6_auto_linklocal = IP6_AUTO_LINKLOCAL;
|
||||
@ -296,9 +300,7 @@ ip6_init(void)
|
||||
printf("%s: WARNING: unable to register pfil hook, "
|
||||
"error %d\n", __func__, i);
|
||||
|
||||
ip6intrq.ifq_maxlen = V_ip6qmaxlen; /* XXX */
|
||||
mtx_init(&ip6intrq.ifq_mtx, "ip6_inq", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_IPV6, ip6_input, &ip6intrq, 0);
|
||||
netisr_register(&ip6_nh);
|
||||
}
|
||||
|
||||
static int
|
||||
|
||||
@ -118,7 +118,6 @@ struct vnet_inet6 {
|
||||
int _icmp6_nodeinfo;
|
||||
int _udp6_sendspace;
|
||||
int _udp6_recvspace;
|
||||
int _ip6qmaxlen;
|
||||
int _ip6_prefer_tempaddr;
|
||||
|
||||
int _nd6_prune;
|
||||
@ -224,7 +223,6 @@ extern struct vnet_inet6 vnet_inet6_0;
|
||||
#define V_ip6_use_tempaddr VNET_INET6(ip6_use_tempaddr)
|
||||
#define V_ip6_v6only VNET_INET6(ip6_v6only)
|
||||
#define V_ip6q VNET_INET6(ip6q)
|
||||
#define V_ip6qmaxlen VNET_INET6(ip6qmaxlen)
|
||||
#define V_ip6stat VNET_INET6(ip6stat)
|
||||
#define V_ip6stealth VNET_INET6(ip6stealth)
|
||||
#define V_llinfo_nd6 VNET_INET6(llinfo_nd6)
|
||||
|
||||
@ -481,7 +481,7 @@ ipsec4_common_input_cb(struct mbuf *m, struct secasvar *sav,
|
||||
/*
|
||||
* Re-dispatch via software interrupt.
|
||||
*/
|
||||
if ((error = netisr_queue(NETISR_IP, m))) {
|
||||
if ((error = netisr_queue_src(NETISR_IP, (uintptr_t)sav, m))) {
|
||||
IPSEC_ISTAT(sproto, V_espstat.esps_qfull, V_ahstat.ahs_qfull,
|
||||
V_ipcompstat.ipcomps_qfull);
|
||||
|
||||
|
||||
@ -100,6 +100,11 @@ static int ipxnetbios = 0;
|
||||
SYSCTL_INT(_net_ipx, OID_AUTO, ipxnetbios, CTLFLAG_RW,
|
||||
&ipxnetbios, 0, "Propagate netbios over ipx");
|
||||
|
||||
static int ipx_do_route(struct ipx_addr *src, struct route *ro);
|
||||
static void ipx_undo_route(struct route *ro);
|
||||
static void ipx_forward(struct mbuf *m);
|
||||
static void ipxintr(struct mbuf *m);
|
||||
|
||||
const union ipx_net ipx_zeronet;
|
||||
const union ipx_host ipx_zerohost;
|
||||
|
||||
@ -119,16 +124,15 @@ struct mtx ipxpcb_list_mtx;
|
||||
struct ipxpcbhead ipxpcb_list;
|
||||
struct ipxpcbhead ipxrawpcb_list;
|
||||
|
||||
static int ipxqmaxlen = IFQ_MAXLEN;
|
||||
static struct ifqueue ipxintrq;
|
||||
static struct netisr_handler ipx_nh = {
|
||||
.nh_name = "ipx",
|
||||
.nh_handler = ipxintr,
|
||||
.nh_proto = NETISR_IPX,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
long ipx_pexseq; /* Locked with ipxpcb_list_mtx. */
|
||||
|
||||
static int ipx_do_route(struct ipx_addr *src, struct route *ro);
|
||||
static void ipx_undo_route(struct route *ro);
|
||||
static void ipx_forward(struct mbuf *m);
|
||||
static void ipxintr(struct mbuf *m);
|
||||
|
||||
/*
|
||||
* IPX initialization.
|
||||
*/
|
||||
@ -151,9 +155,7 @@ ipx_init(void)
|
||||
ipx_hostmask.sipx_addr.x_net = ipx_broadnet;
|
||||
ipx_hostmask.sipx_addr.x_host = ipx_broadhost;
|
||||
|
||||
ipxintrq.ifq_maxlen = ipxqmaxlen;
|
||||
mtx_init(&ipxintrq.ifq_mtx, "ipx_inq", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_IPX, ipxintr, &ipxintrq, 0);
|
||||
netisr_register(&ipx_nh);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -88,8 +88,14 @@ static struct domain natmdomain = {
|
||||
.dom_protoswNPROTOSW = &natmsw[sizeof(natmsw)/sizeof(natmsw[0])],
|
||||
};
|
||||
|
||||
static int natmqmaxlen = 1000 /* IFQ_MAXLEN */; /* max # of packets on queue */
|
||||
static struct ifqueue natmintrq;
|
||||
static struct netisr_handler natm_nh = {
|
||||
.nh_name = "natm",
|
||||
.nh_handler = natmintr,
|
||||
.nh_proto = NETISR_NATM,
|
||||
.nh_qlimit = 1000,
|
||||
.nh_policy = NETISR_POLICY_SOURCE,
|
||||
};
|
||||
|
||||
#ifdef NATM_STAT
|
||||
u_int natm_sodropcnt; /* # mbufs dropped due to full sb */
|
||||
u_int natm_sodropbytes; /* # of bytes dropped */
|
||||
@ -101,11 +107,8 @@ static void
|
||||
natm_init(void)
|
||||
{
|
||||
LIST_INIT(&natm_pcbs);
|
||||
bzero(&natmintrq, sizeof(natmintrq));
|
||||
natmintrq.ifq_maxlen = natmqmaxlen;
|
||||
NATM_LOCK_INIT();
|
||||
mtx_init(&natmintrq.ifq_mtx, "natm_inq", NULL, MTX_DEF);
|
||||
netisr_register(NETISR_NATM, natmintr, &natmintrq, 0);
|
||||
netisr_register(&natm_nh);
|
||||
}
|
||||
|
||||
DOMAIN_SET(natm);
|
||||
|
||||
@ -57,7 +57,7 @@
|
||||
* is created, otherwise 1.
|
||||
*/
|
||||
#undef __FreeBSD_version
|
||||
#define __FreeBSD_version 800095 /* Master, propagated to newvers */
|
||||
#define __FreeBSD_version 800096 /* Master, propagated to newvers */
|
||||
|
||||
#ifndef LOCORE
|
||||
#include <sys/types.h>
|
||||
|
||||
@ -86,6 +86,7 @@ struct pcpu {
|
||||
struct vmmeter pc_cnt; /* VM stats counters */
|
||||
long pc_cp_time[CPUSTATES]; /* statclock ticks */
|
||||
struct device *pc_device;
|
||||
void *pc_netisr; /* netisr SWI cookie. */
|
||||
|
||||
/*
|
||||
* Stuff for read mostly lock
|
||||
|
||||
Loading…
Reference in New Issue
Block a user