Massive sync with HEAD:
- Instead of direct manipulation on queue and worklist mutexes, bring macros for doing this job. This change will make it easy to migrate from using spinning locks to adaptive ones. - We don't need spinning locks here. Change them to the adaptive mutexes. This change should bring no performance decrease, as it did not in my tests. - Despite several examples in the kernel, the third argument of sysctl_handle_int is not sizeof the int type you want to export. - Implement stack protection based on GET_STACK_USAGE() macro. This fixes system panics possible with complicated netgraph setups and allows to avoid unneded extra queueing for stack unwrapping. - Cleanup and tune ng_snd_item() function as it is one of the most busy netgraph functions.
This commit is contained in:
mav
parent
b53943d8df
commit
089bb21733
@ -127,6 +127,7 @@ struct ng_hook {
|
||||
#define HK_QUEUE 0x0002 /* queue for later delivery */
|
||||
#define HK_FORCE_WRITER 0x0004 /* Incoming data queued as a writer */
|
||||
#define HK_DEAD 0x0008 /* This is the dead hook.. don't free */
|
||||
#define HK_HI_STACK 0x0010 /* Hook has hi stack usage */
|
||||
|
||||
/*
|
||||
* Public Methods for hook
|
||||
@ -147,6 +148,7 @@ void ng_unref_hook(hook_p hook); /* don't move this */
|
||||
#define _NG_HOOK_FORCE_WRITER(hook) \
|
||||
do { hook->hk_flags |= HK_FORCE_WRITER; } while (0)
|
||||
#define _NG_HOOK_FORCE_QUEUE(hook) do { hook->hk_flags |= HK_QUEUE; } while (0)
|
||||
#define _NG_HOOK_HI_STACK(hook) do { hook->hk_flags |= HK_HI_STACK; } while (0)
|
||||
|
||||
/* Some shortcuts */
|
||||
#define NG_PEER_NODE(hook) NG_HOOK_NODE(NG_HOOK_PEER(hook))
|
||||
@ -277,6 +279,13 @@ _ng_hook_force_queue(hook_p hook, char * file, int line)
|
||||
_NG_HOOK_FORCE_QUEUE(hook);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
_ng_hook_hi_stack(hook_p hook, char * file, int line)
|
||||
{
|
||||
_chkhook(hook, file, line);
|
||||
_NG_HOOK_HI_STACK(hook);
|
||||
}
|
||||
|
||||
|
||||
#define NG_HOOK_REF(hook) _ng_hook_ref(hook, _NN_)
|
||||
#define NG_HOOK_NAME(hook) _ng_hook_name(hook, _NN_)
|
||||
@ -291,6 +300,7 @@ _ng_hook_force_queue(hook_p hook, char * file, int line)
|
||||
#define NG_HOOK_PEER(hook) _ng_hook_peer(hook, _NN_)
|
||||
#define NG_HOOK_FORCE_WRITER(hook) _ng_hook_force_writer(hook, _NN_)
|
||||
#define NG_HOOK_FORCE_QUEUE(hook) _ng_hook_force_queue(hook, _NN_)
|
||||
#define NG_HOOK_HI_STACK(hook) _ng_hook_hi_stack(hook, _NN_)
|
||||
|
||||
#else /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
|
||||
|
||||
@ -307,6 +317,7 @@ _ng_hook_force_queue(hook_p hook, char * file, int line)
|
||||
#define NG_HOOK_PEER(hook) _NG_HOOK_PEER(hook)
|
||||
#define NG_HOOK_FORCE_WRITER(hook) _NG_HOOK_FORCE_WRITER(hook)
|
||||
#define NG_HOOK_FORCE_QUEUE(hook) _NG_HOOK_FORCE_QUEUE(hook)
|
||||
#define NG_HOOK_HI_STACK(hook) _NG_HOOK_HI_STACK(hook)
|
||||
|
||||
#endif /* NETGRAPH_DEBUG */ /*----------------------------------------------*/
|
||||
|
||||
@ -360,6 +371,7 @@ struct ng_node {
|
||||
#define NG_CLOSING NGF_CLOSING /* compat for old code */
|
||||
#define NGF_REALLY_DIE 0x00000010 /* "persistent" node is unloading */
|
||||
#define NG_REALLY_DIE NGF_REALLY_DIE /* compat for old code */
|
||||
#define NGF_HI_STACK 0x00000020 /* node has hi stack usage */
|
||||
#define NGF_TYPE1 0x10000000 /* reserved for type specific storage */
|
||||
#define NGF_TYPE2 0x20000000 /* reserved for type specific storage */
|
||||
#define NGF_TYPE3 0x40000000 /* reserved for type specific storage */
|
||||
@ -382,6 +394,8 @@ int ng_unref_node(node_p node); /* don't move this */
|
||||
#define _NG_NODE_NUMHOOKS(node) ((node)->nd_numhooks + 0) /* rvalue */
|
||||
#define _NG_NODE_FORCE_WRITER(node) \
|
||||
do{ node->nd_flags |= NGF_FORCE_WRITER; }while (0)
|
||||
#define _NG_NODE_HI_STACK(node) \
|
||||
do{ node->nd_flags |= NGF_HI_STACK; }while (0)
|
||||
#define _NG_NODE_REALLY_DIE(node) \
|
||||
do{ node->nd_flags |= (NGF_REALLY_DIE|NGF_INVALID); }while (0)
|
||||
#define _NG_NODE_REVIVE(node) \
|
||||
@ -512,6 +526,13 @@ _ng_node_force_writer(node_p node, char *file, int line)
|
||||
_NG_NODE_FORCE_WRITER(node);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
_ng_node_hi_stack(node_p node, char *file, int line)
|
||||
{
|
||||
_chknode(node, file, line);
|
||||
_NG_NODE_HI_STACK(node);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
_ng_node_really_die(node_p node, char *file, int line)
|
||||
{
|
||||
@ -546,6 +567,7 @@ _ng_node_foreach_hook(node_p node, ng_fn_eachhook *fn, void *arg,
|
||||
#define NG_NODE_IS_VALID(node) _ng_node_is_valid(node, _NN_)
|
||||
#define NG_NODE_NOT_VALID(node) _ng_node_not_valid(node, _NN_)
|
||||
#define NG_NODE_FORCE_WRITER(node) _ng_node_force_writer(node, _NN_)
|
||||
#define NG_NODE_HI_STACK(node) _ng_node_hi_stack(node, _NN_)
|
||||
#define NG_NODE_REALLY_DIE(node) _ng_node_really_die(node, _NN_)
|
||||
#define NG_NODE_NUMHOOKS(node) _ng_node_numhooks(node, _NN_)
|
||||
#define NG_NODE_REVIVE(node) _ng_node_revive(node, _NN_)
|
||||
@ -566,6 +588,7 @@ _ng_node_foreach_hook(node_p node, ng_fn_eachhook *fn, void *arg,
|
||||
#define NG_NODE_IS_VALID(node) _NG_NODE_IS_VALID(node)
|
||||
#define NG_NODE_NOT_VALID(node) _NG_NODE_NOT_VALID(node)
|
||||
#define NG_NODE_FORCE_WRITER(node) _NG_NODE_FORCE_WRITER(node)
|
||||
#define NG_NODE_HI_STACK(node) _NG_NODE_HI_STACK(node)
|
||||
#define NG_NODE_REALLY_DIE(node) _NG_NODE_REALLY_DIE(node)
|
||||
#define NG_NODE_NUMHOOKS(node) _NG_NODE_NUMHOOKS(node)
|
||||
#define NG_NODE_REVIVE(node) _NG_NODE_REVIVE(node)
|
||||
|
||||
@ -60,6 +60,7 @@
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/syslog.h>
|
||||
#include <sys/refcount.h>
|
||||
#include <sys/proc.h>
|
||||
|
||||
#include <net/netisr.h>
|
||||
|
||||
@ -229,6 +230,19 @@ MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage");
|
||||
#define _NG_ALLOC_NODE(node) \
|
||||
MALLOC(node, node_p, sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO)
|
||||
|
||||
#define NG_QUEUE_LOCK_INIT(n) \
|
||||
mtx_init(&(n)->q_mtx, "ng_node", NULL, MTX_DEF)
|
||||
#define NG_QUEUE_LOCK(n) \
|
||||
mtx_lock(&(n)->q_mtx)
|
||||
#define NG_QUEUE_UNLOCK(n) \
|
||||
mtx_unlock(&(n)->q_mtx)
|
||||
#define NG_WORKLIST_LOCK_INIT() \
|
||||
mtx_init(&ng_worklist_mtx, "ng_worklist", NULL, MTX_DEF)
|
||||
#define NG_WORKLIST_LOCK() \
|
||||
mtx_lock(&ng_worklist_mtx)
|
||||
#define NG_WORKLIST_UNLOCK() \
|
||||
mtx_unlock(&ng_worklist_mtx)
|
||||
|
||||
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
|
||||
/*
|
||||
* In debug mode:
|
||||
@ -607,7 +621,7 @@ ng_make_node_common(struct ng_type *type, node_p *nodepp)
|
||||
NG_NODE_REF(node); /* note reference */
|
||||
type->refs++;
|
||||
|
||||
mtx_init(&node->nd_input_queue.q_mtx, "ng_node", NULL, MTX_SPIN);
|
||||
NG_QUEUE_LOCK_INIT(&node->nd_input_queue);
|
||||
node->nd_input_queue.queue = NULL;
|
||||
node->nd_input_queue.last = &node->nd_input_queue.queue;
|
||||
node->nd_input_queue.q_flags = 0;
|
||||
@ -2056,7 +2070,7 @@ ng_acquire_read(struct ng_queue *ngq, item_p item)
|
||||
atomic_subtract_long(&ngq->q_flags, READER_INCREMENT);
|
||||
|
||||
/* ######### End Hack alert ######### */
|
||||
mtx_lock_spin((&ngq->q_mtx));
|
||||
NG_QUEUE_LOCK(ngq);
|
||||
/*
|
||||
* Try again. Another processor (or interrupt for that matter) may
|
||||
* have removed the last queued item that was stopping us from
|
||||
@ -2067,7 +2081,7 @@ ng_acquire_read(struct ng_queue *ngq, item_p item)
|
||||
*/
|
||||
if ((ngq->q_flags & NGQ_RMASK) == 0) {
|
||||
atomic_add_long(&ngq->q_flags, READER_INCREMENT);
|
||||
mtx_unlock_spin((&ngq->q_mtx));
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
CTR4(KTR_NET, "%20s: node [%x] (%p) slow acquired item %p",
|
||||
__func__, ngq->q_node->nd_ID, ngq->q_node, item);
|
||||
return (item);
|
||||
@ -2077,7 +2091,7 @@ ng_acquire_read(struct ng_queue *ngq, item_p item)
|
||||
* and queue the request for later.
|
||||
*/
|
||||
ng_queue_rw(ngq, item, NGQRW_R);
|
||||
mtx_unlock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
|
||||
return (NULL);
|
||||
}
|
||||
@ -2089,7 +2103,7 @@ ng_acquire_write(struct ng_queue *ngq, item_p item)
|
||||
("%s: working on deadnode", __func__));
|
||||
|
||||
restart:
|
||||
mtx_lock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_LOCK(ngq);
|
||||
/*
|
||||
* If there are no readers, no writer, and no pending packets, then
|
||||
* we can just go ahead. In all other situations we need to queue the
|
||||
@ -2098,7 +2112,7 @@ ng_acquire_write(struct ng_queue *ngq, item_p item)
|
||||
if ((ngq->q_flags & NGQ_WMASK) == 0) {
|
||||
/* collision could happen *HERE* */
|
||||
atomic_add_long(&ngq->q_flags, WRITER_ACTIVE);
|
||||
mtx_unlock_spin((&ngq->q_mtx));
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
if (ngq->q_flags & READER_MASK) {
|
||||
/* Collision with fast-track reader */
|
||||
atomic_subtract_long(&ngq->q_flags, WRITER_ACTIVE);
|
||||
@ -2113,7 +2127,7 @@ ng_acquire_write(struct ng_queue *ngq, item_p item)
|
||||
* and queue the request for later.
|
||||
*/
|
||||
ng_queue_rw(ngq, item, NGQRW_W);
|
||||
mtx_unlock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
|
||||
return (NULL);
|
||||
}
|
||||
@ -2135,7 +2149,7 @@ ng_flush_input_queue(struct ng_queue * ngq)
|
||||
{
|
||||
item_p item;
|
||||
|
||||
mtx_lock_spin(&ngq->q_mtx);
|
||||
NG_QUEUE_LOCK(ngq);
|
||||
while (ngq->queue) {
|
||||
item = ngq->queue;
|
||||
ngq->queue = item->el_next;
|
||||
@ -2143,21 +2157,22 @@ ng_flush_input_queue(struct ng_queue * ngq)
|
||||
ngq->last = &(ngq->queue);
|
||||
atomic_add_long(&ngq->q_flags, -OP_PENDING);
|
||||
}
|
||||
mtx_unlock_spin(&ngq->q_mtx);
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
|
||||
/* If the item is supplying a callback, call it with an error */
|
||||
if (item->apply != NULL &&
|
||||
refcount_release(&item->apply->refs)) {
|
||||
(*item->apply->apply)(item->apply->context, ENOENT);
|
||||
}
|
||||
NG_FREE_ITEM(item);
|
||||
mtx_lock_spin(&ngq->q_mtx);
|
||||
NG_QUEUE_LOCK(ngq);
|
||||
}
|
||||
/*
|
||||
* Take us off the work queue if we are there.
|
||||
* We definately have no work to be done.
|
||||
*/
|
||||
ng_worklist_remove(ngq->q_node);
|
||||
mtx_unlock_spin(&ngq->q_mtx);
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
}
|
||||
|
||||
/***********************************************************************
|
||||
@ -2192,109 +2207,81 @@ ng_snd_item(item_p item, int flags)
|
||||
struct ng_queue *ngq;
|
||||
int error = 0;
|
||||
|
||||
if (item == NULL) {
|
||||
TRAP_ERROR();
|
||||
return (EINVAL); /* failed to get queue element */
|
||||
}
|
||||
/* We are sending item, so it must be present! */
|
||||
KASSERT(item != NULL, ("ng_snd_item: item is NULL"));
|
||||
|
||||
#ifdef NETGRAPH_DEBUG
|
||||
_ngi_check(item, __FILE__, __LINE__);
|
||||
#endif
|
||||
|
||||
/* Item was sent once more, postpone apply() call. */
|
||||
if (item->apply)
|
||||
refcount_acquire(&item->apply->refs);
|
||||
|
||||
hook = NGI_HOOK(item);
|
||||
node = NGI_NODE(item);
|
||||
ngq = &node->nd_input_queue;
|
||||
if (node == NULL) {
|
||||
TRAP_ERROR();
|
||||
ERROUT(EINVAL); /* No address */
|
||||
}
|
||||
/* Node is never optional. */
|
||||
KASSERT(node != NULL, ("ng_snd_item: node is NULL"));
|
||||
|
||||
queue = (flags & NG_QUEUE) ? 1 : 0;
|
||||
|
||||
switch(item->el_flags & NGQF_TYPE) {
|
||||
case NGQF_DATA:
|
||||
/*
|
||||
* DATA MESSAGE
|
||||
* Delivered to a node via a non-optional hook.
|
||||
* Both should be present in the item even though
|
||||
* the node is derivable from the hook.
|
||||
* References are held on both by the item.
|
||||
*/
|
||||
|
||||
/* Protect nodes from sending NULL pointers
|
||||
* to each other
|
||||
*/
|
||||
hook = NGI_HOOK(item);
|
||||
/* Valid hook and mbuf are mandatory for data. */
|
||||
if ((item->el_flags & NGQF_TYPE) == NGQF_DATA) {
|
||||
KASSERT(hook != NULL, ("ng_snd_item: hook for data is NULL"));
|
||||
if (NGI_M(item) == NULL)
|
||||
ERROUT(EINVAL);
|
||||
|
||||
CHECK_DATA_MBUF(NGI_M(item));
|
||||
if (hook == NULL) {
|
||||
TRAP_ERROR();
|
||||
ERROUT(EINVAL);
|
||||
}
|
||||
if ((NG_HOOK_NOT_VALID(hook))
|
||||
|| (NG_NODE_NOT_VALID(NG_HOOK_NODE(hook)))) {
|
||||
ERROUT(ENOTCONN);
|
||||
}
|
||||
if ((hook->hk_flags & HK_QUEUE)) {
|
||||
queue = 1;
|
||||
}
|
||||
break;
|
||||
case NGQF_MESG:
|
||||
/*
|
||||
* CONTROL MESSAGE
|
||||
* Delivered to a node.
|
||||
* Hook is optional.
|
||||
* References are held by the item on the node and
|
||||
* the hook if it is present.
|
||||
*/
|
||||
if (hook && (hook->hk_flags & HK_QUEUE)) {
|
||||
queue = 1;
|
||||
}
|
||||
break;
|
||||
case NGQF_FN:
|
||||
case NGQF_FN2:
|
||||
break;
|
||||
default:
|
||||
TRAP_ERROR();
|
||||
ERROUT(EINVAL);
|
||||
}
|
||||
switch(item->el_flags & NGQF_RW) {
|
||||
case NGQF_READER:
|
||||
rw = NGQRW_R;
|
||||
break;
|
||||
case NGQF_WRITER:
|
||||
rw = NGQRW_W;
|
||||
break;
|
||||
default:
|
||||
panic("%s: invalid item flags %lx", __func__, item->el_flags);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the node specifies single threading, force writer semantics.
|
||||
* Similarly, the node may say one hook always produces writers.
|
||||
* These are overrides.
|
||||
* If the item or the node specifies single threading, force
|
||||
* writer semantics. Similarly, the node may say one hook always
|
||||
* produces writers. These are overrides.
|
||||
*/
|
||||
if ((node->nd_flags & NGF_FORCE_WRITER)
|
||||
|| (hook && (hook->hk_flags & HK_FORCE_WRITER)))
|
||||
rw = NGQRW_W;
|
||||
if (((item->el_flags & NGQF_RW) == NGQRW_W) ||
|
||||
(node->nd_flags & NGF_FORCE_WRITER) ||
|
||||
(hook && (hook->hk_flags & HK_FORCE_WRITER))) {
|
||||
rw = NGQRW_W;
|
||||
} else {
|
||||
rw = NGQRW_R;
|
||||
}
|
||||
|
||||
/*
|
||||
* If sender or receiver requests queued delivery or stack usage
|
||||
* level is dangerous - enqueue message.
|
||||
*/
|
||||
if ((flags & NG_QUEUE) || (hook && (hook->hk_flags & HK_QUEUE))) {
|
||||
queue = 1;
|
||||
} else {
|
||||
queue = 0;
|
||||
#ifdef GET_STACK_USAGE
|
||||
/*
|
||||
* Most of netgraph nodes have small stack consumption and
|
||||
* for them 25% of free stack space is more than enough.
|
||||
* Nodes/hooks with higher stack usage should be marked as
|
||||
* HI_STACK. For them 50% of stack will be guaranteed then.
|
||||
* XXX: Values 25% and 50% are completely empirical.
|
||||
*/
|
||||
size_t st, su, sl;
|
||||
GET_STACK_USAGE(st, su);
|
||||
sl = st - su;
|
||||
if ((sl * 4 < st) ||
|
||||
((sl * 2 < st) && ((node->nd_flags & NGF_HI_STACK) ||
|
||||
(hook && (hook->hk_flags & HK_HI_STACK))))) {
|
||||
queue = 1;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
ngq = &node->nd_input_queue;
|
||||
if (queue) {
|
||||
/* Put it on the queue for that node*/
|
||||
#ifdef NETGRAPH_DEBUG
|
||||
_ngi_check(item, __FILE__, __LINE__);
|
||||
#endif
|
||||
mtx_lock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_LOCK(ngq);
|
||||
ng_queue_rw(ngq, item, rw);
|
||||
mtx_unlock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
|
||||
if (flags & NG_PROGRESS)
|
||||
return (EINPROGRESS);
|
||||
else
|
||||
return (0);
|
||||
return ((flags & NG_PROGRESS) ? EINPROGRESS : 0);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -2307,12 +2294,9 @@ ng_snd_item(item_p item, int flags)
|
||||
item = ng_acquire_write(ngq, item);
|
||||
|
||||
|
||||
if (item == NULL) {
|
||||
if (flags & NG_PROGRESS)
|
||||
return (EINPROGRESS);
|
||||
else
|
||||
return (0);
|
||||
}
|
||||
/* Item was queued while trying to get permission. */
|
||||
if (item == NULL)
|
||||
return ((flags & NG_PROGRESS) ? EINPROGRESS : 0);
|
||||
|
||||
#ifdef NETGRAPH_DEBUG
|
||||
_ngi_check(item, __FILE__, __LINE__);
|
||||
@ -2327,23 +2311,21 @@ ng_snd_item(item_p item, int flags)
|
||||
* whatever we just did caused it.. whatever we do, DO NOT
|
||||
* access the node again!
|
||||
*/
|
||||
if (NG_NODE_UNREF(node) == 0) {
|
||||
if (NG_NODE_UNREF(node) == 0)
|
||||
return (error);
|
||||
}
|
||||
|
||||
mtx_lock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_LOCK(ngq);
|
||||
if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq))
|
||||
ng_setisr(ngq->q_node);
|
||||
mtx_unlock_spin(&(ngq->q_mtx));
|
||||
NG_QUEUE_UNLOCK(ngq);
|
||||
|
||||
return (error);
|
||||
|
||||
done:
|
||||
/* Apply callback. */
|
||||
if (item->apply != NULL &&
|
||||
refcount_release(&item->apply->refs)) {
|
||||
/* If was not sent, apply callback here. */
|
||||
if (item->apply != NULL && refcount_release(&item->apply->refs))
|
||||
(*item->apply->apply)(item->apply->context, error);
|
||||
}
|
||||
|
||||
NG_FREE_ITEM(item);
|
||||
return (error);
|
||||
}
|
||||
@ -2363,6 +2345,10 @@ ng_apply_item(node_p node, item_p item, int rw)
|
||||
ng_rcvmsg_t *rcvmsg;
|
||||
struct ng_apply_info *apply;
|
||||
|
||||
/* Node and item are never optional. */
|
||||
KASSERT(node != NULL, ("ng_apply_item: node is NULL"));
|
||||
KASSERT(item != NULL, ("ng_apply_item: item is NULL"));
|
||||
|
||||
NGI_GET_HOOK(item, hook); /* clears stored hook */
|
||||
#ifdef NETGRAPH_DEBUG
|
||||
_ngi_check(item, __FILE__, __LINE__);
|
||||
@ -2375,9 +2361,9 @@ ng_apply_item(node_p node, item_p item, int rw)
|
||||
/*
|
||||
* Check things are still ok as when we were queued.
|
||||
*/
|
||||
if ((hook == NULL)
|
||||
|| NG_HOOK_NOT_VALID(hook)
|
||||
|| NG_NODE_NOT_VALID(node) ) {
|
||||
KASSERT(hook != NULL, ("ng_apply_item: hook for data is NULL"));
|
||||
if (NG_HOOK_NOT_VALID(hook) ||
|
||||
NG_NODE_NOT_VALID(node)) {
|
||||
error = EIO;
|
||||
NG_FREE_ITEM(item);
|
||||
break;
|
||||
@ -2395,16 +2381,14 @@ ng_apply_item(node_p node, item_p item, int rw)
|
||||
error = (*rcvdata)(hook, item);
|
||||
break;
|
||||
case NGQF_MESG:
|
||||
if (hook) {
|
||||
if (NG_HOOK_NOT_VALID(hook)) {
|
||||
/*
|
||||
* The hook has been zapped then we can't
|
||||
* use it. Immediately drop its reference.
|
||||
* The message may not need it.
|
||||
*/
|
||||
NG_HOOK_UNREF(hook);
|
||||
hook = NULL;
|
||||
}
|
||||
if (hook && NG_HOOK_NOT_VALID(hook)) {
|
||||
/*
|
||||
* The hook has been zapped then we can't use it.
|
||||
* Immediately drop its reference.
|
||||
* The message may not need it.
|
||||
*/
|
||||
NG_HOOK_UNREF(hook);
|
||||
hook = NULL;
|
||||
}
|
||||
/*
|
||||
* Similarly, if the node is a zombie there is
|
||||
@ -2414,42 +2398,31 @@ ng_apply_item(node_p node, item_p item, int rw)
|
||||
TRAP_ERROR();
|
||||
error = EINVAL;
|
||||
NG_FREE_ITEM(item);
|
||||
} else {
|
||||
/*
|
||||
* Call the appropriate message handler for the object.
|
||||
* It is up to the message handler to free the message.
|
||||
* If it's a generic message, handle it generically,
|
||||
* otherwise call the type's message handler
|
||||
* (if it exists)
|
||||
* XXX (race). Remember that a queued message may
|
||||
* reference a node or hook that has just been
|
||||
* invalidated. It will exist as the queue code
|
||||
* is holding a reference, but..
|
||||
*/
|
||||
|
||||
struct ng_mesg *msg = NGI_MSG(item);
|
||||
|
||||
/*
|
||||
* check if the generic handler owns it.
|
||||
*/
|
||||
if ((msg->header.typecookie == NGM_GENERIC_COOKIE)
|
||||
&& ((msg->header.flags & NGF_RESP) == 0)) {
|
||||
error = ng_generic_msg(node, item, hook);
|
||||
break;
|
||||
}
|
||||
/*
|
||||
* Now see if there is a handler (hook or node specific)
|
||||
* in the target node. If none, silently discard.
|
||||
*/
|
||||
if (((!hook) || (!(rcvmsg = hook->hk_rcvmsg)))
|
||||
&& (!(rcvmsg = node->nd_type->rcvmsg))) {
|
||||
TRAP_ERROR();
|
||||
error = 0;
|
||||
NG_FREE_ITEM(item);
|
||||
break;
|
||||
}
|
||||
error = (*rcvmsg)(node, item, hook);
|
||||
break;
|
||||
}
|
||||
/*
|
||||
* Call the appropriate message handler for the object.
|
||||
* It is up to the message handler to free the message.
|
||||
* If it's a generic message, handle it generically,
|
||||
* otherwise call the type's message handler (if it exists).
|
||||
* XXX (race). Remember that a queued message may
|
||||
* reference a node or hook that has just been
|
||||
* invalidated. It will exist as the queue code
|
||||
* is holding a reference, but..
|
||||
*/
|
||||
if ((NGI_MSG(item)->header.typecookie == NGM_GENERIC_COOKIE) &&
|
||||
((NGI_MSG(item)->header.flags & NGF_RESP) == 0)) {
|
||||
error = ng_generic_msg(node, item, hook);
|
||||
break;
|
||||
}
|
||||
if (((!hook) || (!(rcvmsg = hook->hk_rcvmsg))) &&
|
||||
(!(rcvmsg = node->nd_type->rcvmsg))) {
|
||||
TRAP_ERROR();
|
||||
error = 0;
|
||||
NG_FREE_ITEM(item);
|
||||
break;
|
||||
}
|
||||
error = (*rcvmsg)(node, item, hook);
|
||||
break;
|
||||
case NGQF_FN:
|
||||
case NGQF_FN2:
|
||||
@ -2480,21 +2453,17 @@ ng_apply_item(node_p node, item_p item, int rw)
|
||||
* that we took from the item. Now that we have
|
||||
* finished doing everything, drop those references.
|
||||
*/
|
||||
if (hook) {
|
||||
if (hook)
|
||||
NG_HOOK_UNREF(hook);
|
||||
}
|
||||
|
||||
if (rw == NGQRW_R) {
|
||||
if (rw == NGQRW_R)
|
||||
ng_leave_read(&node->nd_input_queue);
|
||||
} else {
|
||||
else
|
||||
ng_leave_write(&node->nd_input_queue);
|
||||
}
|
||||
|
||||
/* Apply callback. */
|
||||
if (apply != NULL &&
|
||||
refcount_release(&apply->refs)) {
|
||||
if (apply != NULL && refcount_release(&apply->refs))
|
||||
(*apply->apply)(apply->context, error);
|
||||
}
|
||||
|
||||
return (error);
|
||||
}
|
||||
@ -3092,7 +3061,7 @@ ngb_mod_event(module_t mod, int event, void *data)
|
||||
switch (event) {
|
||||
case MOD_LOAD:
|
||||
/* Initialize everything. */
|
||||
mtx_init(&ng_worklist_mtx, "ng_worklist", NULL, MTX_SPIN);
|
||||
NG_WORKLIST_LOCK_INIT();
|
||||
mtx_init(&ng_typelist_mtx, "netgraph types mutex", NULL,
|
||||
MTX_DEF);
|
||||
mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", NULL,
|
||||
@ -3181,7 +3150,7 @@ dumpitem(item_p item, char *file, int line)
|
||||
item->body.fn.fn_arg2);
|
||||
break;
|
||||
case NGQF_FN2:
|
||||
printf(" - fn@%p (%p, %p, %p, %d (%x))\n",
|
||||
printf(" - fn2@%p (%p, %p, %p, %d (%x))\n",
|
||||
_NGI_FN2(item),
|
||||
_NGI_NODE(item),
|
||||
_NGI_HOOK(item),
|
||||
@ -3245,7 +3214,7 @@ sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS)
|
||||
|
||||
val = allocated;
|
||||
i = 1;
|
||||
error = sysctl_handle_int(oidp, &val, sizeof(int), req);
|
||||
error = sysctl_handle_int(oidp, &val, 0, req);
|
||||
if (error != 0 || req->newptr == NULL)
|
||||
return (error);
|
||||
if (val == 42) {
|
||||
@ -3277,15 +3246,15 @@ ngintr(void)
|
||||
node_p node = NULL;
|
||||
|
||||
for (;;) {
|
||||
mtx_lock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_LOCK();
|
||||
node = TAILQ_FIRST(&ng_worklist);
|
||||
if (!node) {
|
||||
mtx_unlock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_UNLOCK();
|
||||
break;
|
||||
}
|
||||
node->nd_flags &= ~NGF_WORKQ;
|
||||
TAILQ_REMOVE(&ng_worklist, node, nd_work);
|
||||
mtx_unlock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_UNLOCK();
|
||||
CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist",
|
||||
__func__, node->nd_ID, node);
|
||||
/*
|
||||
@ -3304,13 +3273,13 @@ ngintr(void)
|
||||
for (;;) {
|
||||
int rw;
|
||||
|
||||
mtx_lock_spin(&node->nd_input_queue.q_mtx);
|
||||
NG_QUEUE_LOCK(&node->nd_input_queue);
|
||||
item = ng_dequeue(&node->nd_input_queue, &rw);
|
||||
if (item == NULL) {
|
||||
mtx_unlock_spin(&node->nd_input_queue.q_mtx);
|
||||
NG_QUEUE_UNLOCK(&node->nd_input_queue);
|
||||
break; /* go look for another node */
|
||||
} else {
|
||||
mtx_unlock_spin(&node->nd_input_queue.q_mtx);
|
||||
NG_QUEUE_UNLOCK(&node->nd_input_queue);
|
||||
NGI_GET_NODE(item, node); /* zaps stored node */
|
||||
ng_apply_item(node, item, rw);
|
||||
NG_NODE_UNREF(node);
|
||||
@ -3325,16 +3294,16 @@ ng_worklist_remove(node_p node)
|
||||
{
|
||||
mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED);
|
||||
|
||||
mtx_lock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_LOCK();
|
||||
if (node->nd_flags & NGF_WORKQ) {
|
||||
node->nd_flags &= ~NGF_WORKQ;
|
||||
TAILQ_REMOVE(&ng_worklist, node, nd_work);
|
||||
mtx_unlock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_UNLOCK();
|
||||
NG_NODE_UNREF(node);
|
||||
CTR3(KTR_NET, "%20s: node [%x] (%p) removed from worklist",
|
||||
__func__, node->nd_ID, node);
|
||||
} else {
|
||||
mtx_unlock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_UNLOCK();
|
||||
}
|
||||
}
|
||||
|
||||
@ -3355,9 +3324,9 @@ ng_setisr(node_p node)
|
||||
* then put us on.
|
||||
*/
|
||||
node->nd_flags |= NGF_WORKQ;
|
||||
mtx_lock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_LOCK();
|
||||
TAILQ_INSERT_TAIL(&ng_worklist, node, nd_work);
|
||||
mtx_unlock_spin(&ng_worklist_mtx);
|
||||
NG_WORKLIST_UNLOCK();
|
||||
NG_NODE_REF(node); /* XXX fafe in mutex? */
|
||||
CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__,
|
||||
node->nd_ID, node);
|
||||
@ -3414,7 +3383,6 @@ ng_package_data(struct mbuf *m, int flags)
|
||||
}
|
||||
ITEM_DEBUG_CHECKS;
|
||||
item->el_flags = NGQF_DATA | NGQF_READER;
|
||||
item->el_next = NULL;
|
||||
NGI_M(item) = m;
|
||||
return (item);
|
||||
}
|
||||
@ -3441,7 +3409,6 @@ ng_package_msg(struct ng_mesg *msg, int flags)
|
||||
item->el_flags = NGQF_MESG | NGQF_READER;
|
||||
else
|
||||
item->el_flags = NGQF_MESG | NGQF_WRITER;
|
||||
item->el_next = NULL;
|
||||
/*
|
||||
* Set the current lasthook into the queue item
|
||||
*/
|
||||
@ -3576,7 +3543,6 @@ ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg)
|
||||
|
||||
/* Fill out the contents */
|
||||
item->el_flags = NGQF_MESG | NGQF_WRITER;
|
||||
item->el_next = NULL;
|
||||
NG_NODE_REF(here);
|
||||
NGI_SET_NODE(item, here);
|
||||
if (hook) {
|
||||
|
||||
@ -573,6 +573,7 @@ ng_iface_newhook(node_p node, hook_p hook, const char *name)
|
||||
if (*hookptr != NULL)
|
||||
return (EISCONN);
|
||||
*hookptr = hook;
|
||||
NG_HOOK_HI_STACK(hook);
|
||||
return (0);
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user