- Use counter(9) for node stats updated at a high rate.

- Use simple ++ for rare events.
- Use uma_zone_get_cur() to get knowledge about space left in cache.
- Convert many fields of struct ng_netflow_info to 64 bit.

Tested by:	Viktor Velichkin <avisom yandex.ru>
Sponsored by:	Nginx, Inc.
This commit is contained in:
Gleb Smirnoff 2014-01-01 21:48:04 +00:00
parent 0492757c70
commit 7ee35ac9f3
Notes: svn2git 2020-12-20 02:59:44 +00:00
svn path=/head/; revision=260169
4 changed files with 135 additions and 131 deletions

View File

@ -34,16 +34,13 @@ __FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_route.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/counter.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/mbuf.h>
#include <sys/syslog.h>
#include <sys/systm.h>
#include <sys/socket.h>
#include <sys/endian.h>
#include <machine/atomic.h>
#include <machine/stdarg.h>
#include <net/if.h>
#include <net/if_var.h>
@ -80,8 +77,8 @@ __FBSDID("$FreeBSD$");
/* Macros to shorten logical constructions */
/* XXX: priv must exist in namespace */
#define INACTIVE(fle) (time_uptime - fle->f.last > priv->info.nfinfo_inact_t)
#define AGED(fle) (time_uptime - fle->f.first > priv->info.nfinfo_act_t)
#define INACTIVE(fle) (time_uptime - fle->f.last > priv->nfinfo_inact_t)
#define AGED(fle) (time_uptime - fle->f.first > priv->nfinfo_act_t)
#define ISFREE(fle) (fle->f.packets == 0)
/*
@ -149,54 +146,6 @@ ip6_hash(struct flow6_rec *r)
}
#endif
/* This is callback from uma(9), called on alloc. */
static int
uma_ctor_flow(void *mem, int size, void *arg, int how)
{
priv_p priv = (priv_p )arg;
if (atomic_load_acq_32(&priv->info.nfinfo_used) >= CACHESIZE)
return (ENOMEM);
atomic_add_32(&priv->info.nfinfo_used, 1);
return (0);
}
/* This is callback from uma(9), called on free. */
static void
uma_dtor_flow(void *mem, int size, void *arg)
{
priv_p priv = (priv_p )arg;
atomic_subtract_32(&priv->info.nfinfo_used, 1);
}
#ifdef INET6
/* This is callback from uma(9), called on alloc. */
static int
uma_ctor_flow6(void *mem, int size, void *arg, int how)
{
priv_p priv = (priv_p )arg;
if (atomic_load_acq_32(&priv->info.nfinfo_used6) >= CACHESIZE)
return (ENOMEM);
atomic_add_32(&priv->info.nfinfo_used6, 1);
return (0);
}
/* This is callback from uma(9), called on free. */
static void
uma_dtor_flow6(void *mem, int size, void *arg)
{
priv_p priv = (priv_p )arg;
atomic_subtract_32(&priv->info.nfinfo_used6, 1);
}
#endif
/*
* Detach export datagram from priv, if there is any.
* If there is no, allocate a new one.
@ -267,9 +216,9 @@ expire_flow(priv_p priv, fib_export_p fe, struct flow_entry *fle, int flags)
if ((priv->export != NULL) && (version == IPVERSION)) {
exp.item = get_export_dgram(priv, fe);
if (exp.item == NULL) {
atomic_add_32(&priv->info.nfinfo_export_failed, 1);
priv->nfinfo_export_failed++;
if (priv->export9 != NULL)
atomic_add_32(&priv->info.nfinfo_export9_failed, 1);
priv->nfinfo_export9_failed++;
/* fle definitely contains IPv4 flow. */
uma_zfree_arg(priv->zone, fle, priv);
return;
@ -284,7 +233,7 @@ expire_flow(priv_p priv, fib_export_p fe, struct flow_entry *fle, int flags)
if (priv->export9 != NULL) {
exp.item9 = get_export9_dgram(priv, fe, &exp.item9_opt);
if (exp.item9 == NULL) {
atomic_add_32(&priv->info.nfinfo_export9_failed, 1);
priv->nfinfo_export9_failed++;
if (version == IPVERSION)
uma_zfree_arg(priv->zone, fle, priv);
#ifdef INET6
@ -317,8 +266,27 @@ void
ng_netflow_copyinfo(priv_p priv, struct ng_netflow_info *i)
{
/* XXX: atomic */
memcpy((void *)i, (void *)&priv->info, sizeof(priv->info));
i->nfinfo_bytes = counter_u64_fetch(priv->nfinfo_bytes);
i->nfinfo_packets = counter_u64_fetch(priv->nfinfo_packets);
i->nfinfo_bytes6 = counter_u64_fetch(priv->nfinfo_bytes6);
i->nfinfo_packets6 = counter_u64_fetch(priv->nfinfo_packets6);
i->nfinfo_sbytes = counter_u64_fetch(priv->nfinfo_sbytes);
i->nfinfo_spackets = counter_u64_fetch(priv->nfinfo_spackets);
i->nfinfo_sbytes6 = counter_u64_fetch(priv->nfinfo_sbytes6);
i->nfinfo_spackets6 = counter_u64_fetch(priv->nfinfo_spackets6);
i->nfinfo_act_exp = counter_u64_fetch(priv->nfinfo_act_exp);
i->nfinfo_inact_exp = counter_u64_fetch(priv->nfinfo_inact_exp);
i->nfinfo_used = uma_zone_get_cur(priv->zone);
i->nfinfo_used6 = uma_zone_get_cur(priv->zone6);
i->nfinfo_alloc_failed = priv->nfinfo_alloc_failed;
i->nfinfo_export_failed = priv->nfinfo_export_failed;
i->nfinfo_export9_failed = priv->nfinfo_export9_failed;
i->nfinfo_realloc_mbuf = priv->nfinfo_realloc_mbuf;
i->nfinfo_alloc_fibs = priv->nfinfo_alloc_fibs;
i->nfinfo_inact_t = priv->nfinfo_inact_t;
i->nfinfo_act_t = priv->nfinfo_act_t;
}
/*
@ -343,7 +311,7 @@ hash_insert(priv_p priv, struct flow_hash_entry *hsh, struct flow_rec *r,
fle = uma_zalloc_arg(priv->zone, priv, M_NOWAIT);
if (fle == NULL) {
atomic_add_32(&priv->info.nfinfo_alloc_failed, 1);
priv->nfinfo_alloc_failed++;
return (ENOMEM);
}
@ -433,7 +401,7 @@ hash6_insert(priv_p priv, struct flow_hash_entry *hsh6, struct flow6_rec *r,
fle6 = uma_zalloc_arg(priv->zone6, priv, M_NOWAIT);
if (fle6 == NULL) {
atomic_add_32(&priv->info.nfinfo_alloc_failed, 1);
priv->nfinfo_alloc_failed++;
return (ENOMEM);
}
@ -526,13 +494,13 @@ ng_netflow_cache_init(priv_p priv)
/* Initialize cache UMA zone. */
priv->zone = uma_zcreate("NetFlow IPv4 cache",
sizeof(struct flow_entry), uma_ctor_flow, uma_dtor_flow, NULL,
NULL, UMA_ALIGN_CACHE, 0);
sizeof(struct flow_entry), NULL, NULL, NULL, NULL,
UMA_ALIGN_CACHE, 0);
uma_zone_set_max(priv->zone, CACHESIZE);
#ifdef INET6
priv->zone6 = uma_zcreate("NetFlow IPv6 cache",
sizeof(struct flow6_entry), uma_ctor_flow6, uma_dtor_flow6, NULL,
NULL, UMA_ALIGN_CACHE, 0);
sizeof(struct flow6_entry), NULL, NULL, NULL, NULL,
UMA_ALIGN_CACHE, 0);
uma_zone_set_max(priv->zone6, CACHESIZE);
#endif
@ -558,6 +526,17 @@ ng_netflow_cache_init(priv_p priv)
}
#endif
priv->nfinfo_bytes = counter_u64_alloc(M_WAITOK);
priv->nfinfo_packets = counter_u64_alloc(M_WAITOK);
priv->nfinfo_bytes6 = counter_u64_alloc(M_WAITOK);
priv->nfinfo_packets6 = counter_u64_alloc(M_WAITOK);
priv->nfinfo_sbytes = counter_u64_alloc(M_WAITOK);
priv->nfinfo_spackets = counter_u64_alloc(M_WAITOK);
priv->nfinfo_sbytes6 = counter_u64_alloc(M_WAITOK);
priv->nfinfo_spackets6 = counter_u64_alloc(M_WAITOK);
priv->nfinfo_act_exp = counter_u64_alloc(M_WAITOK);
priv->nfinfo_inact_exp = counter_u64_alloc(M_WAITOK);
ng_netflow_v9_cache_init(priv);
CTR0(KTR_NET, "ng_netflow startup()");
}
@ -594,7 +573,7 @@ ng_netflow_fib_init(priv_p priv, int fib)
/* Increase counter for statistics */
CTR3(KTR_NET, "ng_netflow(): fib %d setup to %p (%p)",
fib, fe, priv_to_fib(priv, fib));
atomic_fetchadd_32(&priv->info.nfinfo_alloc_fibs, 1);
priv->nfinfo_alloc_fibs++;
}
return (0);
@ -667,6 +646,17 @@ ng_netflow_cache_flush(priv_p priv)
free(fe, M_NETGRAPH);
}
counter_u64_free(priv->nfinfo_bytes);
counter_u64_free(priv->nfinfo_packets);
counter_u64_free(priv->nfinfo_bytes6);
counter_u64_free(priv->nfinfo_packets6);
counter_u64_free(priv->nfinfo_sbytes);
counter_u64_free(priv->nfinfo_spackets);
counter_u64_free(priv->nfinfo_sbytes6);
counter_u64_free(priv->nfinfo_spackets6);
counter_u64_free(priv->nfinfo_act_exp);
counter_u64_free(priv->nfinfo_inact_exp);
ng_netflow_v9_cache_flush(priv);
}
@ -734,9 +724,8 @@ ng_netflow_flow_add(priv_p priv, fib_export_p fe, struct ip *ip,
break;
}
atomic_fetchadd_32(&priv->info.nfinfo_packets, 1);
/* XXX: atomic */
priv->info.nfinfo_bytes += plen;
counter_u64_add(priv->nfinfo_packets, 1);
counter_u64_add(priv->nfinfo_bytes, plen);
/* Find hash slot. */
hsh = &priv->hash[ip_hash(&r)];
@ -756,7 +745,7 @@ ng_netflow_flow_add(priv_p priv, fib_export_p fe, struct ip *ip,
TAILQ_REMOVE(&hsh->head, fle, fle_hash);
expire_flow(priv, priv_to_fib(priv, fle->f.r.fib),
fle, NG_QUEUE);
atomic_add_32(&priv->info.nfinfo_act_exp, 1);
counter_u64_add(priv->nfinfo_act_exp, 1);
}
}
@ -778,7 +767,7 @@ ng_netflow_flow_add(priv_p priv, fib_export_p fe, struct ip *ip,
TAILQ_REMOVE(&hsh->head, fle, fle_hash);
expire_flow(priv, priv_to_fib(priv, fle->f.r.fib),
fle, NG_QUEUE);
atomic_add_32(&priv->info.nfinfo_act_exp, 1);
counter_u64_add(priv->nfinfo_act_exp, 1);
} else {
/*
* It is the newest, move it to the tail,
@ -853,9 +842,8 @@ ng_netflow_flow6_add(priv_p priv, fib_export_p fe, struct ip6_hdr *ip6,
r.r_ip_p = upper_proto;
r.r_i_ifx = src_if_index;
atomic_fetchadd_32(&priv->info.nfinfo_packets6, 1);
/* XXX: atomic */
priv->info.nfinfo_bytes6 += plen;
counter_u64_add(priv->nfinfo_packets6, 1);
counter_u64_add(priv->nfinfo_bytes6, plen);
/* Find hash slot. */
hsh = &priv->hash6[ip6_hash(&r)];
@ -878,7 +866,7 @@ ng_netflow_flow6_add(priv_p priv, fib_export_p fe, struct ip6_hdr *ip6,
TAILQ_REMOVE(&hsh->head, fle, fle_hash);
expire_flow(priv, priv_to_fib(priv, fle->f.r.fib), fle,
NG_QUEUE);
atomic_add_32(&priv->info.nfinfo_act_exp, 1);
counter_u64_add(priv->nfinfo_act_exp, 1);
}
}
@ -901,7 +889,7 @@ ng_netflow_flow6_add(priv_p priv, fib_export_p fe, struct ip6_hdr *ip6,
TAILQ_REMOVE(&hsh->head, fle, fle_hash);
expire_flow(priv, priv_to_fib(priv, fle->f.r.fib), fle,
NG_QUEUE);
atomic_add_32(&priv->info.nfinfo_act_exp, 1);
counter_u64_add(priv->nfinfo_act_exp, 1);
} else {
/*
* It is the newest, move it to the tail,
@ -1119,12 +1107,12 @@ ng_netflow_expire(void *arg)
struct flow_entry *fle, *fle1;
struct flow_hash_entry *hsh;
priv_p priv = (priv_p )arg;
uint32_t used;
int i;
int used, i;
/*
* Going through all the cache.
*/
used = uma_zone_get_cur(priv->zone);
for (hsh = priv->hash, i = 0; i < NBUCKETS; hsh++, i++) {
/*
* Skip entries, that are already being worked on.
@ -1132,7 +1120,6 @@ ng_netflow_expire(void *arg)
if (mtx_trylock(&hsh->mtx) == 0)
continue;
used = atomic_load_acq_32(&priv->info.nfinfo_used);
TAILQ_FOREACH_SAFE(fle, &hsh->head, fle_hash, fle1) {
/*
* Interrupt thread wants this entry!
@ -1154,13 +1141,14 @@ ng_netflow_expire(void *arg)
expire_flow(priv, priv_to_fib(priv,
fle->f.r.fib), fle, NG_NOFLAGS);
used--;
atomic_add_32(&priv->info.nfinfo_inact_exp, 1);
counter_u64_add(priv->nfinfo_inact_exp, 1);
}
}
mtx_unlock(&hsh->mtx);
}
#ifdef INET6
used = uma_zone_get_cur(priv->zone6);
for (hsh = priv->hash6, i = 0; i < NBUCKETS; hsh++, i++) {
struct flow6_entry *fle6;
@ -1170,7 +1158,6 @@ ng_netflow_expire(void *arg)
if (mtx_trylock(&hsh->mtx) == 0)
continue;
used = atomic_load_acq_32(&priv->info.nfinfo_used6);
TAILQ_FOREACH_SAFE(fle, &hsh->head, fle_hash, fle1) {
fle6 = (struct flow6_entry *)fle;
/*
@ -1193,7 +1180,7 @@ ng_netflow_expire(void *arg)
expire_flow(priv, priv_to_fib(priv,
fle->f.r.fib), fle, NG_NOFLAGS);
used--;
atomic_add_32(&priv->info.nfinfo_inact_exp, 1);
counter_u64_add(priv->nfinfo_inact_exp, 1);
}
}
mtx_unlock(&hsh->mtx);

View File

@ -32,16 +32,13 @@ __FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_route.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/counter.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/mbuf.h>
#include <sys/syslog.h>
#include <sys/systm.h>
#include <sys/socket.h>
#include <sys/endian.h>
#include <machine/atomic.h>
#include <machine/stdarg.h>
#include <net/if.h>
#include <net/route.h>

View File

@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/counter.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/mbuf.h>
@ -250,8 +251,8 @@ ng_netflow_constructor(node_p node)
priv->node = node;
/* Initialize timeouts to default values */
priv->info.nfinfo_inact_t = INACTIVE_TIMEOUT;
priv->info.nfinfo_act_t = ACTIVE_TIMEOUT;
priv->nfinfo_inact_t = INACTIVE_TIMEOUT;
priv->nfinfo_act_t = ACTIVE_TIMEOUT;
/* Set default config */
for (i = 0; i < NG_NETFLOW_MAXIFACES; i++)
@ -475,8 +476,8 @@ ng_netflow_rcvmsg (node_p node, item_p item, hook_p lasthook)
set = (struct ng_netflow_settimeouts *)msg->data;
priv->info.nfinfo_inact_t = set->inactive_timeout;
priv->info.nfinfo_act_t = set->active_timeout;
priv->nfinfo_inact_t = set->inactive_timeout;
priv->nfinfo_act_t = set->active_timeout;
break;
}
@ -895,7 +896,7 @@ ng_netflow_rcvdata (hook_p hook, item_p item)
#endif
/* Just in case of real reallocation in M_CHECK() / m_pullup() */
if (m != m_old) {
atomic_fetchadd_32(&priv->info.nfinfo_realloc_mbuf, 1);
priv->nfinfo_realloc_mbuf++;
/* Restore ip/ipv6 pointer */
if (ip != NULL)
ip = (struct ip *)(mtod(m, caddr_t) + l3_off);
@ -949,13 +950,13 @@ ng_netflow_rcvdata (hook_p hook, item_p item)
if (acct == 0) {
/* Accounting failure */
if (ip != NULL) {
atomic_fetchadd_32(&priv->info.nfinfo_spackets,
1);
priv->info.nfinfo_sbytes += m_length(m, NULL);
counter_u64_add(priv->nfinfo_spackets, 1);
counter_u64_add(priv->nfinfo_sbytes,
m->m_pkthdr.len);
} else if (ip6 != NULL) {
atomic_fetchadd_32(&priv->info.nfinfo_spackets6,
1);
priv->info.nfinfo_sbytes6 += m_length(m, NULL);
counter_u64_add(priv->nfinfo_spackets6, 1);
counter_u64_add(priv->nfinfo_sbytes6,
m->m_pkthdr.len);
}
}

View File

@ -33,7 +33,7 @@
#define _NG_NETFLOW_H_
#define NG_NETFLOW_NODE_TYPE "netflow"
#define NGM_NETFLOW_COOKIE 1309868867
#define NGM_NETFLOW_COOKIE 1365756954
#define NGM_NETFLOW_V9_COOKIE 1349865386
#define NG_NETFLOW_MAXIFACES USHRT_MAX
@ -65,13 +65,15 @@ enum {
/* This structure is returned by the NGM_NETFLOW_INFO message */
struct ng_netflow_info {
uint64_t nfinfo_bytes; /* accounted IPv4 bytes */
uint32_t nfinfo_packets; /* accounted IPv4 packets */
uint64_t nfinfo_packets; /* accounted IPv4 packets */
uint64_t nfinfo_bytes6; /* accounted IPv6 bytes */
uint32_t nfinfo_packets6; /* accounted IPv6 packets */
uint64_t nfinfo_packets6; /* accounted IPv6 packets */
uint64_t nfinfo_sbytes; /* skipped IPv4 bytes */
uint32_t nfinfo_spackets; /* skipped IPv4 packets */
uint64_t nfinfo_spackets; /* skipped IPv4 packets */
uint64_t nfinfo_sbytes6; /* skipped IPv6 bytes */
uint32_t nfinfo_spackets6; /* skipped IPv6 packets */
uint64_t nfinfo_spackets6; /* skipped IPv6 packets */
uint64_t nfinfo_act_exp; /* active expiries */
uint64_t nfinfo_inact_exp; /* inactive expiries */
uint32_t nfinfo_used; /* used cache records */
uint32_t nfinfo_used6; /* used IPv6 cache records */
uint32_t nfinfo_alloc_failed; /* failed allocations */
@ -79,12 +81,35 @@ struct ng_netflow_info {
uint32_t nfinfo_export9_failed; /* failed exports */
uint32_t nfinfo_realloc_mbuf; /* reallocated mbufs */
uint32_t nfinfo_alloc_fibs; /* fibs allocated */
uint32_t nfinfo_act_exp; /* active expiries */
uint32_t nfinfo_inact_exp; /* inactive expiries */
uint32_t nfinfo_inact_t; /* flow inactive timeout */
uint32_t nfinfo_act_t; /* flow active timeout */
};
/* Parse the info structure */
#define NG_NETFLOW_INFO_TYPE { \
{ "IPv4 bytes", &ng_parse_uint64_type },\
{ "IPv4 packets", &ng_parse_uint64_type },\
{ "IPv6 bytes", &ng_parse_uint64_type },\
{ "IPv6 packets", &ng_parse_uint64_type },\
{ "IPv4 skipped bytes", &ng_parse_uint64_type },\
{ "IPv4 skipped packets", &ng_parse_uint64_type },\
{ "IPv6 skipped bytes", &ng_parse_uint64_type },\
{ "IPv6 skipped packets", &ng_parse_uint64_type },\
{ "Active expiries", &ng_parse_uint64_type },\
{ "Inactive expiries", &ng_parse_uint64_type },\
{ "IPv4 records used", &ng_parse_uint32_type },\
{ "IPv6 records used", &ng_parse_uint32_type },\
{ "Failed allocations", &ng_parse_uint32_type },\
{ "V5 failed exports", &ng_parse_uint32_type },\
{ "V9 failed exports", &ng_parse_uint32_type },\
{ "mbuf reallocations", &ng_parse_uint32_type },\
{ "fibs allocated", &ng_parse_uint32_type },\
{ "Inactive timeout", &ng_parse_uint32_type },\
{ "Active timeout", &ng_parse_uint32_type },\
{ NULL } \
}
/* This structure is returned by the NGM_NETFLOW_IFINFO message */
struct ng_netflow_ifinfo {
uint32_t ifinfo_packets; /* number of packets for this iface */
@ -282,30 +307,6 @@ struct flow6_entry {
};
/* Parsing declarations */
/* Parse the info structure */
#define NG_NETFLOW_INFO_TYPE { \
{ "IPv4 bytes", &ng_parse_uint64_type },\
{ "IPv4 packets", &ng_parse_uint32_type },\
{ "IPv6 bytes", &ng_parse_uint64_type },\
{ "IPv6 packets", &ng_parse_uint32_type },\
{ "IPv4 skipped bytes", &ng_parse_uint64_type },\
{ "IPv4 skipped packets", &ng_parse_uint32_type },\
{ "IPv6 skipped bytes", &ng_parse_uint64_type },\
{ "IPv6 skipped packets", &ng_parse_uint32_type },\
{ "IPv4 records used", &ng_parse_uint32_type },\
{ "IPv6 records used", &ng_parse_uint32_type },\
{ "Failed allocations", &ng_parse_uint32_type },\
{ "V5 failed exports", &ng_parse_uint32_type },\
{ "V9 failed exports", &ng_parse_uint32_type },\
{ "mbuf reallocations", &ng_parse_uint32_type },\
{ "fibs allocated", &ng_parse_uint32_type },\
{ "Active expiries", &ng_parse_uint32_type },\
{ "Inactive expiries", &ng_parse_uint32_type },\
{ "Inactive timeout", &ng_parse_uint32_type },\
{ "Active timeout", &ng_parse_uint32_type },\
{ NULL } \
}
/* Parse the ifinfo structure */
#define NG_NETFLOW_IFINFO_TYPE { \
{ "packets", &ng_parse_uint32_type },\
@ -408,7 +409,6 @@ struct netflow {
node_p node; /* link to the node itself */
hook_p export; /* export data goes there */
hook_p export9; /* Netflow V9 export data goes there */
struct ng_netflow_info info;
struct callout exp_callout; /* expiry periodic job */
/*
@ -440,6 +440,25 @@ struct netflow {
struct flow_hash_entry *hash6;
#endif
/* Statistics. */
counter_u64_t nfinfo_bytes; /* accounted IPv4 bytes */
counter_u64_t nfinfo_packets; /* accounted IPv4 packets */
counter_u64_t nfinfo_bytes6; /* accounted IPv6 bytes */
counter_u64_t nfinfo_packets6; /* accounted IPv6 packets */
counter_u64_t nfinfo_sbytes; /* skipped IPv4 bytes */
counter_u64_t nfinfo_spackets; /* skipped IPv4 packets */
counter_u64_t nfinfo_sbytes6; /* skipped IPv6 bytes */
counter_u64_t nfinfo_spackets6; /* skipped IPv6 packets */
counter_u64_t nfinfo_act_exp; /* active expiries */
counter_u64_t nfinfo_inact_exp; /* inactive expiries */
uint32_t nfinfo_alloc_failed; /* failed allocations */
uint32_t nfinfo_export_failed; /* failed exports */
uint32_t nfinfo_export9_failed; /* failed exports */
uint32_t nfinfo_realloc_mbuf; /* reallocated mbufs */
uint32_t nfinfo_alloc_fibs; /* fibs allocated */
uint32_t nfinfo_inact_t; /* flow inactive timeout */
uint32_t nfinfo_act_t; /* flow active timeout */
/* Multiple FIB support */
fib_export_p *fib_data; /* vector to per-fib data */
uint16_t maxfibs; /* number of allocated fibs */