2009-06-24 22:21:30 +00:00
|
|
|
/*-
|
2017-11-27 15:23:17 +00:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
|
|
*
|
2009-06-24 22:21:30 +00:00
|
|
|
* Copyright (c) 2008 The FreeBSD Foundation
|
2021-10-09 14:09:04 +00:00
|
|
|
* Copyright (c) 2009-2021 Bjoern A. Zeeb <bz@FreeBSD.org>
|
2009-06-24 22:21:30 +00:00
|
|
|
*
|
|
|
|
* This software was developed by CK Software GmbH under sponsorship
|
|
|
|
* from the FreeBSD Foundation.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
2009-07-26 12:20:07 +00:00
|
|
|
* A pair of virtual back-to-back connected ethernet like interfaces
|
|
|
|
* (``two interfaces with a virtual cross-over cable'').
|
|
|
|
*
|
2009-06-24 22:21:30 +00:00
|
|
|
* This is mostly intended to be used to provide connectivity between
|
|
|
|
* different virtual network stack instances.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2021-12-09 13:24:13 +00:00
|
|
|
#include "opt_rss.h"
|
2022-03-03 09:44:43 +00:00
|
|
|
#include "opt_inet.h"
|
|
|
|
#include "opt_inet6.h"
|
2021-12-09 13:24:13 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <sys/param.h>
|
2023-03-13 13:27:47 +00:00
|
|
|
#include <sys/bus.h>
|
2018-05-23 13:10:57 +00:00
|
|
|
#include <sys/hash.h>
|
2023-03-13 13:27:47 +00:00
|
|
|
#include <sys/interrupt.h>
|
2018-05-23 13:10:57 +00:00
|
|
|
#include <sys/jail.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <sys/kernel.h>
|
2018-05-23 13:10:57 +00:00
|
|
|
#include <sys/libkern.h>
|
2016-02-01 17:41:21 +00:00
|
|
|
#include <sys/malloc.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <sys/mbuf.h>
|
|
|
|
#include <sys/module.h>
|
2018-05-23 13:10:57 +00:00
|
|
|
#include <sys/proc.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <sys/queue.h>
|
2021-12-09 13:24:13 +00:00
|
|
|
#include <sys/sched.h>
|
2009-07-26 12:20:07 +00:00
|
|
|
#include <sys/smp.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <sys/socket.h>
|
|
|
|
#include <sys/sockio.h>
|
2021-12-09 13:24:13 +00:00
|
|
|
#include <sys/taskqueue.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
|
|
|
|
#include <net/bpf.h>
|
|
|
|
#include <net/ethernet.h>
|
|
|
|
#include <net/if.h>
|
2013-10-26 17:58:36 +00:00
|
|
|
#include <net/if_var.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <net/if_clone.h>
|
2011-05-24 07:57:28 +00:00
|
|
|
#include <net/if_media.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <net/if_var.h>
|
2023-01-13 15:43:51 +00:00
|
|
|
#include <net/if_private.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
#include <net/if_types.h>
|
|
|
|
#include <net/netisr.h>
|
2021-12-09 13:24:13 +00:00
|
|
|
#ifdef RSS
|
|
|
|
#include <net/rss_config.h>
|
2022-03-03 09:44:43 +00:00
|
|
|
#ifdef INET
|
2021-12-09 13:24:13 +00:00
|
|
|
#include <netinet/in_rss.h>
|
2022-03-03 09:44:43 +00:00
|
|
|
#endif
|
|
|
|
#ifdef INET6
|
2021-12-09 13:24:13 +00:00
|
|
|
#include <netinet6/in6_rss.h>
|
|
|
|
#endif
|
2022-03-03 09:44:43 +00:00
|
|
|
#endif
|
2009-08-01 19:26:27 +00:00
|
|
|
#include <net/vnet.h>
|
2009-06-24 22:21:30 +00:00
|
|
|
|
2012-10-16 13:37:54 +00:00
|
|
|
static const char epairname[] = "epair";
|
2021-10-09 14:09:04 +00:00
|
|
|
#define RXRSIZE 4096 /* Probably overkill by 4-8x. */
|
2012-10-16 13:37:54 +00:00
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
static MALLOC_DEFINE(M_EPAIR, epairname,
|
|
|
|
"Pair of virtual cross-over connected Ethernet-like interfaces");
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
VNET_DEFINE_STATIC(struct if_clone *, epair_cloner);
|
|
|
|
#define V_epair_cloner VNET(epair_cloner)
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
static unsigned int next_index = 0;
|
|
|
|
#define EPAIR_LOCK_INIT() mtx_init(&epair_n_index_mtx, "epairidx", \
|
|
|
|
NULL, MTX_DEF)
|
|
|
|
#define EPAIR_LOCK_DESTROY() mtx_destroy(&epair_n_index_mtx)
|
|
|
|
#define EPAIR_LOCK() mtx_lock(&epair_n_index_mtx)
|
|
|
|
#define EPAIR_UNLOCK() mtx_unlock(&epair_n_index_mtx)
|
|
|
|
|
2021-12-09 13:24:13 +00:00
|
|
|
struct epair_softc;
|
|
|
|
struct epair_queue {
|
2023-03-01 20:21:30 +00:00
|
|
|
struct mtx mtx;
|
|
|
|
struct mbufq q;
|
2021-12-09 13:24:13 +00:00
|
|
|
int id;
|
2023-03-01 20:21:30 +00:00
|
|
|
enum {
|
|
|
|
EPAIR_QUEUE_IDLE,
|
|
|
|
EPAIR_QUEUE_WAKING,
|
|
|
|
EPAIR_QUEUE_RUNNING,
|
|
|
|
} state;
|
2021-12-09 13:24:13 +00:00
|
|
|
struct task tx_task;
|
|
|
|
struct epair_softc *sc;
|
|
|
|
};
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
static struct mtx epair_n_index_mtx;
|
2009-06-24 22:21:30 +00:00
|
|
|
struct epair_softc {
|
2021-12-09 13:24:13 +00:00
|
|
|
struct ifnet *ifp; /* This ifp. */
|
|
|
|
struct ifnet *oifp; /* other ifp of pair. */
|
|
|
|
int num_queues;
|
|
|
|
struct epair_queue *queues;
|
|
|
|
struct ifmedia media; /* Media config (fake). */
|
2021-10-09 14:09:04 +00:00
|
|
|
STAILQ_ENTRY(epair_softc) entry;
|
2009-06-24 22:21:30 +00:00
|
|
|
};
|
|
|
|
|
2021-12-09 13:24:13 +00:00
|
|
|
struct epair_tasks_t {
|
|
|
|
int tasks;
|
|
|
|
struct taskqueue *tq[MAXCPU];
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct epair_tasks_t epair_tasks;
|
|
|
|
|
2021-10-26 07:57:56 +00:00
|
|
|
static void
|
|
|
|
epair_clear_mbuf(struct mbuf *m)
|
|
|
|
{
|
|
|
|
/* Remove any CSUM_SND_TAG as ether_input will barf. */
|
|
|
|
if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) {
|
|
|
|
m_snd_tag_rele(m->m_pkthdr.snd_tag);
|
|
|
|
m->m_pkthdr.snd_tag = NULL;
|
|
|
|
m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
|
|
|
|
}
|
|
|
|
|
|
|
|
m_tag_delete_nonpersistent(m);
|
|
|
|
}
|
|
|
|
|
2009-07-26 12:20:07 +00:00
|
|
|
static void
|
2023-03-01 20:21:30 +00:00
|
|
|
epair_tx_start_deferred(void *arg, int pending)
|
2009-07-26 12:20:07 +00:00
|
|
|
{
|
2023-03-01 20:21:30 +00:00
|
|
|
struct epair_queue *q = (struct epair_queue *)arg;
|
|
|
|
if_t ifp;
|
|
|
|
struct mbuf *m, *n;
|
|
|
|
bool resched;
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
ifp = q->sc->ifp;
|
|
|
|
|
|
|
|
if_ref(ifp);
|
2021-12-09 13:24:13 +00:00
|
|
|
CURVNET_SET(ifp->if_vnet);
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
mtx_lock(&q->mtx);
|
|
|
|
m = mbufq_flush(&q->q);
|
|
|
|
q->state = EPAIR_QUEUE_RUNNING;
|
|
|
|
mtx_unlock(&q->mtx);
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
while (m != NULL) {
|
|
|
|
n = STAILQ_NEXT(m, m_stailqpkt);
|
|
|
|
m->m_nextpkt = NULL;
|
|
|
|
if_input(ifp, m);
|
|
|
|
m = n;
|
2021-12-09 13:24:13 +00:00
|
|
|
}
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
/*
|
|
|
|
* Avoid flushing the queue more than once per task. We can otherwise
|
|
|
|
* end up starving ourselves in a multi-epair routing configuration.
|
|
|
|
*/
|
|
|
|
mtx_lock(&q->mtx);
|
|
|
|
if (mbufq_len(&q->q) > 0) {
|
|
|
|
resched = true;
|
|
|
|
q->state = EPAIR_QUEUE_WAKING;
|
|
|
|
} else {
|
|
|
|
resched = false;
|
|
|
|
q->state = EPAIR_QUEUE_IDLE;
|
|
|
|
}
|
|
|
|
mtx_unlock(&q->mtx);
|
|
|
|
|
|
|
|
if (resched)
|
2021-12-09 13:24:13 +00:00
|
|
|
taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task);
|
2009-06-24 22:21:30 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
CURVNET_RESTORE();
|
|
|
|
if_rele(ifp);
|
2009-07-26 12:20:07 +00:00
|
|
|
}
|
|
|
|
|
2022-09-27 13:24:29 +00:00
|
|
|
static struct epair_queue *
|
2022-10-03 15:02:55 +00:00
|
|
|
epair_select_queue(struct epair_softc *sc, struct mbuf *m)
|
2009-06-24 22:21:30 +00:00
|
|
|
{
|
2021-12-09 13:24:13 +00:00
|
|
|
uint32_t bucket;
|
|
|
|
#ifdef RSS
|
|
|
|
struct ether_header *eh;
|
2022-10-03 15:02:55 +00:00
|
|
|
int ret;
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2021-12-09 13:24:13 +00:00
|
|
|
ret = rss_m2bucket(m, &bucket);
|
|
|
|
if (ret) {
|
|
|
|
/* Actually hash the packet. */
|
|
|
|
eh = mtod(m, struct ether_header *);
|
|
|
|
|
|
|
|
switch (ntohs(eh->ether_type)) {
|
2022-03-03 09:44:43 +00:00
|
|
|
#ifdef INET
|
2021-12-09 13:24:13 +00:00
|
|
|
case ETHERTYPE_IP:
|
|
|
|
rss_soft_m2cpuid_v4(m, 0, &bucket);
|
|
|
|
break;
|
2022-03-03 09:44:43 +00:00
|
|
|
#endif
|
|
|
|
#ifdef INET6
|
2021-12-09 13:24:13 +00:00
|
|
|
case ETHERTYPE_IPV6:
|
|
|
|
rss_soft_m2cpuid_v6(m, 0, &bucket);
|
|
|
|
break;
|
2022-03-03 09:44:43 +00:00
|
|
|
#endif
|
2021-12-09 13:24:13 +00:00
|
|
|
default:
|
|
|
|
bucket = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2022-09-27 13:24:29 +00:00
|
|
|
bucket %= sc->num_queues;
|
2021-12-09 13:24:13 +00:00
|
|
|
#else
|
|
|
|
bucket = 0;
|
|
|
|
#endif
|
2022-09-27 13:24:29 +00:00
|
|
|
return (&sc->queues[bucket]);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_prepare_mbuf(struct mbuf *m, struct ifnet *src_ifp)
|
|
|
|
{
|
|
|
|
M_ASSERTPKTHDR(m);
|
|
|
|
epair_clear_mbuf(m);
|
|
|
|
if_setrcvif(m, src_ifp);
|
|
|
|
M_SETFIB(m, src_ifp->if_fib);
|
|
|
|
|
|
|
|
MPASS(m->m_nextpkt == NULL);
|
|
|
|
MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_menq(struct mbuf *m, struct epair_softc *osc)
|
|
|
|
{
|
2023-03-01 20:21:30 +00:00
|
|
|
struct epair_queue *q;
|
2022-09-27 13:24:29 +00:00
|
|
|
struct ifnet *ifp, *oifp;
|
2023-03-01 20:21:30 +00:00
|
|
|
int error, len;
|
2023-03-06 14:39:17 +00:00
|
|
|
bool mcast;
|
2022-09-27 13:24:29 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* I know this looks weird. We pass the "other sc" as we need that one
|
|
|
|
* and can get both ifps from it as well.
|
|
|
|
*/
|
|
|
|
oifp = osc->ifp;
|
|
|
|
ifp = osc->oifp;
|
|
|
|
|
|
|
|
epair_prepare_mbuf(m, oifp);
|
|
|
|
|
|
|
|
/* Save values as once the mbuf is queued, it's not ours anymore. */
|
|
|
|
len = m->m_pkthdr.len;
|
2023-03-06 14:39:17 +00:00
|
|
|
mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0;
|
2022-09-27 13:24:29 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
q = epair_select_queue(osc, m);
|
2021-12-09 13:24:13 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
mtx_lock(&q->mtx);
|
|
|
|
if (q->state == EPAIR_QUEUE_IDLE) {
|
|
|
|
q->state = EPAIR_QUEUE_WAKING;
|
|
|
|
taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task);
|
2021-10-09 14:09:04 +00:00
|
|
|
}
|
2023-03-01 20:21:30 +00:00
|
|
|
error = mbufq_enqueue(&q->q, m);
|
|
|
|
mtx_unlock(&q->mtx);
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
if (error != 0) {
|
|
|
|
m_freem(m);
|
|
|
|
if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
|
|
|
|
} else {
|
|
|
|
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
|
|
|
|
if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
|
|
|
|
if (mcast)
|
|
|
|
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
|
|
|
|
if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1);
|
|
|
|
}
|
2009-07-26 12:20:07 +00:00
|
|
|
}
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
static void
|
2021-10-09 14:09:04 +00:00
|
|
|
epair_start(struct ifnet *ifp)
|
2009-06-24 22:21:30 +00:00
|
|
|
{
|
|
|
|
struct mbuf *m;
|
|
|
|
struct epair_softc *sc;
|
|
|
|
struct ifnet *oifp;
|
|
|
|
|
|
|
|
/*
|
2017-04-08 04:45:52 +00:00
|
|
|
* We get packets here from ether_output via if_handoff()
|
2017-04-08 04:50:50 +00:00
|
|
|
* and need to put them into the input queue of the oifp
|
2021-10-09 14:09:04 +00:00
|
|
|
* and will put the packet into the receive-queue (rxq) of the
|
|
|
|
* other interface (oifp) of our pair.
|
2009-06-24 22:21:30 +00:00
|
|
|
*/
|
2021-10-09 14:09:04 +00:00
|
|
|
sc = ifp->if_softc;
|
2009-06-24 22:21:30 +00:00
|
|
|
oifp = sc->oifp;
|
|
|
|
sc = oifp->if_softc;
|
|
|
|
for (;;) {
|
|
|
|
IFQ_DEQUEUE(&ifp->if_snd, m);
|
|
|
|
if (m == NULL)
|
|
|
|
break;
|
2021-10-09 14:09:04 +00:00
|
|
|
M_ASSERTPKTHDR(m);
|
2009-06-24 22:21:30 +00:00
|
|
|
BPF_MTAP(ifp, m);
|
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
/* In case either interface is not usable drop the packet. */
|
|
|
|
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
|
|
|
|
(ifp->if_flags & IFF_UP) == 0 ||
|
|
|
|
(oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
|
|
|
|
(oifp->if_flags & IFF_UP) == 0) {
|
2009-06-24 22:21:30 +00:00
|
|
|
m_freem(m);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2022-09-27 13:24:29 +00:00
|
|
|
epair_menq(m, sc);
|
2021-10-09 14:09:04 +00:00
|
|
|
}
|
2009-06-24 22:21:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2021-10-09 14:09:04 +00:00
|
|
|
epair_transmit(struct ifnet *ifp, struct mbuf *m)
|
2009-06-24 22:21:30 +00:00
|
|
|
{
|
|
|
|
struct epair_softc *sc;
|
|
|
|
struct ifnet *oifp;
|
2021-11-24 21:28:54 +00:00
|
|
|
#ifdef ALTQ
|
|
|
|
int len;
|
2023-03-06 14:39:17 +00:00
|
|
|
bool mcast;
|
2021-11-24 21:28:54 +00:00
|
|
|
#endif
|
2009-06-24 22:21:30 +00:00
|
|
|
|
|
|
|
if (m == NULL)
|
|
|
|
return (0);
|
2021-10-09 14:09:04 +00:00
|
|
|
M_ASSERTPKTHDR(m);
|
2020-09-01 21:19:14 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
/*
|
|
|
|
* We are not going to use the interface en/dequeue mechanism
|
|
|
|
* on the TX side. We are called from ether_output_frame()
|
2021-10-09 14:09:04 +00:00
|
|
|
* and will put the packet into the receive-queue (rxq) of the
|
|
|
|
* other interface (oifp) of our pair.
|
2009-06-24 22:21:30 +00:00
|
|
|
*/
|
|
|
|
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
|
|
|
|
m_freem(m);
|
2021-10-09 14:09:04 +00:00
|
|
|
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
|
2009-06-24 22:21:30 +00:00
|
|
|
return (ENXIO);
|
|
|
|
}
|
|
|
|
if ((ifp->if_flags & IFF_UP) == 0) {
|
|
|
|
m_freem(m);
|
2021-10-09 14:09:04 +00:00
|
|
|
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
|
2009-06-24 22:21:30 +00:00
|
|
|
return (ENETDOWN);
|
|
|
|
}
|
|
|
|
|
|
|
|
BPF_MTAP(ifp, m);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* In case the outgoing interface is not usable,
|
|
|
|
* drop the packet.
|
|
|
|
*/
|
2021-10-09 14:09:04 +00:00
|
|
|
sc = ifp->if_softc;
|
2009-06-24 22:21:30 +00:00
|
|
|
oifp = sc->oifp;
|
|
|
|
if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
|
2021-10-09 14:09:04 +00:00
|
|
|
(oifp->if_flags & IFF_UP) == 0) {
|
2014-09-19 10:39:58 +00:00
|
|
|
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
|
2009-06-24 22:21:30 +00:00
|
|
|
m_freem(m);
|
|
|
|
return (0);
|
|
|
|
}
|
2021-11-24 21:28:54 +00:00
|
|
|
|
|
|
|
#ifdef ALTQ
|
2009-06-24 22:21:30 +00:00
|
|
|
len = m->m_pkthdr.len;
|
2023-03-06 14:39:17 +00:00
|
|
|
mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0;
|
2022-09-27 13:24:29 +00:00
|
|
|
int error = 0;
|
2009-06-24 22:21:30 +00:00
|
|
|
|
2016-05-03 18:05:43 +00:00
|
|
|
/* Support ALTQ via the classic if_start() path. */
|
2009-06-24 22:21:30 +00:00
|
|
|
IF_LOCK(&ifp->if_snd);
|
|
|
|
if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
|
|
|
|
ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error);
|
|
|
|
if (error)
|
2014-09-19 10:39:58 +00:00
|
|
|
if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
|
2009-06-24 22:21:30 +00:00
|
|
|
IF_UNLOCK(&ifp->if_snd);
|
|
|
|
if (!error) {
|
2014-09-19 10:39:58 +00:00
|
|
|
if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
|
2023-03-06 14:39:17 +00:00
|
|
|
if (mcast)
|
2014-09-19 10:39:58 +00:00
|
|
|
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
|
2021-10-09 14:09:04 +00:00
|
|
|
epair_start(ifp);
|
2009-06-24 22:21:30 +00:00
|
|
|
}
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
IF_UNLOCK(&ifp->if_snd);
|
|
|
|
#endif
|
|
|
|
|
2022-09-27 13:24:29 +00:00
|
|
|
epair_menq(m, oifp->if_softc);
|
|
|
|
return (0);
|
2009-06-24 22:21:30 +00:00
|
|
|
}
|
|
|
|
|
2022-09-22 09:01:03 +00:00
|
|
|
static void
|
|
|
|
epair_qflush(struct ifnet *ifp __unused)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2011-05-24 07:57:28 +00:00
|
|
|
static int
|
|
|
|
epair_media_change(struct ifnet *ifp __unused)
|
|
|
|
{
|
|
|
|
|
|
|
|
/* Do nothing. */
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr)
|
|
|
|
{
|
|
|
|
|
|
|
|
imr->ifm_status = IFM_AVALID | IFM_ACTIVE;
|
|
|
|
imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
|
|
|
|
}
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
static int
|
|
|
|
epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
|
|
|
|
{
|
2011-05-24 07:57:28 +00:00
|
|
|
struct epair_softc *sc;
|
2009-06-24 22:21:30 +00:00
|
|
|
struct ifreq *ifr;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
ifr = (struct ifreq *)data;
|
|
|
|
switch (cmd) {
|
|
|
|
case SIOCSIFFLAGS:
|
|
|
|
case SIOCADDMULTI:
|
|
|
|
case SIOCDELMULTI:
|
|
|
|
error = 0;
|
|
|
|
break;
|
|
|
|
|
2011-05-24 07:57:28 +00:00
|
|
|
case SIOCSIFMEDIA:
|
|
|
|
case SIOCGIFMEDIA:
|
|
|
|
sc = ifp->if_softc;
|
|
|
|
error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
|
|
|
|
break;
|
|
|
|
|
2009-07-26 12:20:07 +00:00
|
|
|
case SIOCSIFMTU:
|
|
|
|
/* We basically allow all kinds of MTUs. */
|
|
|
|
ifp->if_mtu = ifr->ifr_mtu;
|
|
|
|
error = 0;
|
|
|
|
break;
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
default:
|
|
|
|
/* Let the common ethernet handler process this. */
|
|
|
|
error = ether_ioctl(ifp, cmd, data);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_init(void *dummy __unused)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Interface cloning functions.
|
|
|
|
* We use our private ones so that we can create/destroy our secondary
|
|
|
|
* device along with the primary one.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
epair_clone_match(struct if_clone *ifc, const char *name)
|
|
|
|
{
|
|
|
|
const char *cp;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Our base name is epair.
|
|
|
|
* Our interfaces will be named epair<n>[ab].
|
|
|
|
* So accept anything of the following list:
|
|
|
|
* - epair
|
|
|
|
* - epair<n>
|
|
|
|
* but not the epair<n>[ab] versions.
|
|
|
|
*/
|
2012-10-16 13:37:54 +00:00
|
|
|
if (strncmp(epairname, name, sizeof(epairname)-1) != 0)
|
2009-06-24 22:21:30 +00:00
|
|
|
return (0);
|
|
|
|
|
2012-10-16 13:37:54 +00:00
|
|
|
for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) {
|
2009-06-24 22:21:30 +00:00
|
|
|
if (*cp < '0' || *cp > '9')
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (1);
|
|
|
|
}
|
|
|
|
|
2020-01-28 22:44:24 +00:00
|
|
|
static void
|
|
|
|
epair_clone_add(struct if_clone *ifc, struct epair_softc *scb)
|
|
|
|
{
|
|
|
|
struct ifnet *ifp;
|
|
|
|
uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */
|
|
|
|
|
|
|
|
ifp = scb->ifp;
|
|
|
|
/* Copy epairNa etheraddr and change the last byte. */
|
|
|
|
memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN);
|
|
|
|
eaddr[5] = 0x0b;
|
|
|
|
ether_ifattach(ifp, eaddr);
|
|
|
|
|
|
|
|
if_clone_addif(ifc, ifp);
|
|
|
|
}
|
|
|
|
|
2022-09-22 09:01:03 +00:00
|
|
|
static struct epair_softc *
|
|
|
|
epair_alloc_sc(struct if_clone *ifc)
|
|
|
|
{
|
|
|
|
struct epair_softc *sc;
|
|
|
|
|
|
|
|
struct ifnet *ifp = if_alloc(IFT_ETHER);
|
|
|
|
if (ifp == NULL)
|
|
|
|
return (NULL);
|
|
|
|
|
|
|
|
sc = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO);
|
|
|
|
sc->ifp = ifp;
|
|
|
|
sc->num_queues = epair_tasks.tasks;
|
|
|
|
sc->queues = mallocarray(sc->num_queues, sizeof(struct epair_queue),
|
|
|
|
M_EPAIR, M_WAITOK);
|
|
|
|
for (int i = 0; i < sc->num_queues; i++) {
|
|
|
|
struct epair_queue *q = &sc->queues[i];
|
|
|
|
q->id = i;
|
2023-03-01 20:21:30 +00:00
|
|
|
q->state = EPAIR_QUEUE_IDLE;
|
|
|
|
mtx_init(&q->mtx, "epairq", NULL, MTX_DEF | MTX_NEW);
|
|
|
|
mbufq_init(&q->q, RXRSIZE);
|
2022-09-22 09:01:03 +00:00
|
|
|
q->sc = sc;
|
|
|
|
NET_TASK_INIT(&q->tx_task, 0, epair_tx_start_deferred, q);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Initialise pseudo media types. */
|
|
|
|
ifmedia_init(&sc->media, 0, epair_media_change, epair_media_status);
|
|
|
|
ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL);
|
|
|
|
ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T);
|
|
|
|
|
|
|
|
return (sc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_setup_ifp(struct epair_softc *sc, char *name, int unit)
|
|
|
|
{
|
|
|
|
struct ifnet *ifp = sc->ifp;
|
|
|
|
|
|
|
|
ifp->if_softc = sc;
|
|
|
|
strlcpy(ifp->if_xname, name, IFNAMSIZ);
|
|
|
|
ifp->if_dname = epairname;
|
|
|
|
ifp->if_dunit = unit;
|
|
|
|
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
|
|
|
|
ifp->if_flags |= IFF_KNOWSEPOCH;
|
|
|
|
ifp->if_capabilities = IFCAP_VLAN_MTU;
|
|
|
|
ifp->if_capenable = IFCAP_VLAN_MTU;
|
|
|
|
ifp->if_transmit = epair_transmit;
|
|
|
|
ifp->if_qflush = epair_qflush;
|
|
|
|
ifp->if_start = epair_start;
|
|
|
|
ifp->if_ioctl = epair_ioctl;
|
|
|
|
ifp->if_init = epair_init;
|
|
|
|
if_setsendqlen(ifp, ifqmaxlen);
|
|
|
|
if_setsendqready(ifp);
|
|
|
|
|
|
|
|
ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_generate_mac(struct epair_softc *sc, uint8_t *eaddr)
|
|
|
|
{
|
|
|
|
uint32_t key[3];
|
|
|
|
uint32_t hash;
|
|
|
|
uint64_t hostid;
|
|
|
|
|
|
|
|
EPAIR_LOCK();
|
|
|
|
#ifdef SMP
|
|
|
|
/* Get an approximate distribution. */
|
|
|
|
hash = next_index % mp_ncpus;
|
|
|
|
#else
|
|
|
|
hash = 0;
|
|
|
|
#endif
|
|
|
|
EPAIR_UNLOCK();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate the etheraddr hashing the hostid and the
|
|
|
|
* interface index. The result would be hopefully unique.
|
|
|
|
* Note that the "a" component of an epair instance may get moved
|
|
|
|
* to a different VNET after creation. In that case its index
|
|
|
|
* will be freed and the index can get reused by new epair instance.
|
|
|
|
* Make sure we do not create same etheraddr again.
|
|
|
|
*/
|
|
|
|
getcredhostid(curthread->td_ucred, (unsigned long *)&hostid);
|
|
|
|
if (hostid == 0)
|
|
|
|
arc4rand(&hostid, sizeof(hostid), 0);
|
|
|
|
|
|
|
|
struct ifnet *ifp = sc->ifp;
|
|
|
|
EPAIR_LOCK();
|
|
|
|
if (ifp->if_index > next_index)
|
|
|
|
next_index = ifp->if_index;
|
|
|
|
else
|
|
|
|
next_index++;
|
|
|
|
|
|
|
|
key[0] = (uint32_t)next_index;
|
|
|
|
EPAIR_UNLOCK();
|
|
|
|
key[1] = (uint32_t)(hostid & 0xffffffff);
|
|
|
|
key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff);
|
|
|
|
hash = jenkins_hash32(key, 3, 0);
|
|
|
|
|
|
|
|
eaddr[0] = 0x02;
|
|
|
|
memcpy(&eaddr[1], &hash, 4);
|
|
|
|
eaddr[5] = 0x0a;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
epair_free_sc(struct epair_softc *sc)
|
|
|
|
{
|
|
|
|
if (sc == NULL)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if_free(sc->ifp);
|
|
|
|
ifmedia_removeall(&sc->media);
|
|
|
|
for (int i = 0; i < sc->num_queues; i++) {
|
|
|
|
struct epair_queue *q = &sc->queues[i];
|
2023-03-01 20:21:30 +00:00
|
|
|
mtx_destroy(&q->mtx);
|
2022-09-22 09:01:03 +00:00
|
|
|
}
|
|
|
|
free(sc->queues, M_EPAIR);
|
|
|
|
free(sc, M_EPAIR);
|
|
|
|
}
|
|
|
|
|
2022-09-27 13:24:29 +00:00
|
|
|
static void
|
|
|
|
epair_set_state(struct ifnet *ifp, bool running)
|
|
|
|
{
|
|
|
|
if (running) {
|
|
|
|
ifp->if_drv_flags |= IFF_DRV_RUNNING;
|
|
|
|
if_link_state_change(ifp, LINK_STATE_UP);
|
|
|
|
} else {
|
|
|
|
if_link_state_change(ifp, LINK_STATE_DOWN);
|
|
|
|
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
static int
|
2022-09-27 13:24:29 +00:00
|
|
|
epair_handle_unit(struct if_clone *ifc, char *name, size_t len, int *punit)
|
2009-06-24 22:21:30 +00:00
|
|
|
{
|
2022-09-27 13:24:29 +00:00
|
|
|
int error = 0, unit, wildcard;
|
2009-06-24 22:21:30 +00:00
|
|
|
char *dp;
|
|
|
|
|
|
|
|
/* Try to see if a special unit was requested. */
|
|
|
|
error = ifc_name2unit(name, &unit);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
wildcard = (unit < 0);
|
|
|
|
|
|
|
|
error = ifc_alloc_unit(ifc, &unit);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If no unit had been given, we need to adjust the ifName.
|
|
|
|
* Also make sure there is space for our extra [ab] suffix.
|
|
|
|
*/
|
|
|
|
for (dp = name; *dp != '\0'; dp++);
|
|
|
|
if (wildcard) {
|
2022-09-27 13:24:29 +00:00
|
|
|
int slen = snprintf(dp, len - (dp - name), "%d", unit);
|
|
|
|
if (slen > len - (dp - name) - 1) {
|
2009-06-24 22:21:30 +00:00
|
|
|
/* ifName too long. */
|
2022-09-27 13:24:29 +00:00
|
|
|
error = ENOSPC;
|
|
|
|
goto done;
|
2009-06-24 22:21:30 +00:00
|
|
|
}
|
2022-09-27 13:24:29 +00:00
|
|
|
dp += slen;
|
2009-06-24 22:21:30 +00:00
|
|
|
}
|
|
|
|
if (len - (dp - name) - 1 < 1) {
|
|
|
|
/* No space left for our [ab] suffix. */
|
2022-09-27 13:24:29 +00:00
|
|
|
error = ENOSPC;
|
|
|
|
goto done;
|
2009-06-24 22:21:30 +00:00
|
|
|
}
|
2014-10-10 06:45:13 +00:00
|
|
|
*dp = 'b';
|
2009-06-24 22:21:30 +00:00
|
|
|
/* Must not change dp so we can replace 'a' by 'b' later. */
|
|
|
|
*(dp+1) = '\0';
|
|
|
|
|
2014-10-10 06:45:13 +00:00
|
|
|
/* Check if 'a' and 'b' interfaces already exist. */
|
2022-09-27 13:24:29 +00:00
|
|
|
if (ifunit(name) != NULL) {
|
|
|
|
error = EEXIST;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
2014-10-10 06:45:13 +00:00
|
|
|
*dp = 'a';
|
2022-09-27 13:24:29 +00:00
|
|
|
if (ifunit(name) != NULL) {
|
|
|
|
error = EEXIST;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
*punit = unit;
|
|
|
|
done:
|
|
|
|
if (error != 0)
|
|
|
|
ifc_free_unit(ifc, unit);
|
|
|
|
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
epair_clone_create(struct if_clone *ifc, char *name, size_t len,
|
|
|
|
struct ifc_data *ifd, struct ifnet **ifpp)
|
|
|
|
{
|
|
|
|
struct epair_softc *sca, *scb;
|
|
|
|
struct ifnet *ifp;
|
|
|
|
char *dp;
|
|
|
|
int error, unit;
|
|
|
|
uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */
|
|
|
|
|
|
|
|
error = epair_handle_unit(ifc, name, len, &unit);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
2014-10-10 06:45:13 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
/* Allocate memory for both [ab] interfaces */
|
2022-09-22 09:01:03 +00:00
|
|
|
sca = epair_alloc_sc(ifc);
|
|
|
|
scb = epair_alloc_sc(ifc);
|
|
|
|
if (sca == NULL || scb == NULL) {
|
|
|
|
epair_free_sc(sca);
|
|
|
|
epair_free_sc(scb);
|
2009-06-24 22:21:30 +00:00
|
|
|
ifc_free_unit(ifc, unit);
|
|
|
|
return (ENOSPC);
|
|
|
|
}
|
2020-09-01 21:19:14 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
/*
|
|
|
|
* Cross-reference the interfaces so we will be able to free both.
|
|
|
|
*/
|
|
|
|
sca->oifp = scb->ifp;
|
|
|
|
scb->oifp = sca->ifp;
|
2009-07-26 12:20:07 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
/* Finish initialization of interface <n>a. */
|
|
|
|
ifp = sca->ifp;
|
2022-09-22 09:01:03 +00:00
|
|
|
epair_setup_ifp(sca, name, unit);
|
|
|
|
epair_generate_mac(sca, eaddr);
|
2018-05-23 13:10:57 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
ether_ifattach(ifp, eaddr);
|
|
|
|
|
|
|
|
/* Swap the name and finish initialization of interface <n>b. */
|
2022-09-27 13:24:29 +00:00
|
|
|
dp = name + strlen(name) - 1;
|
2009-06-24 22:21:30 +00:00
|
|
|
*dp = 'b';
|
|
|
|
|
2022-09-22 09:01:03 +00:00
|
|
|
epair_setup_ifp(scb, name, unit);
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
ifp = scb->ifp;
|
|
|
|
/* We need to play some tricks here for the second interface. */
|
2012-10-16 13:37:54 +00:00
|
|
|
strlcpy(name, epairname, len);
|
2020-01-28 22:44:24 +00:00
|
|
|
/* Correctly set the name for the cloner list. */
|
|
|
|
strlcpy(name, scb->ifp->if_xname, len);
|
|
|
|
|
2022-09-22 09:01:03 +00:00
|
|
|
epair_clone_add(ifc, scb);
|
2009-06-24 22:21:30 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Restore name to <n>a as the ifp for this will go into the
|
|
|
|
* cloner list for the initial call.
|
|
|
|
*/
|
|
|
|
strlcpy(name, sca->ifp->if_xname, len);
|
|
|
|
|
|
|
|
/* Tell the world, that we are ready to rock. */
|
2022-09-27 13:24:29 +00:00
|
|
|
epair_set_state(sca->ifp, true);
|
|
|
|
epair_set_state(scb->ifp, true);
|
2009-06-24 22:21:30 +00:00
|
|
|
|
2022-09-22 12:30:09 +00:00
|
|
|
*ifpp = sca->ifp;
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
static void
|
|
|
|
epair_drain_rings(struct epair_softc *sc)
|
|
|
|
{
|
2023-03-01 20:21:30 +00:00
|
|
|
for (int i = 0; i < sc->num_queues; i++) {
|
|
|
|
struct epair_queue *q;
|
|
|
|
struct mbuf *m, *n;
|
2021-10-09 14:09:04 +00:00
|
|
|
|
2023-03-01 20:21:30 +00:00
|
|
|
q = &sc->queues[i];
|
|
|
|
mtx_lock(&q->mtx);
|
|
|
|
m = mbufq_flush(&q->q);
|
|
|
|
mtx_unlock(&q->mtx);
|
|
|
|
|
|
|
|
for (; m != NULL; m = n) {
|
|
|
|
n = m->m_nextpkt;
|
|
|
|
m_freem(m);
|
2021-12-09 13:24:13 +00:00
|
|
|
}
|
2021-10-09 14:09:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
static int
|
2022-09-22 12:30:09 +00:00
|
|
|
epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
|
2009-06-24 22:21:30 +00:00
|
|
|
{
|
|
|
|
struct ifnet *oifp;
|
|
|
|
struct epair_softc *sca, *scb;
|
|
|
|
int unit, error;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* In case we called into if_clone_destroyif() ourselves
|
|
|
|
* again to remove the second interface, the softc will be
|
|
|
|
* NULL. In that case so not do anything but return success.
|
|
|
|
*/
|
|
|
|
if (ifp->if_softc == NULL)
|
|
|
|
return (0);
|
2020-09-01 21:19:14 +00:00
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
unit = ifp->if_dunit;
|
|
|
|
sca = ifp->if_softc;
|
|
|
|
oifp = sca->oifp;
|
|
|
|
scb = oifp->if_softc;
|
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
/* Frist get the interfaces down and detached. */
|
2022-09-27 13:24:29 +00:00
|
|
|
epair_set_state(ifp, false);
|
|
|
|
epair_set_state(oifp, false);
|
2012-07-09 20:38:18 +00:00
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
ether_ifdetach(ifp);
|
2009-06-24 22:21:30 +00:00
|
|
|
ether_ifdetach(oifp);
|
2021-10-09 14:09:04 +00:00
|
|
|
|
|
|
|
/* Third free any queued packets and all the resources. */
|
|
|
|
CURVNET_SET_QUIET(oifp->if_vnet);
|
|
|
|
epair_drain_rings(scb);
|
2009-06-24 22:21:30 +00:00
|
|
|
oifp->if_softc = NULL;
|
|
|
|
error = if_clone_destroyif(ifc, oifp);
|
|
|
|
if (error)
|
|
|
|
panic("%s: if_clone_destroyif() for our 2nd iface failed: %d",
|
|
|
|
__func__, error);
|
2022-09-22 09:01:03 +00:00
|
|
|
epair_free_sc(scb);
|
2012-07-09 20:38:18 +00:00
|
|
|
CURVNET_RESTORE();
|
2009-06-24 22:21:30 +00:00
|
|
|
|
2021-10-09 14:09:04 +00:00
|
|
|
epair_drain_rings(sca);
|
2022-09-22 09:01:03 +00:00
|
|
|
epair_free_sc(sca);
|
2021-10-09 14:09:04 +00:00
|
|
|
|
|
|
|
/* Last free the cloner unit. */
|
2009-06-24 22:21:30 +00:00
|
|
|
ifc_free_unit(ifc, unit);
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2014-10-10 06:45:13 +00:00
|
|
|
static void
|
|
|
|
vnet_epair_init(const void *unused __unused)
|
|
|
|
{
|
2022-09-22 12:30:09 +00:00
|
|
|
struct if_clone_addreq req = {
|
|
|
|
.match_f = epair_clone_match,
|
|
|
|
.create_f = epair_clone_create,
|
|
|
|
.destroy_f = epair_clone_destroy,
|
|
|
|
};
|
|
|
|
V_epair_cloner = ifc_attach_cloner(epairname, &req);
|
2014-10-10 06:45:13 +00:00
|
|
|
}
|
Get closer to a VIMAGE network stack teardown from top to bottom rather
than removing the network interfaces first. This change is rather larger
and convoluted as the ordering requirements cannot be separated.
Move the pfil(9) framework to SI_SUB_PROTO_PFIL, move Firewalls and
related modules to their own SI_SUB_PROTO_FIREWALL.
Move initialization of "physical" interfaces to SI_SUB_DRIVERS,
move virtual (cloned) interfaces to SI_SUB_PSEUDO.
Move Multicast to SI_SUB_PROTO_MC.
Re-work parts of multicast initialisation and teardown, not taking the
huge amount of memory into account if used as a module yet.
For interface teardown we try to do as many of them as we can on
SI_SUB_INIT_IF, but for some this makes no sense, e.g., when tunnelling
over a higher layer protocol such as IP. In that case the interface
has to go along (or before) the higher layer protocol is shutdown.
Kernel hhooks need to go last on teardown as they may be used at various
higher layers and we cannot remove them before we cleaned up the higher
layers.
For interface teardown there are multiple paths:
(a) a cloned interface is destroyed (inside a VIMAGE or in the base system),
(b) any interface is moved from a virtual network stack to a different
network stack ("vmove"), or (c) a virtual network stack is being shut down.
All code paths go through if_detach_internal() where we, depending on the
vmove flag or the vnet state, make a decision on how much to shut down;
in case we are destroying a VNET the individual protocol layers will
cleanup their own parts thus we cannot do so again for each interface as
we end up with, e.g., double-frees, destroying locks twice or acquiring
already destroyed locks.
When calling into protocol cleanups we equally have to tell them
whether they need to detach upper layer protocols ("ulp") or not
(e.g., in6_ifdetach()).
Provide or enahnce helper functions to do proper cleanup at a protocol
rather than at an interface level.
Approved by: re (hrs)
Obtained from: projects/vnet
Reviewed by: gnn, jhb
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D6747
2016-06-21 13:48:49 +00:00
|
|
|
VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
|
2014-10-10 06:45:13 +00:00
|
|
|
vnet_epair_init, NULL);
|
|
|
|
|
|
|
|
static void
|
|
|
|
vnet_epair_uninit(const void *unused __unused)
|
|
|
|
{
|
|
|
|
|
2022-09-22 12:30:09 +00:00
|
|
|
ifc_detach_cloner(V_epair_cloner);
|
2014-10-10 06:45:13 +00:00
|
|
|
}
|
Get closer to a VIMAGE network stack teardown from top to bottom rather
than removing the network interfaces first. This change is rather larger
and convoluted as the ordering requirements cannot be separated.
Move the pfil(9) framework to SI_SUB_PROTO_PFIL, move Firewalls and
related modules to their own SI_SUB_PROTO_FIREWALL.
Move initialization of "physical" interfaces to SI_SUB_DRIVERS,
move virtual (cloned) interfaces to SI_SUB_PSEUDO.
Move Multicast to SI_SUB_PROTO_MC.
Re-work parts of multicast initialisation and teardown, not taking the
huge amount of memory into account if used as a module yet.
For interface teardown we try to do as many of them as we can on
SI_SUB_INIT_IF, but for some this makes no sense, e.g., when tunnelling
over a higher layer protocol such as IP. In that case the interface
has to go along (or before) the higher layer protocol is shutdown.
Kernel hhooks need to go last on teardown as they may be used at various
higher layers and we cannot remove them before we cleaned up the higher
layers.
For interface teardown there are multiple paths:
(a) a cloned interface is destroyed (inside a VIMAGE or in the base system),
(b) any interface is moved from a virtual network stack to a different
network stack ("vmove"), or (c) a virtual network stack is being shut down.
All code paths go through if_detach_internal() where we, depending on the
vmove flag or the vnet state, make a decision on how much to shut down;
in case we are destroying a VNET the individual protocol layers will
cleanup their own parts thus we cannot do so again for each interface as
we end up with, e.g., double-frees, destroying locks twice or acquiring
already destroyed locks.
When calling into protocol cleanups we equally have to tell them
whether they need to detach upper layer protocols ("ulp") or not
(e.g., in6_ifdetach()).
Provide or enahnce helper functions to do proper cleanup at a protocol
rather than at an interface level.
Approved by: re (hrs)
Obtained from: projects/vnet
Reviewed by: gnn, jhb
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D6747
2016-06-21 13:48:49 +00:00
|
|
|
VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
|
2014-10-10 06:45:13 +00:00
|
|
|
vnet_epair_uninit, NULL);
|
|
|
|
|
2021-12-09 13:24:13 +00:00
|
|
|
static int
|
2022-02-15 13:45:22 +00:00
|
|
|
epair_mod_init(void)
|
2021-12-09 13:24:13 +00:00
|
|
|
{
|
|
|
|
char name[32];
|
|
|
|
epair_tasks.tasks = 0;
|
|
|
|
|
|
|
|
#ifdef RSS
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
CPU_FOREACH(cpu) {
|
|
|
|
cpuset_t cpu_mask;
|
|
|
|
|
|
|
|
/* Pin to this CPU so we get appropriate NUMA allocations. */
|
|
|
|
thread_lock(curthread);
|
|
|
|
sched_bind(curthread, cpu);
|
|
|
|
thread_unlock(curthread);
|
|
|
|
|
|
|
|
snprintf(name, sizeof(name), "epair_task_%d", cpu);
|
|
|
|
|
|
|
|
epair_tasks.tq[cpu] = taskqueue_create(name, M_WAITOK,
|
|
|
|
taskqueue_thread_enqueue,
|
|
|
|
&epair_tasks.tq[cpu]);
|
|
|
|
CPU_SETOF(cpu, &cpu_mask);
|
|
|
|
taskqueue_start_threads_cpuset(&epair_tasks.tq[cpu], 1, PI_NET,
|
|
|
|
&cpu_mask, "%s", name);
|
|
|
|
|
|
|
|
epair_tasks.tasks++;
|
|
|
|
}
|
2022-05-07 15:15:34 +00:00
|
|
|
thread_lock(curthread);
|
|
|
|
sched_unbind(curthread);
|
|
|
|
thread_unlock(curthread);
|
2021-12-09 13:24:13 +00:00
|
|
|
#else
|
|
|
|
snprintf(name, sizeof(name), "epair_task");
|
|
|
|
|
|
|
|
epair_tasks.tq[0] = taskqueue_create(name, M_WAITOK,
|
|
|
|
taskqueue_thread_enqueue,
|
|
|
|
&epair_tasks.tq[0]);
|
|
|
|
taskqueue_start_threads(&epair_tasks.tq[0], 1, PI_NET, "%s", name);
|
|
|
|
|
|
|
|
epair_tasks.tasks = 1;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2022-02-15 13:45:22 +00:00
|
|
|
epair_mod_cleanup(void)
|
2021-12-09 13:24:13 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
for (int i = 0; i < epair_tasks.tasks; i++) {
|
|
|
|
taskqueue_drain_all(epair_tasks.tq[i]);
|
|
|
|
taskqueue_free(epair_tasks.tq[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-06-24 22:21:30 +00:00
|
|
|
static int
|
|
|
|
epair_modevent(module_t mod, int type, void *data)
|
|
|
|
{
|
2021-12-09 13:24:13 +00:00
|
|
|
int ret;
|
2009-06-24 22:21:30 +00:00
|
|
|
|
|
|
|
switch (type) {
|
|
|
|
case MOD_LOAD:
|
2021-10-09 14:09:04 +00:00
|
|
|
EPAIR_LOCK_INIT();
|
2021-12-09 13:24:13 +00:00
|
|
|
ret = epair_mod_init();
|
|
|
|
if (ret != 0)
|
|
|
|
return (ret);
|
2009-06-24 22:21:30 +00:00
|
|
|
if (bootverbose)
|
2021-10-09 14:09:04 +00:00
|
|
|
printf("%s: %s initialized.\n", __func__, epairname);
|
2009-06-24 22:21:30 +00:00
|
|
|
break;
|
|
|
|
case MOD_UNLOAD:
|
2021-12-09 13:24:13 +00:00
|
|
|
epair_mod_cleanup();
|
2021-10-09 14:09:04 +00:00
|
|
|
EPAIR_LOCK_DESTROY();
|
|
|
|
if (bootverbose)
|
|
|
|
printf("%s: %s unloaded.\n", __func__, epairname);
|
2009-06-24 22:21:30 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return (EOPNOTSUPP);
|
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static moduledata_t epair_mod = {
|
|
|
|
"if_epair",
|
|
|
|
epair_modevent,
|
2012-10-10 08:36:38 +00:00
|
|
|
0
|
2009-06-24 22:21:30 +00:00
|
|
|
};
|
|
|
|
|
Get closer to a VIMAGE network stack teardown from top to bottom rather
than removing the network interfaces first. This change is rather larger
and convoluted as the ordering requirements cannot be separated.
Move the pfil(9) framework to SI_SUB_PROTO_PFIL, move Firewalls and
related modules to their own SI_SUB_PROTO_FIREWALL.
Move initialization of "physical" interfaces to SI_SUB_DRIVERS,
move virtual (cloned) interfaces to SI_SUB_PSEUDO.
Move Multicast to SI_SUB_PROTO_MC.
Re-work parts of multicast initialisation and teardown, not taking the
huge amount of memory into account if used as a module yet.
For interface teardown we try to do as many of them as we can on
SI_SUB_INIT_IF, but for some this makes no sense, e.g., when tunnelling
over a higher layer protocol such as IP. In that case the interface
has to go along (or before) the higher layer protocol is shutdown.
Kernel hhooks need to go last on teardown as they may be used at various
higher layers and we cannot remove them before we cleaned up the higher
layers.
For interface teardown there are multiple paths:
(a) a cloned interface is destroyed (inside a VIMAGE or in the base system),
(b) any interface is moved from a virtual network stack to a different
network stack ("vmove"), or (c) a virtual network stack is being shut down.
All code paths go through if_detach_internal() where we, depending on the
vmove flag or the vnet state, make a decision on how much to shut down;
in case we are destroying a VNET the individual protocol layers will
cleanup their own parts thus we cannot do so again for each interface as
we end up with, e.g., double-frees, destroying locks twice or acquiring
already destroyed locks.
When calling into protocol cleanups we equally have to tell them
whether they need to detach upper layer protocols ("ulp") or not
(e.g., in6_ifdetach()).
Provide or enahnce helper functions to do proper cleanup at a protocol
rather than at an interface level.
Approved by: re (hrs)
Obtained from: projects/vnet
Reviewed by: gnn, jhb
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D6747
2016-06-21 13:48:49 +00:00
|
|
|
DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE);
|
2021-10-09 14:09:04 +00:00
|
|
|
MODULE_VERSION(if_epair, 3);
|