d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make tcp_output() require network epoch.

Browse Source 
Enter the epoch before calling into tcp_output() from those
functions, that didn't do that before.

This eliminates a bunch of epoch recursions in TCP.
This commit is contained in:
Gleb Smirnoff 2020-01-22 05:53:16 +00:00
parent b955545386
commit 109eb549e1
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=356975
6 changed files with 38 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sys/netinet/tcp_output.c
View File

@ -193,7 +193,6 @@ cc_after_idle(struct tcpcb *tp)
int
tcp_output(struct tcpcb *tp)
{
struct epoch_tracker et;
struct socket *so = tp->t_inpcb->inp_socket;
int32_t len;
uint32_t recwin, sendwin;
@ -233,6 +232,7 @@ tcp_output(struct tcpcb *tp)
const bool hw_tls = false;
#endif
NET_EPOCH_ASSERT();
INP_WLOCK_ASSERT(tp->t_inpcb);
#ifdef TCP_OFFLOAD
@ -1372,7 +1372,6 @@ tcp_output(struct tcpcb *tp)
* m->m_pkthdr.len should have been set before checksum calculation,
* because in6_cksum() need it.
*/
NET_EPOCH_ENTER(et);
#ifdef INET6
if (isipv6) {
/*
@ -1458,7 +1457,6 @@ tcp_output(struct tcpcb *tp)
mtu = tp->t_inpcb->inp_route.ro_rt->rt_mtu;
}
#endif /* INET */
NET_EPOCH_EXIT(et);
out:
/*

5
sys/netinet/tcp_stacks/bbr.c
View File

@ -12091,7 +12091,6 @@ bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, i
static int
bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
{
struct epoch_tracker et;
struct socket *so;
int32_t len;
uint32_t cts;
@ -13938,7 +13937,6 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
* m->m_pkthdr.len should have been set before cksum calcuration,
* because in6_cksum() need it.
*/
NET_EPOCH_ENTER(et);
#ifdef INET6
if (isipv6) {
/*
@ -14016,7 +14014,6 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
mtu = inp->inp_route.ro_rt->rt_mtu;
}
#endif /* INET */
NET_EPOCH_EXIT(et);
out:
if (lgb) {
@ -14465,6 +14462,8 @@ bbr_output(struct tcpcb *tp)
struct timeval tv;
struct tcp_bbr *bbr;
NET_EPOCH_ASSERT();
bbr = (struct tcp_bbr *)tp->t_fb_ptr;
INP_WLOCK_ASSERT(tp->t_inpcb);
(void)tcp_get_usecs(&tv);

7
sys/netinet/tcp_stacks/rack.c
View File

@ -8091,7 +8091,6 @@ rack_get_pacing_delay(struct tcp_rack *rack, struct tcpcb *tp, uint32_t len)
static int
rack_output(struct tcpcb *tp)
{
struct epoch_tracker et;
struct socket *so;
uint32_t recwin, sendwin;
uint32_t sb_offset;
@ -8155,8 +8154,10 @@ rack_output(struct tcpcb *tp)
#ifdef KERN_TLS
hw_tls = (so->so_snd.sb_flags & SB_TLS_IFNET) != 0;
#endif
NET_EPOCH_ASSERT();
INP_WLOCK_ASSERT(inp);
#ifdef TCP_OFFLOAD
if (tp->t_flags & TF_TOE)
return (tcp_offload_output(tp));
@ -9734,7 +9735,6 @@ rack_output(struct tcpcb *tp)
* m->m_pkthdr.len should have been set before cksum calcuration,
* because in6_cksum() need it.
*/
NET_EPOCH_ENTER(et);
#ifdef INET6
if (isipv6) {
/*
@ -9812,7 +9812,6 @@ rack_output(struct tcpcb *tp)
mtu = inp->inp_route.ro_rt->rt_mtu;
}
#endif /* INET */
NET_EPOCH_EXIT(et);
out:
if (lgb) {

9
sys/netinet/tcp_timer.c
View File

@ -251,6 +251,7 @@ int tcp_totbackoff = 2559; /* sum of tcp_backoff[] */
void
tcp_timer_delack(void *xtp)
{
struct epoch_tracker et;
struct tcpcb *tp = xtp;
struct inpcb *inp;
CURVNET_SET(tp->t_vnet);
@ -272,8 +273,10 @@ tcp_timer_delack(void *xtp)
}
tp->t_flags |= TF_ACKNOW;
TCPSTAT_INC(tcps_delack);
NET_EPOCH_ENTER(et);
(void) tp->t_fb->tfb_tcp_output(tp);
INP_WUNLOCK(inp);
NET_EPOCH_EXIT(et);
CURVNET_RESTORE();
}
@ -570,7 +573,9 @@ tcp_timer_persist(void *xtp)
}
tcp_setpersist(tp);
tp->t_flags |= TF_FORCEDATA;
NET_EPOCH_ENTER(et);
(void) tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
tp->t_flags &= ~TF_FORCEDATA;
#ifdef TCPDEBUG
@ -824,9 +829,9 @@ tcp_timer_rexmt(void * xtp)
tp->t_rtttime = 0;
cc_cong_signal(tp, NULL, CC_RTO);
NET_EPOCH_ENTER(et);
(void) tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
#ifdef TCPDEBUG
if (tp != NULL && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG))
tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,

32
sys/netinet/tcp_usrreq.c
View File

@ -531,6 +531,7 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
static int
tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
{
struct epoch_tracker et;
int error = 0;
struct inpcb *inp;
struct tcpcb *tp = NULL;
@ -571,7 +572,9 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
goto out;
#endif
tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
out:
TCPDEBUG2(PRU_CONNECT);
TCP_PROBE2(debug__user, tp, PRU_CONNECT);
@ -584,6 +587,7 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
static int
tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
{
struct epoch_tracker et;
int error = 0;
struct inpcb *inp;
struct tcpcb *tp = NULL;
@ -654,7 +658,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
(error = tcp_offload_connect(so, nam)) == 0)
goto out;
#endif
NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
goto out;
} else {
if ((inp->inp_vflag & INP_IPV6) == 0) {
@ -677,8 +683,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
goto out;
#endif
tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
out:
/*
* If the implicit bind in the connect call fails, restore
@ -882,6 +889,7 @@ tcp_usr_shutdown(struct socket *so)
static int
tcp_usr_rcvd(struct socket *so, int flags)
{
struct epoch_tracker et;
struct inpcb *inp;
struct tcpcb *tp = NULL;
int error = 0;
@ -911,8 +919,9 @@ tcp_usr_rcvd(struct socket *so, int flags)
tcp_offload_rcvd(tp);
else
#endif
NET_EPOCH_ENTER(et);
tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
out:
TCPDEBUG2(PRU_RCVD);
TCP_PROBE2(debug__user, tp, PRU_RCVD);
@ -953,8 +962,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
* We require the pcbinfo "read lock" if we will close the socket
* as part of this call.
*/
if (flags & PRUS_EOF)
NET_EPOCH_ENTER(et);
NET_EPOCH_ENTER(et);
inp = sotoinpcb(so);
KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL"));
INP_WLOCK(inp);
@ -1240,14 +1248,14 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
INP_WUNLOCK(inp);
if (flags & PRUS_EOF)
NET_EPOCH_EXIT(et);
NET_EPOCH_EXIT(et);
return (error);
}
static int
tcp_usr_ready(struct socket *so, struct mbuf *m, int count)
{
struct epoch_tracker et;
struct inpcb *inp;
struct tcpcb *tp;
int error;
@ -1264,8 +1272,11 @@ tcp_usr_ready(struct socket *so, struct mbuf *m, int count)
SOCKBUF_LOCK(&so->so_snd);
error = sbready(&so->so_snd, m, count);
SOCKBUF_UNLOCK(&so->so_snd);
if (error == 0)
if (error == 0) {
NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
}
INP_WUNLOCK(inp);
return (error);
@ -1921,8 +1932,13 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
tp->t_flags |= TF_NOPUSH;
else if (tp->t_flags & TF_NOPUSH) {
tp->t_flags &= ~TF_NOPUSH;
if (TCPS_HAVEESTABLISHED(tp->t_state))
if (TCPS_HAVEESTABLISHED(tp->t_state)) {
struct epoch_tracker et;
NET_EPOCH_ENTER(et);
error = tp->t_fb->tfb_tcp_output(tp);
NET_EPOCH_EXIT(et);
}
}
goto unlock_and_done;

2
sys/netinet/toecore.c
View File

@ -503,6 +503,7 @@ void
toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err)
{
NET_EPOCH_ASSERT();
INP_WLOCK_ASSERT(inp);
if (!(inp->inp_flags & INP_DROPPED)) {
@ -527,7 +528,6 @@ toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err)
(void) tp->t_fb->tfb_tcp_output(tp);
} else {
NET_EPOCH_ASSERT();
tp = tcp_drop(tp, err);
if (tp == NULL)
INP_WLOCK(inp); /* re-acquire */