Reduce the number of unnecessary unlock-relocks on socket buffer mutexes
associated with performing a wakeup on the socket buffer: - When performing an sbappend*() followed by a so[rw]wakeup(), explicitly acquire the socket buffer lock and use the _locked() variants of both calls. Note that the _locked() sowakeup() versions unlock the mutex on return. This is done in uipc_send(), divert_packet(), mroute socket_send(), raw_append(), tcp_reass(), tcp_input(), and udp_append(). - When the socket buffer lock is dropped before a sowakeup(), remove the explicit unlock and use the _locked() sowakeup() variant. This is done in soisdisconnecting(), soisdisconnected() when setting the can't send/ receive flags and dropping data, and in uipc_rcvd() which adjusting back-pressure on the sockets. For UNIX domain sockets running mpsafe with a contention-intensive SMP mysql benchmark, this results in a 1.6% query rate improvement due to reduce mutex costs.
This commit is contained in:
parent
df68e345c5
commit
1e4d7da707
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=131151
@ -169,13 +169,11 @@ soisdisconnecting(so)
|
||||
SOCK_UNLOCK(so);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
so->so_rcv.sb_state |= SBS_CANTRCVMORE;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
sorwakeup_locked(so);
|
||||
SOCKBUF_LOCK(&so->so_snd);
|
||||
so->so_snd.sb_state |= SBS_CANTSENDMORE;
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sowwakeup_locked(so);
|
||||
wakeup(&so->so_timeo);
|
||||
sowwakeup(so);
|
||||
sorwakeup(so);
|
||||
}
|
||||
|
||||
void
|
||||
@ -188,20 +186,19 @@ soisdisconnected(so)
|
||||
* SOCKBUF_LOCK(&so->so_rcv) even though they are the same mutex to
|
||||
* avoid introducing the assumption that they are the same.
|
||||
*/
|
||||
/* XXXRW: so_state locking? */
|
||||
SOCK_LOCK(so);
|
||||
so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
|
||||
so->so_state |= SS_ISDISCONNECTED;
|
||||
SOCK_UNLOCK(so);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
so->so_rcv.sb_state |= SBS_CANTRCVMORE;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
sorwakeup_locked(so);
|
||||
SOCKBUF_LOCK(&so->so_snd);
|
||||
so->so_snd.sb_state |= SBS_CANTSENDMORE;
|
||||
sbdrop_locked(&so->so_snd, so->so_snd.sb_cc);
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sowwakeup_locked(so);
|
||||
wakeup(&so->so_timeo);
|
||||
sowwakeup(so);
|
||||
sorwakeup(so);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -169,13 +169,11 @@ soisdisconnecting(so)
|
||||
SOCK_UNLOCK(so);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
so->so_rcv.sb_state |= SBS_CANTRCVMORE;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
sorwakeup_locked(so);
|
||||
SOCKBUF_LOCK(&so->so_snd);
|
||||
so->so_snd.sb_state |= SBS_CANTSENDMORE;
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sowwakeup_locked(so);
|
||||
wakeup(&so->so_timeo);
|
||||
sowwakeup(so);
|
||||
sorwakeup(so);
|
||||
}
|
||||
|
||||
void
|
||||
@ -188,20 +186,19 @@ soisdisconnected(so)
|
||||
* SOCKBUF_LOCK(&so->so_rcv) even though they are the same mutex to
|
||||
* avoid introducing the assumption that they are the same.
|
||||
*/
|
||||
/* XXXRW: so_state locking? */
|
||||
SOCK_LOCK(so);
|
||||
so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
|
||||
so->so_state |= SS_ISDISCONNECTED;
|
||||
SOCK_UNLOCK(so);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
so->so_rcv.sb_state |= SBS_CANTRCVMORE;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
sorwakeup_locked(so);
|
||||
SOCKBUF_LOCK(&so->so_snd);
|
||||
so->so_snd.sb_state |= SBS_CANTSENDMORE;
|
||||
sbdrop_locked(&so->so_snd, so->so_snd.sb_cc);
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sowwakeup_locked(so);
|
||||
wakeup(&so->so_timeo);
|
||||
sowwakeup(so);
|
||||
sorwakeup(so);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -294,8 +294,7 @@ uipc_rcvd(struct socket *so, int flags)
|
||||
newhiwat, RLIM_INFINITY);
|
||||
unp->unp_cc = so->so_rcv.sb_cc;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
SOCKBUF_UNLOCK(&so2->so_snd);
|
||||
sowwakeup(so2);
|
||||
sowwakeup_locked(so2);
|
||||
break;
|
||||
|
||||
default:
|
||||
@ -355,8 +354,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
|
||||
from = &sun_noname;
|
||||
SOCKBUF_LOCK(&so2->so_rcv);
|
||||
if (sbappendaddr_locked(&so2->so_rcv, from, m, control)) {
|
||||
SOCKBUF_UNLOCK(&so2->so_rcv);
|
||||
sorwakeup(so2);
|
||||
sorwakeup_locked(so2);
|
||||
m = NULL;
|
||||
control = NULL;
|
||||
} else {
|
||||
@ -412,8 +410,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
|
||||
(void)chgsbsize(so->so_cred->cr_uidinfo, &so->so_snd.sb_hiwat,
|
||||
newhiwat, RLIM_INFINITY);
|
||||
unp->unp_conn->unp_cc = so2->so_rcv.sb_cc;
|
||||
SOCKBUF_UNLOCK(&so2->so_rcv);
|
||||
sorwakeup(so2);
|
||||
sorwakeup_locked(so2);
|
||||
m = NULL;
|
||||
break;
|
||||
|
||||
|
@ -228,12 +228,14 @@ divert_packet(struct mbuf *m, int incoming)
|
||||
/* XXX why does only one socket match? */
|
||||
if (inp->inp_lport == nport) {
|
||||
sa = inp->inp_socket;
|
||||
if (sbappendaddr(&sa->so_rcv,
|
||||
SOCKBUF_LOCK(&sa->so_rcv);
|
||||
if (sbappendaddr_locked(&sa->so_rcv,
|
||||
(struct sockaddr *)&divsrc, m,
|
||||
(struct mbuf *)0) == 0)
|
||||
(struct mbuf *)0) == 0) {
|
||||
sa = NULL; /* force mbuf reclaim below */
|
||||
else
|
||||
sorwakeup(sa);
|
||||
SOCKBUF_UNLOCK(&sa->so_rcv);
|
||||
} else
|
||||
sorwakeup_locked(sa);
|
||||
INP_UNLOCK(inp);
|
||||
break;
|
||||
}
|
||||
|
@ -1307,10 +1307,13 @@ static int
|
||||
socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
|
||||
{
|
||||
if (s) {
|
||||
if (sbappendaddr(&s->so_rcv, (struct sockaddr *)src, mm, NULL) != 0) {
|
||||
sorwakeup(s);
|
||||
SOCKBUF_LOCK(&s->so_rcv);
|
||||
if (sbappendaddr_locked(&s->so_rcv, (struct sockaddr *)src, mm,
|
||||
NULL) != 0) {
|
||||
sorwakeup_locked(s);
|
||||
return 0;
|
||||
}
|
||||
SOCKBUF_UNLOCK(&s->so_rcv);
|
||||
}
|
||||
m_freem(mm);
|
||||
return -1;
|
||||
|
@ -160,18 +160,22 @@ raw_append(struct inpcb *last, struct ip *ip, struct mbuf *n)
|
||||
#endif
|
||||
if (!policyfail) {
|
||||
struct mbuf *opts = NULL;
|
||||
struct socket *so;
|
||||
|
||||
so = last->inp_socket;
|
||||
if ((last->inp_flags & INP_CONTROLOPTS) ||
|
||||
(last->inp_socket->so_options & SO_TIMESTAMP))
|
||||
(so->so_options & SO_TIMESTAMP))
|
||||
ip_savecontrol(last, &opts, ip, n);
|
||||
if (sbappendaddr(&last->inp_socket->so_rcv,
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
if (sbappendaddr_locked(&so->so_rcv,
|
||||
(struct sockaddr *)&ripsrc, n, opts) == 0) {
|
||||
/* should notify about lost packet */
|
||||
m_freem(n);
|
||||
if (opts)
|
||||
m_freem(opts);
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
} else
|
||||
sorwakeup(last->inp_socket);
|
||||
sorwakeup_locked(so);
|
||||
} else
|
||||
m_freem(n);
|
||||
return policyfail;
|
||||
|
@ -353,22 +353,24 @@ tcp_reass(tp, th, tlenp, m)
|
||||
q = LIST_FIRST(&tp->t_segq);
|
||||
if (!q || q->tqe_th->th_seq != tp->rcv_nxt)
|
||||
return (0);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
do {
|
||||
tp->rcv_nxt += q->tqe_len;
|
||||
flags = q->tqe_th->th_flags & TH_FIN;
|
||||
nq = LIST_NEXT(q, tqe_q);
|
||||
LIST_REMOVE(q, tqe_q);
|
||||
/* Unlocked read. */
|
||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
|
||||
m_freem(q->tqe_m);
|
||||
else
|
||||
sbappendstream(&so->so_rcv, q->tqe_m);
|
||||
sbappendstream_locked(&so->so_rcv, q->tqe_m);
|
||||
uma_zfree(tcp_reass_zone, q);
|
||||
tp->t_segqlen--;
|
||||
tcp_reass_qsize--;
|
||||
q = nq;
|
||||
} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
|
||||
ND6_HINT(tp);
|
||||
sorwakeup(so);
|
||||
sorwakeup_locked(so);
|
||||
return (flags);
|
||||
}
|
||||
|
||||
@ -1262,13 +1264,15 @@ tcp_input(m, off0)
|
||||
#endif
|
||||
* Add data to socket buffer.
|
||||
*/
|
||||
/* Unlocked read. */
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
|
||||
m_freem(m);
|
||||
} else {
|
||||
m_adj(m, drop_hdrlen); /* delayed header drop */
|
||||
sbappendstream(&so->so_rcv, m);
|
||||
sbappendstream_locked(&so->so_rcv, m);
|
||||
}
|
||||
sorwakeup(so);
|
||||
sorwakeup_locked(so);
|
||||
if (DELAY_ACK(tp)) {
|
||||
tp->t_flags |= TF_DELACK;
|
||||
} else {
|
||||
@ -2153,8 +2157,7 @@ tcp_input(m, off0)
|
||||
tp->snd_wnd -= acked;
|
||||
ourfinisacked = 0;
|
||||
}
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sowwakeup(so);
|
||||
sowwakeup_locked(so);
|
||||
/* detect una wraparound */
|
||||
if ((tcp_do_newreno || tp->sack_enable) &&
|
||||
!IN_FASTRECOVERY(tp) &&
|
||||
@ -2363,11 +2366,13 @@ tcp_input(m, off0)
|
||||
tcpstat.tcps_rcvpack++;
|
||||
tcpstat.tcps_rcvbyte += tlen;
|
||||
ND6_HINT(tp);
|
||||
/* Unlocked read. */
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
|
||||
m_freem(m);
|
||||
else
|
||||
sbappendstream(&so->so_rcv, m);
|
||||
sorwakeup(so);
|
||||
sbappendstream_locked(&so->so_rcv, m);
|
||||
sorwakeup_locked(so);
|
||||
} else {
|
||||
thflags = tcp_reass(tp, th, &tlen, m);
|
||||
tp->t_flags |= TF_ACKNOW;
|
||||
|
@ -353,22 +353,24 @@ tcp_reass(tp, th, tlenp, m)
|
||||
q = LIST_FIRST(&tp->t_segq);
|
||||
if (!q || q->tqe_th->th_seq != tp->rcv_nxt)
|
||||
return (0);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
do {
|
||||
tp->rcv_nxt += q->tqe_len;
|
||||
flags = q->tqe_th->th_flags & TH_FIN;
|
||||
nq = LIST_NEXT(q, tqe_q);
|
||||
LIST_REMOVE(q, tqe_q);
|
||||
/* Unlocked read. */
|
||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
|
||||
m_freem(q->tqe_m);
|
||||
else
|
||||
sbappendstream(&so->so_rcv, q->tqe_m);
|
||||
sbappendstream_locked(&so->so_rcv, q->tqe_m);
|
||||
uma_zfree(tcp_reass_zone, q);
|
||||
tp->t_segqlen--;
|
||||
tcp_reass_qsize--;
|
||||
q = nq;
|
||||
} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
|
||||
ND6_HINT(tp);
|
||||
sorwakeup(so);
|
||||
sorwakeup_locked(so);
|
||||
return (flags);
|
||||
}
|
||||
|
||||
@ -1262,13 +1264,15 @@ tcp_input(m, off0)
|
||||
#endif
|
||||
* Add data to socket buffer.
|
||||
*/
|
||||
/* Unlocked read. */
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
|
||||
m_freem(m);
|
||||
} else {
|
||||
m_adj(m, drop_hdrlen); /* delayed header drop */
|
||||
sbappendstream(&so->so_rcv, m);
|
||||
sbappendstream_locked(&so->so_rcv, m);
|
||||
}
|
||||
sorwakeup(so);
|
||||
sorwakeup_locked(so);
|
||||
if (DELAY_ACK(tp)) {
|
||||
tp->t_flags |= TF_DELACK;
|
||||
} else {
|
||||
@ -2153,8 +2157,7 @@ tcp_input(m, off0)
|
||||
tp->snd_wnd -= acked;
|
||||
ourfinisacked = 0;
|
||||
}
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sowwakeup(so);
|
||||
sowwakeup_locked(so);
|
||||
/* detect una wraparound */
|
||||
if ((tcp_do_newreno || tp->sack_enable) &&
|
||||
!IN_FASTRECOVERY(tp) &&
|
||||
@ -2363,11 +2366,13 @@ tcp_input(m, off0)
|
||||
tcpstat.tcps_rcvpack++;
|
||||
tcpstat.tcps_rcvbyte += tlen;
|
||||
ND6_HINT(tp);
|
||||
/* Unlocked read. */
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
|
||||
m_freem(m);
|
||||
else
|
||||
sbappendstream(&so->so_rcv, m);
|
||||
sorwakeup(so);
|
||||
sbappendstream_locked(&so->so_rcv, m);
|
||||
sorwakeup_locked(so);
|
||||
} else {
|
||||
thflags = tcp_reass(tp, th, &tlen, m);
|
||||
tp->t_flags |= TF_ACKNOW;
|
||||
|
@ -447,6 +447,7 @@ udp_append(last, ip, n, off)
|
||||
int off;
|
||||
{
|
||||
struct sockaddr *append_sa;
|
||||
struct socket *so;
|
||||
struct mbuf *opts = 0;
|
||||
|
||||
INP_LOCK_ASSERT(last);
|
||||
@ -496,13 +497,17 @@ udp_append(last, ip, n, off)
|
||||
#endif
|
||||
append_sa = (struct sockaddr *)&udp_in;
|
||||
m_adj(n, off);
|
||||
if (sbappendaddr(&last->inp_socket->so_rcv, append_sa, n, opts) == 0) {
|
||||
|
||||
so = last->inp_socket;
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
if (sbappendaddr_locked(&so->so_rcv, append_sa, n, opts) == 0) {
|
||||
m_freem(n);
|
||||
if (opts)
|
||||
m_freem(opts);
|
||||
udpstat.udps_fullsock++;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
} else
|
||||
sorwakeup(last->inp_socket);
|
||||
sorwakeup_locked(so);
|
||||
}
|
||||
|
||||
/*
|
||||
|
Loading…
Reference in New Issue
Block a user