tcp: remove delayed drop KPI
No longer needed after tcp_output() can ask caller to drop. Reviewed by: rrs, tuexen Differential revision: https://reviews.freebsd.org/D33371
This commit is contained in:
Gleb Smirnoff
parent
f64dc2ab5b
commit
a370832bec
@ -620,7 +620,7 @@ in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
|
||||
* If using hpts lets drop a random number in so
|
||||
* not all new connections fall on the same CPU.
|
||||
*/
|
||||
inp->inp_hpts_cpu = inp->inp_dropq_cpu = hpts_random_cpu(inp);
|
||||
inp->inp_hpts_cpu = hpts_random_cpu(inp);
|
||||
#endif
|
||||
refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */
|
||||
INP_WLOCK(inp);
|
||||
@ -1750,7 +1750,6 @@ in_pcbrele_rlocked(struct inpcb *inp)
|
||||
MPASS(inp->inp_flags & INP_FREED);
|
||||
MPASS(inp->inp_socket == NULL);
|
||||
MPASS(inp->inp_in_hpts == 0);
|
||||
MPASS(inp->inp_in_dropq == 0);
|
||||
INP_RUNLOCK(inp);
|
||||
uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
|
||||
return (true);
|
||||
@ -1768,7 +1767,6 @@ in_pcbrele_wlocked(struct inpcb *inp)
|
||||
MPASS(inp->inp_flags & INP_FREED);
|
||||
MPASS(inp->inp_socket == NULL);
|
||||
MPASS(inp->inp_in_hpts == 0);
|
||||
MPASS(inp->inp_in_dropq == 0);
|
||||
INP_WUNLOCK(inp);
|
||||
uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
|
||||
return (true);
|
||||
|
||||
@ -235,19 +235,15 @@ struct inpcb {
|
||||
*/
|
||||
#if defined(__amd64__) || defined(__i386__)
|
||||
uint8_t inp_in_hpts; /* on output hpts (lock b) */
|
||||
uint8_t inp_in_dropq; /* on input hpts (lock b) */
|
||||
#else
|
||||
uint32_t inp_in_hpts; /* on output hpts (lock b) */
|
||||
uint32_t inp_in_dropq; /* on input hpts (lock b) */
|
||||
#endif
|
||||
volatile uint16_t inp_hpts_cpu; /* Lock (i) */
|
||||
volatile uint16_t inp_irq_cpu; /* Set by LRO in behalf of or the driver */
|
||||
u_int inp_refcount; /* (i) refcount */
|
||||
int inp_flags; /* (i) generic IP/datagram flags */
|
||||
int inp_flags2; /* (i) generic IP/datagram flags #2*/
|
||||
uint16_t inp_dropq_cpu; /* Lock (i) */
|
||||
uint8_t inp_hpts_cpu_set :1, /* on output hpts (i) */
|
||||
inp_dropq_cpu_set : 1, /* on input hpts (i) */
|
||||
inp_hpts_calls :1, /* (i) from output hpts */
|
||||
inp_irq_cpu_set :1, /* (i) from LRO/Driver */
|
||||
inp_spare_bits2 : 3;
|
||||
@ -256,8 +252,6 @@ struct inpcb {
|
||||
struct socket *inp_socket; /* (i) back pointer to socket */
|
||||
int32_t inp_hptsslot; /* Hpts wheel slot this tcb is Lock(i&b) */
|
||||
uint32_t inp_hpts_drop_reas; /* reason we are dropping the PCB (lock i&b) */
|
||||
uint32_t inp_dropq_gencnt;
|
||||
TAILQ_ENTRY(inpcb) inp_dropq; /* hpts drop queue next lock(b) */
|
||||
struct inpcbinfo *inp_pcbinfo; /* (c) PCB list info */
|
||||
struct ucred *inp_cred; /* (c) cache of socket cred */
|
||||
u_int32_t inp_flow; /* (i) IPv6 flow information */
|
||||
|
||||
@ -100,25 +100,6 @@ __FBSDID("$FreeBSD$");
|
||||
* function (ctf_do_queued_segments()) requires that
|
||||
* you have defined the tfb_do_segment_nounlock() as
|
||||
* described above.
|
||||
*
|
||||
* Now the second function the tcp_hpts system provides is the ability
|
||||
* to abort a connection later. Why would you want to do this?
|
||||
* To not have to worry about untangling any recursive locks.
|
||||
*
|
||||
* The second feature of the input side of hpts is the
|
||||
* dropping of a connection. This is due to the way that
|
||||
* locking may have occured on the INP_WLOCK. So if
|
||||
* a stack wants to drop a connection it calls:
|
||||
*
|
||||
* tcp_set_inp_to_drop(tp, ETIMEDOUT)
|
||||
*
|
||||
* To schedule the tcp_hpts system to call
|
||||
*
|
||||
* tcp_drop(tp, drop_reason)
|
||||
*
|
||||
* at a future point. This is quite handy to prevent locking
|
||||
* issues when dropping connections.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <sys/param.h>
|
||||
@ -222,14 +203,11 @@ struct tcp_hpts_entry {
|
||||
p_avail:5;
|
||||
uint8_t p_fill[3]; /* Fill to 32 bits */
|
||||
/* Cache line 0x40 */
|
||||
TAILQ_HEAD(, inpcb) p_dropq; /* Delayed drop queue */
|
||||
struct hptsh {
|
||||
TAILQ_HEAD(, inpcb) head;
|
||||
uint32_t count;
|
||||
uint32_t gencnt;
|
||||
} *p_hptss; /* Hptsi wheel */
|
||||
uint32_t p_dropq_cnt; /* Count on drop queue */
|
||||
uint32_t p_dropq_gencnt;
|
||||
uint32_t p_hpts_sleep_time; /* Current sleep interval having a max
|
||||
* of 255ms */
|
||||
uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */
|
||||
@ -473,7 +451,7 @@ tcp_hpts_log(struct tcp_hpts_entry *hpts, struct tcpcb *tp, struct timeval *tv,
|
||||
* Unused logs are
|
||||
* 64 bit - delRate, rttProp, bw_inuse
|
||||
* 16 bit - cwnd_gain
|
||||
* 8 bit - bbr_state, bbr_substate, inhpts, ininput;
|
||||
* 8 bit - bbr_state, bbr_substate, inhpts;
|
||||
*/
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.flex1 = hpts->p_nxt_slot;
|
||||
@ -565,19 +543,6 @@ tcp_hpts_lock(struct inpcb *inp)
|
||||
return (hpts);
|
||||
}
|
||||
|
||||
static struct tcp_hpts_entry *
|
||||
tcp_dropq_lock(struct inpcb *inp)
|
||||
{
|
||||
struct tcp_hpts_entry *hpts;
|
||||
|
||||
INP_LOCK_ASSERT(inp);
|
||||
|
||||
hpts = tcp_pace.rp_ent[inp->inp_dropq_cpu];
|
||||
HPTS_LOCK(hpts);
|
||||
|
||||
return (hpts);
|
||||
}
|
||||
|
||||
static void
|
||||
inp_hpts_release(struct inpcb *inp)
|
||||
{
|
||||
@ -588,69 +553,20 @@ inp_hpts_release(struct inpcb *inp)
|
||||
MPASS(released == false);
|
||||
}
|
||||
|
||||
static void
|
||||
tcp_dropq_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp)
|
||||
{
|
||||
bool released __diagused;
|
||||
|
||||
HPTS_MTX_ASSERT(hpts);
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
|
||||
if (inp->inp_in_dropq != IHPTS_ONQUEUE)
|
||||
return;
|
||||
|
||||
MPASS(hpts->p_cpu == inp->inp_dropq_cpu);
|
||||
if (__predict_true(inp->inp_dropq_gencnt == hpts->p_dropq_gencnt)) {
|
||||
TAILQ_REMOVE(&hpts->p_dropq, inp, inp_dropq);
|
||||
MPASS(hpts->p_dropq_cnt > 0);
|
||||
hpts->p_dropq_cnt--;
|
||||
inp->inp_in_dropq = IHPTS_NONE;
|
||||
released = in_pcbrele_wlocked(inp);
|
||||
MPASS(released == false);
|
||||
} else {
|
||||
/*
|
||||
* tcp_delayed_drop() now owns the TAILQ head of this inp.
|
||||
* Can't TAILQ_REMOVE, just mark it.
|
||||
*/
|
||||
#ifdef INVARIANTS
|
||||
struct inpcb *tmp;
|
||||
|
||||
TAILQ_FOREACH(tmp, &hpts->p_dropq, inp_dropq)
|
||||
MPASS(tmp != inp);
|
||||
#endif
|
||||
inp->inp_in_dropq = IHPTS_MOVING;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Called normally with the INP_LOCKED but it
|
||||
* does not matter, the hpts lock is the key
|
||||
* but the lock order allows us to hold the
|
||||
* INP lock and then get the hpts lock.
|
||||
*
|
||||
* Valid values in the flags are
|
||||
* HPTS_REMOVE_OUTPUT - remove from the output of the hpts.
|
||||
* HPTS_REMOVE_DROPQ - remove from the drop queue of the hpts.
|
||||
* Note that you can use one or both values together
|
||||
* and get two actions.
|
||||
*/
|
||||
void
|
||||
__tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line)
|
||||
tcp_hpts_remove(struct inpcb *inp)
|
||||
{
|
||||
struct tcp_hpts_entry *hpts;
|
||||
struct hptsh *hptsh;
|
||||
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
|
||||
if (flags & HPTS_REMOVE_DROPQ) {
|
||||
hpts = tcp_dropq_lock(inp);
|
||||
tcp_dropq_remove(hpts, inp);
|
||||
mtx_unlock(&hpts->p_mtx);
|
||||
}
|
||||
|
||||
MPASS(flags & HPTS_REMOVE_OUTPUT);
|
||||
|
||||
hpts = tcp_hpts_lock(inp);
|
||||
if (inp->inp_in_hpts == IHPTS_ONQUEUE) {
|
||||
hptsh = &hpts->p_hptss[inp->inp_hptsslot];
|
||||
@ -1074,32 +990,6 @@ tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts
|
||||
return (slot_on);
|
||||
}
|
||||
|
||||
void
|
||||
tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason)
|
||||
{
|
||||
struct tcp_hpts_entry *hpts;
|
||||
struct tcpcb *tp = intotcpcb(inp);
|
||||
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
inp->inp_hpts_drop_reas = reason;
|
||||
if (inp->inp_in_dropq != IHPTS_NONE)
|
||||
return;
|
||||
hpts = tcp_dropq_lock(tp->t_inpcb);
|
||||
MPASS(hpts->p_cpu == inp->inp_dropq_cpu);
|
||||
|
||||
TAILQ_INSERT_TAIL(&hpts->p_dropq, inp, inp_dropq);
|
||||
inp->inp_in_dropq = IHPTS_ONQUEUE;
|
||||
inp->inp_dropq_gencnt = hpts->p_dropq_gencnt;
|
||||
hpts->p_dropq_cnt++;
|
||||
in_pcbref(inp);
|
||||
|
||||
if ((hpts->p_hpts_active == 0) && (hpts->p_on_min_sleep == 0)){
|
||||
hpts->p_direct_wake = 1;
|
||||
tcp_wakehpts(hpts);
|
||||
}
|
||||
HPTS_UNLOCK(hpts);
|
||||
}
|
||||
|
||||
uint16_t
|
||||
hpts_random_cpu(struct inpcb *inp){
|
||||
/*
|
||||
@ -1109,12 +999,9 @@ hpts_random_cpu(struct inpcb *inp){
|
||||
uint32_t ran;
|
||||
|
||||
/*
|
||||
* If one has been set use it i.e. we want both in and out on the
|
||||
* same hpts.
|
||||
* Shortcut if it is already set. XXXGL: does it happen?
|
||||
*/
|
||||
if (inp->inp_dropq_cpu_set) {
|
||||
return (inp->inp_dropq_cpu);
|
||||
} else if (inp->inp_hpts_cpu_set) {
|
||||
if (inp->inp_hpts_cpu_set) {
|
||||
return (inp->inp_hpts_cpu);
|
||||
}
|
||||
/* Nothing set use a random number */
|
||||
@ -1132,13 +1019,7 @@ hpts_cpuid(struct inpcb *inp, int *failed)
|
||||
#endif
|
||||
|
||||
*failed = 0;
|
||||
/*
|
||||
* If one has been set use it i.e. we want both in and out on the
|
||||
* same hpts.
|
||||
*/
|
||||
if (inp->inp_dropq_cpu_set) {
|
||||
return (inp->inp_dropq_cpu);
|
||||
} else if (inp->inp_hpts_cpu_set) {
|
||||
if (inp->inp_hpts_cpu_set) {
|
||||
return (inp->inp_hpts_cpu);
|
||||
}
|
||||
/*
|
||||
@ -1204,57 +1085,6 @@ tcp_drop_in_pkts(struct tcpcb *tp)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Delayed drop functionality is factored out into separate function,
|
||||
* but logic is similar to the logic of tcp_hptsi().
|
||||
*/
|
||||
static void
|
||||
tcp_delayed_drop(struct tcp_hpts_entry *hpts)
|
||||
{
|
||||
TAILQ_HEAD(, inpcb) head = TAILQ_HEAD_INITIALIZER(head);
|
||||
struct inpcb *inp, *tmp;
|
||||
struct tcpcb *tp;
|
||||
|
||||
HPTS_MTX_ASSERT(hpts);
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
TAILQ_SWAP(&head, &hpts->p_dropq, inpcb, inp_dropq);
|
||||
hpts->p_dropq_cnt = 0;
|
||||
hpts->p_dropq_gencnt++;
|
||||
HPTS_UNLOCK(hpts);
|
||||
|
||||
TAILQ_FOREACH_SAFE(inp, &head, inp_dropq, tmp) {
|
||||
INP_WLOCK(inp);
|
||||
MPASS(inp->inp_hpts_drop_reas != 0);
|
||||
if (__predict_false(inp->inp_in_dropq == IHPTS_MOVING)) {
|
||||
inp->inp_in_dropq = IHPTS_NONE;
|
||||
if (in_pcbrele_wlocked(inp) == false)
|
||||
INP_WUNLOCK(inp);
|
||||
continue;
|
||||
}
|
||||
MPASS(inp->inp_in_dropq == IHPTS_ONQUEUE);
|
||||
inp->inp_in_dropq = IHPTS_NONE;
|
||||
if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED))) {
|
||||
if (in_pcbrele_wlocked(inp) == false)
|
||||
INP_WUNLOCK(inp);
|
||||
continue;
|
||||
}
|
||||
CURVNET_SET(inp->inp_vnet);
|
||||
if (__predict_true((tp = intotcpcb(inp)) != NULL)) {
|
||||
MPASS(tp->t_inpcb == inp);
|
||||
tcp_drop_in_pkts(tp);
|
||||
tp = tcp_drop(tp, inp->inp_hpts_drop_reas);
|
||||
if (tp == NULL)
|
||||
INP_WLOCK(inp);
|
||||
}
|
||||
if (in_pcbrele_wlocked(inp) == false)
|
||||
INP_WUNLOCK(inp);
|
||||
CURVNET_RESTORE();
|
||||
}
|
||||
|
||||
mtx_lock(&hpts->p_mtx); /* XXXGL */
|
||||
}
|
||||
|
||||
static void
|
||||
tcp_hpts_set_max_sleep(struct tcp_hpts_entry *hpts, int wrap_loop_cnt)
|
||||
{
|
||||
@ -1392,10 +1222,6 @@ tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout)
|
||||
hpts->p_nxt_slot = hpts->p_prev_slot;
|
||||
hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1);
|
||||
}
|
||||
KASSERT((((TAILQ_EMPTY(&hpts->p_dropq) != 0) && (hpts->p_dropq_cnt == 0)) ||
|
||||
((TAILQ_EMPTY(&hpts->p_dropq) == 0) && (hpts->p_dropq_cnt > 0))),
|
||||
("%s hpts:%p in_hpts cnt:%d and queue state mismatch",
|
||||
__FUNCTION__, hpts, hpts->p_dropq_cnt));
|
||||
if (hpts->p_on_queue_cnt == 0) {
|
||||
goto no_one;
|
||||
}
|
||||
@ -1615,10 +1441,6 @@ tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout)
|
||||
* Check to see if we took an excess amount of time and need to run
|
||||
* more ticks (if we did not hit eno-bufs).
|
||||
*/
|
||||
KASSERT((((TAILQ_EMPTY(&hpts->p_dropq) != 0) && (hpts->p_dropq_cnt == 0)) ||
|
||||
((TAILQ_EMPTY(&hpts->p_dropq) == 0) && (hpts->p_dropq_cnt > 0))),
|
||||
("%s hpts:%p in_hpts cnt:%d queue state mismatch",
|
||||
__FUNCTION__, hpts, hpts->p_dropq_cnt));
|
||||
hpts->p_prev_slot = hpts->p_cur_slot;
|
||||
hpts->p_lasttick = hpts->p_curtick;
|
||||
if ((from_callout == 0) || (loop_cnt > max_pacer_loops)) {
|
||||
@ -1660,11 +1482,6 @@ tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout)
|
||||
* input.
|
||||
*/
|
||||
hpts->p_wheel_complete = 1;
|
||||
/*
|
||||
* Run any input that may be there not covered
|
||||
* in running data.
|
||||
*/
|
||||
tcp_delayed_drop(hpts);
|
||||
/*
|
||||
* Now did we spend too long running input and need to run more ticks?
|
||||
* Note that if wrap_loop_cnt < 2 then we should have the conditions
|
||||
@ -1712,14 +1529,6 @@ __tcp_set_hpts(struct inpcb *inp, int32_t line)
|
||||
inp->inp_hpts_cpu_set = 1;
|
||||
}
|
||||
mtx_unlock(&hpts->p_mtx);
|
||||
hpts = tcp_dropq_lock(inp);
|
||||
if ((inp->inp_dropq_cpu_set == 0) &&
|
||||
(inp->inp_in_dropq == 0)) {
|
||||
inp->inp_dropq_cpu = hpts_cpuid(inp, &failed);
|
||||
if (failed == 0)
|
||||
inp->inp_dropq_cpu_set = 1;
|
||||
}
|
||||
mtx_unlock(&hpts->p_mtx);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -2037,7 +1846,6 @@ tcp_init_hptsi(void *st)
|
||||
*/
|
||||
mtx_init(&hpts->p_mtx, "tcp_hpts_lck",
|
||||
"hpts", MTX_DEF | MTX_DUPOK);
|
||||
TAILQ_INIT(&hpts->p_dropq);
|
||||
for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) {
|
||||
TAILQ_INIT(&hpts->p_hptss[j].head);
|
||||
hpts->p_hptss[j].count = 0;
|
||||
@ -2051,11 +1859,6 @@ tcp_init_hptsi(void *st)
|
||||
unit,
|
||||
CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
|
||||
"");
|
||||
SYSCTL_ADD_INT(&hpts->hpts_ctx,
|
||||
SYSCTL_CHILDREN(hpts->hpts_root),
|
||||
OID_AUTO, "in_qcnt", CTLFLAG_RD,
|
||||
&hpts->p_dropq_cnt, 0,
|
||||
"Count TCB's awaiting delayed drop");
|
||||
SYSCTL_ADD_INT(&hpts->hpts_ctx,
|
||||
SYSCTL_CHILDREN(hpts->hpts_root),
|
||||
OID_AUTO, "out_qcnt", CTLFLAG_RD,
|
||||
|
||||
@ -114,11 +114,7 @@ struct hpts_diag {
|
||||
|
||||
|
||||
#ifdef _KERNEL
|
||||
#define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__)
|
||||
void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line);
|
||||
#define HPTS_REMOVE_DROPQ 0x01
|
||||
#define HPTS_REMOVE_OUTPUT 0x02
|
||||
#define HPTS_REMOVE_ALL (HPTS_REMOVE_DROPQ | HPTS_REMOVE_OUTPUT)
|
||||
void tcp_hpts_remove(struct inpcb *);
|
||||
bool tcp_in_hpts(struct inpcb *);
|
||||
|
||||
/*
|
||||
|
||||
@ -95,7 +95,7 @@ struct tcp_log_bbr {
|
||||
uint8_t bbr_state;
|
||||
uint8_t bbr_substate;
|
||||
uint8_t inhpts;
|
||||
uint8_t ininput;
|
||||
uint8_t __spare;
|
||||
uint8_t use_lt_bw;
|
||||
uint8_t flex8;
|
||||
uint32_t pkt_epoch;
|
||||
|
||||
@ -1353,8 +1353,7 @@ tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le)
|
||||
/* Check if any data mbufs left. */
|
||||
if (le->m_head != NULL) {
|
||||
counter_u64_add(tcp_inp_lro_direct_queue, 1);
|
||||
tcp_lro_log(tp, lc, le, NULL, 22, 1,
|
||||
inp->inp_flags2, inp->inp_in_dropq, 1);
|
||||
tcp_lro_log(tp, lc, le, NULL, 22, 1, inp->inp_flags2, 0, 1);
|
||||
tcp_queue_pkts(inp, tp, le);
|
||||
}
|
||||
if (should_wake) {
|
||||
|
||||
@ -1059,7 +1059,7 @@ bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sock
|
||||
wrong_timer:
|
||||
if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
|
||||
if (tcp_in_hpts(inp))
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(inp);
|
||||
bbr_timer_cancel(bbr, __LINE__, cts);
|
||||
bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
|
||||
0);
|
||||
@ -1884,7 +1884,6 @@ bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t ct
|
||||
l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
|
||||
l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
|
||||
l->inhpts = tcp_in_hpts(bbr->rc_inp);
|
||||
l->ininput = bbr->rc_inp->inp_in_dropq;
|
||||
l->use_lt_bw = bbr->rc_lt_use_bw;
|
||||
l->pkts_out = bbr->r_ctl.rc_flight_at_input;
|
||||
l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;
|
||||
@ -5265,7 +5264,7 @@ bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
|
||||
* must remove ourselves from the hpts.
|
||||
*/
|
||||
hpts_removed = 1;
|
||||
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(bbr->rc_inp);
|
||||
if (bbr->r_ctl.rc_last_delay_val) {
|
||||
/* Update the last hptsi delay too */
|
||||
uint32_t time_since_send;
|
||||
@ -7966,7 +7965,7 @@ bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t li
|
||||
* for our sum's calculations.
|
||||
*/
|
||||
if (tcp_in_hpts(bbr->rc_inp)) {
|
||||
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(bbr->rc_inp);
|
||||
bbr->rc_timer_first = 0;
|
||||
bbr->r_ctl.rc_hpts_flags = 0;
|
||||
bbr->r_ctl.rc_last_delay_val = 0;
|
||||
@ -11656,7 +11655,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
|
||||
;
|
||||
} else {
|
||||
if (tcp_in_hpts(bbr->rc_inp)) {
|
||||
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(bbr->rc_inp);
|
||||
if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
|
||||
(TSTMP_GT(lcts, bbr->rc_pacer_started))) {
|
||||
uint32_t del;
|
||||
@ -12035,7 +12034,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
|
||||
return (0);
|
||||
}
|
||||
}
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(inp);
|
||||
bbr_timer_cancel(bbr, __LINE__, cts);
|
||||
}
|
||||
if (bbr->r_ctl.rc_last_delay_val) {
|
||||
@ -12052,7 +12051,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
|
||||
(tp->t_state < TCPS_ESTABLISHED)) {
|
||||
/* Timeouts or early states are exempt */
|
||||
if (tcp_in_hpts(inp))
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(inp);
|
||||
} else if (tcp_in_hpts(inp)) {
|
||||
if ((bbr->r_ctl.rc_last_delay_val) &&
|
||||
(bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
|
||||
@ -12065,10 +12064,10 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
|
||||
*/
|
||||
counter_u64_add(bbr_out_size[TCP_MSS_ACCT_LATE], 1);
|
||||
bbr->r_ctl.rc_last_delay_val = 0;
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(inp);
|
||||
} else if (tp->t_state == TCPS_CLOSED) {
|
||||
bbr->r_ctl.rc_last_delay_val = 0;
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(inp);
|
||||
} else {
|
||||
/*
|
||||
* On the hpts, you shall not pass! even if ACKNOW
|
||||
|
||||
@ -2295,7 +2295,6 @@ rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t
|
||||
log.u_bbr.flex6 = rsm->r_end;
|
||||
log.u_bbr.flex8 = mod;
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
|
||||
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
|
||||
log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
|
||||
@ -2330,7 +2329,6 @@ rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot
|
||||
else
|
||||
log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
|
||||
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
|
||||
log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
|
||||
@ -2355,7 +2353,6 @@ rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rs
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex8 = to_num;
|
||||
log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt;
|
||||
log.u_bbr.flex2 = rack->rc_rack_rtt;
|
||||
@ -2394,7 +2391,6 @@ rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack,
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.flex8 = flag;
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.cur_del_rate = (uint64_t)prev;
|
||||
log.u_bbr.delRate = (uint64_t)rsm;
|
||||
log.u_bbr.rttProp = (uint64_t)next;
|
||||
@ -2439,7 +2435,6 @@ rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t l
|
||||
struct timeval tv;
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex1 = t;
|
||||
log.u_bbr.flex2 = len;
|
||||
log.u_bbr.flex3 = rack->r_ctl.rc_rack_min_rtt;
|
||||
@ -2589,7 +2584,6 @@ rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick,
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex1 = line;
|
||||
log.u_bbr.flex2 = tick;
|
||||
log.u_bbr.flex3 = tp->t_maxunacktime;
|
||||
@ -2616,7 +2610,6 @@ rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex1 = slot;
|
||||
if (rack->rack_no_prr)
|
||||
log.u_bbr.flex2 = 0;
|
||||
@ -2718,7 +2711,6 @@ rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, ui
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex1 = slot;
|
||||
log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags;
|
||||
log.u_bbr.flex4 = reason;
|
||||
@ -2751,7 +2743,6 @@ rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex1 = line;
|
||||
log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to;
|
||||
log.u_bbr.flex3 = flags_on_entry;
|
||||
@ -4476,7 +4467,7 @@ rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
|
||||
* Stop the pacer and clear up all the aggregate
|
||||
* delays etc.
|
||||
*/
|
||||
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(rack->rc_inp);
|
||||
rack->r_ctl.rc_hpts_flags = 0;
|
||||
rack->r_ctl.rc_last_output_to = 0;
|
||||
}
|
||||
@ -5679,7 +5670,7 @@ static void
|
||||
rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
|
||||
{
|
||||
if (tcp_in_hpts(rack->rc_inp)) {
|
||||
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(rack->rc_inp);
|
||||
rack->r_ctl.rc_hpts_flags = 0;
|
||||
}
|
||||
#ifdef NETFLIX_SHARED_CWND
|
||||
@ -7229,7 +7220,7 @@ rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int lin
|
||||
if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
|
||||
((TSTMP_GEQ(us_cts, rack->r_ctl.rc_last_output_to)) ||
|
||||
((tp->snd_max - tp->snd_una) == 0))) {
|
||||
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(rack->rc_inp);
|
||||
hpts_removed = 1;
|
||||
/* If we were not delayed cancel out the flag. */
|
||||
if ((tp->snd_max - tp->snd_una) == 0)
|
||||
@ -7245,7 +7236,7 @@ rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int lin
|
||||
* paced. We also must remove ourselves from the
|
||||
* hpts.
|
||||
*/
|
||||
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(rack->rc_inp);
|
||||
hpts_removed = 1;
|
||||
}
|
||||
rack->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
|
||||
@ -13231,7 +13222,7 @@ rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb)
|
||||
}
|
||||
rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
|
||||
}
|
||||
tcp_hpts_remove(tp->t_inpcb, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(tp->t_inpcb);
|
||||
}
|
||||
rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
|
||||
rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
|
||||
@ -13317,7 +13308,6 @@ rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent
|
||||
#endif
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
if (rack->rack_no_prr == 0)
|
||||
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
|
||||
else
|
||||
@ -14313,7 +14303,6 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
|
||||
#endif
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
if (rack->rack_no_prr == 0)
|
||||
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
|
||||
else
|
||||
@ -14688,7 +14677,7 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
|
||||
late = 1;
|
||||
rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
|
||||
}
|
||||
tcp_hpts_remove(tp->t_inpcb, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(tp->t_inpcb);
|
||||
}
|
||||
if (late && (did_out == 0)) {
|
||||
/*
|
||||
@ -15605,7 +15594,6 @@ rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
log.u_bbr.flex1 = error;
|
||||
log.u_bbr.flex2 = flags;
|
||||
log.u_bbr.flex3 = rsm_is_null;
|
||||
@ -16121,7 +16109,6 @@ rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendma
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
if (rack->rack_no_prr)
|
||||
log.u_bbr.flex1 = 0;
|
||||
else
|
||||
@ -16622,7 +16609,6 @@ rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val,
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
if (rack->rack_no_prr)
|
||||
log.u_bbr.flex1 = 0;
|
||||
else
|
||||
@ -16909,7 +16895,7 @@ rack_output(struct tcpcb *tp)
|
||||
(tp->t_state < TCPS_ESTABLISHED)) {
|
||||
rack->rc_ack_can_sendout_data = 0;
|
||||
if (tcp_in_hpts(rack->rc_inp))
|
||||
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
|
||||
tcp_hpts_remove(rack->rc_inp);
|
||||
} else if (tcp_in_hpts(rack->rc_inp)) {
|
||||
/*
|
||||
* On the hpts you can't pass even if ACKNOW is on, we will
|
||||
@ -18804,7 +18790,6 @@ rack_output(struct tcpcb *tp)
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
|
||||
log.u_bbr.ininput = rack->rc_inp->inp_in_dropq;
|
||||
if (rack->rack_no_prr)
|
||||
log.u_bbr.flex1 = 0;
|
||||
else
|
||||
|
||||
@ -269,7 +269,7 @@ struct bbr_log {
|
||||
uint8_t n_sackblks;
|
||||
uint8_t applied; /* UU */
|
||||
uint8_t inhpts; /* UU */
|
||||
uint8_t ininput; /* UU */
|
||||
uint8_t __spare; /* UU */
|
||||
uint8_t use_lt_bw; /* UU */
|
||||
};
|
||||
|
||||
|
||||
@ -2103,7 +2103,6 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
|
||||
|
||||
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
|
||||
log.u_bbr.inhpts = tp->t_inpcb->inp_in_hpts;
|
||||
log.u_bbr.ininput = tp->t_inpcb->inp_in_dropq;
|
||||
log.u_bbr.flex8 = 4;
|
||||
log.u_bbr.pkts_out = tp->t_maxseg;
|
||||
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
|
||||
@ -2596,7 +2595,7 @@ tcp_close(struct tcpcb *tp)
|
||||
tp->t_tfo_pending = NULL;
|
||||
}
|
||||
#ifdef TCPHPTS
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_ALL);
|
||||
tcp_hpts_remove(inp);
|
||||
#endif
|
||||
in_pcbdrop(inp);
|
||||
TCPSTAT_INC(tcps_closed);
|
||||
|
||||
@ -347,7 +347,7 @@ tcp_twstart(struct tcpcb *tp)
|
||||
* and might not be needed here any longer.
|
||||
*/
|
||||
#ifdef TCPHPTS
|
||||
tcp_hpts_remove(inp, HPTS_REMOVE_ALL);
|
||||
tcp_hpts_remove(inp);
|
||||
#endif
|
||||
tcp_discardcb(tp);
|
||||
soisdisconnected(so);
|
||||
|
||||
@ -1845,7 +1845,7 @@ tcp_ctloutput_set(struct inpcb *inp, struct sockopt *sopt)
|
||||
}
|
||||
#ifdef TCPHPTS
|
||||
/* Assure that we are not on any hpts */
|
||||
tcp_hpts_remove(tp->t_inpcb, HPTS_REMOVE_ALL);
|
||||
tcp_hpts_remove(tp->t_inpcb);
|
||||
#endif
|
||||
if (blk->tfb_tcp_fb_init) {
|
||||
error = (*blk->tfb_tcp_fb_init)(tp);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user