If the INP lock is uncontested, avoid taking a reference and jumping
through the lock-switching hoops. A few of the INP lookup operations that lock INPs after the lookup do so using this mechanism (to maintain lock ordering): 1. Lock lookup structure. 2. Find INP. 3. Acquire reference on INP. 4. Drop lock on lookup structure. 5. Acquire INP lock. 6. Drop reference on INP. This change provides a slightly shorter path for cases where the INP lock is uncontested: 1. Lock lookup structure. 2. Find INP. 3. Try to acquire the INP lock. 4. If successful, drop lock on lookup structure. Of course, if the INP lock is contested, the functions will need to revert to the previous way of switching locks safely. This saves a few atomic operations when the INP lock is uncontested. Discussed with: gallatin, rrs, rwatson MFC after: 2 weeks Sponsored by: Netflix, Inc. Differential Revision: https://reviews.freebsd.org/D12911
This commit is contained in:
parent
d614c09a82
commit
7fb2986ff6
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=331309
@ -1632,6 +1632,7 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
|
||||
struct inpcbhead *head;
|
||||
struct inpcb *inp, *tmpinp;
|
||||
u_short fport = fport_arg, lport = lport_arg;
|
||||
bool locked;
|
||||
|
||||
/*
|
||||
* First look for an exact match.
|
||||
@ -1818,18 +1819,32 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
|
||||
return (NULL);
|
||||
|
||||
found:
|
||||
in_pcbref(inp);
|
||||
INP_GROUP_UNLOCK(pcbgroup);
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
} else
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
locked = TRY_INP_WLOCK(inp);
|
||||
else if (lookupflags & INPLOOKUP_RLOCKPCB)
|
||||
locked = TRY_INP_RLOCK(inp);
|
||||
else
|
||||
panic("%s: locking bug", __func__);
|
||||
if (!locked)
|
||||
in_pcbref(inp);
|
||||
INP_GROUP_UNLOCK(pcbgroup);
|
||||
if (!locked) {
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
#ifdef INVARIANTS
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
else
|
||||
INP_RLOCK_ASSERT(inp);
|
||||
#endif
|
||||
return (inp);
|
||||
}
|
||||
#endif /* PCBGROUP */
|
||||
@ -1968,23 +1983,38 @@ in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
|
||||
struct ifnet *ifp)
|
||||
{
|
||||
struct inpcb *inp;
|
||||
bool locked;
|
||||
|
||||
INP_HASH_RLOCK(pcbinfo);
|
||||
inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
|
||||
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
|
||||
if (inp != NULL) {
|
||||
in_pcbref(inp);
|
||||
INP_HASH_RUNLOCK(pcbinfo);
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
} else
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
locked = INP_TRY_WLOCK(inp);
|
||||
else if (lookupflags & INPLOOKUP_RLOCKPCB)
|
||||
locked = INP_TRY_RLOCK(inp);
|
||||
else
|
||||
panic("%s: locking bug", __func__);
|
||||
if (!locked)
|
||||
in_pcbref(inp);
|
||||
INP_HASH_RUNLOCK(pcbinfo);
|
||||
if (!locked) {
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
#ifdef INVARIANTS
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
else
|
||||
INP_RLOCK_ASSERT(inp);
|
||||
#endif
|
||||
} else
|
||||
INP_HASH_RUNLOCK(pcbinfo);
|
||||
return (inp);
|
||||
|
@ -868,6 +868,7 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
|
||||
struct inpcbhead *head;
|
||||
struct inpcb *inp, *tmpinp;
|
||||
u_short fport = fport_arg, lport = lport_arg;
|
||||
bool locked;
|
||||
|
||||
/*
|
||||
* First look for an exact match.
|
||||
@ -1026,18 +1027,32 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
|
||||
return (NULL);
|
||||
|
||||
found:
|
||||
in_pcbref(inp);
|
||||
INP_GROUP_UNLOCK(pcbgroup);
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
} else
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
locked = INP_TRY_WLOCK(inp);
|
||||
else if (lookupflags & INPLOOKUP_RLOCKPCB)
|
||||
locked = INP_TRY_RLOCK(inp);
|
||||
else
|
||||
panic("%s: locking buf", __func__);
|
||||
if (!locked)
|
||||
in_pcbref(inp);
|
||||
INP_GROUP_UNLOCK(pcbgroup);
|
||||
if (!locked) {
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
#ifdef INVARIANTS
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
else
|
||||
INP_RLOCK_ASSERT(inp);
|
||||
#endif
|
||||
return (inp);
|
||||
}
|
||||
#endif /* PCBGROUP */
|
||||
@ -1163,23 +1178,38 @@ in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
|
||||
struct ifnet *ifp)
|
||||
{
|
||||
struct inpcb *inp;
|
||||
bool locked;
|
||||
|
||||
INP_HASH_RLOCK(pcbinfo);
|
||||
inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
|
||||
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
|
||||
if (inp != NULL) {
|
||||
in_pcbref(inp);
|
||||
INP_HASH_RUNLOCK(pcbinfo);
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
} else
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
locked = INP_TRY_WLOCK(inp);
|
||||
else if (lookupflags & INPLOOKUP_RLOCKPCB)
|
||||
locked = INP_TRY_RLOCK(inp);
|
||||
else
|
||||
panic("%s: locking bug", __func__);
|
||||
if (!locked)
|
||||
in_pcbref(inp);
|
||||
INP_HASH_RUNLOCK(pcbinfo);
|
||||
if (!locked) {
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB) {
|
||||
INP_WLOCK(inp);
|
||||
if (in_pcbrele_wlocked(inp))
|
||||
return (NULL);
|
||||
} else {
|
||||
INP_RLOCK(inp);
|
||||
if (in_pcbrele_rlocked(inp))
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
#ifdef INVARIANTS
|
||||
if (lookupflags & INPLOOKUP_WLOCKPCB)
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
else
|
||||
INP_RLOCK_ASSERT(inp);
|
||||
#endif
|
||||
} else
|
||||
INP_HASH_RUNLOCK(pcbinfo);
|
||||
return (inp);
|
||||
|
Loading…
Reference in New Issue
Block a user