ipsec: replace SECASVAR mtx by rmlock
This mutex is a significant point of contention in the ipsec code, and can be relatively trivially replaced by a read-mostly lock. It does require a separate lock for the replay protection, which we do here by adding a separate mutex. This improves throughput (without replay protection) by 10-15%. MFC after: 3 weeks Sponsored by: Orange Business Services Differential Revision: https://reviews.freebsd.org/D35763
This commit is contained in:
parent
4eaaacc755
commit
0361f165f2
@ -1196,6 +1196,8 @@ check_window(const struct secreplay *replay, uint64_t seq)
|
|||||||
{
|
{
|
||||||
int index, bit_location;
|
int index, bit_location;
|
||||||
|
|
||||||
|
SECREPLAY_ASSERT(replay);
|
||||||
|
|
||||||
bit_location = seq & IPSEC_BITMAP_LOC_MASK;
|
bit_location = seq & IPSEC_BITMAP_LOC_MASK;
|
||||||
index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
|
index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
|
||||||
& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
|
& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
|
||||||
@ -1210,6 +1212,8 @@ advance_window(const struct secreplay *replay, uint64_t seq)
|
|||||||
int i;
|
int i;
|
||||||
uint64_t index, index_cur, diff;
|
uint64_t index, index_cur, diff;
|
||||||
|
|
||||||
|
SECREPLAY_ASSERT(replay);
|
||||||
|
|
||||||
index_cur = replay->last >> IPSEC_REDUNDANT_BIT_SHIFTS;
|
index_cur = replay->last >> IPSEC_REDUNDANT_BIT_SHIFTS;
|
||||||
index = seq >> IPSEC_REDUNDANT_BIT_SHIFTS;
|
index = seq >> IPSEC_REDUNDANT_BIT_SHIFTS;
|
||||||
diff = index - index_cur;
|
diff = index - index_cur;
|
||||||
@ -1230,6 +1234,8 @@ set_window(const struct secreplay *replay, uint64_t seq)
|
|||||||
{
|
{
|
||||||
int index, bit_location;
|
int index, bit_location;
|
||||||
|
|
||||||
|
SECREPLAY_ASSERT(replay);
|
||||||
|
|
||||||
bit_location = seq & IPSEC_BITMAP_LOC_MASK;
|
bit_location = seq & IPSEC_BITMAP_LOC_MASK;
|
||||||
index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
|
index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
|
||||||
& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
|
& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
|
||||||
@ -1263,12 +1269,17 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
|
|||||||
replay = sav->replay;
|
replay = sav->replay;
|
||||||
|
|
||||||
/* No need to check replay if disabled. */
|
/* No need to check replay if disabled. */
|
||||||
if (replay->wsize == 0)
|
if (replay->wsize == 0) {
|
||||||
return (1);
|
return (1);
|
||||||
|
}
|
||||||
|
|
||||||
|
SECREPLAY_LOCK(replay);
|
||||||
|
|
||||||
/* Zero sequence number is not allowed. */
|
/* Zero sequence number is not allowed. */
|
||||||
if (seq == 0 && replay->last == 0)
|
if (seq == 0 && replay->last == 0) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
|
}
|
||||||
|
|
||||||
window = replay->wsize << 3; /* Size of window */
|
window = replay->wsize << 3; /* Size of window */
|
||||||
tl = (uint32_t)replay->last; /* Top of window, lower part */
|
tl = (uint32_t)replay->last; /* Top of window, lower part */
|
||||||
@ -1286,10 +1297,13 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
|
|||||||
*seqhigh = th;
|
*seqhigh = th;
|
||||||
if (seq <= tl) {
|
if (seq <= tl) {
|
||||||
/* Sequence number inside window - check against replay */
|
/* Sequence number inside window - check against replay */
|
||||||
if (check_window(replay, seq))
|
if (check_window(replay, seq)) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
/* Sequence number above top of window or not found in bitmap */
|
/* Sequence number above top of window or not found in bitmap */
|
||||||
return (1);
|
return (1);
|
||||||
}
|
}
|
||||||
@ -1307,6 +1321,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
|
|||||||
ESPSTAT_INC(esps_wrap);
|
ESPSTAT_INC(esps_wrap);
|
||||||
else if (sav->sah->saidx.proto == IPPROTO_AH)
|
else if (sav->sah->saidx.proto == IPPROTO_AH)
|
||||||
AHSTAT_INC(ahs_wrap);
|
AHSTAT_INC(ahs_wrap);
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1326,8 +1341,11 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
|
|||||||
return (0);
|
return (0);
|
||||||
*seqhigh = th - 1;
|
*seqhigh = th - 1;
|
||||||
seqh = th - 1;
|
seqh = th - 1;
|
||||||
if (check_window(replay, seq))
|
if (check_window(replay, seq)) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
|
}
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1348,6 +1366,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
|
|||||||
ESPSTAT_INC(esps_wrap);
|
ESPSTAT_INC(esps_wrap);
|
||||||
else if (sav->sah->saidx.proto == IPPROTO_AH)
|
else if (sav->sah->saidx.proto == IPPROTO_AH)
|
||||||
AHSTAT_INC(ahs_wrap);
|
AHSTAT_INC(ahs_wrap);
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1356,6 +1375,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
|
|||||||
ipsec_sa2str(sav, buf, sizeof(buf))));
|
ipsec_sa2str(sav, buf, sizeof(buf))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1381,9 +1401,13 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
|
|||||||
if (replay->wsize == 0)
|
if (replay->wsize == 0)
|
||||||
return (0);
|
return (0);
|
||||||
|
|
||||||
|
SECREPLAY_LOCK(replay);
|
||||||
|
|
||||||
/* Zero sequence number is not allowed. */
|
/* Zero sequence number is not allowed. */
|
||||||
if (seq == 0 && replay->last == 0)
|
if (seq == 0 && replay->last == 0) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
|
}
|
||||||
|
|
||||||
window = replay->wsize << 3; /* Size of window */
|
window = replay->wsize << 3; /* Size of window */
|
||||||
tl = (uint32_t)replay->last; /* Top of window, lower part */
|
tl = (uint32_t)replay->last; /* Top of window, lower part */
|
||||||
@ -1401,8 +1425,10 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
|
|||||||
seqh = th;
|
seqh = th;
|
||||||
if (seq <= tl) {
|
if (seq <= tl) {
|
||||||
/* Sequence number inside window - check against replay */
|
/* Sequence number inside window - check against replay */
|
||||||
if (check_window(replay, seq))
|
if (check_window(replay, seq)) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
|
}
|
||||||
set_window(replay, seq);
|
set_window(replay, seq);
|
||||||
} else {
|
} else {
|
||||||
advance_window(replay, ((uint64_t)seqh << 32) | seq);
|
advance_window(replay, ((uint64_t)seqh << 32) | seq);
|
||||||
@ -1412,11 +1438,14 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
|
|||||||
|
|
||||||
/* Sequence number above top of window or not found in bitmap */
|
/* Sequence number above top of window or not found in bitmap */
|
||||||
replay->count++;
|
replay->count++;
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!(sav->flags & SADB_X_SAFLAGS_ESN))
|
if (!(sav->flags & SADB_X_SAFLAGS_ESN)) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Seq is within [bl, 0xffffffff] and bl is within
|
* Seq is within [bl, 0xffffffff] and bl is within
|
||||||
@ -1425,13 +1454,18 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
|
|||||||
* subspace.
|
* subspace.
|
||||||
*/
|
*/
|
||||||
if (tl < window - 1 && seq >= bl) {
|
if (tl < window - 1 && seq >= bl) {
|
||||||
if (th == 0)
|
if (th == 0) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
if (check_window(replay, seq))
|
}
|
||||||
|
if (check_window(replay, seq)) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
|
}
|
||||||
|
|
||||||
set_window(replay, seq);
|
set_window(replay, seq);
|
||||||
replay->count++;
|
replay->count++;
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1442,13 +1476,17 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
|
|||||||
seqh = th + 1;
|
seqh = th + 1;
|
||||||
|
|
||||||
/* Don't let high part wrap. */
|
/* Don't let high part wrap. */
|
||||||
if (seqh == 0)
|
if (seqh == 0) {
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (1);
|
return (1);
|
||||||
|
}
|
||||||
|
|
||||||
advance_window(replay, ((uint64_t)seqh << 32) | seq);
|
advance_window(replay, ((uint64_t)seqh << 32) | seq);
|
||||||
set_window(replay, seq);
|
set_window(replay, seq);
|
||||||
replay->last = ((uint64_t)seqh << 32) | seq;
|
replay->last = ((uint64_t)seqh << 32) | seq;
|
||||||
replay->count++;
|
replay->count++;
|
||||||
|
|
||||||
|
SECREPLAY_UNLOCK(replay);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
int
|
int
|
||||||
@ -1484,17 +1522,17 @@ ipsec_updateid(struct secasvar *sav, crypto_session_t *new,
|
|||||||
printf("%s: SA(%p) moves cryptoid %p -> %p\n",
|
printf("%s: SA(%p) moves cryptoid %p -> %p\n",
|
||||||
__func__, sav, *old, *new));
|
__func__, sav, *old, *new));
|
||||||
KEYDBG(IPSEC_DATA, kdebug_secasv(sav));
|
KEYDBG(IPSEC_DATA, kdebug_secasv(sav));
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_WLOCK(sav);
|
||||||
if (sav->tdb_cryptoid != *old) {
|
if (sav->tdb_cryptoid != *old) {
|
||||||
/* cryptoid was already updated */
|
/* cryptoid was already updated */
|
||||||
tmp = *new;
|
tmp = *new;
|
||||||
*new = sav->tdb_cryptoid;
|
*new = sav->tdb_cryptoid;
|
||||||
*old = tmp;
|
*old = tmp;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_WUNLOCK(sav);
|
||||||
return (1);
|
return (1);
|
||||||
}
|
}
|
||||||
sav->tdb_cryptoid = *new;
|
sav->tdb_cryptoid = *new;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_WUNLOCK(sav);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2965,13 +2965,14 @@ key_newsav(const struct sadb_msghdr *mhp, struct secasindex *saidx,
|
|||||||
*errp = ENOBUFS;
|
*errp = ENOBUFS;
|
||||||
goto done;
|
goto done;
|
||||||
}
|
}
|
||||||
sav->lock = malloc_aligned(max(sizeof(struct mtx), CACHE_LINE_SIZE),
|
sav->lock = malloc_aligned(max(sizeof(struct rmlock),
|
||||||
CACHE_LINE_SIZE, M_IPSEC_MISC, M_NOWAIT | M_ZERO);
|
CACHE_LINE_SIZE), CACHE_LINE_SIZE, M_IPSEC_MISC,
|
||||||
|
M_NOWAIT | M_ZERO);
|
||||||
if (sav->lock == NULL) {
|
if (sav->lock == NULL) {
|
||||||
*errp = ENOBUFS;
|
*errp = ENOBUFS;
|
||||||
goto done;
|
goto done;
|
||||||
}
|
}
|
||||||
mtx_init(sav->lock, "ipsec association", NULL, MTX_DEF);
|
rm_init(sav->lock, "ipsec association");
|
||||||
sav->lft_c = uma_zalloc_pcpu(ipsec_key_lft_zone, M_NOWAIT | M_ZERO);
|
sav->lft_c = uma_zalloc_pcpu(ipsec_key_lft_zone, M_NOWAIT | M_ZERO);
|
||||||
if (sav->lft_c == NULL) {
|
if (sav->lft_c == NULL) {
|
||||||
*errp = ENOBUFS;
|
*errp = ENOBUFS;
|
||||||
@ -3058,7 +3059,7 @@ key_newsav(const struct sadb_msghdr *mhp, struct secasindex *saidx,
|
|||||||
if (*errp != 0) {
|
if (*errp != 0) {
|
||||||
if (sav != NULL) {
|
if (sav != NULL) {
|
||||||
if (sav->lock != NULL) {
|
if (sav->lock != NULL) {
|
||||||
mtx_destroy(sav->lock);
|
rm_destroy(sav->lock);
|
||||||
free(sav->lock, M_IPSEC_MISC);
|
free(sav->lock, M_IPSEC_MISC);
|
||||||
}
|
}
|
||||||
if (sav->lft_c != NULL)
|
if (sav->lft_c != NULL)
|
||||||
@ -3104,6 +3105,7 @@ key_cleansav(struct secasvar *sav)
|
|||||||
sav->key_enc = NULL;
|
sav->key_enc = NULL;
|
||||||
}
|
}
|
||||||
if (sav->replay != NULL) {
|
if (sav->replay != NULL) {
|
||||||
|
mtx_destroy(&sav->replay->lock);
|
||||||
if (sav->replay->bitmap != NULL)
|
if (sav->replay->bitmap != NULL)
|
||||||
free(sav->replay->bitmap, M_IPSEC_MISC);
|
free(sav->replay->bitmap, M_IPSEC_MISC);
|
||||||
free(sav->replay, M_IPSEC_MISC);
|
free(sav->replay, M_IPSEC_MISC);
|
||||||
@ -3138,7 +3140,7 @@ key_delsav(struct secasvar *sav)
|
|||||||
*/
|
*/
|
||||||
key_cleansav(sav);
|
key_cleansav(sav);
|
||||||
if ((sav->flags & SADB_X_EXT_F_CLONED) == 0) {
|
if ((sav->flags & SADB_X_EXT_F_CLONED) == 0) {
|
||||||
mtx_destroy(sav->lock);
|
rm_destroy(sav->lock);
|
||||||
free(sav->lock, M_IPSEC_MISC);
|
free(sav->lock, M_IPSEC_MISC);
|
||||||
uma_zfree_pcpu(ipsec_key_lft_zone, sav->lft_c);
|
uma_zfree_pcpu(ipsec_key_lft_zone, sav->lft_c);
|
||||||
}
|
}
|
||||||
@ -3269,7 +3271,7 @@ key_updatelifetimes(struct secasvar *sav, const struct sadb_msghdr *mhp)
|
|||||||
* key_update() holds reference to this SA,
|
* key_update() holds reference to this SA,
|
||||||
* so it won't be deleted in meanwhile.
|
* so it won't be deleted in meanwhile.
|
||||||
*/
|
*/
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_WLOCK(sav);
|
||||||
tmp = sav->lft_h;
|
tmp = sav->lft_h;
|
||||||
sav->lft_h = lft_h;
|
sav->lft_h = lft_h;
|
||||||
lft_h = tmp;
|
lft_h = tmp;
|
||||||
@ -3277,7 +3279,7 @@ key_updatelifetimes(struct secasvar *sav, const struct sadb_msghdr *mhp)
|
|||||||
tmp = sav->lft_s;
|
tmp = sav->lft_s;
|
||||||
sav->lft_s = lft_s;
|
sav->lft_s = lft_s;
|
||||||
lft_s = tmp;
|
lft_s = tmp;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_WUNLOCK(sav);
|
||||||
if (lft_h != NULL)
|
if (lft_h != NULL)
|
||||||
free(lft_h, M_IPSEC_MISC);
|
free(lft_h, M_IPSEC_MISC);
|
||||||
if (lft_s != NULL)
|
if (lft_s != NULL)
|
||||||
@ -3366,6 +3368,7 @@ key_setsaval(struct secasvar *sav, const struct sadb_msghdr *mhp)
|
|||||||
error = ENOBUFS;
|
error = ENOBUFS;
|
||||||
goto fail;
|
goto fail;
|
||||||
}
|
}
|
||||||
|
mtx_init(&sav->replay->lock, "ipsec replay", NULL, MTX_DEF);
|
||||||
|
|
||||||
if (replay != 0) {
|
if (replay != 0) {
|
||||||
/* number of 32b blocks to be allocated */
|
/* number of 32b blocks to be allocated */
|
||||||
@ -3583,6 +3586,8 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
|
|||||||
};
|
};
|
||||||
uint32_t replay_count;
|
uint32_t replay_count;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
|
m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
|
||||||
if (m == NULL)
|
if (m == NULL)
|
||||||
goto fail;
|
goto fail;
|
||||||
@ -3597,16 +3602,16 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
|
|||||||
goto fail;
|
goto fail;
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case SADB_X_EXT_SA2:
|
case SADB_X_EXT_SA2: {
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
replay_count = sav->replay ? sav->replay->count : 0;
|
replay_count = sav->replay ? sav->replay->count : 0;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
|
m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
|
||||||
sav->sah->saidx.reqid);
|
sav->sah->saidx.reqid);
|
||||||
if (!m)
|
if (!m)
|
||||||
goto fail;
|
goto fail;
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
case SADB_X_EXT_SA_REPLAY:
|
case SADB_X_EXT_SA_REPLAY:
|
||||||
if (sav->replay == NULL ||
|
if (sav->replay == NULL ||
|
||||||
sav->replay->wsize <= UINT8_MAX)
|
sav->replay->wsize <= UINT8_MAX)
|
||||||
@ -4508,6 +4513,8 @@ key_flush_sad(time_t now)
|
|||||||
struct secashead *sah, *nextsah;
|
struct secashead *sah, *nextsah;
|
||||||
struct secasvar *sav, *nextsav;
|
struct secasvar *sav, *nextsav;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
LIST_INIT(&drainq);
|
LIST_INIT(&drainq);
|
||||||
LIST_INIT(&hexpireq);
|
LIST_INIT(&hexpireq);
|
||||||
LIST_INIT(&sexpireq);
|
LIST_INIT(&sexpireq);
|
||||||
@ -4533,13 +4540,13 @@ key_flush_sad(time_t now)
|
|||||||
/* lifetimes aren't specified */
|
/* lifetimes aren't specified */
|
||||||
if (sav->lft_h == NULL)
|
if (sav->lft_h == NULL)
|
||||||
continue;
|
continue;
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
/*
|
/*
|
||||||
* Check again with lock held, because it may
|
* Check again with lock held, because it may
|
||||||
* be updated by SADB_UPDATE.
|
* be updated by SADB_UPDATE.
|
||||||
*/
|
*/
|
||||||
if (sav->lft_h == NULL) {
|
if (sav->lft_h == NULL) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
/*
|
/*
|
||||||
@ -4556,7 +4563,7 @@ key_flush_sad(time_t now)
|
|||||||
now - sav->firstused > sav->lft_h->usetime) ||
|
now - sav->firstused > sav->lft_h->usetime) ||
|
||||||
(sav->lft_h->bytes != 0 && counter_u64_fetch(
|
(sav->lft_h->bytes != 0 && counter_u64_fetch(
|
||||||
sav->lft_c_bytes) > sav->lft_h->bytes)) {
|
sav->lft_c_bytes) > sav->lft_h->bytes)) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
SAV_ADDREF(sav);
|
SAV_ADDREF(sav);
|
||||||
LIST_INSERT_HEAD(&hexpireq, sav, drainq);
|
LIST_INSERT_HEAD(&hexpireq, sav, drainq);
|
||||||
continue;
|
continue;
|
||||||
@ -4573,12 +4580,12 @@ key_flush_sad(time_t now)
|
|||||||
(sav->replay != NULL) && (
|
(sav->replay != NULL) && (
|
||||||
(sav->replay->count > UINT32_80PCT) ||
|
(sav->replay->count > UINT32_80PCT) ||
|
||||||
(sav->replay->last > UINT32_80PCT))))) {
|
(sav->replay->last > UINT32_80PCT))))) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
SAV_ADDREF(sav);
|
SAV_ADDREF(sav);
|
||||||
LIST_INSERT_HEAD(&sexpireq, sav, drainq);
|
LIST_INSERT_HEAD(&sexpireq, sav, drainq);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
SAHTREE_RUNLOCK();
|
SAHTREE_RUNLOCK();
|
||||||
@ -5282,11 +5289,11 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
|
|||||||
* isnew == 0 -> we use the same @sah, that was used by @sav,
|
* isnew == 0 -> we use the same @sah, that was used by @sav,
|
||||||
* and we use its reference for @newsav.
|
* and we use its reference for @newsav.
|
||||||
*/
|
*/
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_WLOCK(sav);
|
||||||
/* XXX: replace cntr with pointer? */
|
/* XXX: replace cntr with pointer? */
|
||||||
newsav->cntr = sav->cntr;
|
newsav->cntr = sav->cntr;
|
||||||
sav->flags |= SADB_X_EXT_F_CLONED;
|
sav->flags |= SADB_X_EXT_F_CLONED;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_WUNLOCK(sav);
|
||||||
|
|
||||||
SAHTREE_WUNLOCK();
|
SAHTREE_WUNLOCK();
|
||||||
|
|
||||||
@ -7310,6 +7317,8 @@ key_expire(struct secasvar *sav, int hard)
|
|||||||
int error, len;
|
int error, len;
|
||||||
uint8_t satype;
|
uint8_t satype;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
IPSEC_ASSERT (sav != NULL, ("null sav"));
|
IPSEC_ASSERT (sav != NULL, ("null sav"));
|
||||||
IPSEC_ASSERT (sav->sah != NULL, ("null sa header"));
|
IPSEC_ASSERT (sav->sah != NULL, ("null sa header"));
|
||||||
|
|
||||||
@ -7336,9 +7345,9 @@ key_expire(struct secasvar *sav, int hard)
|
|||||||
m_cat(result, m);
|
m_cat(result, m);
|
||||||
|
|
||||||
/* create SA extension */
|
/* create SA extension */
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
replay_count = sav->replay ? sav->replay->count : 0;
|
replay_count = sav->replay ? sav->replay->count : 0;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
|
m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
|
||||||
sav->sah->saidx.reqid);
|
sav->sah->saidx.reqid);
|
||||||
|
@ -808,12 +808,15 @@ kdebug_secreplay(struct secreplay *rpl)
|
|||||||
{
|
{
|
||||||
int len, l;
|
int len, l;
|
||||||
|
|
||||||
|
SECREPLAY_LOCK(rpl);
|
||||||
|
|
||||||
IPSEC_ASSERT(rpl != NULL, ("null rpl"));
|
IPSEC_ASSERT(rpl != NULL, ("null rpl"));
|
||||||
printf(" secreplay{ count=%lu bitmap_size=%u wsize=%u last=%lu",
|
printf(" secreplay{ count=%lu bitmap_size=%u wsize=%u last=%lu",
|
||||||
rpl->count, rpl->bitmap_size, rpl->wsize, rpl->last);
|
rpl->count, rpl->bitmap_size, rpl->wsize, rpl->last);
|
||||||
|
|
||||||
if (rpl->bitmap == NULL) {
|
if (rpl->bitmap == NULL) {
|
||||||
printf(" }\n");
|
printf(" }\n");
|
||||||
|
SECREPLAY_UNLOCK(rpl);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -823,6 +826,7 @@ kdebug_secreplay(struct secreplay *rpl)
|
|||||||
printf("%u", (((rpl->bitmap)[len] >> l) & 1) ? 1 : 0);
|
printf("%u", (((rpl->bitmap)[len] >> l) & 1) ? 1 : 0);
|
||||||
}
|
}
|
||||||
printf(" }\n");
|
printf(" }\n");
|
||||||
|
SECREPLAY_UNLOCK(rpl);
|
||||||
}
|
}
|
||||||
#endif /* IPSEC_DEBUG */
|
#endif /* IPSEC_DEBUG */
|
||||||
|
|
||||||
|
@ -39,6 +39,7 @@
|
|||||||
#include <sys/counter.h>
|
#include <sys/counter.h>
|
||||||
#include <sys/lock.h>
|
#include <sys/lock.h>
|
||||||
#include <sys/mutex.h>
|
#include <sys/mutex.h>
|
||||||
|
#include <sys/rmlock.h>
|
||||||
|
|
||||||
#include <netipsec/key_var.h>
|
#include <netipsec/key_var.h>
|
||||||
#include <opencrypto/_cryptodev.h>
|
#include <opencrypto/_cryptodev.h>
|
||||||
@ -157,7 +158,7 @@ struct secasvar {
|
|||||||
struct seckey *key_enc; /* Key for Encryption */
|
struct seckey *key_enc; /* Key for Encryption */
|
||||||
struct secreplay *replay; /* replay prevention */
|
struct secreplay *replay; /* replay prevention */
|
||||||
struct secnatt *natt; /* NAT-T config */
|
struct secnatt *natt; /* NAT-T config */
|
||||||
struct mtx *lock; /* update/access lock */
|
struct rmlock *lock; /* update/access lock */
|
||||||
|
|
||||||
const struct xformsw *tdb_xform; /* transform */
|
const struct xformsw *tdb_xform; /* transform */
|
||||||
const struct enc_xform *tdb_encalgxform;/* encoding algorithm */
|
const struct enc_xform *tdb_encalgxform;/* encoding algorithm */
|
||||||
@ -187,9 +188,13 @@ struct secasvar {
|
|||||||
volatile u_int refcnt; /* reference count */
|
volatile u_int refcnt; /* reference count */
|
||||||
};
|
};
|
||||||
|
|
||||||
#define SECASVAR_LOCK(_sav) mtx_lock((_sav)->lock)
|
#define SECASVAR_RLOCK_TRACKER struct rm_priotracker _secas_tracker
|
||||||
#define SECASVAR_UNLOCK(_sav) mtx_unlock((_sav)->lock)
|
#define SECASVAR_RLOCK(_sav) rm_rlock((_sav)->lock, &_secas_tracker)
|
||||||
#define SECASVAR_LOCK_ASSERT(_sav) mtx_assert((_sav)->lock, MA_OWNED)
|
#define SECASVAR_RUNLOCK(_sav) rm_runlock((_sav)->lock, &_secas_tracker)
|
||||||
|
#define SECASVAR_WLOCK(_sav) rm_wlock((_sav)->lock)
|
||||||
|
#define SECASVAR_WUNLOCK(_sav) rm_wunlock((_sav)->lock)
|
||||||
|
#define SECASVAR_LOCK_ASSERT(_sav) rm_assert((_sav)->lock, RA_LOCKED)
|
||||||
|
#define SECASVAR_LOCK_WASSERT(_sav) rm_assert((_sav)->lock, RA_WLOCKED)
|
||||||
#define SAV_ISGCM(_sav) \
|
#define SAV_ISGCM(_sav) \
|
||||||
((_sav)->alg_enc == SADB_X_EALG_AESGCM8 || \
|
((_sav)->alg_enc == SADB_X_EALG_AESGCM8 || \
|
||||||
(_sav)->alg_enc == SADB_X_EALG_AESGCM12 || \
|
(_sav)->alg_enc == SADB_X_EALG_AESGCM12 || \
|
||||||
@ -204,6 +209,7 @@ struct secasvar {
|
|||||||
* (c) read only except during creation / free
|
* (c) read only except during creation / free
|
||||||
*/
|
*/
|
||||||
struct secreplay {
|
struct secreplay {
|
||||||
|
struct mtx lock;
|
||||||
u_int64_t count; /* (m) */
|
u_int64_t count; /* (m) */
|
||||||
u_int wsize; /* (c) window size, i.g. 4 bytes */
|
u_int wsize; /* (c) window size, i.g. 4 bytes */
|
||||||
u_int64_t last; /* (m) used by receiver */
|
u_int64_t last; /* (m) used by receiver */
|
||||||
@ -212,6 +218,10 @@ struct secreplay {
|
|||||||
int overflow; /* (m) overflow flag */
|
int overflow; /* (m) overflow flag */
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#define SECREPLAY_LOCK(_r) mtx_lock(&(_r)->lock)
|
||||||
|
#define SECREPLAY_UNLOCK(_r) mtx_unlock(&(_r)->lock)
|
||||||
|
#define SECREPLAY_ASSERT(_r) mtx_assert(&(_r)->lock, MA_OWNED)
|
||||||
|
|
||||||
/* socket table due to send PF_KEY messages. */
|
/* socket table due to send PF_KEY messages. */
|
||||||
struct secreg {
|
struct secreg {
|
||||||
LIST_ENTRY(secreg) chain;
|
LIST_ENTRY(secreg) chain;
|
||||||
|
@ -543,6 +543,8 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
int hl, rplen, authsize, ahsize, error;
|
int hl, rplen, authsize, ahsize, error;
|
||||||
uint32_t seqh;
|
uint32_t seqh;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
||||||
IPSEC_ASSERT(sav->key_auth != NULL, ("null authentication key"));
|
IPSEC_ASSERT(sav->key_auth != NULL, ("null authentication key"));
|
||||||
IPSEC_ASSERT(sav->tdb_authalgxform != NULL,
|
IPSEC_ASSERT(sav->tdb_authalgxform != NULL,
|
||||||
@ -563,10 +565,10 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
ah = (struct newah *)(mtod(m, caddr_t) + skip);
|
ah = (struct newah *)(mtod(m, caddr_t) + skip);
|
||||||
|
|
||||||
/* Check replay window, if applicable. */
|
/* Check replay window, if applicable. */
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
if (sav->replay != NULL && sav->replay->wsize != 0 &&
|
if (sav->replay != NULL && sav->replay->wsize != 0 &&
|
||||||
ipsec_chkreplay(ntohl(ah->ah_seq), &seqh, sav) == 0) {
|
ipsec_chkreplay(ntohl(ah->ah_seq), &seqh, sav) == 0) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
AHSTAT_INC(ahs_replay);
|
AHSTAT_INC(ahs_replay);
|
||||||
DPRINTF(("%s: packet replay failure: %s\n", __func__,
|
DPRINTF(("%s: packet replay failure: %s\n", __func__,
|
||||||
ipsec_sa2str(sav, buf, sizeof(buf))));
|
ipsec_sa2str(sav, buf, sizeof(buf))));
|
||||||
@ -574,7 +576,7 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
cryptoid = sav->tdb_cryptoid;
|
cryptoid = sav->tdb_cryptoid;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
/* Verify AH header length. */
|
/* Verify AH header length. */
|
||||||
hl = sizeof(struct ah) + (ah->ah_len * sizeof (u_int32_t));
|
hl = sizeof(struct ah) + (ah->ah_len * sizeof (u_int32_t));
|
||||||
@ -699,6 +701,8 @@ ah_input_cb(struct cryptop *crp)
|
|||||||
int authsize, rplen, ahsize, error, skip, protoff;
|
int authsize, rplen, ahsize, error, skip, protoff;
|
||||||
uint8_t nxt;
|
uint8_t nxt;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
m = crp->crp_buf.cb_mbuf;
|
m = crp->crp_buf.cb_mbuf;
|
||||||
xd = crp->crp_opaque;
|
xd = crp->crp_opaque;
|
||||||
CURVNET_SET(xd->vnet);
|
CURVNET_SET(xd->vnet);
|
||||||
@ -779,14 +783,14 @@ ah_input_cb(struct cryptop *crp)
|
|||||||
|
|
||||||
m_copydata(m, skip + offsetof(struct newah, ah_seq),
|
m_copydata(m, skip + offsetof(struct newah, ah_seq),
|
||||||
sizeof (seq), (caddr_t) &seq);
|
sizeof (seq), (caddr_t) &seq);
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
if (ipsec_updatereplay(ntohl(seq), sav)) {
|
if (ipsec_updatereplay(ntohl(seq), sav)) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
AHSTAT_INC(ahs_replay);
|
AHSTAT_INC(ahs_replay);
|
||||||
error = EACCES;
|
error = EACCES;
|
||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -850,6 +854,8 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
uint8_t prot;
|
uint8_t prot;
|
||||||
uint32_t seqh;
|
uint32_t seqh;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
||||||
ahx = sav->tdb_authalgxform;
|
ahx = sav->tdb_authalgxform;
|
||||||
IPSEC_ASSERT(ahx != NULL, ("null authentication xform"));
|
IPSEC_ASSERT(ahx != NULL, ("null authentication xform"));
|
||||||
@ -939,13 +945,15 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
ipseczeroes);
|
ipseczeroes);
|
||||||
|
|
||||||
/* Insert packet replay counter, as requested. */
|
/* Insert packet replay counter, as requested. */
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
if (sav->replay) {
|
if (sav->replay) {
|
||||||
|
SECREPLAY_LOCK(sav->replay);
|
||||||
if ((sav->replay->count == ~0 ||
|
if ((sav->replay->count == ~0 ||
|
||||||
(!(sav->flags & SADB_X_SAFLAGS_ESN) &&
|
(!(sav->flags & SADB_X_SAFLAGS_ESN) &&
|
||||||
((uint32_t)sav->replay->count) == ~0)) &&
|
((uint32_t)sav->replay->count) == ~0)) &&
|
||||||
(sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
|
(sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECREPLAY_UNLOCK(sav->replay);
|
||||||
|
SECASVAR_RUNLOCK(sav);
|
||||||
DPRINTF(("%s: replay counter wrapped for SA %s/%08lx\n",
|
DPRINTF(("%s: replay counter wrapped for SA %s/%08lx\n",
|
||||||
__func__, ipsec_address(&sav->sah->saidx.dst, buf,
|
__func__, ipsec_address(&sav->sah->saidx.dst, buf,
|
||||||
sizeof(buf)), (u_long) ntohl(sav->spi)));
|
sizeof(buf)), (u_long) ntohl(sav->spi)));
|
||||||
@ -959,9 +967,10 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
#endif
|
#endif
|
||||||
sav->replay->count++;
|
sav->replay->count++;
|
||||||
ah->ah_seq = htonl((uint32_t)sav->replay->count);
|
ah->ah_seq = htonl((uint32_t)sav->replay->count);
|
||||||
|
SECREPLAY_UNLOCK(sav->replay);
|
||||||
}
|
}
|
||||||
cryptoid = sav->tdb_cryptoid;
|
cryptoid = sav->tdb_cryptoid;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
/* Get crypto descriptors. */
|
/* Get crypto descriptors. */
|
||||||
crp = crypto_getreq(cryptoid, M_NOWAIT);
|
crp = crypto_getreq(cryptoid, M_NOWAIT);
|
||||||
@ -1045,8 +1054,10 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
crp->crp_opaque = xd;
|
crp->crp_opaque = xd;
|
||||||
|
|
||||||
if (sav->flags & SADB_X_SAFLAGS_ESN && sav->replay != NULL) {
|
if (sav->flags & SADB_X_SAFLAGS_ESN && sav->replay != NULL) {
|
||||||
|
SECREPLAY_LOCK(sav->replay);
|
||||||
seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
|
seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
|
||||||
memcpy(crp->crp_esn, &seqh, sizeof(seqh));
|
memcpy(crp->crp_esn, &seqh, sizeof(seqh));
|
||||||
|
SECREPLAY_UNLOCK(sav->replay);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* These are passed as-is to the callback. */
|
/* These are passed as-is to the callback. */
|
||||||
|
@ -274,6 +274,8 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
uint32_t seqh;
|
uint32_t seqh;
|
||||||
const struct crypto_session_params *csp;
|
const struct crypto_session_params *csp;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
||||||
IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
|
IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
|
||||||
|
|
||||||
@ -329,10 +331,10 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
/*
|
/*
|
||||||
* Check sequence number.
|
* Check sequence number.
|
||||||
*/
|
*/
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
if (esph != NULL && sav->replay != NULL && sav->replay->wsize != 0) {
|
if (esph != NULL && sav->replay != NULL && sav->replay->wsize != 0) {
|
||||||
if (ipsec_chkreplay(ntohl(esp->esp_seq), &seqh, sav) == 0) {
|
if (ipsec_chkreplay(ntohl(esp->esp_seq), &seqh, sav) == 0) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
DPRINTF(("%s: packet replay check for %s\n", __func__,
|
DPRINTF(("%s: packet replay check for %s\n", __func__,
|
||||||
ipsec_sa2str(sav, buf, sizeof(buf))));
|
ipsec_sa2str(sav, buf, sizeof(buf))));
|
||||||
ESPSTAT_INC(esps_replay);
|
ESPSTAT_INC(esps_replay);
|
||||||
@ -342,7 +344,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
seqh = htonl(seqh);
|
seqh = htonl(seqh);
|
||||||
}
|
}
|
||||||
cryptoid = sav->tdb_cryptoid;
|
cryptoid = sav->tdb_cryptoid;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
/* Update the counters */
|
/* Update the counters */
|
||||||
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
|
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
|
||||||
@ -494,6 +496,8 @@ esp_input_cb(struct cryptop *crp)
|
|||||||
crypto_session_t cryptoid;
|
crypto_session_t cryptoid;
|
||||||
int hlen, skip, protoff, error, alen;
|
int hlen, skip, protoff, error, alen;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
m = crp->crp_buf.cb_mbuf;
|
m = crp->crp_buf.cb_mbuf;
|
||||||
xd = crp->crp_opaque;
|
xd = crp->crp_opaque;
|
||||||
CURVNET_SET(xd->vnet);
|
CURVNET_SET(xd->vnet);
|
||||||
@ -570,16 +574,16 @@ esp_input_cb(struct cryptop *crp)
|
|||||||
|
|
||||||
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
|
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
|
||||||
sizeof (seq), (caddr_t) &seq);
|
sizeof (seq), (caddr_t) &seq);
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
if (ipsec_updatereplay(ntohl(seq), sav)) {
|
if (ipsec_updatereplay(ntohl(seq), sav)) {
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
DPRINTF(("%s: packet replay check for %s\n", __func__,
|
DPRINTF(("%s: packet replay check for %s\n", __func__,
|
||||||
ipsec_sa2str(sav, buf, sizeof(buf))));
|
ipsec_sa2str(sav, buf, sizeof(buf))));
|
||||||
ESPSTAT_INC(esps_replay);
|
ESPSTAT_INC(esps_replay);
|
||||||
error = EACCES;
|
error = EACCES;
|
||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Determine the ESP header length */
|
/* Determine the ESP header length */
|
||||||
@ -694,6 +698,8 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
uint32_t seqh;
|
uint32_t seqh;
|
||||||
const struct crypto_session_params *csp;
|
const struct crypto_session_params *csp;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
||||||
esph = sav->tdb_authalgxform;
|
esph = sav->tdb_authalgxform;
|
||||||
espx = sav->tdb_encalgxform;
|
espx = sav->tdb_encalgxform;
|
||||||
@ -786,10 +792,11 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
/* Initialize ESP header. */
|
/* Initialize ESP header. */
|
||||||
bcopy((caddr_t) &sav->spi, mtod(mo, caddr_t) + roff,
|
bcopy((caddr_t) &sav->spi, mtod(mo, caddr_t) + roff,
|
||||||
sizeof(uint32_t));
|
sizeof(uint32_t));
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
if (sav->replay) {
|
if (sav->replay) {
|
||||||
uint32_t replay;
|
uint32_t replay;
|
||||||
|
|
||||||
|
SECREPLAY_LOCK(sav->replay);
|
||||||
#ifdef REGRESSION
|
#ifdef REGRESSION
|
||||||
/* Emulate replay attack when ipsec_replay is TRUE. */
|
/* Emulate replay attack when ipsec_replay is TRUE. */
|
||||||
if (!V_ipsec_replay)
|
if (!V_ipsec_replay)
|
||||||
@ -801,11 +808,12 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
sizeof(uint32_t), sizeof(uint32_t));
|
sizeof(uint32_t), sizeof(uint32_t));
|
||||||
|
|
||||||
seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
|
seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
|
||||||
|
SECREPLAY_UNLOCK(sav->replay);
|
||||||
}
|
}
|
||||||
cryptoid = sav->tdb_cryptoid;
|
cryptoid = sav->tdb_cryptoid;
|
||||||
if (SAV_ISCTRORGCM(sav))
|
if (SAV_ISCTRORGCM(sav))
|
||||||
cntr = sav->cntr++;
|
cntr = sav->cntr++;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Add padding -- better to do it ourselves than use the crypto engine,
|
* Add padding -- better to do it ourselves than use the crypto engine,
|
||||||
|
@ -205,6 +205,8 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
caddr_t addr;
|
caddr_t addr;
|
||||||
int error, hlen = IPCOMP_HLENGTH;
|
int error, hlen = IPCOMP_HLENGTH;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Check that the next header of the IPComp is not IPComp again, before
|
* Check that the next header of the IPComp is not IPComp again, before
|
||||||
* doing any real work. Given it is not possible to do double
|
* doing any real work. Given it is not possible to do double
|
||||||
@ -226,9 +228,9 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
|
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
cryptoid = sav->tdb_cryptoid;
|
cryptoid = sav->tdb_cryptoid;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
/* Get crypto descriptors */
|
/* Get crypto descriptors */
|
||||||
crp = crypto_getreq(cryptoid, M_NOWAIT);
|
crp = crypto_getreq(cryptoid, M_NOWAIT);
|
||||||
@ -264,9 +266,9 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
xd->vnet = curvnet;
|
xd->vnet = curvnet;
|
||||||
xd->cryptoid = cryptoid;
|
xd->cryptoid = cryptoid;
|
||||||
|
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
|
crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
return crypto_dispatch(crp);
|
return crypto_dispatch(crp);
|
||||||
bad:
|
bad:
|
||||||
@ -405,6 +407,8 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
crypto_session_t cryptoid;
|
crypto_session_t cryptoid;
|
||||||
int error, ralen, maxpacketsize;
|
int error, ralen, maxpacketsize;
|
||||||
|
|
||||||
|
SECASVAR_RLOCK_TRACKER;
|
||||||
|
|
||||||
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
IPSEC_ASSERT(sav != NULL, ("null SA"));
|
||||||
ipcompx = sav->tdb_compalgxform;
|
ipcompx = sav->tdb_compalgxform;
|
||||||
IPSEC_ASSERT(ipcompx != NULL, ("null compression xform"));
|
IPSEC_ASSERT(ipcompx != NULL, ("null compression xform"));
|
||||||
@ -470,9 +474,9 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Ok now, we can pass to the crypto processing. */
|
/* Ok now, we can pass to the crypto processing. */
|
||||||
SECASVAR_LOCK(sav);
|
SECASVAR_RLOCK(sav);
|
||||||
cryptoid = sav->tdb_cryptoid;
|
cryptoid = sav->tdb_cryptoid;
|
||||||
SECASVAR_UNLOCK(sav);
|
SECASVAR_RUNLOCK(sav);
|
||||||
|
|
||||||
/* Get crypto descriptors */
|
/* Get crypto descriptors */
|
||||||
crp = crypto_getreq(cryptoid, M_NOWAIT);
|
crp = crypto_getreq(cryptoid, M_NOWAIT);
|
||||||
|
Loading…
Reference in New Issue
Block a user