d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce contention on per-adapter lock.

Browse Source 
- Move temporary sglists into the session structure and protect them
  with a per-session lock instead of a per-adapter lock.

- Retire an unused session field, and move a debugging field under
  INVARIANTS to avoid using the session lock for completion handling
  when INVARIANTS isn't enabled.

- Use counter_u64 for per-adapter statistics.

Note that this helps for cases where multiple sessions are used
(e.g. multiple IPsec SAs or multiple KTLS connections).  It does not
help for workloads that use a single session (e.g. a single GELI
volume).

Reviewed by:	np
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D25457
This commit is contained in:
John Baldwin 2020-06-26 00:01:31 +00:00
parent dae61c9d09
commit 94578db218
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=362637
1 changed files with 231 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

405
sys/dev/cxgbe/crypto/t4_crypto.c
View File

@ -170,8 +170,9 @@ struct ccr_port {
}; };
struct ccr_session { struct ccr_session {
bool active; #ifdef INVARIANTS
int pending; int pending;
#endif
enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode;
struct ccr_port *port; struct ccr_port *port;
union { union {
@ -180,16 +181,7 @@ struct ccr_session {
struct ccr_session_ccm_mac ccm_mac; struct ccr_session_ccm_mac ccm_mac;
}; };
struct ccr_session_blkcipher blkcipher; struct ccr_session_blkcipher blkcipher;
};
struct ccr_softc {
struct adapter *adapter;
device_t dev;
uint32_t cid;
struct mtx lock; struct mtx lock;
bool detaching;
struct ccr_port ports[MAX_NPORTS];
u_int port_mask;
/* /*
* Pre-allocate S/G lists used when preparing a work request. * Pre-allocate S/G lists used when preparing a work request.
@ -205,6 +197,16 @@ struct ccr_softc {
struct sglist *sg_output; struct sglist *sg_output;
struct sglist *sg_ulptx; struct sglist *sg_ulptx;
struct sglist *sg_dsgl; struct sglist *sg_dsgl;
};
struct ccr_softc {
struct adapter *adapter;
device_t dev;
uint32_t cid;
struct mtx lock;
bool detaching;
struct ccr_port ports[MAX_NPORTS];
u_int port_mask;
/* /*
* Pre-allocate a dummy output buffer for the IV and AAD for * Pre-allocate a dummy output buffer for the IV and AAD for
@ -214,24 +216,23 @@ struct ccr_softc {
struct sglist *sg_iv_aad; struct sglist *sg_iv_aad;
/* Statistics. */ /* Statistics. */
uint64_t stats_blkcipher_encrypt; counter_u64_t stats_blkcipher_encrypt;
uint64_t stats_blkcipher_decrypt; counter_u64_t stats_blkcipher_decrypt;
uint64_t stats_hash; counter_u64_t stats_hash;
uint64_t stats_hmac; counter_u64_t stats_hmac;
uint64_t stats_eta_encrypt; counter_u64_t stats_eta_encrypt;
uint64_t stats_eta_decrypt; counter_u64_t stats_eta_decrypt;
uint64_t stats_gcm_encrypt; counter_u64_t stats_gcm_encrypt;
uint64_t stats_gcm_decrypt; counter_u64_t stats_gcm_decrypt;
uint64_t stats_ccm_encrypt; counter_u64_t stats_ccm_encrypt;
uint64_t stats_ccm_decrypt; counter_u64_t stats_ccm_decrypt;
uint64_t stats_wr_nomem; counter_u64_t stats_wr_nomem;
uint64_t stats_inflight; counter_u64_t stats_inflight;
uint64_t stats_mac_error; counter_u64_t stats_mac_error;
uint64_t stats_pad_error; counter_u64_t stats_pad_error;
uint64_t stats_bad_session; counter_u64_t stats_sglist_error;
uint64_t stats_sglist_error; counter_u64_t stats_process_error;
uint64_t stats_process_error; counter_u64_t stats_sw_fallback;
uint64_t stats_sw_fallback;
}; };
/* /*
@ -307,8 +308,7 @@ ccr_phys_dsgl_len(int nsegs)
} }
static void static void
ccr_write_phys_dsgl(struct ccr_softc *sc, struct ccr_session *s, void *dst, ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs)
int nsegs)
{ {
struct sglist *sg; struct sglist *sg;
struct cpl_rx_phys_dsgl *cpl; struct cpl_rx_phys_dsgl *cpl;
@ -317,7 +317,7 @@ ccr_write_phys_dsgl(struct ccr_softc *sc, struct ccr_session *s, void *dst,
size_t seglen; size_t seglen;
u_int i, j; u_int i, j;
sg = sc->sg_dsgl; sg = s->sg_dsgl;
cpl = dst; cpl = dst;
cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) |
V_CPL_RX_PHYS_DSGL_ISRDMA(0)); V_CPL_RX_PHYS_DSGL_ISRDMA(0));
@ -366,14 +366,14 @@ ccr_ulptx_sgl_len(int nsegs)
} }
static void static void
ccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs) ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs)
{ {
struct ulptx_sgl *usgl; struct ulptx_sgl *usgl;
struct sglist *sg; struct sglist *sg;
struct sglist_seg *ss; struct sglist_seg *ss;
int i; int i;
sg = sc->sg_ulptx; sg = s->sg_ulptx;
MPASS(nsegs == sg->sg_nseg); MPASS(nsegs == sg->sg_nseg);
ss = &sg->sg_segs[0]; ss = &sg->sg_segs[0];
usgl = dst; usgl = dst;
@ -496,12 +496,12 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
@ -510,7 +510,7 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
@ -564,7 +564,7 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
else else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
@ -620,16 +620,16 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
else else
op_type = CHCR_DECRYPT_OP; op_type = CHCR_DECRYPT_OP;
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
@ -650,12 +650,12 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
@ -665,7 +665,7 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
@ -737,7 +737,7 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
} }
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
@ -745,7 +745,7 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
else else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
@ -846,30 +846,30 @@ ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
MAX_REQUEST_SIZE) MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} }
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0,
iv_len + crp->crp_aad_length); iv_len + crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_ENCRYPT_OP) { if (op_type == CHCR_ENCRYPT_OP) {
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
@ -916,29 +916,29 @@ ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
error = sglist_append(sc->sg_ulptx, error = sglist_append(s->sg_ulptx,
crp->crp_aad, crp->crp_aad_length); crp->crp_aad, crp->crp_aad_length);
else else
error = sglist_append_sglist(sc->sg_ulptx, error = sglist_append_sglist(s->sg_ulptx,
sc->sg_input, crp->crp_aad_start, s->sg_input, crp->crp_aad_start,
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
@ -966,7 +966,7 @@ ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
@ -1048,7 +1048,7 @@ ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
memcpy(dst, s->hmac.pads, iopad_size * 2); memcpy(dst, s->hmac.pads, iopad_size * 2);
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
@ -1068,7 +1068,7 @@ ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
} else } else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
@ -1168,30 +1168,30 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
MAX_REQUEST_SIZE) MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} }
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_ENCRYPT_OP) { if (op_type == CHCR_ENCRYPT_OP) {
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
@ -1225,29 +1225,29 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
error = sglist_append(sc->sg_ulptx, error = sglist_append(s->sg_ulptx,
crp->crp_aad, crp->crp_aad_length); crp->crp_aad, crp->crp_aad_length);
else else
error = sglist_append_sglist(sc->sg_ulptx, error = sglist_append_sglist(s->sg_ulptx,
sc->sg_input, crp->crp_aad_start, s->sg_input, crp->crp_aad_start,
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
@ -1274,7 +1274,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
@ -1339,7 +1339,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN);
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
@ -1359,7 +1359,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
} else } else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
@ -1642,30 +1642,30 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
MAX_REQUEST_SIZE) MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} }
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
aad_len); aad_len);
if (error) if (error)
return (error); return (error);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_ENCRYPT_OP) { if (op_type == CHCR_ENCRYPT_OP) {
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
@ -1701,29 +1701,29 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
/* Block 0 is passed as immediate data. */ /* Block 0 is passed as immediate data. */
imm_len = b0_len; imm_len = b0_len;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
error = sglist_append(sc->sg_ulptx, error = sglist_append(s->sg_ulptx,
crp->crp_aad, crp->crp_aad_length); crp->crp_aad, crp->crp_aad_length);
else else
error = sglist_append_sglist(sc->sg_ulptx, error = sglist_append_sglist(s->sg_ulptx,
sc->sg_input, crp->crp_aad_start, s->sg_input, crp->crp_aad_start,
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
@ -1745,7 +1745,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
@ -1809,7 +1809,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
s->blkcipher.enckey, s->blkcipher.key_len); s->blkcipher.enckey, s->blkcipher.key_len);
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
@ -1847,7 +1847,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
idata->len = htobe32(0); idata->len = htobe32(0);
dst = (void *)(idata + 1); dst = (void *)(idata + 1);
} }
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
} }
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
@ -2054,47 +2054,50 @@ ccr_sysctls(struct ccr_softc *sc)
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
children = SYSCTL_CHILDREN(oid); children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD,
&sc->stats_hash, 0, "Hash requests submitted"); &sc->stats_hash, "Hash requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
&sc->stats_hmac, 0, "HMAC requests submitted"); &sc->stats_hmac, "HMAC requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt",
&sc->stats_blkcipher_encrypt, 0, CTLFLAG_RD, &sc->stats_blkcipher_encrypt,
"Cipher encryption requests submitted"); "Cipher encryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt",
&sc->stats_blkcipher_decrypt, 0, CTLFLAG_RD, &sc->stats_blkcipher_decrypt,
"Cipher decryption requests submitted"); "Cipher decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt",
&sc->stats_eta_encrypt, 0, CTLFLAG_RD, &sc->stats_eta_encrypt,
"Combined AES+HMAC encryption requests submitted"); "Combined AES+HMAC encryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt",
&sc->stats_eta_decrypt, 0, CTLFLAG_RD, &sc->stats_eta_decrypt,
"Combined AES+HMAC decryption requests submitted"); "Combined AES+HMAC decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt",
&sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted"); CTLFLAG_RD, &sc->stats_gcm_encrypt,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD, "AES-GCM encryption requests submitted");
&sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt",
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD, CTLFLAG_RD, &sc->stats_gcm_decrypt,
&sc->stats_ccm_encrypt, 0, "AES-CCM encryption requests submitted"); "AES-GCM decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt",
&sc->stats_ccm_decrypt, 0, "AES-CCM decryption requests submitted"); CTLFLAG_RD, &sc->stats_ccm_encrypt,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, "AES-CCM encryption requests submitted");
&sc->stats_wr_nomem, 0, "Work request memory allocation failures"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt",
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, CTLFLAG_RD, &sc->stats_ccm_decrypt,
&sc->stats_inflight, 0, "Requests currently pending"); "AES-CCM decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
&sc->stats_mac_error, 0, "MAC errors"); &sc->stats_wr_nomem, "Work request memory allocation failures");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
&sc->stats_pad_error, 0, "Padding errors"); &sc->stats_inflight, "Requests currently pending");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
&sc->stats_bad_session, 0, "Requests with invalid session ID"); &sc->stats_mac_error, "MAC errors");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
&sc->stats_sglist_error, 0, &sc->stats_pad_error, "Padding errors");
SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error",
CTLFLAG_RD, &sc->stats_sglist_error,
"Requests for which DMA mapping failed"); "Requests for which DMA mapping failed");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error",
&sc->stats_process_error, 0, "Requests failed during queueing"); CTLFLAG_RD, &sc->stats_process_error,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD, "Requests failed during queueing");
&sc->stats_sw_fallback, 0, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback",
CTLFLAG_RD, &sc->stats_sw_fallback,
"Requests processed by falling back to software"); "Requests processed by falling back to software");
/* /*
@ -2157,12 +2160,25 @@ ccr_attach(device_t dev)
sc->adapter->ccr_softc = sc; sc->adapter->ccr_softc = sc;
mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); mtx_init(&sc->lock, "ccr", NULL, MTX_DEF);
sc->sg_input = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
sc->sg_output = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK);
sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK);
sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_hash = counter_u64_alloc(M_WAITOK);
sc->stats_hmac = counter_u64_alloc(M_WAITOK);
sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK);
sc->stats_inflight = counter_u64_alloc(M_WAITOK);
sc->stats_mac_error = counter_u64_alloc(M_WAITOK);
sc->stats_pad_error = counter_u64_alloc(M_WAITOK);
sc->stats_sglist_error = counter_u64_alloc(M_WAITOK);
sc->stats_process_error = counter_u64_alloc(M_WAITOK);
sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK);
ccr_sysctls(sc); ccr_sysctls(sc);
return (0); return (0);
@ -2182,12 +2198,25 @@ ccr_detach(device_t dev)
crypto_unregister_all(sc->cid); crypto_unregister_all(sc->cid);
mtx_destroy(&sc->lock); mtx_destroy(&sc->lock);
counter_u64_free(sc->stats_blkcipher_encrypt);
counter_u64_free(sc->stats_blkcipher_decrypt);
counter_u64_free(sc->stats_hash);
counter_u64_free(sc->stats_hmac);
counter_u64_free(sc->stats_eta_encrypt);
counter_u64_free(sc->stats_eta_decrypt);
counter_u64_free(sc->stats_gcm_encrypt);
counter_u64_free(sc->stats_gcm_decrypt);
counter_u64_free(sc->stats_ccm_encrypt);
counter_u64_free(sc->stats_ccm_decrypt);
counter_u64_free(sc->stats_wr_nomem);
counter_u64_free(sc->stats_inflight);
counter_u64_free(sc->stats_mac_error);
counter_u64_free(sc->stats_pad_error);
counter_u64_free(sc->stats_sglist_error);
counter_u64_free(sc->stats_process_error);
counter_u64_free(sc->stats_sw_fallback);
sglist_free(sc->sg_iv_aad); sglist_free(sc->sg_iv_aad);
free(sc->iv_aad_buf, M_CCR); free(sc->iv_aad_buf, M_CCR);
sglist_free(sc->sg_dsgl);
sglist_free(sc->sg_ulptx);
sglist_free(sc->sg_output);
sglist_free(sc->sg_input);
sc->adapter->ccr_softc = NULL; sc->adapter->ccr_softc = NULL;
return (0); return (0);
} }
@ -2459,6 +2488,16 @@ ccr_choose_port(struct ccr_softc *sc)
return (best); return (best);
} }
static void
ccr_delete_session(struct ccr_session *s)
{
sglist_free(s->sg_input);
sglist_free(s->sg_output);
sglist_free(s->sg_ulptx);
sglist_free(s->sg_dsgl);
mtx_destroy(&s->lock);
}
static int static int
ccr_newsession(device_t dev, crypto_session_t cses, ccr_newsession(device_t dev, crypto_session_t cses,
const struct crypto_session_params *csp) const struct crypto_session_params *csp)
@ -2547,18 +2586,31 @@ ccr_newsession(device_t dev, crypto_session_t cses,
} }
#endif #endif
s = crypto_get_driver_session(cses);
mtx_init(&s->lock, "ccr session", NULL, MTX_DEF);
s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT);
if (s->sg_input == NULL || s->sg_output == NULL ||
s->sg_ulptx == NULL || s->sg_dsgl == NULL) {
ccr_delete_session(s);
return (ENOMEM);
}
sc = device_get_softc(dev); sc = device_get_softc(dev);
mtx_lock(&sc->lock); mtx_lock(&sc->lock);
if (sc->detaching) { if (sc->detaching) {
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
ccr_delete_session(s);
return (ENXIO); return (ENXIO);
} }
s = crypto_get_driver_session(cses);
s->port = ccr_choose_port(sc); s->port = ccr_choose_port(sc);
if (s->port == NULL) { if (s->port == NULL) {
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
ccr_delete_session(s);
return (ENXIO); return (ENXIO);
} }
@ -2619,7 +2671,6 @@ ccr_newsession(device_t dev, crypto_session_t cses,
csp->csp_cipher_klen); csp->csp_cipher_klen);
} }
s->active = true;
s->port->active_sessions++; s->port->active_sessions++;
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
return (0); return (0);
@ -2633,14 +2684,16 @@ ccr_freesession(device_t dev, crypto_session_t cses)
sc = device_get_softc(dev); sc = device_get_softc(dev);
s = crypto_get_driver_session(cses); s = crypto_get_driver_session(cses);
mtx_lock(&sc->lock); #ifdef INVARIANTS
if (s->pending != 0) if (s->pending != 0)
device_printf(dev, device_printf(dev,
"session %p freed with %d pending requests\n", s, "session %p freed with %d pending requests\n", s,
s->pending); s->pending);
s->active = false; #endif
mtx_lock(&sc->lock);
s->port->active_sessions--; s->port->active_sessions--;
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
ccr_delete_session(s);
} }
static int static int
@ -2655,12 +2708,12 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
s = crypto_get_driver_session(crp->crp_session); s = crypto_get_driver_session(crp->crp_session);
sc = device_get_softc(dev); sc = device_get_softc(dev);
mtx_lock(&sc->lock); mtx_lock(&s->lock);
error = ccr_populate_sglist(sc->sg_input, &crp->crp_buf); error = ccr_populate_sglist(s->sg_input, &crp->crp_buf);
if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = ccr_populate_sglist(sc->sg_output, &crp->crp_obuf); error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf);
if (error) { if (error) {
sc->stats_sglist_error++; counter_u64_add(sc->stats_sglist_error, 1);
goto out; goto out;
} }
@ -2668,7 +2721,7 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
case HASH: case HASH:
error = ccr_hash(sc, s, crp); error = ccr_hash(sc, s, crp);
if (error == 0) if (error == 0)
sc->stats_hash++; counter_u64_add(sc->stats_hash, 1);
break; break;
case HMAC: case HMAC:
if (crp->crp_auth_key != NULL) if (crp->crp_auth_key != NULL)
@ -2677,7 +2730,7 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
csp->csp_auth_klen, s->hmac.pads); csp->csp_auth_klen, s->hmac.pads);
error = ccr_hash(sc, s, crp); error = ccr_hash(sc, s, crp);
if (error == 0) if (error == 0)
sc->stats_hmac++; counter_u64_add(sc->stats_hmac, 1);
break; break;
case BLKCIPHER: case BLKCIPHER:
if (crp->crp_cipher_key != NULL) if (crp->crp_cipher_key != NULL)
@ -2686,9 +2739,9 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
error = ccr_blkcipher(sc, s, crp); error = ccr_blkcipher(sc, s, crp);
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_blkcipher_encrypt++; counter_u64_add(sc->stats_blkcipher_encrypt, 1);
else else
sc->stats_blkcipher_decrypt++; counter_u64_add(sc->stats_blkcipher_decrypt, 1);
} }
break; break;
case ETA: case ETA:
@ -2702,9 +2755,9 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
error = ccr_eta(sc, s, crp); error = ccr_eta(sc, s, crp);
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_eta_encrypt++; counter_u64_add(sc->stats_eta_encrypt, 1);
else else
sc->stats_eta_decrypt++; counter_u64_add(sc->stats_eta_decrypt, 1);
} }
break; break;
case GCM: case GCM:
@ -2715,22 +2768,22 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
csp->csp_cipher_klen); csp->csp_cipher_klen);
} }
if (crp->crp_payload_length == 0) { if (crp->crp_payload_length == 0) {
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
ccr_gcm_soft(s, crp); ccr_gcm_soft(s, crp);
return (0); return (0);
} }
error = ccr_gcm(sc, s, crp); error = ccr_gcm(sc, s, crp);
if (error == EMSGSIZE) { if (error == EMSGSIZE) {
sc->stats_sw_fallback++; counter_u64_add(sc->stats_sw_fallback, 1);
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
ccr_gcm_soft(s, crp); ccr_gcm_soft(s, crp);
return (0); return (0);
} }
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_gcm_encrypt++; counter_u64_add(sc->stats_gcm_encrypt, 1);
else else
sc->stats_gcm_decrypt++; counter_u64_add(sc->stats_gcm_decrypt, 1);
} }
break; break;
case CCM: case CCM:
@ -2740,28 +2793,30 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
} }
error = ccr_ccm(sc, s, crp); error = ccr_ccm(sc, s, crp);
if (error == EMSGSIZE) { if (error == EMSGSIZE) {
sc->stats_sw_fallback++; counter_u64_add(sc->stats_sw_fallback, 1);
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
ccr_ccm_soft(s, crp); ccr_ccm_soft(s, crp);
return (0); return (0);
} }
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_ccm_encrypt++; counter_u64_add(sc->stats_ccm_encrypt, 1);
else else
sc->stats_ccm_decrypt++; counter_u64_add(sc->stats_ccm_decrypt, 1);
} }
break; break;
} }
if (error == 0) { if (error == 0) {
#ifdef INVARIANTS
s->pending++; s->pending++;
sc->stats_inflight++; #endif
counter_u64_add(sc->stats_inflight, 1);
} else } else
sc->stats_process_error++; counter_u64_add(sc->stats_process_error, 1);
out: out:
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
if (error) { if (error) {
crp->crp_etype = error; crp->crp_etype = error;
@ -2795,9 +2850,12 @@ do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
else else
error = 0; error = 0;
mtx_lock(&sc->lock); #ifdef INVARIANTS
mtx_lock(&s->lock);
s->pending--; s->pending--;
sc->stats_inflight--; mtx_unlock(&s->lock);
#endif
counter_u64_add(sc->stats_inflight, -1);
switch (s->mode) { switch (s->mode) {
case HASH: case HASH:
@ -2820,11 +2878,10 @@ do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
if (error == EBADMSG) { if (error == EBADMSG) {
if (CHK_MAC_ERR_BIT(status)) if (CHK_MAC_ERR_BIT(status))
sc->stats_mac_error++; counter_u64_add(sc->stats_mac_error, 1);
if (CHK_PAD_ERR_BIT(status)) if (CHK_PAD_ERR_BIT(status))
sc->stats_pad_error++; counter_u64_add(sc->stats_pad_error, 1);
} }
mtx_unlock(&sc->lock);
crp->crp_etype = error; crp->crp_etype = error;
crypto_done(crp); crypto_done(crp);
m_freem(m); m_freem(m);