diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c index 1063aadb01d1..40f86a396834 100644 --- a/sys/netipsec/xform_ah.c +++ b/sys/netipsec/xform_ah.c @@ -687,7 +687,7 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) /* Crypto operation descriptor. */ crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ - crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_NODELAY; + crp->crp_flags = CRYPTO_F_IMBUF; crp->crp_buf = (caddr_t) m; crp->crp_callback = ah_input_cb; crp->crp_sid = sav->tdb_cryptoid; diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c index f91d3cf54b82..649d2d0b0bc5 100644 --- a/sys/netipsec/xform_esp.c +++ b/sys/netipsec/xform_esp.c @@ -395,7 +395,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) /* Crypto operation descriptor */ crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ - crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_NODELAY; + crp->crp_flags = CRYPTO_F_IMBUF; crp->crp_buf = (caddr_t) m; crp->crp_callback = esp_input_cb; crp->crp_sid = sav->tdb_cryptoid; diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c index 108d356b167f..75b3d7583b7e 100644 --- a/sys/netipsec/xform_ipcomp.c +++ b/sys/netipsec/xform_ipcomp.c @@ -173,7 +173,7 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) /* Crypto operation descriptor */ crp->crp_ilen = m->m_pkthdr.len - (skip + hlen); - crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_NODELAY; + crp->crp_flags = CRYPTO_F_IMBUF; crp->crp_buf = (caddr_t) m; crp->crp_callback = ipcomp_input_cb; crp->crp_sid = sav->tdb_cryptoid; diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index c31b383ce812..7f6c71ca2565 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -655,7 +655,6 @@ int crypto_dispatch(struct cryptop *crp) { u_int32_t hid = SESID2HID(crp->crp_sid); - struct cryptocap *cap; int result; cryptostats.cs_ops++; @@ -665,29 +664,54 @@ crypto_dispatch(struct cryptop *crp) binuptime(&crp->crp_tstamp); #endif - CRYPTO_Q_LOCK(); - cap = crypto_checkdriver(hid); - if (cap && !cap->cc_qblocked) { - result = crypto_invoke(crp, 0); - if (result == ERESTART) { + if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { + struct cryptocap *cap; + /* + * Caller marked the request to be processed + * immediately; dispatch it directly to the + * driver unless the driver is currently blocked. + */ + cap = crypto_checkdriver(hid); + if (cap && !cap->cc_qblocked) { + result = crypto_invoke(crp, 0); + if (result == ERESTART) { + /* + * The driver ran out of resources, mark the + * driver ``blocked'' for cryptop's and put + * the request on the queue. + */ + CRYPTO_Q_LOCK(); + crypto_drivers[hid].cc_qblocked = 1; + TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); + CRYPTO_Q_UNLOCK(); + cryptostats.cs_blocks++; + } + } else { /* - * The driver ran out of resources, mark the - * driver ``blocked'' for cryptop's and put - * the request on the queue. + * The driver is blocked, just queue the op until + * it unblocks and the kernel thread gets kicked. */ - crypto_drivers[hid].cc_qblocked = 1; - TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); - cryptostats.cs_blocks++; + CRYPTO_Q_LOCK(); + TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); + CRYPTO_Q_UNLOCK(); + result = 0; } } else { + int wasempty; /* - * The driver is blocked, just queue the op until - * it unblocks and the kernel thread gets kicked. + * Caller marked the request as ``ok to delay''; + * queue it for the dispatch thread. This is desirable + * when the operation is low priority and/or suitable + * for batching. */ + CRYPTO_Q_LOCK(); + wasempty = TAILQ_EMPTY(&crp_q); TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); + if (wasempty) + wakeup_one(&crp_q); + CRYPTO_Q_UNLOCK(); result = 0; } - CRYPTO_Q_UNLOCK(); return result; } @@ -818,8 +842,6 @@ crypto_invoke(struct cryptop *crp, int hint) if (crypto_timing) crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); #endif - mtx_assert(&crypto_q_mtx, MA_OWNED); - /* Sanity checks. */ if (crp == NULL) return EINVAL; @@ -917,21 +939,45 @@ crypto_getreq(int num) void crypto_done(struct cryptop *crp) { - int wasempty; - if (crp->crp_etype != 0) cryptostats.cs_errs++; #ifdef CRYPTO_TIMING if (crypto_timing) crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); #endif - CRYPTO_RETQ_LOCK(); - wasempty = TAILQ_EMPTY(&crp_ret_q); - TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); + if (crp->crp_flags & CRYPTO_F_CBIMM) { + /* + * Do the callback directly. This is ok when the + * callback routine does very little (e.g. the + * /dev/crypto callback method just does a wakeup). + */ +#ifdef CRYPTO_TIMING + if (crypto_timing) { + /* + * NB: We must copy the timestamp before + * doing the callback as the cryptop is + * likely to be reclaimed. + */ + struct bintime t = crp->crp_tstamp; + crypto_tstat(&cryptostats.cs_cb, &t); + crp->crp_callback(crp); + crypto_tstat(&cryptostats.cs_finis, &t); + } else +#endif + crp->crp_callback(crp); + } else { + int wasempty; + /* + * Normal case; queue the callback for the thread. + */ + CRYPTO_RETQ_LOCK(); + wasempty = TAILQ_EMPTY(&crp_ret_q); + TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); - if (wasempty) - wakeup_one(&crp_ret_q); /* shared wait channel */ - CRYPTO_RETQ_UNLOCK(); + if (wasempty) + wakeup_one(&crp_ret_q); /* shared wait channel */ + CRYPTO_RETQ_UNLOCK(); + } } /* @@ -1043,7 +1089,7 @@ crypto_proc(void) break; } else { submit = crp; - if (submit->crp_flags & CRYPTO_F_NODELAY) + if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) break; /* keep scanning for more are q'd */ } diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c index 8b51d698a6c0..d8487f4b1bde 100644 --- a/sys/opencrypto/cryptodev.c +++ b/sys/opencrypto/cryptodev.c @@ -382,7 +382,8 @@ cryptodev_op( } crp->crp_ilen = cop->len; - crp->crp_flags = CRYPTO_F_IOV; + crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM + | (cop->flags & COP_F_BATCH); crp->crp_buf = (caddr_t)&cse->uio; crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb; crp->crp_sid = cse->sid; diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h index 964f383b3f45..b9d22d5c8d9b 100644 --- a/sys/opencrypto/cryptodev.h +++ b/sys/opencrypto/cryptodev.h @@ -122,7 +122,8 @@ struct crypt_op { u_int16_t op; /* i.e. COP_ENCRYPT */ #define COP_ENCRYPT 1 #define COP_DECRYPT 2 - u_int16_t flags; /* always 0 */ + u_int16_t flags; +#define COP_F_BATCH 0x0008 /* Batch op if possible */ u_int len; caddr_t src, dst; /* become iov[] inside kernel */ caddr_t mac; /* must be big enough for chosen MAC */ @@ -261,7 +262,8 @@ struct cryptop { #define CRYPTO_F_IMBUF 0x0001 /* Input/output are mbuf chains, otherwise contig */ #define CRYPTO_F_IOV 0x0002 /* Input/output are uio */ #define CRYPTO_F_REL 0x0004 /* Must return data in same place */ -#define CRYPTO_F_NODELAY 0x0008 /* Dispatch as quickly as possible */ +#define CRYPTO_F_BATCH 0x0008 /* Batch op if possible */ +#define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */ caddr_t crp_buf; /* Data to be processed */ caddr_t crp_opaque; /* Opaque pointer, passed along */