o add a CRYPTO_F_CBIMM flag to symmetric ops to indicate the callback

should be done in crypto_done rather than in the callback thread
o use this flag to mark operations from /dev/crypto since the callback
  routine just does a wakeup; this eliminates the last unneeded ctx switch
o change CRYPTO_F_NODELAY to CRYPTO_F_BATCH with an inverted meaning
  so "0" becomes the default/desired setting (needed for user-mode
  compatibility with openbsd)
o change crypto_dispatch to honor CRYPTO_F_BATCH instead of always
  dispatching immediately
o remove uses of CRYPTO_F_NODELAY
o define COP_F_BATCH for ops submitted through /dev/crypto and pass
  this on to the op that is submitted

Similar changes and more eventually coming for asymmetric ops.

MFC if re gives approval.

sys/netipsec/xform_ah.c

@@ -687,7 +687,7 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_NODELAY;
+	crp->crp_flags = CRYPTO_F_IMBUF;
 	crp->crp_buf = (caddr_t) m;
 	crp->crp_callback = ah_input_cb;
 	crp->crp_sid = sav->tdb_cryptoid;

sys/netipsec/xform_esp.c

@@ -395,7 +395,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_NODELAY;
+	crp->crp_flags = CRYPTO_F_IMBUF;
 	crp->crp_buf = (caddr_t) m;
 	crp->crp_callback = esp_input_cb;
 	crp->crp_sid = sav->tdb_cryptoid;

sys/netipsec/xform_ipcomp.c

@@ -173,7 +173,7 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_NODELAY;
+	crp->crp_flags = CRYPTO_F_IMBUF;
 	crp->crp_buf = (caddr_t) m;
 	crp->crp_callback = ipcomp_input_cb;
 	crp->crp_sid = sav->tdb_cryptoid;

sys/opencrypto/crypto.c

@@ -655,7 +655,6 @@ int
 crypto_dispatch(struct cryptop *crp)
 {
 	u_int32_t hid = SESID2HID(crp->crp_sid);
-	struct cryptocap *cap;
 	int result;
 
 	cryptostats.cs_ops++;
@@ -665,29 +664,54 @@ crypto_dispatch(struct cryptop *crp)
 		binuptime(&crp->crp_tstamp);
 #endif
 
-	CRYPTO_Q_LOCK();
+	if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
-	cap = crypto_checkdriver(hid);
+		struct cryptocap *cap;
-	if (cap && !cap->cc_qblocked) {
+		/*
-		result = crypto_invoke(crp, 0);
+		 * Caller marked the request to be processed
-		if (result == ERESTART) {
+		 * immediately; dispatch it directly to the
+		 * driver unless the driver is currently blocked.
+		 */
+		cap = crypto_checkdriver(hid);
+		if (cap && !cap->cc_qblocked) {
+			result = crypto_invoke(crp, 0);
+			if (result == ERESTART) {
+				/*
+				 * The driver ran out of resources, mark the
+				 * driver ``blocked'' for cryptop's and put
+				 * the request on the queue.
+				 */
+				CRYPTO_Q_LOCK();
+				crypto_drivers[hid].cc_qblocked = 1;
+				TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
+				CRYPTO_Q_UNLOCK();
+				cryptostats.cs_blocks++;
+			}
+		} else {
 			/*
-			 * The driver ran out of resources, mark the
+			 * The driver is blocked, just queue the op until
-			 * driver ``blocked'' for cryptop's and put
+			 * it unblocks and the kernel thread gets kicked.
-			 * the request on the queue.
 			 */
-			crypto_drivers[hid].cc_qblocked = 1;
+			CRYPTO_Q_LOCK();
-			TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
+			TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
-			cryptostats.cs_blocks++;
+			CRYPTO_Q_UNLOCK();
+			result = 0;
 		}
 	} else {
+		int wasempty;
 		/*
-		 * The driver is blocked, just queue the op until
+		 * Caller marked the request as ``ok to delay'';
-		 * it unblocks and the kernel thread gets kicked.
+		 * queue it for the dispatch thread.  This is desirable
+		 * when the operation is low priority and/or suitable
+		 * for batching.
 		 */
+		CRYPTO_Q_LOCK();
+		wasempty = TAILQ_EMPTY(&crp_q);
 		TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
+		if (wasempty)
+			wakeup_one(&crp_q);
+		CRYPTO_Q_UNLOCK();
 		result = 0;
 	}
-	CRYPTO_Q_UNLOCK();
 
 	return result;
 }
@@ -818,8 +842,6 @@ crypto_invoke(struct cryptop *crp, int hint)
 	if (crypto_timing)
 		crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
 #endif
-	mtx_assert(&crypto_q_mtx, MA_OWNED);
-
 	/* Sanity checks. */
 	if (crp == NULL)
 		return EINVAL;
@@ -917,21 +939,45 @@ crypto_getreq(int num)
 void
 crypto_done(struct cryptop *crp)
 {
-	int wasempty;
-
 	if (crp->crp_etype != 0)
 		cryptostats.cs_errs++;
 #ifdef CRYPTO_TIMING
 	if (crypto_timing)
 		crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
 #endif
-	CRYPTO_RETQ_LOCK();
+	if (crp->crp_flags & CRYPTO_F_CBIMM) {
-	wasempty = TAILQ_EMPTY(&crp_ret_q);
+		/*
-	TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
+		 * Do the callback directly.  This is ok when the
+		 * callback routine does very little (e.g. the
+		 * /dev/crypto callback method just does a wakeup).
+		 */
+#ifdef CRYPTO_TIMING
+		if (crypto_timing) {
+			/*
+			 * NB: We must copy the timestamp before
+			 * doing the callback as the cryptop is
+			 * likely to be reclaimed.
+			 */
+			struct bintime t = crp->crp_tstamp;
+			crypto_tstat(&cryptostats.cs_cb, &t);
+			crp->crp_callback(crp);
+			crypto_tstat(&cryptostats.cs_finis, &t);
+		} else
+#endif
+			crp->crp_callback(crp);
+	} else {
+		int wasempty;
+		/*
+		 * Normal case; queue the callback for the thread.
+		 */
+		CRYPTO_RETQ_LOCK();
+		wasempty = TAILQ_EMPTY(&crp_ret_q);
+		TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
 
-	if (wasempty)
+		if (wasempty)
-		wakeup_one(&crp_ret_q);		/* shared wait channel */
+			wakeup_one(&crp_ret_q);	/* shared wait channel */
-	CRYPTO_RETQ_UNLOCK();
+		CRYPTO_RETQ_UNLOCK();
+	}
 }
 
 /*
@@ -1043,7 +1089,7 @@ crypto_proc(void)
 					break;
 				} else {
 					submit = crp;
-					if (submit->crp_flags & CRYPTO_F_NODELAY)
+					if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
 						break;
 					/* keep scanning for more are q'd */
 				}

sys/opencrypto/cryptodev.c

@@ -382,7 +382,8 @@ cryptodev_op(
 	}
 
 	crp->crp_ilen = cop->len;
-	crp->crp_flags = CRYPTO_F_IOV;
+	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
+		       | (cop->flags & COP_F_BATCH);
 	crp->crp_buf = (caddr_t)&cse->uio;
 	crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb;
 	crp->crp_sid = cse->sid;

sys/opencrypto/cryptodev.h

@@ -122,7 +122,8 @@ struct crypt_op {
 	u_int16_t	op;		/* i.e. COP_ENCRYPT */
 #define COP_ENCRYPT	1
 #define COP_DECRYPT	2
-	u_int16_t	flags;		/* always 0 */
+	u_int16_t	flags;
+#define	COP_F_BATCH	0x0008		/* Batch op if possible */
 	u_int		len;
 	caddr_t		src, dst;	/* become iov[] inside kernel */
 	caddr_t		mac;		/* must be big enough for chosen MAC */
@@ -261,7 +262,8 @@ struct cryptop {
 #define CRYPTO_F_IMBUF	0x0001	/* Input/output are mbuf chains, otherwise contig */
 #define CRYPTO_F_IOV	0x0002	/* Input/output are uio */
 #define CRYPTO_F_REL	0x0004	/* Must return data in same place */
-#define	CRYPTO_F_NODELAY 0x0008	/* Dispatch as quickly as possible */
+#define	CRYPTO_F_BATCH	0x0008	/* Batch op if possible */
+#define	CRYPTO_F_CBIMM	0x0010	/* Do callback immediately */
 
 	caddr_t		crp_buf;	/* Data to be processed */
 	caddr_t		crp_opaque;	/* Opaque pointer, passed along */