crypto(9) is called from ipsec in CRYPTO_F_CBIFSYNC mode. This is working
fine when a lot of different flows to be ciphered/deciphered are involved. However, when a software crypto driver is used, there are situations where we could benefit from making crypto(9) multi threaded: - a single flow is to be ciphered: only one thread is used to cipher it, - a single ESP flow is to be deciphered: only one thread is used to decipher it. The idea here is to call crypto(9) using a new mode (CRYPTO_F_ASYNC) to dispatch the crypto jobs on multiple threads, if the underlying crypto driver is working in synchronous mode. Another flag is added (CRYPTO_F_ASYNC_KEEPORDER) to make crypto(9) dispatch the crypto jobs in the order they are received (an additional queue/thread is used), so that the packets are reinjected in the network using the same order they were posted. A new sysctl net.inet.ipsec.async_crypto can be used to activate this new behavior (disabled by default). Submitted by: Emeric Poupon <emeric.poupon@stormshield.eu> Reviewed by: ae, jmg, jhb Differential Revision: https://reviews.freebsd.org/D10680 Sponsored by: Stormshield
This commit is contained in:
parent
ca23f27560
commit
39bbca6ffd
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=325354
@ -331,6 +331,17 @@ Do callback immediately if operation is synchronous (that the driver
|
|||||||
specified the
|
specified the
|
||||||
.Dv CRYPTOCAP_F_SYNC
|
.Dv CRYPTOCAP_F_SYNC
|
||||||
flag).
|
flag).
|
||||||
|
.It Dv CRYPTO_F_ASYNC
|
||||||
|
Try to do the crypto operation in a pool of workers
|
||||||
|
if the operation is synchronous (that the driver specified the
|
||||||
|
.Dv CRYPTOCAP_F_SYNC
|
||||||
|
flag). It aims to speed up processing by dispatching crypto operations
|
||||||
|
on different processors.
|
||||||
|
.It Dv CRYPTO_F_ASYNC_KEEPORDER
|
||||||
|
Dispatch callbacks in the same order they are posted.
|
||||||
|
Only relevant if the
|
||||||
|
.Dv CRYPTO_F_ASYNC
|
||||||
|
flag is set and if the operation is synchronous.
|
||||||
.El
|
.El
|
||||||
.It Va crp_buf
|
.It Va crp_buf
|
||||||
Points to the input buffer.
|
Points to the input buffer.
|
||||||
|
@ -149,6 +149,15 @@ sysctl_def_policy(SYSCTL_HANDLER_ARGS)
|
|||||||
* 0 take anything
|
* 0 take anything
|
||||||
*/
|
*/
|
||||||
VNET_DEFINE(int, crypto_support) = CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE;
|
VNET_DEFINE(int, crypto_support) = CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Use asynchronous mode to parallelize crypto jobs:
|
||||||
|
*
|
||||||
|
* 0 - disabled
|
||||||
|
* 1 - enabled
|
||||||
|
*/
|
||||||
|
VNET_DEFINE(int, async_crypto) = 0;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* TCP/UDP checksum handling policy for transport mode NAT-T (RFC3948)
|
* TCP/UDP checksum handling policy for transport mode NAT-T (RFC3948)
|
||||||
*
|
*
|
||||||
@ -195,6 +204,9 @@ SYSCTL_INT(_net_inet_ipsec, IPSECCTL_ECN, ecn,
|
|||||||
SYSCTL_INT(_net_inet_ipsec, OID_AUTO, crypto_support,
|
SYSCTL_INT(_net_inet_ipsec, OID_AUTO, crypto_support,
|
||||||
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(crypto_support), 0,
|
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(crypto_support), 0,
|
||||||
"Crypto driver selection.");
|
"Crypto driver selection.");
|
||||||
|
SYSCTL_INT(_net_inet_ipsec, OID_AUTO, async_crypto,
|
||||||
|
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0,
|
||||||
|
"Use asynchronous mode to parallelize crypto jobs.");
|
||||||
SYSCTL_INT(_net_inet_ipsec, OID_AUTO, check_policy_history,
|
SYSCTL_INT(_net_inet_ipsec, OID_AUTO, check_policy_history,
|
||||||
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(check_policy_history), 0,
|
CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(check_policy_history), 0,
|
||||||
"Use strict check of inbound packets to security policy compliance.");
|
"Use strict check of inbound packets to security policy compliance.");
|
||||||
|
@ -283,6 +283,7 @@ VNET_DECLARE(int, ip4_ipsec_dfbit);
|
|||||||
VNET_DECLARE(int, ip4_ipsec_ecn);
|
VNET_DECLARE(int, ip4_ipsec_ecn);
|
||||||
VNET_DECLARE(int, ip4_esp_randpad);
|
VNET_DECLARE(int, ip4_esp_randpad);
|
||||||
VNET_DECLARE(int, crypto_support);
|
VNET_DECLARE(int, crypto_support);
|
||||||
|
VNET_DECLARE(int, async_crypto);
|
||||||
VNET_DECLARE(int, natt_cksum_policy);
|
VNET_DECLARE(int, natt_cksum_policy);
|
||||||
|
|
||||||
#define IPSECSTAT_INC(name) \
|
#define IPSECSTAT_INC(name) \
|
||||||
@ -296,6 +297,7 @@ VNET_DECLARE(int, natt_cksum_policy);
|
|||||||
#define V_ip4_ipsec_ecn VNET(ip4_ipsec_ecn)
|
#define V_ip4_ipsec_ecn VNET(ip4_ipsec_ecn)
|
||||||
#define V_ip4_esp_randpad VNET(ip4_esp_randpad)
|
#define V_ip4_esp_randpad VNET(ip4_esp_randpad)
|
||||||
#define V_crypto_support VNET(crypto_support)
|
#define V_crypto_support VNET(crypto_support)
|
||||||
|
#define V_async_crypto VNET(async_crypto)
|
||||||
#define V_natt_cksum_policy VNET(natt_cksum_policy)
|
#define V_natt_cksum_policy VNET(natt_cksum_policy)
|
||||||
|
|
||||||
#define ipseclog(x) do { if (V_ipsec_debug) log x; } while (0)
|
#define ipseclog(x) do { if (V_ipsec_debug) log x; } while (0)
|
||||||
|
@ -657,6 +657,8 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
/* Crypto operation descriptor. */
|
/* Crypto operation descriptor. */
|
||||||
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
|
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
|
||||||
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
||||||
|
if (V_async_crypto)
|
||||||
|
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
|
||||||
crp->crp_buf = (caddr_t) m;
|
crp->crp_buf = (caddr_t) m;
|
||||||
crp->crp_callback = ah_input_cb;
|
crp->crp_callback = ah_input_cb;
|
||||||
crp->crp_sid = cryptoid;
|
crp->crp_sid = cryptoid;
|
||||||
@ -1033,6 +1035,8 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
/* Crypto operation descriptor. */
|
/* Crypto operation descriptor. */
|
||||||
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
|
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
|
||||||
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
||||||
|
if (V_async_crypto)
|
||||||
|
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
|
||||||
crp->crp_buf = (caddr_t) m;
|
crp->crp_buf = (caddr_t) m;
|
||||||
crp->crp_callback = ah_output_cb;
|
crp->crp_callback = ah_output_cb;
|
||||||
crp->crp_sid = cryptoid;
|
crp->crp_sid = cryptoid;
|
||||||
|
@ -385,6 +385,8 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
|
|||||||
/* Crypto operation descriptor */
|
/* Crypto operation descriptor */
|
||||||
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
|
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
|
||||||
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
||||||
|
if (V_async_crypto)
|
||||||
|
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
|
||||||
crp->crp_buf = (caddr_t) m;
|
crp->crp_buf = (caddr_t) m;
|
||||||
crp->crp_callback = esp_input_cb;
|
crp->crp_callback = esp_input_cb;
|
||||||
crp->crp_sid = cryptoid;
|
crp->crp_sid = cryptoid;
|
||||||
@ -841,6 +843,8 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
|
|||||||
/* Crypto operation descriptor. */
|
/* Crypto operation descriptor. */
|
||||||
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
|
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
|
||||||
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
|
||||||
|
if (V_async_crypto)
|
||||||
|
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
|
||||||
crp->crp_buf = (caddr_t) m;
|
crp->crp_buf = (caddr_t) m;
|
||||||
crp->crp_callback = esp_output_cb;
|
crp->crp_callback = esp_output_cb;
|
||||||
crp->crp_opaque = (caddr_t) xd;
|
crp->crp_opaque = (caddr_t) xd;
|
||||||
|
@ -70,7 +70,9 @@ __FBSDID("$FreeBSD$");
|
|||||||
#include <sys/malloc.h>
|
#include <sys/malloc.h>
|
||||||
#include <sys/proc.h>
|
#include <sys/proc.h>
|
||||||
#include <sys/sdt.h>
|
#include <sys/sdt.h>
|
||||||
|
#include <sys/smp.h>
|
||||||
#include <sys/sysctl.h>
|
#include <sys/sysctl.h>
|
||||||
|
#include <sys/taskqueue.h>
|
||||||
|
|
||||||
#include <ddb/ddb.h>
|
#include <ddb/ddb.h>
|
||||||
|
|
||||||
@ -135,26 +137,51 @@ static int crypto_drivers_num = 0;
|
|||||||
* operations.
|
* operations.
|
||||||
*/
|
*/
|
||||||
static int crp_sleep = 0;
|
static int crp_sleep = 0;
|
||||||
static TAILQ_HEAD(,cryptop) crp_q; /* request queues */
|
static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */
|
||||||
static TAILQ_HEAD(,cryptkop) crp_kq;
|
static TAILQ_HEAD(,cryptkop) crp_kq;
|
||||||
static struct mtx crypto_q_mtx;
|
static struct mtx crypto_q_mtx;
|
||||||
#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
|
#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
|
||||||
#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
|
#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* There are two queues for processing completed crypto requests; one
|
* Taskqueue used to dispatch the crypto requests
|
||||||
* for the symmetric and one for the asymmetric ops. We only need one
|
* that have the CRYPTO_F_ASYNC flag
|
||||||
* but have two to avoid type futzing (cryptop vs. cryptkop). A single
|
|
||||||
* mutex is used to lock access to both queues. Note that this lock
|
|
||||||
* must be separate from the lock on request queues to insure driver
|
|
||||||
* callbacks don't generate lock order reversals.
|
|
||||||
*/
|
*/
|
||||||
static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */
|
static struct taskqueue *crypto_tq;
|
||||||
static TAILQ_HEAD(,cryptkop) crp_ret_kq;
|
|
||||||
static struct mtx crypto_ret_q_mtx;
|
/*
|
||||||
#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx)
|
* Crypto seq numbers are operated on with modular arithmetic
|
||||||
#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx)
|
*/
|
||||||
#define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq))
|
#define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0)
|
||||||
|
|
||||||
|
struct crypto_ret_worker {
|
||||||
|
struct mtx crypto_ret_mtx;
|
||||||
|
|
||||||
|
TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
|
||||||
|
TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */
|
||||||
|
TAILQ_HEAD(,cryptkop) crp_ret_kq; /* callback queue for asym jobs */
|
||||||
|
|
||||||
|
u_int32_t reorder_ops; /* total ordered sym jobs received */
|
||||||
|
u_int32_t reorder_cur_seq; /* current sym job dispatched */
|
||||||
|
|
||||||
|
struct proc *cryptoretproc;
|
||||||
|
};
|
||||||
|
static struct crypto_ret_worker *crypto_ret_workers = NULL;
|
||||||
|
|
||||||
|
#define CRYPTO_RETW(i) (&crypto_ret_workers[i])
|
||||||
|
#define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers)
|
||||||
|
#define FOREACH_CRYPTO_RETW(w) \
|
||||||
|
for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
|
||||||
|
|
||||||
|
#define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx)
|
||||||
|
#define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx)
|
||||||
|
#define CRYPTO_RETW_EMPTY(w) \
|
||||||
|
(TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
|
||||||
|
|
||||||
|
static int crypto_workers_num = 0;
|
||||||
|
SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
|
||||||
|
&crypto_workers_num, 0,
|
||||||
|
"Number of crypto workers used to dispatch crypto jobs");
|
||||||
|
|
||||||
static uma_zone_t cryptop_zone;
|
static uma_zone_t cryptop_zone;
|
||||||
static uma_zone_t cryptodesc_zone;
|
static uma_zone_t cryptodesc_zone;
|
||||||
@ -172,11 +199,12 @@ MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
|
|||||||
|
|
||||||
static void crypto_proc(void);
|
static void crypto_proc(void);
|
||||||
static struct proc *cryptoproc;
|
static struct proc *cryptoproc;
|
||||||
static void crypto_ret_proc(void);
|
static void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
|
||||||
static struct proc *cryptoretproc;
|
|
||||||
static void crypto_destroy(void);
|
static void crypto_destroy(void);
|
||||||
static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
|
static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
|
||||||
static int crypto_kinvoke(struct cryptkop *krp, int flags);
|
static int crypto_kinvoke(struct cryptkop *krp, int flags);
|
||||||
|
static void crypto_task_invoke(void *ctx, int pending);
|
||||||
|
static void crypto_batch_enqueue(struct cryptop *crp);
|
||||||
|
|
||||||
static struct cryptostats cryptostats;
|
static struct cryptostats cryptostats;
|
||||||
SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
|
SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
|
||||||
@ -222,6 +250,7 @@ struct keybuf * get_keybuf(void) {
|
|||||||
static int
|
static int
|
||||||
crypto_init(void)
|
crypto_init(void)
|
||||||
{
|
{
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
int error;
|
int error;
|
||||||
|
|
||||||
mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
|
mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
|
||||||
@ -231,10 +260,6 @@ crypto_init(void)
|
|||||||
TAILQ_INIT(&crp_kq);
|
TAILQ_INIT(&crp_kq);
|
||||||
mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
|
mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
|
||||||
|
|
||||||
TAILQ_INIT(&crp_ret_q);
|
|
||||||
TAILQ_INIT(&crp_ret_kq);
|
|
||||||
mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF);
|
|
||||||
|
|
||||||
cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
|
cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
|
||||||
0, 0, 0, 0,
|
0, 0, 0, 0,
|
||||||
UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
|
UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
|
||||||
@ -256,6 +281,20 @@ crypto_init(void)
|
|||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
|
||||||
|
crypto_workers_num = mp_ncpus;
|
||||||
|
|
||||||
|
crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
|
||||||
|
taskqueue_thread_enqueue, &crypto_tq);
|
||||||
|
if (crypto_tq == NULL) {
|
||||||
|
printf("crypto init: cannot setup crypto taskqueue\n");
|
||||||
|
error = ENOMEM;
|
||||||
|
goto bad;
|
||||||
|
}
|
||||||
|
|
||||||
|
taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
|
||||||
|
"crypto");
|
||||||
|
|
||||||
error = kproc_create((void (*)(void *)) crypto_proc, NULL,
|
error = kproc_create((void (*)(void *)) crypto_proc, NULL,
|
||||||
&cryptoproc, 0, 0, "crypto");
|
&cryptoproc, 0, 0, "crypto");
|
||||||
if (error) {
|
if (error) {
|
||||||
@ -264,15 +303,35 @@ crypto_init(void)
|
|||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
|
|
||||||
error = kproc_create((void (*)(void *)) crypto_ret_proc, NULL,
|
crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
|
||||||
&cryptoretproc, 0, 0, "crypto returns");
|
M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
|
||||||
if (error) {
|
if (crypto_ret_workers == NULL) {
|
||||||
printf("crypto_init: cannot start cryptoret thread; error %d",
|
error = ENOMEM;
|
||||||
error);
|
printf("crypto_init: cannot allocate ret workers\n");
|
||||||
goto bad;
|
goto bad;
|
||||||
}
|
}
|
||||||
|
|
||||||
keybuf_init();
|
|
||||||
|
FOREACH_CRYPTO_RETW(ret_worker) {
|
||||||
|
TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
|
||||||
|
TAILQ_INIT(&ret_worker->crp_ret_q);
|
||||||
|
TAILQ_INIT(&ret_worker->crp_ret_kq);
|
||||||
|
|
||||||
|
ret_worker->reorder_ops = 0;
|
||||||
|
ret_worker->reorder_cur_seq = 0;
|
||||||
|
|
||||||
|
mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
|
||||||
|
|
||||||
|
error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
|
||||||
|
&ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
|
||||||
|
if (error) {
|
||||||
|
printf("crypto_init: cannot start cryptoret thread; error %d",
|
||||||
|
error);
|
||||||
|
goto bad;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
keybuf_init();
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
bad:
|
bad:
|
||||||
@ -308,12 +367,17 @@ crypto_terminate(struct proc **pp, void *q)
|
|||||||
static void
|
static void
|
||||||
crypto_destroy(void)
|
crypto_destroy(void)
|
||||||
{
|
{
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Terminate any crypto threads.
|
* Terminate any crypto threads.
|
||||||
*/
|
*/
|
||||||
|
if (crypto_tq != NULL)
|
||||||
|
taskqueue_drain_all(crypto_tq);
|
||||||
CRYPTO_DRIVER_LOCK();
|
CRYPTO_DRIVER_LOCK();
|
||||||
crypto_terminate(&cryptoproc, &crp_q);
|
crypto_terminate(&cryptoproc, &crp_q);
|
||||||
crypto_terminate(&cryptoretproc, &crp_ret_q);
|
FOREACH_CRYPTO_RETW(ret_worker)
|
||||||
|
crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
|
||||||
CRYPTO_DRIVER_UNLOCK();
|
CRYPTO_DRIVER_UNLOCK();
|
||||||
|
|
||||||
/* XXX flush queues??? */
|
/* XXX flush queues??? */
|
||||||
@ -329,7 +393,11 @@ crypto_destroy(void)
|
|||||||
if (cryptop_zone != NULL)
|
if (cryptop_zone != NULL)
|
||||||
uma_zdestroy(cryptop_zone);
|
uma_zdestroy(cryptop_zone);
|
||||||
mtx_destroy(&crypto_q_mtx);
|
mtx_destroy(&crypto_q_mtx);
|
||||||
mtx_destroy(&crypto_ret_q_mtx);
|
FOREACH_CRYPTO_RETW(ret_worker)
|
||||||
|
mtx_destroy(&ret_worker->crypto_ret_mtx);
|
||||||
|
free(crypto_ret_workers, M_CRYPTO_DATA);
|
||||||
|
if (crypto_tq != NULL)
|
||||||
|
taskqueue_free(crypto_tq);
|
||||||
mtx_destroy(&crypto_drivers_mtx);
|
mtx_destroy(&crypto_drivers_mtx);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -828,9 +896,26 @@ crypto_dispatch(struct cryptop *crp)
|
|||||||
binuptime(&crp->crp_tstamp);
|
binuptime(&crp->crp_tstamp);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
hid = CRYPTO_SESID2HID(crp->crp_sid);
|
if (CRYPTOP_ASYNC(crp)) {
|
||||||
|
if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
|
|
||||||
|
crp->crp_retw_id = crp->crp_sid % crypto_workers_num;
|
||||||
|
ret_worker = CRYPTO_RETW(crp->crp_retw_id);
|
||||||
|
|
||||||
|
CRYPTO_RETW_LOCK(ret_worker);
|
||||||
|
crp->crp_seq = ret_worker->reorder_ops++;
|
||||||
|
CRYPTO_RETW_UNLOCK(ret_worker);
|
||||||
|
}
|
||||||
|
|
||||||
|
TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
|
||||||
|
taskqueue_enqueue(crypto_tq, &crp->crp_task);
|
||||||
|
return (0);
|
||||||
|
}
|
||||||
|
|
||||||
if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
|
if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
|
||||||
|
hid = CRYPTO_SESID2HID(crp->crp_sid);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Caller marked the request to be processed
|
* Caller marked the request to be processed
|
||||||
* immediately; dispatch it directly to the
|
* immediately; dispatch it directly to the
|
||||||
@ -849,12 +934,19 @@ crypto_dispatch(struct cryptop *crp)
|
|||||||
*/
|
*/
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
crypto_batch_enqueue(crp);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
crypto_batch_enqueue(struct cryptop *crp)
|
||||||
|
{
|
||||||
|
|
||||||
CRYPTO_Q_LOCK();
|
CRYPTO_Q_LOCK();
|
||||||
TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
|
TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
|
||||||
if (crp_sleep)
|
if (crp_sleep)
|
||||||
wakeup_one(&crp_q);
|
wakeup_one(&crp_q);
|
||||||
CRYPTO_Q_UNLOCK();
|
CRYPTO_Q_UNLOCK();
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -1035,6 +1127,23 @@ crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
static void
|
||||||
|
crypto_task_invoke(void *ctx, int pending)
|
||||||
|
{
|
||||||
|
struct cryptocap *cap;
|
||||||
|
struct cryptop *crp;
|
||||||
|
int hid, result;
|
||||||
|
|
||||||
|
crp = (struct cryptop *)ctx;
|
||||||
|
|
||||||
|
hid = CRYPTO_SESID2HID(crp->crp_sid);
|
||||||
|
cap = crypto_checkdriver(hid);
|
||||||
|
|
||||||
|
result = crypto_invoke(cap, crp, 0);
|
||||||
|
if (result == ERESTART)
|
||||||
|
crypto_batch_enqueue(crp);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Dispatch a crypto request to the appropriate crypto devices.
|
* Dispatch a crypto request to the appropriate crypto devices.
|
||||||
*/
|
*/
|
||||||
@ -1097,6 +1206,7 @@ crypto_freereq(struct cryptop *crp)
|
|||||||
#ifdef DIAGNOSTIC
|
#ifdef DIAGNOSTIC
|
||||||
{
|
{
|
||||||
struct cryptop *crp2;
|
struct cryptop *crp2;
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
|
|
||||||
CRYPTO_Q_LOCK();
|
CRYPTO_Q_LOCK();
|
||||||
TAILQ_FOREACH(crp2, &crp_q, crp_next) {
|
TAILQ_FOREACH(crp2, &crp_q, crp_next) {
|
||||||
@ -1105,13 +1215,16 @@ crypto_freereq(struct cryptop *crp)
|
|||||||
crp));
|
crp));
|
||||||
}
|
}
|
||||||
CRYPTO_Q_UNLOCK();
|
CRYPTO_Q_UNLOCK();
|
||||||
CRYPTO_RETQ_LOCK();
|
|
||||||
TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
|
FOREACH_CRYPTO_RETW(ret_worker) {
|
||||||
KASSERT(crp2 != crp,
|
CRYPTO_RETW_LOCK(ret_worker);
|
||||||
("Freeing cryptop from the return queue (%p).",
|
TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
|
||||||
crp));
|
KASSERT(crp2 != crp,
|
||||||
|
("Freeing cryptop from the return queue (%p).",
|
||||||
|
crp));
|
||||||
|
}
|
||||||
|
CRYPTO_RETW_UNLOCK(ret_worker);
|
||||||
}
|
}
|
||||||
CRYPTO_RETQ_UNLOCK();
|
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -1169,9 +1282,10 @@ crypto_done(struct cryptop *crp)
|
|||||||
* doing extraneous context switches; the latter is mostly
|
* doing extraneous context switches; the latter is mostly
|
||||||
* used with the software crypto driver.
|
* used with the software crypto driver.
|
||||||
*/
|
*/
|
||||||
if ((crp->crp_flags & CRYPTO_F_CBIMM) ||
|
if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
|
||||||
|
((crp->crp_flags & CRYPTO_F_CBIMM) ||
|
||||||
((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
|
((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
|
||||||
(CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) {
|
(CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC)))) {
|
||||||
/*
|
/*
|
||||||
* Do the callback directly. This is ok when the
|
* Do the callback directly. This is ok when the
|
||||||
* callback routine does very little (e.g. the
|
* callback routine does very little (e.g. the
|
||||||
@ -1192,14 +1306,45 @@ crypto_done(struct cryptop *crp)
|
|||||||
#endif
|
#endif
|
||||||
crp->crp_callback(crp);
|
crp->crp_callback(crp);
|
||||||
} else {
|
} else {
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
|
bool wake;
|
||||||
|
|
||||||
|
ret_worker = CRYPTO_RETW(crp->crp_retw_id);
|
||||||
|
wake = false;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Normal case; queue the callback for the thread.
|
* Normal case; queue the callback for the thread.
|
||||||
*/
|
*/
|
||||||
CRYPTO_RETQ_LOCK();
|
CRYPTO_RETW_LOCK(ret_worker);
|
||||||
if (CRYPTO_RETQ_EMPTY())
|
if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
|
||||||
wakeup_one(&crp_ret_q); /* shared wait channel */
|
struct cryptop *tmp;
|
||||||
TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
|
|
||||||
CRYPTO_RETQ_UNLOCK();
|
TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
|
||||||
|
cryptop_q, crp_next) {
|
||||||
|
if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
|
||||||
|
TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
|
||||||
|
tmp, crp, crp_next);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (tmp == NULL) {
|
||||||
|
TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
|
||||||
|
crp, crp_next);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (crp->crp_seq == ret_worker->reorder_cur_seq)
|
||||||
|
wake = true;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
if (CRYPTO_RETW_EMPTY(ret_worker))
|
||||||
|
wake = true;
|
||||||
|
|
||||||
|
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (wake)
|
||||||
|
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
|
||||||
|
CRYPTO_RETW_UNLOCK(ret_worker);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1209,6 +1354,7 @@ crypto_done(struct cryptop *crp)
|
|||||||
void
|
void
|
||||||
crypto_kdone(struct cryptkop *krp)
|
crypto_kdone(struct cryptkop *krp)
|
||||||
{
|
{
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
struct cryptocap *cap;
|
struct cryptocap *cap;
|
||||||
|
|
||||||
if (krp->krp_status != 0)
|
if (krp->krp_status != 0)
|
||||||
@ -1223,11 +1369,14 @@ crypto_kdone(struct cryptkop *krp)
|
|||||||
crypto_remove(cap);
|
crypto_remove(cap);
|
||||||
}
|
}
|
||||||
CRYPTO_DRIVER_UNLOCK();
|
CRYPTO_DRIVER_UNLOCK();
|
||||||
CRYPTO_RETQ_LOCK();
|
|
||||||
if (CRYPTO_RETQ_EMPTY())
|
ret_worker = CRYPTO_RETW(0);
|
||||||
wakeup_one(&crp_ret_q); /* shared wait channel */
|
|
||||||
TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
|
CRYPTO_RETW_LOCK(ret_worker);
|
||||||
CRYPTO_RETQ_UNLOCK();
|
if (CRYPTO_RETW_EMPTY(ret_worker))
|
||||||
|
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
|
||||||
|
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
|
||||||
|
CRYPTO_RETW_UNLOCK(ret_worker);
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
@ -1427,24 +1576,36 @@ crypto_proc(void)
|
|||||||
* callbacks typically are expensive and would slow interrupt handling.
|
* callbacks typically are expensive and would slow interrupt handling.
|
||||||
*/
|
*/
|
||||||
static void
|
static void
|
||||||
crypto_ret_proc(void)
|
crypto_ret_proc(struct crypto_ret_worker *ret_worker)
|
||||||
{
|
{
|
||||||
struct cryptop *crpt;
|
struct cryptop *crpt;
|
||||||
struct cryptkop *krpt;
|
struct cryptkop *krpt;
|
||||||
|
|
||||||
CRYPTO_RETQ_LOCK();
|
CRYPTO_RETW_LOCK(ret_worker);
|
||||||
for (;;) {
|
for (;;) {
|
||||||
/* Harvest return q's for completed ops */
|
/* Harvest return q's for completed ops */
|
||||||
crpt = TAILQ_FIRST(&crp_ret_q);
|
crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
|
||||||
if (crpt != NULL)
|
if (crpt != NULL) {
|
||||||
TAILQ_REMOVE(&crp_ret_q, crpt, crp_next);
|
if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
|
||||||
|
TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
|
||||||
|
ret_worker->reorder_cur_seq++;
|
||||||
|
} else {
|
||||||
|
crpt = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
krpt = TAILQ_FIRST(&crp_ret_kq);
|
if (crpt == NULL) {
|
||||||
|
crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
|
||||||
|
if (crpt != NULL)
|
||||||
|
TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
|
||||||
|
}
|
||||||
|
|
||||||
|
krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
|
||||||
if (krpt != NULL)
|
if (krpt != NULL)
|
||||||
TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next);
|
TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
|
||||||
|
|
||||||
if (crpt != NULL || krpt != NULL) {
|
if (crpt != NULL || krpt != NULL) {
|
||||||
CRYPTO_RETQ_UNLOCK();
|
CRYPTO_RETW_UNLOCK(ret_worker);
|
||||||
/*
|
/*
|
||||||
* Run callbacks unlocked.
|
* Run callbacks unlocked.
|
||||||
*/
|
*/
|
||||||
@ -1466,22 +1627,22 @@ crypto_ret_proc(void)
|
|||||||
}
|
}
|
||||||
if (krpt != NULL)
|
if (krpt != NULL)
|
||||||
krpt->krp_callback(krpt);
|
krpt->krp_callback(krpt);
|
||||||
CRYPTO_RETQ_LOCK();
|
CRYPTO_RETW_LOCK(ret_worker);
|
||||||
} else {
|
} else {
|
||||||
/*
|
/*
|
||||||
* Nothing more to be processed. Sleep until we're
|
* Nothing more to be processed. Sleep until we're
|
||||||
* woken because there are more returns to process.
|
* woken because there are more returns to process.
|
||||||
*/
|
*/
|
||||||
msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT,
|
msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
|
||||||
"crypto_ret_wait", 0);
|
"crypto_ret_wait", 0);
|
||||||
if (cryptoretproc == NULL)
|
if (ret_worker->cryptoretproc == NULL)
|
||||||
break;
|
break;
|
||||||
cryptostats.cs_rets++;
|
cryptostats.cs_rets++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
CRYPTO_RETQ_UNLOCK();
|
CRYPTO_RETW_UNLOCK(ret_worker);
|
||||||
|
|
||||||
crypto_finis(&crp_ret_q);
|
crypto_finis(&ret_worker->crp_ret_q);
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef DDB
|
#ifdef DDB
|
||||||
@ -1516,6 +1677,7 @@ db_show_drivers(void)
|
|||||||
DB_SHOW_COMMAND(crypto, db_show_crypto)
|
DB_SHOW_COMMAND(crypto, db_show_crypto)
|
||||||
{
|
{
|
||||||
struct cryptop *crp;
|
struct cryptop *crp;
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
|
|
||||||
db_show_drivers();
|
db_show_drivers();
|
||||||
db_printf("\n");
|
db_printf("\n");
|
||||||
@ -1534,16 +1696,19 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
|
|||||||
, crp->crp_callback
|
, crp->crp_callback
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!TAILQ_EMPTY(&crp_ret_q)) {
|
FOREACH_CRYPTO_RETW(ret_worker) {
|
||||||
db_printf("\n%4s %4s %4s %8s\n",
|
db_printf("\n%8s %4s %4s %4s %8s\n",
|
||||||
"HID", "Etype", "Flags", "Callback");
|
"ret_worker", "HID", "Etype", "Flags", "Callback");
|
||||||
TAILQ_FOREACH(crp, &crp_ret_q, crp_next) {
|
if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
|
||||||
db_printf("%4u %4u %04x %8p\n"
|
TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
|
||||||
, (int) CRYPTO_SESID2HID(crp->crp_sid)
|
db_printf("%8td %4u %4u %04x %8p\n"
|
||||||
, crp->crp_etype
|
, CRYPTO_RETW_ID(ret_worker)
|
||||||
, crp->crp_flags
|
, (int) CRYPTO_SESID2HID(crp->crp_sid)
|
||||||
, crp->crp_callback
|
, crp->crp_etype
|
||||||
);
|
, crp->crp_flags
|
||||||
|
, crp->crp_callback
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1551,6 +1716,7 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
|
|||||||
DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
|
DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
|
||||||
{
|
{
|
||||||
struct cryptkop *krp;
|
struct cryptkop *krp;
|
||||||
|
struct crypto_ret_worker *ret_worker;
|
||||||
|
|
||||||
db_show_drivers();
|
db_show_drivers();
|
||||||
db_printf("\n");
|
db_printf("\n");
|
||||||
@ -1566,10 +1732,12 @@ DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
|
|||||||
, krp->krp_callback
|
, krp->krp_callback
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!TAILQ_EMPTY(&crp_ret_q)) {
|
|
||||||
|
ret_worker = CRYPTO_RETW(0);
|
||||||
|
if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
|
||||||
db_printf("%4s %5s %8s %4s %8s\n",
|
db_printf("%4s %5s %8s %4s %8s\n",
|
||||||
"Op", "Status", "CRID", "HID", "Callback");
|
"Op", "Status", "CRID", "HID", "Callback");
|
||||||
TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) {
|
TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
|
||||||
db_printf("%4u %5u %08x %4u %8p\n"
|
db_printf("%4u %5u %08x %4u %8p\n"
|
||||||
, krp->krp_op
|
, krp->krp_op
|
||||||
, krp->krp_status
|
, krp->krp_status
|
||||||
|
@ -63,6 +63,7 @@
|
|||||||
#define _CRYPTO_CRYPTO_H_
|
#define _CRYPTO_CRYPTO_H_
|
||||||
|
|
||||||
#include <sys/ioccom.h>
|
#include <sys/ioccom.h>
|
||||||
|
#include <sys/_task.h>
|
||||||
|
|
||||||
/* Some initial values */
|
/* Some initial values */
|
||||||
#define CRYPTO_DRIVERS_INITIAL 4
|
#define CRYPTO_DRIVERS_INITIAL 4
|
||||||
@ -391,6 +392,8 @@ struct cryptodesc {
|
|||||||
struct cryptop {
|
struct cryptop {
|
||||||
TAILQ_ENTRY(cryptop) crp_next;
|
TAILQ_ENTRY(cryptop) crp_next;
|
||||||
|
|
||||||
|
struct task crp_task;
|
||||||
|
|
||||||
u_int64_t crp_sid; /* Session ID */
|
u_int64_t crp_sid; /* Session ID */
|
||||||
int crp_ilen; /* Input data total length */
|
int crp_ilen; /* Input data total length */
|
||||||
int crp_olen; /* Result total length */
|
int crp_olen; /* Result total length */
|
||||||
@ -413,6 +416,14 @@ struct cryptop {
|
|||||||
#define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */
|
#define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */
|
||||||
#define CRYPTO_F_DONE 0x0020 /* Operation completed */
|
#define CRYPTO_F_DONE 0x0020 /* Operation completed */
|
||||||
#define CRYPTO_F_CBIFSYNC 0x0040 /* Do CBIMM if op is synchronous */
|
#define CRYPTO_F_CBIFSYNC 0x0040 /* Do CBIMM if op is synchronous */
|
||||||
|
#define CRYPTO_F_ASYNC 0x0080 /* Dispatch crypto jobs on several threads
|
||||||
|
* if op is synchronous
|
||||||
|
*/
|
||||||
|
#define CRYPTO_F_ASYNC_KEEPORDER 0x0100 /*
|
||||||
|
* Dispatch the crypto jobs in the same
|
||||||
|
* order there are submitted. Applied only
|
||||||
|
* if CRYPTO_F_ASYNC flags is set
|
||||||
|
*/
|
||||||
|
|
||||||
caddr_t crp_buf; /* Data to be processed */
|
caddr_t crp_buf; /* Data to be processed */
|
||||||
caddr_t crp_opaque; /* Opaque pointer, passed along */
|
caddr_t crp_opaque; /* Opaque pointer, passed along */
|
||||||
@ -421,8 +432,20 @@ struct cryptop {
|
|||||||
int (*crp_callback)(struct cryptop *); /* Callback function */
|
int (*crp_callback)(struct cryptop *); /* Callback function */
|
||||||
|
|
||||||
struct bintime crp_tstamp; /* performance time stamp */
|
struct bintime crp_tstamp; /* performance time stamp */
|
||||||
|
uint32_t crp_seq; /* used for ordered dispatch */
|
||||||
|
uint32_t crp_retw_id; /*
|
||||||
|
* the return worker to be used,
|
||||||
|
* used for ordered dispatch
|
||||||
|
*/
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#define CRYPTOP_ASYNC(crp) \
|
||||||
|
(((crp)->crp_flags & CRYPTO_F_ASYNC) && \
|
||||||
|
CRYPTO_SESID2CAPS((crp)->crp_sid) & CRYPTOCAP_F_SYNC)
|
||||||
|
#define CRYPTOP_ASYNC_KEEPORDER(crp) \
|
||||||
|
(CRYPTOP_ASYNC(crp) && \
|
||||||
|
(crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER)
|
||||||
|
|
||||||
#define CRYPTO_BUF_CONTIG 0x0
|
#define CRYPTO_BUF_CONTIG 0x0
|
||||||
#define CRYPTO_BUF_IOV 0x1
|
#define CRYPTO_BUF_IOV 0x1
|
||||||
#define CRYPTO_BUF_MBUF 0x2
|
#define CRYPTO_BUF_MBUF 0x2
|
||||||
|
Loading…
Reference in New Issue
Block a user