d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ccp(4): experimental driver for AMD Crypto Co-Processor

Browse Source 
* Registers TRNG source for random(4)
* Finds available queues, LSBs; allocates static objects
* Allocates a shared MSI-X for all queues.  The hardware does not have
  separate interrupts per queue.  Working interrupt mode driver.
* Computes SHA hashes, HMAC.  Passes cryptotest.py, cryptocheck tests.
* Does AES-CBC, CTR mode, and XTS.  cryptotest.py and cryptocheck pass.
* Support for "authenc" (AES + HMAC).  (SHA1 seems to result in
  "unaligned" cleartext inputs from cryptocheck -- which the engine
  cannot handle.  SHA2 seems to work fine.)
* GCM passes for block-multiple AAD, input lengths

Largely based on ccr(4), part of cxgbe(4).

Rough performance averages on AMD Ryzen 1950X (4kB buffer):
aesni:      SHA1: ~8300 Mb/s    SHA256: ~8000 Mb/s
ccp:               ~630 Mb/s    SHA256:  ~660 Mb/s  SHA512:  ~700 Mb/s
cryptosoft:       ~1800 Mb/s    SHA256: ~1800 Mb/s  SHA512: ~2700 Mb/s

As you can see, performance is poor in comparison to aesni(4) and even
cryptosoft (due to high setup cost).  At a larger buffer size (128kB),
throughput is a little better (but still worse than aesni(4)):

aesni:      SHA1:~10400 Mb/s    SHA256: ~9950 Mb/s
ccp:              ~2200 Mb/s    SHA256: ~2600 Mb/s  SHA512: ~3800 Mb/s
cryptosoft:       ~1750 Mb/s    SHA256: ~1800 Mb/s  SHA512: ~2700 Mb/s

AES performance has a similar story:

aesni:      4kB: ~11250 Mb/s    128kB: ~11250 Mb/s
ccp:               ~350 Mb/s    128kB:  ~4600 Mb/s
cryptosoft:       ~1750 Mb/s    128kB:  ~1700 Mb/s

This driver is EXPERIMENTAL.  You should verify cryptographic results on
typical and corner case inputs from your application against a known- good
implementation.

Sponsored by:	Dell EMC Isilon
Differential Revision:	https://reviews.freebsd.org/D12723
This commit is contained in:
Conrad Meyer 2018-01-18 22:01:30 +00:00
parent 024469e429
commit 844d9543dc
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=328150
9 changed files with 3965 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

928
sys/crypto/ccp/ccp.c Normal file
View File

@ -0,0 +1,928 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2017 Chelsio Communications, Inc.
* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
* All rights reserved.
* Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include <sys/types.h>
#include <sys/bus.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/module.h>
#include <sys/random.h>
#include <sys/sglist.h>
#include <sys/sysctl.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <dev/pci/pcivar.h>
#include <dev/random/randomdev.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>
#include "cryptodev_if.h"
#include "ccp.h"
#include "ccp_hardware.h"
MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
/*
* Need a global softc available for garbage random_source API, which lacks any
* context pointer. It's also handy for debugging.
*/
struct ccp_softc *g_ccp_softc;
bool g_debug_print = false;
SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
"Set to enable debugging log messages");
static struct pciid {
uint32_t devid;
const char *desc;
} ccp_ids[] = {
{ 0x14561022, "AMD CCP-5a" },
{ 0x14681022, "AMD CCP-5b" },
};
MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, sizeof(ccp_ids[0]),
nitems(ccp_ids));
static struct random_source random_ccp = {
.rs_ident = "AMD CCP TRNG",
.rs_source = RANDOM_PURE_CCP,
.rs_read = random_ccp_read,
};
/*
* ccp_populate_sglist() generates a scatter/gather list that covers the entire
* crypto operation buffer.
*/
static int
ccp_populate_sglist(struct sglist *sg, struct cryptop *crp)
{
int error;
sglist_reset(sg);
if (crp->crp_flags & CRYPTO_F_IMBUF)
error = sglist_append_mbuf(sg, crp->crp_mbuf);
else if (crp->crp_flags & CRYPTO_F_IOV)
error = sglist_append_uio(sg, crp->crp_uio);
else
error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
return (error);
}
/*
* Handle a GCM request with an empty payload by performing the
* operation in software. Derived from swcr_authenc().
*/
static void
ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp,
struct cryptodesc *crda, struct cryptodesc *crde)
{
struct aes_gmac_ctx gmac_ctx;
char block[GMAC_BLOCK_LEN];
char digest[GMAC_DIGEST_LEN];
char iv[AES_BLOCK_LEN];
int i, len;
/*
* This assumes a 12-byte IV from the crp. See longer comment
* above in ccp_gcm() for more details.
*/
if (crde->crd_flags & CRD_F_ENCRYPT) {
if (crde->crd_flags & CRD_F_IV_EXPLICIT)
memcpy(iv, crde->crd_iv, 12);
else
arc4rand(iv, 12, 0);
} else {
if (crde->crd_flags & CRD_F_IV_EXPLICIT)
memcpy(iv, crde->crd_iv, 12);
else
crypto_copydata(crp->crp_flags, crp->crp_buf,
crde->crd_inject, 12, iv);
}
*(uint32_t *)&iv[12] = htobe32(1);
/* Initialize the MAC. */
AES_GMAC_Init(&gmac_ctx);
AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));
/* MAC the AAD. */
for (i = 0; i < crda->crd_len; i += sizeof(block)) {
len = imin(crda->crd_len - i, sizeof(block));
crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
i, len, block);
bzero(block + len, sizeof(block) - len);
AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
}
/* Length block. */
bzero(block, sizeof(block));
((uint32_t *)block)[1] = htobe32(crda->crd_len * 8);
AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
AES_GMAC_Final(digest, &gmac_ctx);
if (crde->crd_flags & CRD_F_ENCRYPT) {
crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
sizeof(digest), digest);
crp->crp_etype = 0;
} else {
char digest2[GMAC_DIGEST_LEN];
crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
sizeof(digest2), digest2);
if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
crp->crp_etype = 0;
else
crp->crp_etype = EBADMSG;
}
crypto_done(crp);
}
static int
ccp_probe(device_t dev)
{
struct pciid *ip;
uint32_t id;
id = pci_get_devid(dev);
for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
if (id == ip->devid) {
device_set_desc(dev, ip->desc);
return (0);
}
}
return (ENXIO);
}
static void
ccp_initialize_queues(struct ccp_softc *sc)
{
struct ccp_queue *qp;
size_t i;
for (i = 0; i < nitems(sc->queues); i++) {
qp = &sc->queues[i];
qp->cq_softc = sc;
qp->cq_qindex = i;
mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
/* XXX - arbitrarily chosen sizes */
qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
/* Two more SGEs than sg_crp to accommodate ipad. */
qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
}
}
static void
ccp_free_queues(struct ccp_softc *sc)
{
struct ccp_queue *qp;
size_t i;
for (i = 0; i < nitems(sc->queues); i++) {
qp = &sc->queues[i];
mtx_destroy(&qp->cq_lock);
sglist_free(qp->cq_sg_crp);
sglist_free(qp->cq_sg_ulptx);
sglist_free(qp->cq_sg_dst);
}
}
static int
ccp_attach(device_t dev)
{
struct ccp_softc *sc;
int error;
sc = device_get_softc(dev);
sc->dev = dev;
sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
if (sc->cid < 0) {
device_printf(dev, "could not get crypto driver id\n");
return (ENXIO);
}
error = ccp_hw_attach(dev);
if (error != 0)
return (error);
mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
ccp_initialize_queues(sc);
if (g_ccp_softc == NULL) {
g_ccp_softc = sc;
if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
random_source_register(&random_ccp);
}
if ((sc->hw_features & VERSION_CAP_AES) != 0) {
crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
}
if ((sc->hw_features & VERSION_CAP_SHA) != 0) {
crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0);
}
return (0);
}
static int
ccp_detach(device_t dev)
{
struct ccp_softc *sc;
int i;
sc = device_get_softc(dev);
mtx_lock(&sc->lock);
for (i = 0; i < sc->nsessions; i++) {
if (sc->sessions[i].active || sc->sessions[i].pending != 0) {
mtx_unlock(&sc->lock);
return (EBUSY);
}
}
sc->detaching = true;
mtx_unlock(&sc->lock);
crypto_unregister_all(sc->cid);
if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
random_source_deregister(&random_ccp);
ccp_hw_detach(dev);
ccp_free_queues(sc);
if (g_ccp_softc == sc)
g_ccp_softc = NULL;
free(sc->sessions, M_CCP);
mtx_destroy(&sc->lock);
return (0);
}
static void
ccp_init_hmac_digest(struct ccp_session *s, int cri_alg, char *key,
int klen)
{
union authctx auth_ctx;
struct auth_hash *axf;
u_int i;
/*
* If the key is larger than the block size, use the digest of
* the key as the key instead.
*/
axf = s->hmac.auth_hash;
klen /= 8;
if (klen > axf->blocksize) {
axf->Init(&auth_ctx);
axf->Update(&auth_ctx, key, klen);
axf->Final(s->hmac.ipad, &auth_ctx);
explicit_bzero(&auth_ctx, sizeof(auth_ctx));
klen = axf->hashsize;
} else
memcpy(s->hmac.ipad, key, klen);
memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
for (i = 0; i < axf->blocksize; i++) {
s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
s->hmac.opad[i] ^= HMAC_OPAD_VAL;
}
}
static int
ccp_aes_check_keylen(int alg, int klen)
{
switch (klen) {
case 128:
case 192:
if (alg == CRYPTO_AES_XTS)
return (EINVAL);
break;
case 256:
break;
case 512:
if (alg != CRYPTO_AES_XTS)
return (EINVAL);
break;
default:
return (EINVAL);
}
return (0);
}
static void
ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
{
unsigned kbits;
if (alg == CRYPTO_AES_XTS)
kbits = klen / 2;
else
kbits = klen;
switch (kbits) {
case 128:
s->blkcipher.cipher_type = CCP_AES_TYPE_128;
break;
case 192:
s->blkcipher.cipher_type = CCP_AES_TYPE_192;
break;
case 256:
s->blkcipher.cipher_type = CCP_AES_TYPE_256;
break;
default:
panic("should not get here");
}
s->blkcipher.key_len = klen / 8;
memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
}
static int
ccp_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
{
struct ccp_softc *sc;
struct ccp_session *s;
struct auth_hash *auth_hash;
struct cryptoini *c, *hash, *cipher;
enum ccp_aes_mode cipher_mode;
unsigned auth_mode, iv_len;
unsigned partial_digest_len;
unsigned q;
int error, i, sess;
bool gcm_hash;
if (sidp == NULL || cri == NULL)
return (EINVAL);
gcm_hash = false;
cipher = NULL;
hash = NULL;
auth_hash = NULL;
/* XXX reconcile auth_mode with use by ccp_sha */
auth_mode = 0;
cipher_mode = CCP_AES_MODE_ECB;
iv_len = 0;
partial_digest_len = 0;
for (c = cri; c != NULL; c = c->cri_next) {
switch (c->cri_alg) {
case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:
if (hash)
return (EINVAL);
hash = c;
switch (c->cri_alg) {
case CRYPTO_SHA1_HMAC:
auth_hash = &auth_hash_hmac_sha1;
auth_mode = SHA1;
partial_digest_len = SHA1_HASH_LEN;
break;
case CRYPTO_SHA2_256_HMAC:
auth_hash = &auth_hash_hmac_sha2_256;
auth_mode = SHA2_256;
partial_digest_len = SHA2_256_HASH_LEN;
break;
case CRYPTO_SHA2_384_HMAC:
auth_hash = &auth_hash_hmac_sha2_384;
auth_mode = SHA2_384;
partial_digest_len = SHA2_512_HASH_LEN;
break;
case CRYPTO_SHA2_512_HMAC:
auth_hash = &auth_hash_hmac_sha2_512;
auth_mode = SHA2_512;
partial_digest_len = SHA2_512_HASH_LEN;
break;
case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:
gcm_hash = true;
#if 0
auth_mode = CHCR_SCMD_AUTH_MODE_GHASH;
#endif
break;
}
break;
case CRYPTO_AES_CBC:
case CRYPTO_AES_ICM:
case CRYPTO_AES_NIST_GCM_16:
case CRYPTO_AES_XTS:
if (cipher)
return (EINVAL);
cipher = c;
switch (c->cri_alg) {
case CRYPTO_AES_CBC:
cipher_mode = CCP_AES_MODE_CBC;
iv_len = AES_BLOCK_LEN;
break;
case CRYPTO_AES_ICM:
cipher_mode = CCP_AES_MODE_CTR;
iv_len = AES_BLOCK_LEN;
break;
case CRYPTO_AES_NIST_GCM_16:
cipher_mode = CCP_AES_MODE_GCTR;
iv_len = AES_GCM_IV_LEN;
break;
case CRYPTO_AES_XTS:
cipher_mode = CCP_AES_MODE_XTS;
iv_len = AES_BLOCK_LEN;
break;
}
if (c->cri_key != NULL) {
error = ccp_aes_check_keylen(c->cri_alg,
c->cri_klen);
if (error != 0)
return (error);
}
break;
default:
return (EINVAL);
}
}
if (gcm_hash != (cipher_mode == CCP_AES_MODE_GCTR))
return (EINVAL);
if (hash == NULL && cipher == NULL)
return (EINVAL);
if (hash != NULL && hash->cri_key == NULL)
return (EINVAL);
sc = device_get_softc(dev);
mtx_lock(&sc->lock);
if (sc->detaching) {
mtx_unlock(&sc->lock);
return (ENXIO);
}
sess = -1;
for (i = 0; i < sc->nsessions; i++) {
if (!sc->sessions[i].active && sc->sessions[i].pending == 0) {
sess = i;
break;
}
}
if (sess == -1) {
s = malloc(sizeof(*s) * (sc->nsessions + 1), M_CCP,
M_NOWAIT | M_ZERO);
if (s == NULL) {
mtx_unlock(&sc->lock);
return (ENOMEM);
}
if (sc->sessions != NULL)
memcpy(s, sc->sessions, sizeof(*s) * sc->nsessions);
sess = sc->nsessions;
free(sc->sessions, M_CCP);
sc->sessions = s;
sc->nsessions++;
}
s = &sc->sessions[sess];
/* Just grab the first usable queue for now. */
for (q = 0; q < nitems(sc->queues); q++)
if ((sc->valid_queues & (1 << q)) != 0)
break;
if (q == nitems(sc->queues)) {
mtx_unlock(&sc->lock);
return (ENXIO);
}
s->queue = q;
if (gcm_hash)
s->mode = GCM;
else if (hash != NULL && cipher != NULL)
s->mode = AUTHENC;
else if (hash != NULL)
s->mode = HMAC;
else {
MPASS(cipher != NULL);
s->mode = BLKCIPHER;
}
if (gcm_hash) {
if (hash->cri_mlen == 0)
s->gmac.hash_len = AES_GMAC_HASH_LEN;
else
s->gmac.hash_len = hash->cri_mlen;
} else if (hash != NULL) {
s->hmac.auth_hash = auth_hash;
s->hmac.auth_mode = auth_mode;
s->hmac.partial_digest_len = partial_digest_len;
if (hash->cri_mlen == 0)
s->hmac.hash_len = auth_hash->hashsize;
else
s->hmac.hash_len = hash->cri_mlen;
ccp_init_hmac_digest(s, hash->cri_alg, hash->cri_key,
hash->cri_klen);
}
if (cipher != NULL) {
s->blkcipher.cipher_mode = cipher_mode;
s->blkcipher.iv_len = iv_len;
if (cipher->cri_key != NULL)
ccp_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
cipher->cri_klen);
}
s->active = true;
mtx_unlock(&sc->lock);
*sidp = sess;
return (0);
}
static int
ccp_freesession(device_t dev, uint64_t tid)
{
struct ccp_softc *sc;
uint32_t sid;
int error;
sc = device_get_softc(dev);
sid = CRYPTO_SESID2LID(tid);
mtx_lock(&sc->lock);
if (sid >= sc->nsessions || !sc->sessions[sid].active)
error = EINVAL;
else {
if (sc->sessions[sid].pending != 0)
device_printf(dev,
"session %d freed with %d pending requests\n", sid,
sc->sessions[sid].pending);
sc->sessions[sid].active = false;
error = 0;
}
mtx_unlock(&sc->lock);
return (error);
}
static int
ccp_process(device_t dev, struct cryptop *crp, int hint)
{
struct ccp_softc *sc;
struct ccp_queue *qp;
struct ccp_session *s;
struct cryptodesc *crd, *crda, *crde;
uint32_t sid;
int error;
bool qpheld;
qpheld = false;
qp = NULL;
if (crp == NULL)
return (EINVAL);
crd = crp->crp_desc;
sid = CRYPTO_SESID2LID(crp->crp_sid);
sc = device_get_softc(dev);
mtx_lock(&sc->lock);
if (sid >= sc->nsessions || !sc->sessions[sid].active) {
mtx_unlock(&sc->lock);
error = EINVAL;
goto out;
}
s = &sc->sessions[sid];
qp = &sc->queues[s->queue];
mtx_unlock(&sc->lock);
error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
if (error != 0)
goto out;
qpheld = true;
error = ccp_populate_sglist(qp->cq_sg_crp, crp);
if (error != 0)
goto out;
switch (s->mode) {
case HMAC:
if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
ccp_init_hmac_digest(s, crd->crd_alg, crd->crd_key,
crd->crd_klen);
error = ccp_hmac(qp, s, crp);
break;
case BLKCIPHER:
if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
error = ccp_aes_check_keylen(crd->crd_alg,
crd->crd_klen);
if (error != 0)
break;
ccp_aes_setkey(s, crd->crd_alg, crd->crd_key,
crd->crd_klen);
}
error = ccp_blkcipher(qp, s, crp);
break;
case AUTHENC:
error = 0;
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
case CRYPTO_AES_ICM:
case CRYPTO_AES_XTS:
/* Only encrypt-then-authenticate supported. */
crde = crd;
crda = crd->crd_next;
if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
error = EINVAL;
break;
}
s->cipher_first = true;
break;
default:
crda = crd;
crde = crd->crd_next;
if (crde->crd_flags & CRD_F_ENCRYPT) {
error = EINVAL;
break;
}
s->cipher_first = false;
break;
}
if (error != 0)
break;
if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
ccp_init_hmac_digest(s, crda->crd_alg, crda->crd_key,
crda->crd_klen);
if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
error = ccp_aes_check_keylen(crde->crd_alg,
crde->crd_klen);
if (error != 0)
break;
ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
crde->crd_klen);
}
error = ccp_authenc(qp, s, crp, crda, crde);
break;
case GCM:
error = 0;
if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
crde = crd;
crda = crd->crd_next;
s->cipher_first = true;
} else {
crda = crd;
crde = crd->crd_next;
s->cipher_first = false;
}
if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
error = ccp_aes_check_keylen(crde->crd_alg,
crde->crd_klen);
if (error != 0)
break;
ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
crde->crd_klen);
}
if (crde->crd_len == 0) {
mtx_unlock(&qp->cq_lock);
ccp_gcm_soft(s, crp, crda, crde);
return (0);
}
error = ccp_gcm(qp, s, crp, crda, crde);
break;
}
if (error == 0)
s->pending++;
out:
if (qpheld) {
if (error != 0) {
/*
* Squash EAGAIN so callers don't uselessly and
* expensively retry if the ring was full.
*/
if (error == EAGAIN)
error = ENOMEM;
ccp_queue_abort(qp);
} else
ccp_queue_release(qp);
}
if (error != 0) {
DPRINTF(dev, "%s: early error:%d\n", __func__, error);
crp->crp_etype = error;
crypto_done(crp);
}
return (0);
}
static device_method_t ccp_methods[] = {
DEVMETHOD(device_probe, ccp_probe),
DEVMETHOD(device_attach, ccp_attach),
DEVMETHOD(device_detach, ccp_detach),
DEVMETHOD(cryptodev_newsession, ccp_newsession),
DEVMETHOD(cryptodev_freesession, ccp_freesession),
DEVMETHOD(cryptodev_process, ccp_process),
DEVMETHOD_END
};
static driver_t ccp_driver = {
"ccp",
ccp_methods,
sizeof(struct ccp_softc)
};
static devclass_t ccp_devclass;
DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
MODULE_VERSION(ccp, 1);
MODULE_DEPEND(ccp, crypto, 1, 1, 1);
MODULE_DEPEND(ccp, random_device, 1, 1, 1);
static int
ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
{
struct ccp_softc *sc;
mtx_assert(&qp->cq_lock, MA_OWNED);
sc = qp->cq_softc;
if (n < 1 || n >= (1 << sc->ring_size_order))
return (EINVAL);
while (true) {
if (ccp_queue_get_ring_space(qp) >= n)
return (0);
if ((mflags & M_WAITOK) == 0)
return (EAGAIN);
qp->cq_waiting = true;
msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
}
}
int
ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
{
int error;
mtx_lock(&qp->cq_lock);
qp->cq_acq_tail = qp->cq_tail;
error = ccp_queue_reserve_space(qp, n, mflags);
if (error != 0)
mtx_unlock(&qp->cq_lock);
return (error);
}
void
ccp_queue_release(struct ccp_queue *qp)
{
mtx_assert(&qp->cq_lock, MA_OWNED);
if (qp->cq_tail != qp->cq_acq_tail) {
wmb();
ccp_queue_write_tail(qp);
}
mtx_unlock(&qp->cq_lock);
}
void
ccp_queue_abort(struct ccp_queue *qp)
{
unsigned i;
mtx_assert(&qp->cq_lock, MA_OWNED);
/* Wipe out any descriptors associated with this aborted txn. */
for (i = qp->cq_acq_tail; i != qp->cq_tail;
i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
}
qp->cq_tail = qp->cq_acq_tail;
mtx_unlock(&qp->cq_lock);
}
#ifdef DDB
#define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo)
#define db_show_lock(lk) _db_show_lock(&(lk)->lock_object)
static void
db_show_ccp_sc(struct ccp_softc *sc)
{
db_printf("ccp softc at %p\n", sc);
db_printf(" cid: %d\n", (int)sc->cid);
db_printf(" nsessions: %d\n", sc->nsessions);
db_printf(" lock: ");
db_show_lock(&sc->lock);
db_printf(" detaching: %d\n", (int)sc->detaching);
db_printf(" ring_size_order: %u\n", sc->ring_size_order);
db_printf(" hw_version: %d\n", (int)sc->hw_version);
db_printf(" hw_features: %b\n", (int)sc->hw_features,
"\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
"\11SHA\0103DES\07AES");
db_printf(" hw status:\n");
db_ccp_show_hw(sc);
}
static void
db_show_ccp_qp(struct ccp_queue *qp)
{
db_printf(" lock: ");
db_show_lock(&qp->cq_lock);
db_printf(" cq_qindex: %u\n", qp->cq_qindex);
db_printf(" cq_softc: %p\n", qp->cq_softc);
db_printf(" head: %u\n", qp->cq_head);
db_printf(" tail: %u\n", qp->cq_tail);
db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
db_printf(" desc_ring: %p\n", qp->desc_ring);
db_printf(" completions_ring: %p\n", qp->completions_ring);
db_printf(" descriptors (phys): 0x%jx\n",
(uintmax_t)qp->desc_ring_bus_addr);
db_printf(" hw status:\n");
db_ccp_show_queue_hw(qp);
}
DB_SHOW_COMMAND(ccp, db_show_ccp)
{
struct ccp_softc *sc;
unsigned unit, qindex;
if (!have_addr)
goto usage;
unit = (unsigned)addr;
sc = devclass_get_softc(ccp_devclass, unit);
if (sc == NULL) {
db_printf("No such device ccp%u\n", unit);
goto usage;
}
if (count == -1) {
db_show_ccp_sc(sc);
return;
}
qindex = (unsigned)count;
if (qindex >= nitems(sc->queues)) {
db_printf("No such queue %u\n", qindex);
goto usage;
}
db_show_ccp_qp(&sc->queues[qindex]);
return;
usage:
db_printf("usage: show ccp <unit>[,<qindex>]\n");
return;
}
#endif /* DDB */

261
sys/crypto/ccp/ccp.h Normal file
View File

@ -0,0 +1,261 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#pragma once
/*
* Keccak SHAKE128 (if supported by the device?) uses a 1344 bit block.
* SHA3-224 is the next largest block size, at 1152 bits. However, crypto(4)
* doesn't support any SHA3 hash, so SHA2 is the constraint:
*/
#define CCP_HASH_MAX_BLOCK_SIZE (SHA2_512_HMAC_BLOCK_LEN)
#define CCP_AES_MAX_KEY_LEN (AES_XTS_MAX_KEY)
#define CCP_MAX_CRYPTO_IV_LEN 32 /* GCM IV + GHASH context */
#define MAX_HW_QUEUES 5
#define MAX_LSB_REGIONS 8
#ifndef __must_check
#define __must_check __attribute__((__warn_unused_result__))
#endif
/*
* Internal data structures.
*/
enum sha_version {
SHA1,
#if 0
SHA2_224,
#endif
SHA2_256, SHA2_384, SHA2_512
};
struct ccp_session_hmac {
struct auth_hash *auth_hash;
int hash_len;
unsigned int partial_digest_len;
unsigned int auth_mode;
unsigned int mk_size;
char ipad[CCP_HASH_MAX_BLOCK_SIZE];
char opad[CCP_HASH_MAX_BLOCK_SIZE];
};
struct ccp_session_gmac {
int hash_len;
char final_block[GMAC_BLOCK_LEN];
};
struct ccp_session_blkcipher {
unsigned cipher_mode;
unsigned cipher_type;
unsigned key_len;
unsigned iv_len;
char enckey[CCP_AES_MAX_KEY_LEN];
char iv[CCP_MAX_CRYPTO_IV_LEN];
};
struct ccp_session {
bool active : 1;
bool cipher_first : 1;
int pending;
enum { HMAC, BLKCIPHER, AUTHENC, GCM } mode;
unsigned queue;
union {
struct ccp_session_hmac hmac;
struct ccp_session_gmac gmac;
};
struct ccp_session_blkcipher blkcipher;
};
struct ccp_softc;
struct ccp_queue {
struct mtx cq_lock;
unsigned cq_qindex;
struct ccp_softc *cq_softc;
/* Host memory and tracking structures for descriptor ring. */
bus_dma_tag_t ring_desc_tag;
bus_dmamap_t ring_desc_map;
struct ccp_desc *desc_ring;
bus_addr_t desc_ring_bus_addr;
/* Callbacks and arguments ring; indices correspond to above ring. */
struct ccp_completion_ctx *completions_ring;
uint32_t qcontrol; /* Cached register value */
unsigned lsb_mask; /* LSBs available to queue */
int private_lsb; /* Reserved LSB #, or -1 */
unsigned cq_head;
unsigned cq_tail;
unsigned cq_acq_tail;
bool cq_waiting; /* Thread waiting for space */
struct sglist *cq_sg_crp;
struct sglist *cq_sg_ulptx;
struct sglist *cq_sg_dst;
};
struct ccp_completion_ctx {
void (*callback_fn)(struct ccp_queue *qp, struct ccp_session *s,
void *arg, int error);
void *callback_arg;
struct ccp_session *session;
};
struct ccp_softc {
device_t dev;
int32_t cid;
struct ccp_session *sessions;
int nsessions;
struct mtx lock;
bool detaching;
unsigned ring_size_order;
/*
* Each command queue is either public or private. "Private"
* (PSP-only) by default. PSP grants access to some queues to host via
* QMR (Queue Mask Register). Set bits are host accessible.
*/
uint8_t valid_queues;
uint8_t hw_version;
uint8_t num_queues;
uint16_t hw_features;
uint16_t num_lsb_entries;
/* Primary BAR (RID 2) used for register access */
bus_space_tag_t pci_bus_tag;
bus_space_handle_t pci_bus_handle;
int pci_resource_id;
struct resource *pci_resource;
/* Secondary BAR (RID 5) apparently used for MSI-X */
int pci_resource_id_msix;
struct resource *pci_resource_msix;
/* Interrupt resources */
void *intr_tag[2];
struct resource *intr_res[2];
unsigned intr_count;
struct ccp_queue queues[MAX_HW_QUEUES];
};
/* Internal globals */
SYSCTL_DECL(_hw_ccp);
MALLOC_DECLARE(M_CCP);
extern bool g_debug_print;
extern struct ccp_softc *g_ccp_softc;
/*
* Debug macros.
*/
#define DPRINTF(dev, ...) do { \
if (!g_debug_print) \
break; \
if ((dev) != NULL) \
device_printf((dev), "XXX " __VA_ARGS__); \
else \
printf("ccpXXX: " __VA_ARGS__); \
} while (0)
#if 0
#define INSECURE_DEBUG(dev, ...) do { \
if (!g_debug_print) \
break; \
if ((dev) != NULL) \
device_printf((dev), "XXX " __VA_ARGS__); \
else \
printf("ccpXXX: " __VA_ARGS__); \
} while (0)
#else
#define INSECURE_DEBUG(dev, ...)
#endif
/*
* Internal hardware manipulation routines.
*/
int ccp_hw_attach(device_t dev);
void ccp_hw_detach(device_t dev);
void ccp_queue_write_tail(struct ccp_queue *qp);
#ifdef DDB
void db_ccp_show_hw(struct ccp_softc *sc);
void db_ccp_show_queue_hw(struct ccp_queue *qp);
#endif
/*
* Internal hardware crypt-op submission routines.
*/
int ccp_authenc(struct ccp_queue *sc, struct ccp_session *s,
struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde)
__must_check;
int ccp_blkcipher(struct ccp_queue *sc, struct ccp_session *s,
struct cryptop *crp) __must_check;
int ccp_gcm(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp,
struct cryptodesc *crda, struct cryptodesc *crde) __must_check;
int ccp_hmac(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp)
__must_check;
/*
* Internal hardware TRNG read routine.
*/
u_int random_ccp_read(void *v, u_int c);
/* XXX */
int ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
__must_check;
void ccp_queue_abort(struct ccp_queue *qp);
void ccp_queue_release(struct ccp_queue *qp);
/*
* Internal inline routines.
*/
static inline unsigned
ccp_queue_get_active(struct ccp_queue *qp)
{
struct ccp_softc *sc;
sc = qp->cq_softc;
return ((qp->cq_tail - qp->cq_head) & ((1 << sc->ring_size_order) - 1));
}
static inline unsigned
ccp_queue_get_ring_space(struct ccp_queue *qp)
{
struct ccp_softc *sc;
sc = qp->cq_softc;
return ((1 << sc->ring_size_order) - ccp_queue_get_active(qp) - 1);
}

2142
sys/crypto/ccp/ccp_hardware.c Normal file
View File

File diff suppressed because it is too large Load Diff

432
sys/crypto/ccp/ccp_hardware.h Normal file
View File

@ -0,0 +1,432 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#pragma once
#define CMD_QUEUE_MASK_OFFSET 0x000
#define CMD_QUEUE_PRIO_OFFSET 0x004
#define CMD_REQID_CONFIG_OFFSET 0x008
#define TRNG_OUT_OFFSET 0x00C
#define CMD_CMD_TIMEOUT_OFFSET 0x010
#define LSB_PUBLIC_MASK_LO_OFFSET 0x018
#define LSB_PUBLIC_MASK_HI_OFFSET 0x01C
#define LSB_PRIVATE_MASK_LO_OFFSET 0x020
#define LSB_PRIVATE_MASK_HI_OFFSET 0x024
#define VERSION_REG 0x100
#define VERSION_NUM_MASK 0x3F
#define VERSION_CAP_MASK 0x7FC0
#define VERSION_CAP_AES (1 << 6)
#define VERSION_CAP_3DES (1 << 7)
#define VERSION_CAP_SHA (1 << 8)
#define VERSION_CAP_RSA (1 << 9)
#define VERSION_CAP_ECC (1 << 10)
#define VERSION_CAP_ZDE (1 << 11)
#define VERSION_CAP_ZCE (1 << 12)
#define VERSION_CAP_TRNG (1 << 13)
#define VERSION_CAP_ELFC (1 << 14)
#define VERSION_NUMVQM_SHIFT 15
#define VERSION_NUMVQM_MASK 0xF
#define VERSION_LSBSIZE_SHIFT 19
#define VERSION_LSBSIZE_MASK 0x3FF
#define CMD_Q_CONTROL_BASE 0x000
#define CMD_Q_TAIL_LO_BASE 0x004
#define CMD_Q_HEAD_LO_BASE 0x008
#define CMD_Q_INT_ENABLE_BASE 0x00C
#define CMD_Q_INTERRUPT_STATUS_BASE 0x010
#define CMD_Q_STATUS_BASE 0x100
#define CMD_Q_INT_STATUS_BASE 0x104
#define CMD_Q_STATUS_INCR 0x1000
/* Don't think there's much point in keeping these -- OS can't access: */
#define CMD_CONFIG_0_OFFSET 0x6000
#define CMD_TRNG_CTL_OFFSET 0x6008
#define CMD_AES_MASK_OFFSET 0x6010
#define CMD_CLK_GATE_CTL_OFFSET 0x603C
/* CMD_Q_CONTROL_BASE bits */
#define CMD_Q_RUN (1 << 0)
#define CMD_Q_HALTED (1 << 1)
#define CMD_Q_MEM_LOCATION (1 << 2)
#define CMD_Q_SIZE_SHIFT 3
#define CMD_Q_SIZE_MASK 0x1F
#define CMD_Q_PTR_HI_SHIFT 16
#define CMD_Q_PTR_HI_MASK 0xFFFF
/*
* The following bits are used for both CMD_Q_INT_ENABLE_BASE and
* CMD_Q_INTERRUPT_STATUS_BASE.
*/
#define INT_COMPLETION (1 << 0)
#define INT_ERROR (1 << 1)
#define INT_QUEUE_STOPPED (1 << 2)
#define INT_QUEUE_EMPTY (1 << 3)
#define ALL_INTERRUPTS (INT_COMPLETION | \
INT_ERROR | \
INT_QUEUE_STOPPED | \
INT_QUEUE_EMPTY)
#define STATUS_ERROR_MASK 0x3F
#define STATUS_JOBSTATUS_SHIFT 7
#define STATUS_JOBSTATUS_MASK 0x7
#define STATUS_ERRORSOURCE_SHIFT 10
#define STATUS_ERRORSOURCE_MASK 0x3
#define STATUS_VLSB_FAULTBLOCK_SHIFT 12
#define STATUS_VLSB_FAULTBLOCK_MASK 0x7
/* From JOBSTATUS field in STATUS register above */
#define JOBSTATUS_IDLE 0
#define JOBSTATUS_ACTIVE_WAITING 1
#define JOBSTATUS_ACTIVE 2
#define JOBSTATUS_WAIT_ABORT 3
#define JOBSTATUS_DYN_ERROR 4
#define JOBSTATUS_PREPARE_HALT 5
/* From ERRORSOURCE field in STATUS register */
#define ERRORSOURCE_INPUT_MEMORY 0
#define ERRORSOURCE_CMD_DESCRIPTOR 1
#define ERRORSOURCE_INPUT_DATA 2
#define ERRORSOURCE_KEY_DATA 3
#define Q_DESC_SIZE sizeof(struct ccp_desc)
enum ccp_aes_mode {
CCP_AES_MODE_ECB = 0,
CCP_AES_MODE_CBC,
CCP_AES_MODE_OFB,
CCP_AES_MODE_CFB,
CCP_AES_MODE_CTR,
CCP_AES_MODE_CMAC,
CCP_AES_MODE_GHASH,
CCP_AES_MODE_GCTR,
CCP_AES_MODE_IAPM_NIST,
CCP_AES_MODE_IAPM_IPSEC,
/* Not a real hardware mode; used as a sentinel value internally. */
CCP_AES_MODE_XTS,
};
enum ccp_aes_ghash_mode {
CCP_AES_MODE_GHASH_AAD = 0,
CCP_AES_MODE_GHASH_FINAL,
};
enum ccp_aes_type {
CCP_AES_TYPE_128 = 0,
CCP_AES_TYPE_192,
CCP_AES_TYPE_256,
};
enum ccp_des_mode {
CCP_DES_MODE_ECB = 0,
CCP_DES_MODE_CBC,
CCP_DES_MODE_CFB,
};
enum ccp_des_type {
CCP_DES_TYPE_128 = 0, /* 112 + 16 parity */
CCP_DES_TYPE_192, /* 168 + 24 parity */
};
enum ccp_sha_type {
CCP_SHA_TYPE_1 = 1,
CCP_SHA_TYPE_224,
CCP_SHA_TYPE_256,
CCP_SHA_TYPE_384,
CCP_SHA_TYPE_512,
CCP_SHA_TYPE_RSVD1,
CCP_SHA_TYPE_RSVD2,
CCP_SHA3_TYPE_224,
CCP_SHA3_TYPE_256,
CCP_SHA3_TYPE_384,
CCP_SHA3_TYPE_512,
};
enum ccp_cipher_algo {
CCP_CIPHER_ALGO_AES_CBC = 0,
CCP_CIPHER_ALGO_AES_ECB,
CCP_CIPHER_ALGO_AES_CTR,
CCP_CIPHER_ALGO_AES_GCM,
CCP_CIPHER_ALGO_3DES_CBC,
};
enum ccp_cipher_dir {
CCP_CIPHER_DIR_DECRYPT = 0,
CCP_CIPHER_DIR_ENCRYPT = 1,
};
enum ccp_hash_algo {
CCP_AUTH_ALGO_SHA1 = 0,
CCP_AUTH_ALGO_SHA1_HMAC,
CCP_AUTH_ALGO_SHA224,
CCP_AUTH_ALGO_SHA224_HMAC,
CCP_AUTH_ALGO_SHA3_224,
CCP_AUTH_ALGO_SHA3_224_HMAC,
CCP_AUTH_ALGO_SHA256,
CCP_AUTH_ALGO_SHA256_HMAC,
CCP_AUTH_ALGO_SHA3_256,
CCP_AUTH_ALGO_SHA3_256_HMAC,
CCP_AUTH_ALGO_SHA384,
CCP_AUTH_ALGO_SHA384_HMAC,
CCP_AUTH_ALGO_SHA3_384,
CCP_AUTH_ALGO_SHA3_384_HMAC,
CCP_AUTH_ALGO_SHA512,
CCP_AUTH_ALGO_SHA512_HMAC,
CCP_AUTH_ALGO_SHA3_512,
CCP_AUTH_ALGO_SHA3_512_HMAC,
CCP_AUTH_ALGO_AES_CMAC,
CCP_AUTH_ALGO_AES_GCM,
};
enum ccp_hash_op {
CCP_AUTH_OP_GENERATE = 0,
CCP_AUTH_OP_VERIFY = 1,
};
enum ccp_engine {
CCP_ENGINE_AES = 0,
CCP_ENGINE_XTS_AES,
CCP_ENGINE_3DES,
CCP_ENGINE_SHA,
CCP_ENGINE_RSA,
CCP_ENGINE_PASSTHRU,
CCP_ENGINE_ZLIB_DECOMPRESS,
CCP_ENGINE_ECC,
};
enum ccp_xts_unitsize {
CCP_XTS_AES_UNIT_SIZE_16 = 0,
CCP_XTS_AES_UNIT_SIZE_512,
CCP_XTS_AES_UNIT_SIZE_1024,
CCP_XTS_AES_UNIT_SIZE_2048,
CCP_XTS_AES_UNIT_SIZE_4096,
};
enum ccp_passthru_bitwise {
CCP_PASSTHRU_BITWISE_NOOP = 0,
CCP_PASSTHRU_BITWISE_AND,
CCP_PASSTHRU_BITWISE_OR,
CCP_PASSTHRU_BITWISE_XOR,
CCP_PASSTHRU_BITWISE_MASK,
};
enum ccp_passthru_byteswap {
CCP_PASSTHRU_BYTESWAP_NOOP = 0,
CCP_PASSTHRU_BYTESWAP_32BIT,
CCP_PASSTHRU_BYTESWAP_256BIT,
};
/**
* descriptor for version 5 CPP commands
* 8 32-bit words:
* word 0: function; engine; control bits
* word 1: length of source data
* word 2: low 32 bits of source pointer
* word 3: upper 16 bits of source pointer; source memory type
* word 4: low 32 bits of destination pointer
* word 5: upper 16 bits of destination pointer; destination memory
* type
* word 6: low 32 bits of key pointer
* word 7: upper 16 bits of key pointer; key memory type
*/
struct ccp_desc {
union dword0 {
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t size:7;
uint32_t encrypt:1;
uint32_t mode:5;
uint32_t type:2;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_2:7;
} aes;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t size:7;
uint32_t encrypt:1;
uint32_t mode:5;
uint32_t type:2;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_2:7;
} des;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t size:7;
uint32_t encrypt:1;
uint32_t reserved_2:5;
uint32_t type:2;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_3:7;
} aes_xts;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t reserved_2:10;
uint32_t type:4;
uint32_t reserved_3:1;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_4:7;
} sha;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t mode:3;
uint32_t size:12;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_2:7;
} rsa;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t byteswap:2;
uint32_t bitwise:3;
uint32_t reflect:2;
uint32_t reserved_2:8;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_3:7;
} pt;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t reserved_2:13;
uint32_t reserved_3:2;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_4:7;
} zlib;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t size:10;
uint32_t type:2;
uint32_t mode:3;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_2:7;
} ecc;
struct {
uint32_t hoc:1; /* Halt on completion */
uint32_t ioc:1; /* Intr. on completion */
uint32_t reserved_1:1;
uint32_t som:1; /* Start of message */
uint32_t eom:1; /* End " */
uint32_t function:15;
uint32_t engine:4;
uint32_t prot:1;
uint32_t reserved_2:7;
} /* generic */;
};
uint32_t length;
uint32_t src_lo;
struct dword3 {
uint32_t src_hi:16;
uint32_t src_mem:2;
uint32_t lsb_ctx_id:8;
uint32_t reserved_3:5;
uint32_t src_fixed:1;
};
union dword4 {
uint32_t dst_lo; /* NON-SHA */
uint32_t sha_len_lo; /* SHA */
};
union dword5 {
struct {
uint32_t dst_hi:16;
uint32_t dst_mem:2;
uint32_t reserved_4:13;
uint32_t dst_fixed:1;
};
uint32_t sha_len_hi;
};
uint32_t key_lo;
struct dword7 {
uint32_t key_hi:16;
uint32_t key_mem:2;
uint32_t reserved_5:14;
};
};
enum ccp_memtype {
CCP_MEMTYPE_SYSTEM = 0,
CCP_MEMTYPE_SB,
CCP_MEMTYPE_LOCAL,
};
enum ccp_cmd_order {
CCP_CMD_CIPHER = 0,
CCP_CMD_AUTH,
CCP_CMD_CIPHER_HASH,
CCP_CMD_HASH_CIPHER,
CCP_CMD_COMBINED,
CCP_CMD_NOT_SUPPORTED,
};

99
sys/crypto/ccp/ccp_lsb.c Normal file
View File

@ -0,0 +1,99 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/types.h>
#include <sys/bus.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <opencrypto/xform.h>
#include "ccp.h"
#include "ccp_lsb.h"
void
ccp_queue_decode_lsb_regions(struct ccp_softc *sc, uint64_t lsbmask,
unsigned queue)
{
struct ccp_queue *qp;
unsigned i;
qp = &sc->queues[queue];
qp->lsb_mask = 0;
for (i = 0; i < MAX_LSB_REGIONS; i++) {
if (((1 << queue) & lsbmask) != 0)
qp->lsb_mask |= (1 << i);
lsbmask >>= MAX_HW_QUEUES;
}
/*
* Ignore region 0, which has special entries that cannot be used
* generally.
*/
qp->lsb_mask &= ~(1 << 0);
}
/*
* Look for a private LSB for each queue. There are 7 general purpose LSBs
* total and 5 queues. PSP will reserve some of both. Firmware limits some
* queues' access to some LSBs; we hope it is fairly sane and just use a dumb
* greedy algorithm to assign LSBs to queues.
*/
void
ccp_assign_lsb_regions(struct ccp_softc *sc, uint64_t lsbmask)
{
unsigned q, i;
for (q = 0; q < nitems(sc->queues); q++) {
if (((1 << q) & sc->valid_queues) == 0)
continue;
sc->queues[q].private_lsb = -1;
/* Intentionally skip specialized 0th LSB */
for (i = 1; i < MAX_LSB_REGIONS; i++) {
if ((lsbmask &
(1ull << (q + (MAX_HW_QUEUES * i)))) != 0) {
sc->queues[q].private_lsb = i;
lsbmask &= ~(0x1Full << (MAX_HW_QUEUES * i));
break;
}
}
if (i == MAX_LSB_REGIONS) {
device_printf(sc->dev,
"Ignoring queue %u with no private LSB\n", q);
sc->valid_queues &= ~(1 << q);
}
}
}

45
sys/crypto/ccp/ccp_lsb.h Normal file
View File

@ -0,0 +1,45 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#pragma once
#define LSB_ENTRY_SIZE 32 /* bytes, or 256 bits */
#define LSB_REGION_LENGTH 16 /* entries */
/* For now, just statically allocate some LSB entries for specific purposes. */
#define LSB_ENTRY_KEY 0
#define LSB_ENTRY_IV 2
#define LSB_ENTRY_SHA 4
#define LSB_ENTRY_GHASH 6
#define LSB_ENTRY_GHASH_IN 7
void ccp_queue_decode_lsb_regions(struct ccp_softc *sc, uint64_t lsbmask,
unsigned queue);
void ccp_assign_lsb_regions(struct ccp_softc *sc, uint64_t lsbmask);

2
sys/modules/Makefile
View File

@ -79,6 +79,7 @@ SUBDIR= \
cas \
${_cbb} \
cc \
${_ccp} \
cd9660 \
cd9660_iconv \
${_ce} \
@ -574,6 +575,7 @@ _bxe= bxe
.endif
_cardbus= cardbus
_cbb= cbb
_ccp= ccp
_cpuctl= cpuctl
_cpufreq= cpufreq
_cs= cs

21
sys/modules/ccp/Makefile Normal file
View File

@ -0,0 +1,21 @@
# $FreeBSD$
.PATH: ${SRCTOP}/sys/crypto/ccp
KMOD= ccp
SRCS= ccp.c ccp_hardware.c ccp_lsb.c
SRCS+= ccp.h ccp_hardware.h ccp_lsb.h
SRCS+= opt_ddb.h
SRCS+= bus_if.h
SRCS+= device_if.h
SRCS+= cryptodev_if.h
SRCS+= pci_if.h
CFLAGS+= -fms-extensions
CFLAGS.clang+= -Wno-microsoft-anon-tag
MFILES= kern/bus_if.m kern/device_if.m opencrypto/cryptodev_if.m \
dev/pci/pci_if.m
.include <bsd.kmod.mk>

45
tests/sys/opencrypto/cryptotest.py
View File

@ -45,9 +45,9 @@ def katg(base, glob):
assert os.path.exists(os.path.join(katdir, base)), "Please 'pkg install nist-kat'"
return iglob(os.path.join(katdir, base, glob))
aesmodules = [ 'cryptosoft0', 'aesni0', 'ccr0' ]
aesmodules = [ 'cryptosoft0', 'aesni0', 'ccr0', 'ccp0' ]
desmodules = [ 'cryptosoft0', ]
shamodules = [ 'cryptosoft0', 'aesni0', 'ccr0' ]
shamodules = [ 'cryptosoft0', 'aesni0', 'ccr0', 'ccp0' ]
def GenTestCase(cname):
try:
@ -108,13 +108,25 @@ def runGCM(self, fname, mode):
# XXX - isn't supported
continue
c = Crypto(cryptodev.CRYPTO_AES_NIST_GCM_16,
cipherkey,
mac=self._gmacsizes[len(cipherkey)],
mackey=cipherkey, crid=crid)
try:
c = Crypto(cryptodev.CRYPTO_AES_NIST_GCM_16,
cipherkey,
mac=self._gmacsizes[len(cipherkey)],
mackey=cipherkey, crid=crid)
except EnvironmentError, e:
# Can't test algorithms the driver does not support.
if e.errno != errno.EOPNOTSUPP:
raise
continue
if mode == 'ENCRYPT':
rct, rtag = c.encrypt(pt, iv, aad)
try:
rct, rtag = c.encrypt(pt, iv, aad)
except EnvironmentError, e:
# Can't test inputs the driver does not support.
if e.errno != errno.EINVAL:
raise
continue
rtag = rtag[:len(tag)]
data['rct'] = rct.encode('hex')
data['rtag'] = rtag.encode('hex')
@ -128,7 +140,13 @@ def runGCM(self, fname, mode):
self.assertRaises(IOError,
c.decrypt, *args)
else:
rpt, rtag = c.decrypt(*args)
try:
rpt, rtag = c.decrypt(*args)
except EnvironmentError, e:
# Can't test inputs the driver does not support.
if e.errno != errno.EINVAL:
raise
continue
data['rpt'] = rpt.encode('hex')
data['rtag'] = rtag.encode('hex')
self.assertEqual(rpt, pt,
@ -189,8 +207,14 @@ def runXTS(self, fname, meth):
if swapptct:
pt, ct = ct, pt
# run the fun
c = Crypto(meth, cipherkey, crid=crid)
r = curfun(c, pt, iv)
try:
c = Crypto(meth, cipherkey, crid=crid)
r = curfun(c, pt, iv)
except EnvironmentError, e:
# Can't test hashes the driver does not support.
if e.errno != errno.EOPNOTSUPP:
raise
continue
self.assertEqual(r, ct)
###############
@ -309,6 +333,7 @@ def runSHA1HMAC(self, fname):
cryptosoft = GenTestCase('cryptosoft0')
aesni = GenTestCase('aesni0')
ccr = GenTestCase('ccr0')
ccp = GenTestCase('ccp0')
if __name__ == '__main__':
unittest.main()