d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
4516c7eed9
freebsd-dev/sys/dev/cesa/cesa.c
John Baldwin 9b6dc28176 Explicitly zero the temporary auth context used to generate HMAC state. 
Reviewed by:	delphij
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D25439
2020-06-25 20:22:44 +00:00

1793 lines
45 KiB
C
Raw Blame History

/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2009-2011 Semihalf.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* CESA SRAM Memory Map:
*
* +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE
* | |
* | DATA |
* | |
* +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0)
* | struct cesa_sa_data |
* +------------------------+
* | struct cesa_sa_hdesc |
* +------------------------+ <= sc->sc_sram_base_va
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/endian.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/rman.h>
#include <machine/bus.h>
#include <machine/intr.h>
#include <machine/resource.h>
#include <machine/fdt.h>
#include <dev/fdt/simplebus.h>
#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include <crypto/sha1.h>
#include <crypto/sha2/sha256.h>
#include <crypto/rijndael/rijndael.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>
#include "cryptodev_if.h"
#include <arm/mv/mvreg.h>
#include <arm/mv/mvvar.h>
#include "cesa.h"
static int cesa_probe(device_t);
static int cesa_attach(device_t);
static int cesa_attach_late(device_t);
static int cesa_detach(device_t);
static void cesa_intr(void *);
static int cesa_probesession(device_t,
const struct crypto_session_params *);
static int cesa_newsession(device_t, crypto_session_t,
const struct crypto_session_params *);
static int cesa_process(device_t, struct cryptop *, int);
static struct resource_spec cesa_res_spec[] = {
{ SYS_RES_MEMORY, 0, RF_ACTIVE },
{ SYS_RES_MEMORY, 1, RF_ACTIVE },
{ SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE },
{ -1, 0 }
};
static device_method_t cesa_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, cesa_probe),
DEVMETHOD(device_attach, cesa_attach),
DEVMETHOD(device_detach, cesa_detach),
/* Crypto device methods */
DEVMETHOD(cryptodev_probesession, cesa_probesession),
DEVMETHOD(cryptodev_newsession, cesa_newsession),
DEVMETHOD(cryptodev_process, cesa_process),
DEVMETHOD_END
};
static driver_t cesa_driver = {
"cesa",
cesa_methods,
sizeof (struct cesa_softc)
};
static devclass_t cesa_devclass;
DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0);
MODULE_DEPEND(cesa, crypto, 1, 1, 1);
static void
cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd)
{
#ifdef DEBUG
device_t dev;
dev = sc->sc_dev;
device_printf(dev, "CESA SA Hardware Descriptor:\n");
device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config);
device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src);
device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst);
device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen);
device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key);
device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv);
device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf);
device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src);
device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst);
device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen);
device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen);
device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in);
device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out);
#endif
}
static void
cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
struct cesa_dma_mem *cdm;
if (error)
return;
KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1."));
cdm = arg;
cdm->cdm_paddr = segs->ds_addr;
}
static int
cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm,
bus_size_t size)
{
int error;
KASSERT(cdm->cdm_vaddr == NULL,
("%s(): DMA memory descriptor in use.", __func__));
error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
PAGE_SIZE, 0, /* alignment, boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filtfunc, filtfuncarg */
size, 1, /* maxsize, nsegments */
size, 0, /* maxsegsz, flags */
NULL, NULL, /* lockfunc, lockfuncarg */
&cdm->cdm_tag); /* dmat */
if (error) {
device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
" %i!\n", error);
goto err1;
}
error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr,
BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map);
if (error) {
device_printf(sc->sc_dev, "failed to allocate DMA safe"
" memory, error %i!\n", error);
goto err2;
}
error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr,
size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT);
if (error) {
device_printf(sc->sc_dev, "cannot get address of the DMA"
" memory, error %i\n", error);
goto err3;
}
return (0);
err3:
bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
err2:
bus_dma_tag_destroy(cdm->cdm_tag);
err1:
cdm->cdm_vaddr = NULL;
return (error);
}
static void
cesa_free_dma_mem(struct cesa_dma_mem *cdm)
{
bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map);
bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
bus_dma_tag_destroy(cdm->cdm_tag);
cdm->cdm_vaddr = NULL;
}
static void
cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op)
{
/* Sync only if dma memory is valid */
if (cdm->cdm_vaddr != NULL)
bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op);
}
static void
cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op)
{
cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op);
cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op);
cesa_sync_dma_mem(&sc->sc_requests_cdm, op);
}
static struct cesa_request *
cesa_alloc_request(struct cesa_softc *sc)
{
struct cesa_request *cr;
CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests);
if (!cr)
return (NULL);
STAILQ_INIT(&cr->cr_tdesc);
STAILQ_INIT(&cr->cr_sdesc);
return (cr);
}
static void
cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr)
{
/* Free TDMA descriptors assigned to this request */
CESA_LOCK(sc, tdesc);
STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc);
CESA_UNLOCK(sc, tdesc);
/* Free SA descriptors assigned to this request */
CESA_LOCK(sc, sdesc);
STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
CESA_UNLOCK(sc, sdesc);
/* Unload DMA memory associated with request */
if (cr->cr_dmap_loaded) {
bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap);
cr->cr_dmap_loaded = 0;
}
CESA_GENERIC_FREE_LOCKED(sc, cr, requests);
}
static void
cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr)
{
CESA_LOCK(sc, requests);
STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq);
CESA_UNLOCK(sc, requests);
}
static struct cesa_tdma_desc *
cesa_alloc_tdesc(struct cesa_softc *sc)
{
struct cesa_tdma_desc *ctd;
CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc);
if (!ctd)
device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. "
"Consider increasing CESA_TDMA_DESCRIPTORS.\n");
return (ctd);
}
static struct cesa_sa_desc *
cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr)
{
struct cesa_sa_desc *csd;
CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
if (!csd) {
device_printf(sc->sc_dev, "SA descriptors pool exhaused. "
"Consider increasing CESA_SA_DESCRIPTORS.\n");
return (NULL);
}
STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
/* Fill-in SA descriptor with default values */
csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
csd->csd_cshd->cshd_enc_src = 0;
csd->csd_cshd->cshd_enc_dst = 0;
csd->csd_cshd->cshd_enc_dlen = 0;
csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
csd->csd_cshd->cshd_mac_src = 0;
csd->csd_cshd->cshd_mac_dlen = 0;
return (csd);
}
static struct cesa_tdma_desc *
cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src,
bus_size_t size)
{
struct cesa_tdma_desc *ctd;
ctd = cesa_alloc_tdesc(sc);
if (!ctd)
return (NULL);
ctd->ctd_cthd->cthd_dst = dst;
ctd->ctd_cthd->cthd_src = src;
ctd->ctd_cthd->cthd_byte_count = size;
/* Handle special control packet */
if (size != 0)
ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED;
else
ctd->ctd_cthd->cthd_flags = 0;
return (ctd);
}
static struct cesa_tdma_desc *
cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
{
return (cesa_tdma_copy(sc, sc->sc_sram_base_pa +
sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr,
sizeof(struct cesa_sa_data)));
}
static struct cesa_tdma_desc *
cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
{
return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa +
sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data)));
}
static struct cesa_tdma_desc *
cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd)
{
return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr,
sizeof(struct cesa_sa_hdesc)));
}
static void
cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd)
{
struct cesa_tdma_desc *ctd_prev;
if (!STAILQ_EMPTY(&cr->cr_tdesc)) {
ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq);
ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr;
}
ctd->ctd_cthd->cthd_next = 0;
STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq);
}
static int
cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
struct cesa_packet *cp, struct cesa_sa_desc *csd)
{
struct cesa_tdma_desc *ctd, *tmp;
/* Copy SA descriptor for this packet */
ctd = cesa_tdma_copy_sdesc(sc, csd);
if (!ctd)
return (ENOMEM);
cesa_append_tdesc(cr, ctd);
/* Copy data to be processed */
STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp)
cesa_append_tdesc(cr, ctd);
STAILQ_INIT(&cp->cp_copyin);
/* Insert control descriptor */
ctd = cesa_tdma_copy(sc, 0, 0, 0);
if (!ctd)
return (ENOMEM);
cesa_append_tdesc(cr, ctd);
/* Copy back results */
STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp)
cesa_append_tdesc(cr, ctd);
STAILQ_INIT(&cp->cp_copyout);
return (0);
}
static void
cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
{
union authctx auth_ctx;
uint32_t *hout;
uint32_t *hin;
int i;
hin = (uint32_t *)cs->cs_hiv_in;
hout = (uint32_t *)cs->cs_hiv_out;
switch (alg) {
case CRYPTO_SHA1_HMAC:
hmac_init_ipad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
memcpy(hin, auth_ctx.sha1ctx.h.b32,
sizeof(auth_ctx.sha1ctx.h.b32));
hmac_init_opad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
memcpy(hout, auth_ctx.sha1ctx.h.b32,
sizeof(auth_ctx.sha1ctx.h.b32));
break;
case CRYPTO_SHA2_256_HMAC:
hmac_init_ipad(&auth_hash_hmac_sha2_256, mkey, mklen,
&auth_ctx);
memcpy(hin, auth_ctx.sha256ctx.state,
sizeof(auth_ctx.sha256ctx.state));
hmac_init_opad(&auth_hash_hmac_sha2_256, mkey, mklen,
&auth_ctx);
memcpy(hout, auth_ctx.sha256ctx.state,
sizeof(auth_ctx.sha256ctx.state));
break;
default:
panic("shouldn't get here");
}
for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
hin[i] = htobe32(hin[i]);
hout[i] = htobe32(hout[i]);
}
explicit_bzero(&auth_ctx, sizeof(auth_ctx));
}
static int
cesa_prep_aes_key(struct cesa_session *cs,
const struct crypto_session_params *csp)
{
uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
uint32_t *dkey;
int i;
rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8);
cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
dkey = (uint32_t *)cs->cs_aes_dkey;
switch (csp->csp_cipher_klen) {
case 16:
cs->cs_config |= CESA_CSH_AES_KLEN_128;
for (i = 0; i < 4; i++)
*dkey++ = htobe32(ek[4 * 10 + i]);
break;
case 24:
cs->cs_config |= CESA_CSH_AES_KLEN_192;
for (i = 0; i < 4; i++)
*dkey++ = htobe32(ek[4 * 12 + i]);
for (i = 0; i < 2; i++)
*dkey++ = htobe32(ek[4 * 11 + 2 + i]);
break;
case 32:
cs->cs_config |= CESA_CSH_AES_KLEN_256;
for (i = 0; i < 4; i++)
*dkey++ = htobe32(ek[4 * 14 + i]);
for (i = 0; i < 4; i++)
*dkey++ = htobe32(ek[4 * 13 + i]);
break;
default:
return (EINVAL);
}
return (0);
}
static void
cesa_start_packet(struct cesa_packet *cp, unsigned int size)
{
cp->cp_size = size;
cp->cp_offset = 0;
STAILQ_INIT(&cp->cp_copyin);
STAILQ_INIT(&cp->cp_copyout);
}
static int
cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp,
bus_dma_segment_t *seg)
{
struct cesa_tdma_desc *ctd;
unsigned int bsize;
/* Calculate size of block copy */
bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset);
if (bsize > 0) {
ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa +
CESA_DATA(cp->cp_offset), seg->ds_addr, bsize);
if (!ctd)
return (-ENOMEM);
STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq);
ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa +
CESA_DATA(cp->cp_offset), bsize);
if (!ctd)
return (-ENOMEM);
STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq);
seg->ds_len -= bsize;
seg->ds_addr += bsize;
cp->cp_offset += bsize;
}
return (bsize);
}
static void
cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
unsigned int mpsize, fragmented;
unsigned int mlen, mskip, tmlen;
struct cesa_chain_info *cci;
unsigned int elen, eskip;
unsigned int skip, len;
struct cesa_sa_desc *csd;
struct cesa_request *cr;
struct cryptop *crp;
struct cesa_softc *sc;
struct cesa_packet cp;
bus_dma_segment_t seg;
uint32_t config;
int size;
cci = arg;
sc = cci->cci_sc;
cr = cci->cci_cr;
crp = cr->cr_crp;
if (error) {
cci->cci_error = error;
return;
}
/*
* Only do a combined op if the AAD is adjacent to the payload
* and the AAD length is a multiple of the IV length. The
* checks against 'config' are to avoid recursing when the
* logic below invokes separate operations.
*/
config = cci->cci_config;
if (((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC ||
(config & CESA_CSHD_OP_MASK) == CESA_CSHD_ENC_AND_MAC) &&
crp->crp_aad_length != 0 &&
(crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) {
/*
* Data alignment in the request does not meet CESA requiremnts
* for combined encryption/decryption and hashing. We have to
* split the request to separate operations and process them
* one by one.
*/
if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
config &= ~CESA_CSHD_OP_MASK;
cci->cci_config = config | CESA_CSHD_MAC;
cesa_create_chain_cb(cci, segs, nseg, 0);
cci->cci_config = config | CESA_CSHD_ENC;
cesa_create_chain_cb(cci, segs, nseg, 0);
} else {
config &= ~CESA_CSHD_OP_MASK;
cci->cci_config = config | CESA_CSHD_ENC;
cesa_create_chain_cb(cci, segs, nseg, 0);
cci->cci_config = config | CESA_CSHD_MAC;
cesa_create_chain_cb(cci, segs, nseg, 0);
}
return;
}
mskip = mlen = eskip = elen = 0;
if (crp->crp_aad_length == 0) {
skip = crp->crp_payload_start;
len = crp->crp_payload_length;
switch (config & CESA_CSHD_OP_MASK) {
case CESA_CSHD_ENC:
eskip = skip;
elen = len;
break;
case CESA_CSHD_MAC:
mskip = skip;
mlen = len;
break;
default:
eskip = skip;
elen = len;
mskip = skip;
mlen = len;
break;
}
} else {
/*
* For an encryption-only separate request, only
* process the payload. For combined requests and
* hash-only requests, process the entire region.
*/
switch (config & CESA_CSHD_OP_MASK) {
case CESA_CSHD_ENC:
skip = crp->crp_payload_start;
len = crp->crp_payload_length;
eskip = skip;
elen = len;
break;
case CESA_CSHD_MAC:
skip = crp->crp_aad_start;
len = crp->crp_aad_length + crp->crp_payload_length;
mskip = skip;
mlen = len;
break;
default:
skip = crp->crp_aad_start;
len = crp->crp_aad_length + crp->crp_payload_length;
mskip = skip;
mlen = len;
eskip = crp->crp_payload_start;
elen = crp->crp_payload_length;
break;
}
}
tmlen = mlen;
fragmented = 0;
mpsize = CESA_MAX_PACKET_SIZE;
mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
/* Start first packet in chain */
cesa_start_packet(&cp, MIN(mpsize, len));
while (nseg-- && len > 0) {
seg = *(segs++);
/*
* Skip data in buffer on which neither ENC nor MAC operation
* is requested.
*/
if (skip > 0) {
size = MIN(skip, seg.ds_len);
skip -= size;
seg.ds_addr += size;
seg.ds_len -= size;
if (eskip > 0)
eskip -= size;
if (mskip > 0)
mskip -= size;
if (seg.ds_len == 0)
continue;
}
while (1) {
/*
* Fill in current packet with data. Break if there is
* no more data in current DMA segment or an error
* occurred.
*/
size = cesa_fill_packet(sc, &cp, &seg);
if (size <= 0) {
error = -size;
break;
}
len -= size;
/* If packet is full, append it to the chain */
if (cp.cp_size == cp.cp_offset) {
csd = cesa_alloc_sdesc(sc, cr);
if (!csd) {
error = ENOMEM;
break;
}
/* Create SA descriptor for this packet */
csd->csd_cshd->cshd_config = cci->cci_config;
csd->csd_cshd->cshd_mac_total_dlen = tmlen;
/*
* Enable fragmentation if request will not fit
* into one packet.
*/
if (len > 0) {
if (!fragmented) {
fragmented = 1;
csd->csd_cshd->cshd_config |=
CESA_CSHD_FRAG_FIRST;
} else
csd->csd_cshd->cshd_config |=
CESA_CSHD_FRAG_MIDDLE;
} else if (fragmented)
csd->csd_cshd->cshd_config |=
CESA_CSHD_FRAG_LAST;
if (eskip < cp.cp_size && elen > 0) {
csd->csd_cshd->cshd_enc_src =
CESA_DATA(eskip);
csd->csd_cshd->cshd_enc_dst =
CESA_DATA(eskip);
csd->csd_cshd->cshd_enc_dlen =
MIN(elen, cp.cp_size - eskip);
}
if (mskip < cp.cp_size && mlen > 0) {
csd->csd_cshd->cshd_mac_src =
CESA_DATA(mskip);
csd->csd_cshd->cshd_mac_dlen =
MIN(mlen, cp.cp_size - mskip);
}
elen -= csd->csd_cshd->cshd_enc_dlen;
eskip -= MIN(eskip, cp.cp_size);
mlen -= csd->csd_cshd->cshd_mac_dlen;
mskip -= MIN(mskip, cp.cp_size);
cesa_dump_cshd(sc, csd->csd_cshd);
/* Append packet to the request */
error = cesa_append_packet(sc, cr, &cp, csd);
if (error)
break;
/* Start a new packet, as current is full */
cesa_start_packet(&cp, MIN(mpsize, len));
}
}
if (error)
break;
}
if (error) {
/*
* Move all allocated resources to the request. They will be
* freed later.
*/
STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin);
STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout);
cci->cci_error = error;
}
}
static int
cesa_create_chain(struct cesa_softc *sc,
const struct crypto_session_params *csp, struct cesa_request *cr)
{
struct cesa_chain_info cci;
struct cesa_tdma_desc *ctd;
uint32_t config;
int error;
error = 0;
CESA_LOCK_ASSERT(sc, sessions);
/* Create request metadata */
if (csp->csp_cipher_klen != 0) {
if (csp->csp_cipher_alg == CRYPTO_AES_CBC &&
!CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
csp->csp_cipher_klen);
else
memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
csp->csp_cipher_klen);
}
if (csp->csp_auth_klen != 0) {
memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
CESA_MAX_HASH_LEN);
memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
CESA_MAX_HASH_LEN);
}
ctd = cesa_tdma_copyin_sa_data(sc, cr);
if (!ctd)
return (ENOMEM);
cesa_append_tdesc(cr, ctd);
/* Prepare SA configuration */
config = cr->cr_cs->cs_config;
if (csp->csp_cipher_alg != 0 &&
!CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
config |= CESA_CSHD_DECRYPT;
switch (csp->csp_mode) {
case CSP_MODE_CIPHER:
config |= CESA_CSHD_ENC;
break;
case CSP_MODE_DIGEST:
config |= CESA_CSHD_MAC;
break;
case CSP_MODE_ETA:
config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
CESA_CSHD_ENC_AND_MAC;
break;
}
/* Create data packets */
cci.cci_sc = sc;
cci.cci_cr = cr;
cci.cci_config = config;
cci.cci_error = 0;
error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp,
cesa_create_chain_cb, &cci, BUS_DMA_NOWAIT);
if (!error)
cr->cr_dmap_loaded = 1;
if (cci.cci_error)
error = cci.cci_error;
if (error)
return (error);
/* Read back request metadata */
ctd = cesa_tdma_copyout_sa_data(sc, cr);
if (!ctd)
return (ENOMEM);
cesa_append_tdesc(cr, ctd);
return (0);
}
static void
cesa_execute(struct cesa_softc *sc)
{
struct cesa_tdma_desc *prev_ctd, *ctd;
struct cesa_request *prev_cr, *cr;
CESA_LOCK(sc, requests);
/*
* If ready list is empty, there is nothing to execute. If queued list
* is not empty, the hardware is busy and we cannot start another
* execution.
*/
if (STAILQ_EMPTY(&sc->sc_ready_requests) ||
!STAILQ_EMPTY(&sc->sc_queued_requests)) {
CESA_UNLOCK(sc, requests);
return;
}
/* Move all ready requests to queued list */
STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests);
STAILQ_INIT(&sc->sc_ready_requests);
/* Create one execution chain from all requests on the list */
if (STAILQ_FIRST(&sc->sc_queued_requests) !=
STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) {
prev_cr = NULL;
cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD |
BUS_DMASYNC_POSTWRITE);
STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
if (prev_cr) {
ctd = STAILQ_FIRST(&cr->cr_tdesc);
prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc,
cesa_tdma_desc, ctd_stq);
prev_ctd->ctd_cthd->cthd_next =
ctd->ctd_cthd_paddr;
}
prev_cr = cr;
}
cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD |
BUS_DMASYNC_PREWRITE);
}
/* Start chain execution in hardware */
cr = STAILQ_FIRST(&sc->sc_queued_requests);
ctd = STAILQ_FIRST(&cr->cr_tdesc);
CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr);
if (sc->sc_soc_id == MV_DEV_88F6828 ||
sc->sc_soc_id == MV_DEV_88F6820 ||
sc->sc_soc_id == MV_DEV_88F6810)
CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2);
else
CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE);
CESA_UNLOCK(sc, requests);
}
static int
cesa_setup_sram(struct cesa_softc *sc)
{
phandle_t sram_node;
ihandle_t sram_ihandle;
pcell_t sram_handle, sram_reg[2];
void *sram_va;
int rv;
rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle",
(void *)&sram_handle, sizeof(sram_handle));
if (rv <= 0)
return (rv);
sram_ihandle = (ihandle_t)sram_handle;
sram_node = OF_instance_to_package(sram_ihandle);
rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg));
if (rv <= 0)
return (rv);
sc->sc_sram_base_pa = sram_reg[0];
/* Store SRAM size to be able to unmap in detach() */
sc->sc_sram_size = sram_reg[1];
if (sc->sc_soc_id != MV_DEV_88F6828 &&
sc->sc_soc_id != MV_DEV_88F6820 &&
sc->sc_soc_id != MV_DEV_88F6810)
return (0);
/* SRAM memory was not mapped in platform_sram_devmap(), map it now */
sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
if (sram_va == NULL)
return (ENOMEM);
sc->sc_sram_base_va = (vm_offset_t)sram_va;
return (0);
}
/*
* Function: device_from_node
* This function returns appropriate device_t to phandle_t
* Parameters:
* root - device where you want to start search
* if you provide NULL here, function will take
* "root0" device as root.
* node - we are checking every device_t to be
* appropriate with this.
*/
static device_t
device_from_node(device_t root, phandle_t node)
{
device_t *children, retval;
int nkid, i;
/* Nothing matches no node */
if (node == -1)
return (NULL);
if (root == NULL)
/* Get root of device tree */
if ((root = device_lookup_by_name("root0")) == NULL)
return (NULL);
if (device_get_children(root, &children, &nkid) != 0)
return (NULL);
retval = NULL;
for (i = 0; i < nkid; i++) {
/* Check if device and node matches */
if (OFW_BUS_GET_NODE(root, children[i]) == node) {
retval = children[i];
break;
}
/* or go deeper */
if ((retval = device_from_node(children[i], node)) != NULL)
break;
}
free(children, M_TEMP);
return (retval);
}
static int
cesa_setup_sram_armada(struct cesa_softc *sc)
{
phandle_t sram_node;
ihandle_t sram_ihandle;
pcell_t sram_handle[2];
void *sram_va;
int rv, j;
struct resource_list rl;
struct resource_list_entry *rle;
struct simplebus_softc *ssc;
device_t sdev;
/* Get refs to SRAMS from CESA node */
rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "marvell,crypto-srams",
(void *)sram_handle, sizeof(sram_handle));
if (rv <= 0)
return (rv);
if (sc->sc_cesa_engine_id >= 2)
return (ENXIO);
/* Get SRAM node on the basis of sc_cesa_engine_id */
sram_ihandle = (ihandle_t)sram_handle[sc->sc_cesa_engine_id];
sram_node = OF_instance_to_package(sram_ihandle);
/* Get device_t of simplebus (sram_node parent) */
sdev = device_from_node(NULL, OF_parent(sram_node));
if (!sdev)
return (ENXIO);
ssc = device_get_softc(sdev);
resource_list_init(&rl);
/* Parse reg property to resource list */
ofw_bus_reg_to_rl(sdev, sram_node, ssc->acells,
ssc->scells, &rl);
/* We expect only one resource */
rle = resource_list_find(&rl, SYS_RES_MEMORY, 0);
if (rle == NULL)
return (ENXIO);
/* Remap through ranges property */
for (j = 0; j < ssc->nranges; j++) {
if (rle->start >= ssc->ranges[j].bus &&
rle->end < ssc->ranges[j].bus + ssc->ranges[j].size) {
rle->start -= ssc->ranges[j].bus;
rle->start += ssc->ranges[j].host;
rle->end -= ssc->ranges[j].bus;
rle->end += ssc->ranges[j].host;
}
}
sc->sc_sram_base_pa = rle->start;
sc->sc_sram_size = rle->count;
/* SRAM memory was not mapped in platform_sram_devmap(), map it now */
sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
if (sram_va == NULL)
return (ENOMEM);
sc->sc_sram_base_va = (vm_offset_t)sram_va;
return (0);
}
struct ofw_compat_data cesa_devices[] = {
{ "mrvl,cesa", (uintptr_t)true },
{ "marvell,armada-38x-crypto", (uintptr_t)true },
{ NULL, 0 }
};
static int
cesa_probe(device_t dev)
{
if (!ofw_bus_status_okay(dev))
return (ENXIO);
if (!ofw_bus_search_compatible(dev, cesa_devices)->ocd_data)
return (ENXIO);
device_set_desc(dev, "Marvell Cryptographic Engine and Security "
"Accelerator");
return (BUS_PROBE_DEFAULT);
}
static int
cesa_attach(device_t dev)
{
static int engine_idx = 0;
struct simplebus_devinfo *ndi;
struct resource_list *rl;
struct cesa_softc *sc;
if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto"))
return (cesa_attach_late(dev));
/*
* Get simplebus_devinfo which contains
* resource list filled with adresses and
* interrupts read form FDT.
* Let's correct it by splitting resources
* for each engine.
*/
if ((ndi = device_get_ivars(dev)) == NULL)
return (ENXIO);
rl = &ndi->rl;
switch (engine_idx) {
case 0:
/* Update regs values */
resource_list_add(rl, SYS_RES_MEMORY, 0, CESA0_TDMA_ADDR,
CESA0_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE);
resource_list_add(rl, SYS_RES_MEMORY, 1, CESA0_CESA_ADDR,
CESA0_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE);
/* Remove unused interrupt */
resource_list_delete(rl, SYS_RES_IRQ, 1);
break;
case 1:
/* Update regs values */
resource_list_add(rl, SYS_RES_MEMORY, 0, CESA1_TDMA_ADDR,
CESA1_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE);
resource_list_add(rl, SYS_RES_MEMORY, 1, CESA1_CESA_ADDR,
CESA1_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE);
/* Remove unused interrupt */
resource_list_delete(rl, SYS_RES_IRQ, 0);
resource_list_find(rl, SYS_RES_IRQ, 1)->rid = 0;
break;
default:
device_printf(dev, "Bad cesa engine_idx\n");
return (ENXIO);
}
sc = device_get_softc(dev);
sc->sc_cesa_engine_id = engine_idx;
/*
* Call simplebus_add_device only once.
* It will create second cesa driver instance
* with the same FDT node as first instance.
* When second driver reach this function,
* it will be configured to use second cesa engine
*/
if (engine_idx == 0)
simplebus_add_device(device_get_parent(dev), ofw_bus_get_node(dev),
0, "cesa", 1, NULL);
engine_idx++;
return (cesa_attach_late(dev));
}
static int
cesa_attach_late(device_t dev)
{
struct cesa_softc *sc;
uint32_t d, r, val;
int error;
int i;
sc = device_get_softc(dev);
sc->sc_blocked = 0;
sc->sc_error = 0;
sc->sc_dev = dev;
soc_id(&d, &r);
switch (d) {
case MV_DEV_88F6281:
case MV_DEV_88F6282:
/* Check if CESA peripheral device has power turned on */
if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) ==
CPU_PM_CTRL_CRYPTO) {
device_printf(dev, "not powered on\n");
return (ENXIO);
}
sc->sc_tperr = 0;
break;
case MV_DEV_88F6828:
case MV_DEV_88F6820:
case MV_DEV_88F6810:
sc->sc_tperr = 0;
break;
case MV_DEV_MV78100:
case MV_DEV_MV78100_Z0:
/* Check if CESA peripheral device has power turned on */
if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) !=
CPU_PM_CTRL_CRYPTO) {
device_printf(dev, "not powered on\n");
return (ENXIO);
}
sc->sc_tperr = CESA_ICR_TPERR;
break;
default:
return (ENXIO);
}
sc->sc_soc_id = d;
/* Initialize mutexes */
mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev),
"CESA Shared Data", MTX_DEF);
mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev),
"CESA TDMA Descriptors Pool", MTX_DEF);
mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev),
"CESA SA Descriptors Pool", MTX_DEF);
mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev),
"CESA Requests Pool", MTX_DEF);
mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
"CESA Sessions Pool", MTX_DEF);
/* Allocate I/O and IRQ resources */
error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res);
if (error) {
device_printf(dev, "could not allocate resources\n");
goto err0;
}
/* Acquire SRAM base address */
if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto"))
error = cesa_setup_sram(sc);
else
error = cesa_setup_sram_armada(sc);
if (error) {
device_printf(dev, "could not setup SRAM\n");
goto err1;
}
/* Setup interrupt handler */
error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET |
INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie));
if (error) {
device_printf(dev, "could not setup engine completion irq\n");
goto err2;
}
/* Create DMA tag for processed data */
error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1, 0, /* alignment, boundary */
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filtfunc, filtfuncarg */
CESA_MAX_REQUEST_SIZE, /* maxsize */
CESA_MAX_FRAGMENTS, /* nsegments */
CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */
NULL, NULL, /* lockfunc, lockfuncarg */
&sc->sc_data_dtag); /* dmat */
if (error)
goto err3;
/* Initialize data structures: TDMA Descriptors Pool */
error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm,
CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc));
if (error)
goto err4;
STAILQ_INIT(&sc->sc_free_tdesc);
for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) {
sc->sc_tdesc[i].ctd_cthd =
(struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i;
sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr +
(i * sizeof(struct cesa_tdma_hdesc));
STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i],
ctd_stq);
}
/* Initialize data structures: SA Descriptors Pool */
error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm,
CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc));
if (error)
goto err5;
STAILQ_INIT(&sc->sc_free_sdesc);
for (i = 0; i < CESA_SA_DESCRIPTORS; i++) {
sc->sc_sdesc[i].csd_cshd =
(struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i;
sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr +
(i * sizeof(struct cesa_sa_hdesc));
STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i],
csd_stq);
}
/* Initialize data structures: Requests Pool */
error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm,
CESA_REQUESTS * sizeof(struct cesa_sa_data));
if (error)
goto err6;
STAILQ_INIT(&sc->sc_free_requests);
STAILQ_INIT(&sc->sc_ready_requests);
STAILQ_INIT(&sc->sc_queued_requests);
for (i = 0; i < CESA_REQUESTS; i++) {
sc->sc_requests[i].cr_csd =
(struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i;
sc->sc_requests[i].cr_csd_paddr =
sc->sc_requests_cdm.cdm_paddr +
(i * sizeof(struct cesa_sa_data));
/* Preallocate DMA maps */
error = bus_dmamap_create(sc->sc_data_dtag, 0,
&sc->sc_requests[i].cr_dmap);
if (error && i > 0) {
i--;
do {
bus_dmamap_destroy(sc->sc_data_dtag,
sc->sc_requests[i].cr_dmap);
} while (i--);
goto err7;
}
STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i],
cr_stq);
}
/*
* Initialize TDMA:
* - Burst limit: 128 bytes,
* - Outstanding reads enabled,
* - No byte-swap.
*/
val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 |
CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE;
if (sc->sc_soc_id == MV_DEV_88F6828 ||
sc->sc_soc_id == MV_DEV_88F6820 ||
sc->sc_soc_id == MV_DEV_88F6810)
val |= CESA_TDMA_NUM_OUTSTAND;
CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val);
/*
* Initialize SA:
* - SA descriptor is present at beginning of CESA SRAM,
* - Multi-packet chain mode,
* - Cooperation with TDMA enabled.
*/
CESA_REG_WRITE(sc, CESA_SA_DPR, 0);
CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA |
CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE);
/* Unmask interrupts */
CESA_REG_WRITE(sc, CESA_ICR, 0);
CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr);
CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS |
CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT |
CESA_TDMA_EMR_DATA_ERROR);
/* Register in OCF */
sc->sc_cid = crypto_get_driverid(dev, sizeof(struct cesa_session),
CRYPTOCAP_F_HARDWARE);
if (sc->sc_cid < 0) {
device_printf(dev, "could not get crypto driver id\n");
goto err8;
}
return (0);
err8:
for (i = 0; i < CESA_REQUESTS; i++)
bus_dmamap_destroy(sc->sc_data_dtag,
sc->sc_requests[i].cr_dmap);
err7:
cesa_free_dma_mem(&sc->sc_requests_cdm);
err6:
cesa_free_dma_mem(&sc->sc_sdesc_cdm);
err5:
cesa_free_dma_mem(&sc->sc_tdesc_cdm);
err4:
bus_dma_tag_destroy(sc->sc_data_dtag);
err3:
bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
err2:
if (sc->sc_soc_id == MV_DEV_88F6828 ||
sc->sc_soc_id == MV_DEV_88F6820 ||
sc->sc_soc_id == MV_DEV_88F6810)
pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
err1:
bus_release_resources(dev, cesa_res_spec, sc->sc_res);
err0:
mtx_destroy(&sc->sc_sessions_lock);
mtx_destroy(&sc->sc_requests_lock);
mtx_destroy(&sc->sc_sdesc_lock);
mtx_destroy(&sc->sc_tdesc_lock);
mtx_destroy(&sc->sc_sc_lock);
return (ENXIO);
}
static int
cesa_detach(device_t dev)
{
struct cesa_softc *sc;
int i;
sc = device_get_softc(dev);
/* TODO: Wait for queued requests completion before shutdown. */
/* Mask interrupts */
CESA_REG_WRITE(sc, CESA_ICM, 0);
CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0);
/* Unregister from OCF */
crypto_unregister_all(sc->sc_cid);
/* Free DMA Maps */
for (i = 0; i < CESA_REQUESTS; i++)
bus_dmamap_destroy(sc->sc_data_dtag,
sc->sc_requests[i].cr_dmap);
/* Free DMA Memory */
cesa_free_dma_mem(&sc->sc_requests_cdm);
cesa_free_dma_mem(&sc->sc_sdesc_cdm);
cesa_free_dma_mem(&sc->sc_tdesc_cdm);
/* Free DMA Tag */
bus_dma_tag_destroy(sc->sc_data_dtag);
/* Stop interrupt */
bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
/* Relase I/O and IRQ resources */
bus_release_resources(dev, cesa_res_spec, sc->sc_res);
/* Unmap SRAM memory */
if (sc->sc_soc_id == MV_DEV_88F6828 ||
sc->sc_soc_id == MV_DEV_88F6820 ||
sc->sc_soc_id == MV_DEV_88F6810)
pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
/* Destroy mutexes */
mtx_destroy(&sc->sc_sessions_lock);
mtx_destroy(&sc->sc_requests_lock);
mtx_destroy(&sc->sc_sdesc_lock);
mtx_destroy(&sc->sc_tdesc_lock);
mtx_destroy(&sc->sc_sc_lock);
return (0);
}
static void
cesa_intr(void *arg)
{
STAILQ_HEAD(, cesa_request) requests;
struct cesa_request *cr, *tmp;
struct cesa_softc *sc;
uint32_t ecr, icr;
uint8_t hash[HASH_MAX_LEN];
int blocked;
sc = arg;
/* Ack interrupt */
ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR);
CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
icr = CESA_REG_READ(sc, CESA_ICR);
CESA_REG_WRITE(sc, CESA_ICR, 0);
/* Check for TDMA errors */
if (ecr & CESA_TDMA_ECR_MISS) {
device_printf(sc->sc_dev, "TDMA Miss error detected!\n");
sc->sc_error = EIO;
}
if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) {
device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n");
sc->sc_error = EIO;
}
if (ecr & CESA_TDMA_ECR_BOTH_HIT) {
device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n");
sc->sc_error = EIO;
}
if (ecr & CESA_TDMA_ECR_DATA_ERROR) {
device_printf(sc->sc_dev, "TDMA Data error detected!\n");
sc->sc_error = EIO;
}
/* Check for CESA errors */
if (icr & sc->sc_tperr) {
device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n");
sc->sc_error = EIO;
}
/* If there is nothing more to do, return */
if ((icr & CESA_ICR_ACCTDMA) == 0)
return;
/* Get all finished requests */
CESA_LOCK(sc, requests);
STAILQ_INIT(&requests);
STAILQ_CONCAT(&requests, &sc->sc_queued_requests);
STAILQ_INIT(&sc->sc_queued_requests);
CESA_UNLOCK(sc, requests);
/* Execute all ready requests */
cesa_execute(sc);
/* Process completed requests */
cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD |
BUS_DMASYNC_POSTWRITE);
STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) {
bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap,
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
cr->cr_crp->crp_etype = sc->sc_error;
if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) {
if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
crypto_copydata(cr->cr_crp,
cr->cr_crp->crp_digest_start,
cr->cr_cs->cs_hlen, hash);
if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash,
cr->cr_cs->cs_hlen) != 0)
cr->cr_crp->crp_etype = EBADMSG;
} else
crypto_copyback(cr->cr_crp,
cr->cr_crp->crp_digest_start,
cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
}
crypto_done(cr->cr_crp);
cesa_free_request(sc, cr);
}
cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD |
BUS_DMASYNC_PREWRITE);
sc->sc_error = 0;
/* Unblock driver if it ran out of resources */
CESA_LOCK(sc, sc);
blocked = sc->sc_blocked;
sc->sc_blocked = 0;
CESA_UNLOCK(sc, sc);
if (blocked)
crypto_unblock(sc->sc_cid, blocked);
}
static bool
cesa_cipher_supported(const struct crypto_session_params *csp)
{
switch (csp->csp_cipher_alg) {
case CRYPTO_AES_CBC:
if (csp->csp_ivlen != AES_BLOCK_LEN)
return (false);
break;
default:
return (false);
}
if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN)
return (false);
return (true);
}
static bool
cesa_auth_supported(struct cesa_softc *sc,
const struct crypto_session_params *csp)
{
switch (csp->csp_auth_alg) {
case CRYPTO_SHA2_256_HMAC:
if (!(sc->sc_soc_id == MV_DEV_88F6828 ||
sc->sc_soc_id == MV_DEV_88F6820 ||
sc->sc_soc_id == MV_DEV_88F6810))
return (false);
/* FALLTHROUGH */
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
break;
default:
return (false);
}
if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN)
return (false);
return (true);
}
static int
cesa_probesession(device_t dev, const struct crypto_session_params *csp)
{
struct cesa_softc *sc;
sc = device_get_softc(dev);
if (csp->csp_flags != 0)
return (EINVAL);
switch (csp->csp_mode) {
case CSP_MODE_DIGEST:
if (!cesa_auth_supported(sc, csp))
return (EINVAL);
break;
case CSP_MODE_CIPHER:
if (!cesa_cipher_supported(csp))
return (EINVAL);
break;
case CSP_MODE_ETA:
if (!cesa_auth_supported(sc, csp) ||
!cesa_cipher_supported(csp))
return (EINVAL);
break;
default:
return (EINVAL);
}
return (CRYPTODEV_PROBE_HARDWARE);
}
static int
cesa_newsession(device_t dev, crypto_session_t cses,
const struct crypto_session_params *csp)
{
struct cesa_session *cs;
struct cesa_softc *sc;
int error;
sc = device_get_softc(dev);
error = 0;
/* Allocate session */
cs = crypto_get_driver_session(cses);
/* Prepare CESA configuration */
cs->cs_config = 0;
cs->cs_ivlen = 1;
cs->cs_mblen = 1;
switch (csp->csp_cipher_alg) {
case CRYPTO_AES_CBC:
cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
cs->cs_ivlen = AES_BLOCK_LEN;
break;
}
switch (csp->csp_auth_alg) {
case CRYPTO_SHA1:
cs->cs_mblen = 1;
cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
csp->csp_auth_mlen;
cs->cs_config |= CESA_CSHD_SHA1;
break;
case CRYPTO_SHA1_HMAC:
cs->cs_mblen = SHA1_BLOCK_LEN;
cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
csp->csp_auth_mlen;
cs->cs_config |= CESA_CSHD_SHA1_HMAC;
if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
break;
case CRYPTO_SHA2_256_HMAC:
cs->cs_mblen = SHA2_256_BLOCK_LEN;
cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN :
csp->csp_auth_mlen;
cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
break;
}
/* Save cipher key */
if (csp->csp_cipher_key != NULL) {
memcpy(cs->cs_key, csp->csp_cipher_key,
csp->csp_cipher_klen);
if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
error = cesa_prep_aes_key(cs, csp);
}
/* Save digest key */
if (csp->csp_auth_key != NULL)
cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key,
csp->csp_auth_klen);
return (error);
}
static int
cesa_process(device_t dev, struct cryptop *crp, int hint)
{
const struct crypto_session_params *csp;
struct cesa_request *cr;
struct cesa_session *cs;
struct cesa_softc *sc;
int error;
sc = device_get_softc(dev);
error = 0;
cs = crypto_get_driver_session(crp->crp_session);
csp = crypto_get_params(crp->crp_session);
/* Check and parse input */
if (crypto_buffer_len(&crp->crp_buf) > CESA_MAX_REQUEST_SIZE) {
crp->crp_etype = E2BIG;
crypto_done(crp);
return (0);
}
/*
* For requests with AAD, only requests where the AAD is
* immediately adjacent to the payload are supported.
*/
if (crp->crp_aad_length != 0 &&
(crp->crp_aad_start + crp->crp_aad_length) !=
crp->crp_payload_start) {
crp->crp_etype = EINVAL;
crypto_done(crp);
return (0);
}
/*
* Get request descriptor. Block driver if there is no free
* descriptors in pool.
*/
cr = cesa_alloc_request(sc);
if (!cr) {
CESA_LOCK(sc, sc);
sc->sc_blocked = CRYPTO_SYMQ;
CESA_UNLOCK(sc, sc);
return (ERESTART);
}
/* Prepare request */
cr->cr_crp = crp;
cr->cr_cs = cs;
CESA_LOCK(sc, sessions);
cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
if (csp->csp_cipher_alg != 0)
crypto_read_iv(crp, cr->cr_csd->csd_iv);
if (crp->crp_cipher_key != NULL) {
memcpy(cs->cs_key, crp->crp_cipher_key,
csp->csp_cipher_klen);
if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
error = cesa_prep_aes_key(cs, csp);
}
if (!error && crp->crp_auth_key != NULL)
cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key,
csp->csp_auth_klen);
/* Convert request to chain of TDMA and SA descriptors */
if (!error)
error = cesa_create_chain(sc, csp, cr);
cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
CESA_UNLOCK(sc, sessions);
if (error) {
cesa_free_request(sc, cr);
crp->crp_etype = error;
crypto_done(crp);
return (0);
}
bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD |
BUS_DMASYNC_PREWRITE);
/* Enqueue request to execution */
cesa_enqueue_request(sc, cr);
/* Start execution, if we have no more requests in queue */
if ((hint & CRYPTO_HINT_MORE) == 0)
cesa_execute(sc);
return (0);
}
Reference in New Issue View Git Blame Copy Permalink