diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile index b3cdb95fb953..fe715f48288d 100644 --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -449,6 +449,7 @@ MAN= aac.4 \ rue.4 \ sa.4 \ safe.4 \ + safexcel.4 \ sbp.4 \ sbp_targ.4 \ scc.4 \ diff --git a/share/man/man4/safexcel.4 b/share/man/man4/safexcel.4 new file mode 100644 index 000000000000..6751570713f9 --- /dev/null +++ b/share/man/man4/safexcel.4 @@ -0,0 +1,84 @@ +.\"- +.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate) +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd June 23, 2020 +.Dt SAFEXCEL 4 +.Os +.Sh NAME +.Nm safexcel +.Nd Inside Secure SafeXcel-IP-97 security packet engine +.Sh SYNOPSIS +To compile this driver into the kernel, +place the following lines in your +kernel configuration file: +.Bd -ragged -offset indent +.Cd "device crypto" +.Cd "device cryptodev" +.Cd "device safexcel" +.Ed +.Pp +Alternatively, to load the driver as a +module at boot time, place the following line in +.Xr loader.conf 5 : +.Bd -literal -offset indent +safexcel_load="YES" +.Ed +.Sh DESCRIPTION +The +.Nm +driver implements +.Xr crypto 4 +support for the cryptographic acceleration functions of the EIP-97 device +found on some Marvell systems-on-chip. +The driver can accelerate the following AES modes: +.Pp +.Bl -bullet -compact +.It +AES-CBC +.It +AES-CTR +.It +AES-XTS +.It +AES-GCM +.It +AES-CCM +.El +.Pp +.Nm +also implements SHA1 and SHA2 transforms, and can combine AES-CBC and AES-CTR +with SHA1-HMAC and SHA2-HMAC for encrypt-then-authenticate operations. +.Sh SEE ALSO +.Xr crypto 4 , +.Xr ipsec 4 , +.Xr random 4 , +.Xr geli 8 , +.Xr crypto 9 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 13.0 . diff --git a/sys/arm64/conf/GENERIC b/sys/arm64/conf/GENERIC index 5e16c8f41990..30d1a8a0d9ad 100644 --- a/sys/arm64/conf/GENERIC +++ b/sys/arm64/conf/GENERIC @@ -283,6 +283,9 @@ device mv_ap806_sei # Marvell AP806 SEI device aw_rtc # Allwinner Real-time Clock device mv_rtc # Marvell Real-time Clock +# Crypto accelerators +device safexcel # Inside Secure EIP-97 + # Watchdog controllers device aw_wdog # Allwinner Watchdog diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index f17c53a78699..a538da81f1c4 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -324,6 +324,7 @@ dev/pci/pci_dw_if.m optional pci fdt dev/psci/psci.c standard dev/psci/smccc_arm64.S standard dev/psci/smccc.c standard +dev/safexcel/safexcel.c optional safexcel fdt dev/sdhci/sdhci_xenon.c optional sdhci_xenon sdhci fdt dev/uart/uart_cpu_arm64.c optional uart dev/uart/uart_dev_mu.c optional uart uart_mu diff --git a/sys/dev/safexcel/safexcel.c b/sys/dev/safexcel/safexcel.c new file mode 100644 index 000000000000..d7fee9b42a29 --- /dev/null +++ b/sys/dev/safexcel/safexcel.c @@ -0,0 +1,2637 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "cryptodev_if.h" + +#include "safexcel_reg.h" +#include "safexcel_var.h" + +static MALLOC_DEFINE(M_SAFEXCEL, "safexcel_req", "safexcel request buffers"); + +/* + * We only support the EIP97 for now. + */ +static struct ofw_compat_data safexcel_compat[] = { + { "inside-secure,safexcel-eip97ies", (uintptr_t)97 }, + { "inside-secure,safexcel-eip97", (uintptr_t)97 }, + { NULL, 0 } +}; + +const struct safexcel_reg_offsets eip97_regs_offset = { + .hia_aic = SAFEXCEL_EIP97_HIA_AIC_BASE, + .hia_aic_g = SAFEXCEL_EIP97_HIA_AIC_G_BASE, + .hia_aic_r = SAFEXCEL_EIP97_HIA_AIC_R_BASE, + .hia_aic_xdr = SAFEXCEL_EIP97_HIA_AIC_xDR_BASE, + .hia_dfe = SAFEXCEL_EIP97_HIA_DFE_BASE, + .hia_dfe_thr = SAFEXCEL_EIP97_HIA_DFE_THR_BASE, + .hia_dse = SAFEXCEL_EIP97_HIA_DSE_BASE, + .hia_dse_thr = SAFEXCEL_EIP97_HIA_DSE_THR_BASE, + .hia_gen_cfg = SAFEXCEL_EIP97_HIA_GEN_CFG_BASE, + .pe = SAFEXCEL_EIP97_PE_BASE, +}; + +const struct safexcel_reg_offsets eip197_regs_offset = { + .hia_aic = SAFEXCEL_EIP197_HIA_AIC_BASE, + .hia_aic_g = SAFEXCEL_EIP197_HIA_AIC_G_BASE, + .hia_aic_r = SAFEXCEL_EIP197_HIA_AIC_R_BASE, + .hia_aic_xdr = SAFEXCEL_EIP197_HIA_AIC_xDR_BASE, + .hia_dfe = SAFEXCEL_EIP197_HIA_DFE_BASE, + .hia_dfe_thr = SAFEXCEL_EIP197_HIA_DFE_THR_BASE, + .hia_dse = SAFEXCEL_EIP197_HIA_DSE_BASE, + .hia_dse_thr = SAFEXCEL_EIP197_HIA_DSE_THR_BASE, + .hia_gen_cfg = SAFEXCEL_EIP197_HIA_GEN_CFG_BASE, + .pe = SAFEXCEL_EIP197_PE_BASE, +}; + +static struct safexcel_cmd_descr * +safexcel_cmd_descr_next(struct safexcel_cmd_descr_ring *ring) +{ + struct safexcel_cmd_descr *cdesc; + + if (ring->write == ring->read) + return (NULL); + cdesc = &ring->desc[ring->read]; + ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; + return (cdesc); +} + +static struct safexcel_res_descr * +safexcel_res_descr_next(struct safexcel_res_descr_ring *ring) +{ + struct safexcel_res_descr *rdesc; + + if (ring->write == ring->read) + return (NULL); + rdesc = &ring->desc[ring->read]; + ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; + return (rdesc); +} + +static struct safexcel_request * +safexcel_alloc_request(struct safexcel_softc *sc, struct safexcel_ring *ring) +{ + struct safexcel_request *req; + + mtx_assert(&ring->mtx, MA_OWNED); + + if ((req = STAILQ_FIRST(&ring->free_requests)) != NULL) + STAILQ_REMOVE_HEAD(&ring->free_requests, link); + return (req); +} + +static void +safexcel_free_request(struct safexcel_ring *ring, struct safexcel_request *req) +{ + struct safexcel_context_record *ctx; + + mtx_assert(&ring->mtx, MA_OWNED); + + if (req->dmap_loaded) { + bus_dmamap_unload(ring->data_dtag, req->dmap); + req->dmap_loaded = false; + } + ctx = (struct safexcel_context_record *)req->ctx.vaddr; + explicit_bzero(ctx->data, sizeof(ctx->data)); + explicit_bzero(req->iv, sizeof(req->iv)); + STAILQ_INSERT_TAIL(&ring->free_requests, req, link); +} + +static void +safexcel_enqueue_request(struct safexcel_softc *sc, struct safexcel_ring *ring, + struct safexcel_request *req) +{ + mtx_assert(&ring->mtx, MA_OWNED); + + STAILQ_INSERT_TAIL(&ring->ready_requests, req, link); +} + +static void +safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx) +{ + struct safexcel_cmd_descr *cdesc; + struct safexcel_res_descr *rdesc; + struct safexcel_request *req; + struct safexcel_ring *ring; + uint32_t error, i, ncdescs, nrdescs, nreqs; + + ring = &sc->sc_ring[ringidx]; + + mtx_lock(&ring->mtx); + nreqs = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT); + nreqs >>= SAFEXCEL_xDR_PROC_xD_PKT_OFFSET; + nreqs &= SAFEXCEL_xDR_PROC_xD_PKT_MASK; + if (nreqs == 0) { + SAFEXCEL_DPRINTF(sc, 1, + "zero pending requests on ring %d\n", ringidx); + goto out; + } + + ring = &sc->sc_ring[ringidx]; + bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + ncdescs = nrdescs = 0; + for (i = 0; i < nreqs; i++) { + req = STAILQ_FIRST(&ring->queued_requests); + KASSERT(req != NULL, ("%s: expected %d pending requests", + __func__, nreqs)); + STAILQ_REMOVE_HEAD(&ring->queued_requests, link); + mtx_unlock(&ring->mtx); + + bus_dmamap_sync(req->ctx.tag, req->ctx.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(ring->data_dtag, req->dmap, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + ncdescs += req->cdescs; + while (req->cdescs-- > 0) { + cdesc = safexcel_cmd_descr_next(&ring->cdr); + KASSERT(cdesc != NULL, + ("%s: missing control descriptor", __func__)); + if (req->cdescs == 0) + KASSERT(cdesc->last_seg, + ("%s: chain is not terminated", __func__)); + } + nrdescs += req->rdescs; + while (req->rdescs-- > 0) { + rdesc = safexcel_res_descr_next(&ring->rdr); + error = rdesc->result_data.error_code; + if (error != 0) { + if (error == SAFEXCEL_RESULT_ERR_AUTH_FAILED && + req->crp->crp_etype == 0) { + req->crp->crp_etype = EBADMSG; + } else { + SAFEXCEL_DPRINTF(sc, 1, + "error code %#x\n", error); + req->crp->crp_etype = EIO; + } + } + } + + crypto_done(req->crp); + mtx_lock(&ring->mtx); + safexcel_free_request(ring, req); + } + + if (nreqs != 0) { + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_xD_PKT(nreqs) | + (sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t))); + } +out: + if (!STAILQ_EMPTY(&ring->queued_requests)) { + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, + SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | 1); + } + mtx_unlock(&ring->mtx); +} + +static void +safexcel_ring_intr(void *arg) +{ + struct safexcel_softc *sc; + struct safexcel_intr_handle *ih; + uint32_t status, stat; + int ring; + bool blocked, rdrpending; + + ih = arg; + sc = ih->sc; + ring = ih->ring; + + status = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + + SAFEXCEL_HIA_AIC_R_ENABLED_STAT(ring)); + /* CDR interrupts */ + if (status & SAFEXCEL_CDR_IRQ(ring)) { + stat = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, + stat & SAFEXCEL_CDR_INTR_MASK); + } + /* RDR interrupts */ + rdrpending = false; + if (status & SAFEXCEL_RDR_IRQ(ring)) { + stat = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); + if ((stat & SAFEXCEL_xDR_ERR) == 0) + rdrpending = true; + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, + stat & SAFEXCEL_RDR_INTR_MASK); + } + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ACK(ring), + status); + + if (rdrpending) + safexcel_rdr_intr(sc, ring); + + mtx_lock(&sc->sc_mtx); + blocked = sc->sc_blocked; + sc->sc_blocked = 0; + mtx_unlock(&sc->sc_mtx); + + if (blocked) + crypto_unblock(sc->sc_cid, blocked); +} + +static int +safexcel_configure(struct safexcel_softc *sc) +{ + uint32_t i, mask, pemask, reg; + device_t dev; + + if (sc->sc_type == 197) { + sc->sc_offsets = eip197_regs_offset; + pemask = SAFEXCEL_N_PES_MASK; + } else { + sc->sc_offsets = eip97_regs_offset; + pemask = EIP97_N_PES_MASK; + } + + dev = sc->sc_dev; + + /* Scan for valid ring interrupt controllers. */ + for (i = 0; i < SAFEXCEL_MAX_RING_AIC; i++) { + reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + + SAFEXCEL_HIA_AIC_R_VERSION(i)); + if (SAFEXCEL_REG_LO16(reg) != EIP201_VERSION_LE) + break; + } + sc->sc_config.aic_rings = i; + if (sc->sc_config.aic_rings == 0) + return (-1); + + reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_OPTIONS); + /* Check for 64bit addressing. */ + if ((reg & SAFEXCEL_OPT_ADDR_64) == 0) + return (-1); + /* Check alignment constraints (which we do not support). */ + if (((reg & SAFEXCEL_OPT_TGT_ALIGN_MASK) >> + SAFEXCEL_OPT_TGT_ALIGN_OFFSET) != 0) + return (-1); + + sc->sc_config.hdw = + (reg & SAFEXCEL_xDR_HDW_MASK) >> SAFEXCEL_xDR_HDW_OFFSET; + mask = (1 << sc->sc_config.hdw) - 1; + + sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK; + /* Limit the number of rings to the number of the AIC Rings. */ + sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings); + + sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET; + + sc->sc_config.cd_size = + sizeof(struct safexcel_cmd_descr) / sizeof(uint32_t); + sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask; + + sc->sc_config.rd_size = + sizeof(struct safexcel_res_descr) / sizeof(uint32_t); + sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask; + + sc->sc_config.atok_offset = + (SAFEXCEL_MAX_ATOKENS * sizeof(struct safexcel_instr) + mask) & + ~mask; + + return (0); +} + +static void +safexcel_init_hia_bus_access(struct safexcel_softc *sc) +{ + uint32_t version, val; + + /* Determine endianness and configure byte swap. */ + version = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_VERSION); + val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); + if (SAFEXCEL_REG_HI16(version) == SAFEXCEL_HIA_VERSION_BE) { + val = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); + val = val ^ (SAFEXCEL_MST_CTRL_NO_BYTE_SWAP >> 24); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL, + val); + } + + /* Configure wr/rd cache values. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_GEN_CFG(sc) + SAFEXCEL_HIA_MST_CTRL, + SAFEXCEL_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | + SAFEXCEL_MST_CTRL_WD_CACHE(WR_CACHE_4BITS)); +} + +static void +safexcel_disable_global_interrupts(struct safexcel_softc *sc) +{ + /* Disable and clear pending interrupts. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ENABLE_CTRL, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, + SAFEXCEL_AIC_G_ACK_ALL_MASK); +} + +/* + * Configure the data fetch engine. This component parses command descriptors + * and sets up DMA transfers from host memory to the corresponding processing + * engine. + */ +static void +safexcel_configure_dfe_engine(struct safexcel_softc *sc, int pe) +{ + /* Reset all DFE threads. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_RESET_PE); + + /* Deassert the DFE reset. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), 0); + + /* DMA transfer size to use. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE(sc) + SAFEXCEL_HIA_DFE_CFG(pe), + SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG | + SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(6) | + SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(9) | + SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | + SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(7) | + SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS) | + SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS)); + + /* Configure the PE DMA transfer thresholds. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_DBUF_THRES(pe), + SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | + SAFEXCEL_PE_IN_xBUF_THRES_MAX(9)); + SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_TBUF_THRES(pe), + SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | + SAFEXCEL_PE_IN_xBUF_THRES_MAX(7)); +} + +/* + * Configure the data store engine. This component parses result descriptors + * and sets up DMA transfers from the processing engine to host memory. + */ +static int +safexcel_configure_dse(struct safexcel_softc *sc, int pe) +{ + uint32_t val; + int count; + + /* Disable and reset all DSE threads. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_RESET_PE); + + /* Wait for a second for threads to go idle. */ + for (count = 0;;) { + val = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_STAT(pe)); + if ((val & SAFEXCEL_DSE_THR_RDR_ID_MASK) == + SAFEXCEL_DSE_THR_RDR_ID_MASK) + break; + if (count++ > 10000) { + device_printf(sc->sc_dev, "DSE reset timeout\n"); + return (-1); + } + DELAY(100); + } + + /* Exit the reset state. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), 0); + + /* DMA transfer size to use */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE(sc) + SAFEXCEL_HIA_DSE_CFG(pe), + SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG | + SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(7) | + SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(8) | + SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS) | + SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE); + + /* Configure the procesing engine thresholds */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_PE(sc) + SAFEXCEL_PE_OUT_DBUF_THRES(pe), + SAFEXCEL_PE_OUT_DBUF_THRES_MIN(7) | + SAFEXCEL_PE_OUT_DBUF_THRES_MAX(8)); + + return (0); +} + +static void +safexcel_hw_prepare_rings(struct safexcel_softc *sc) +{ + int i; + + for (i = 0; i < sc->sc_config.rings; i++) { + /* + * Command descriptors. + */ + + /* Clear interrupts for this ring. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), + SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK); + + /* Disable external triggering. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, + SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset * + sizeof(uint32_t)); + + /* + * Result descriptors. + */ + + /* Disable external triggering. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + /* Ring size. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, + SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset * + sizeof(uint32_t)); + } +} + +static void +safexcel_hw_setup_rings(struct safexcel_softc *sc) +{ + struct safexcel_ring *ring; + uint32_t cd_size_rnd, mask, rd_size_rnd, val; + int i; + + mask = (1 << sc->sc_config.hdw) - 1; + cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw; + val = (sizeof(struct safexcel_res_descr) - + sizeof(struct safexcel_res_data)) / sizeof(uint32_t); + rd_size_rnd = (val + mask) >> sc->sc_config.hdw; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + + /* + * Command descriptors. + */ + + /* Ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, + SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr)); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, + SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr)); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, + SAFEXCEL_xDR_DESC_MODE_64BIT | SAFEXCEL_CDR_DESC_MODE_ADCP | + (sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | + sc->sc_config.cd_size); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, + ((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) << + SAFEXCEL_xDR_xD_FETCH_THRESH) | + (SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset)); + + /* Configure DMA tx control. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, + SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | + SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS)); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_CDR_INTR_MASK); + + /* + * Result descriptors. + */ + + /* Ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, + SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr)); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, + SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr)); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, + SAFEXCEL_xDR_DESC_MODE_64BIT | + (sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | + sc->sc_config.rd_size); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, + ((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) << + SAFEXCEL_xDR_xD_FETCH_THRESH) | + (SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset)); + + /* Configure DMA tx control. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, + SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | + SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS) | + SAFEXCEL_HIA_xDR_WR_RES_BUF | SAFEXCEL_HIA_xDR_WR_CTRL_BUF); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_RDR_INTR_MASK); + + /* Enable ring interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(i), + SAFEXCEL_RDR_IRQ(i)); + } +} + +/* Reset the command and result descriptor rings. */ +static void +safexcel_hw_reset_rings(struct safexcel_softc *sc) +{ + int i; + + for (i = 0; i < sc->sc_config.rings; i++) { + /* + * Result descriptor ring operations. + */ + + /* Reset ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_RDR_INTR_MASK); + + /* Disable ring interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), + SAFEXCEL_RDR_IRQ(i)); + + /* + * Command descriptor ring operations. + */ + + /* Reset ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_CDR_INTR_MASK); + } +} + +static void +safexcel_enable_pe_engine(struct safexcel_softc *sc, int pe) +{ + int i, ring_mask; + + for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) { + ring_mask <<= 1; + ring_mask |= 1; + } + + /* Enable command descriptor rings. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); + + /* Enable result descriptor rings. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); + + /* Clear any HIA interrupt. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, + SAFEXCEL_AIC_G_ACK_HIA_MASK); +} + +static void +safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring, + struct safexcel_request *req) +{ + uint32_t ncdescs, nrdescs, nreqs; + int ringidx; + bool busy; + + mtx_assert(&ring->mtx, MA_OWNED); + + ringidx = req->sess->ringidx; + if (STAILQ_EMPTY(&ring->ready_requests)) + return; + busy = !STAILQ_EMPTY(&ring->queued_requests); + ncdescs = nrdescs = nreqs = 0; + while ((req = STAILQ_FIRST(&ring->ready_requests)) != NULL && + req->cdescs + ncdescs <= SAFEXCEL_MAX_BATCH_SIZE && + req->rdescs + nrdescs <= SAFEXCEL_MAX_BATCH_SIZE) { + STAILQ_REMOVE_HEAD(&ring->ready_requests, link); + STAILQ_INSERT_TAIL(&ring->queued_requests, req, link); + ncdescs += req->cdescs; + nrdescs += req->rdescs; + nreqs++; + } + + if (!busy) { + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, + SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | nreqs); + } + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, + nrdescs * sc->sc_config.rd_offset * sizeof(uint32_t)); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, + ncdescs * sc->sc_config.cd_offset * sizeof(uint32_t)); +} + +static void +safexcel_init_rings(struct safexcel_softc *sc) +{ + struct safexcel_cmd_descr *cdesc; + struct safexcel_ring *ring; + char buf[32]; + uint64_t atok; + int i, j; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + + snprintf(buf, sizeof(buf), "safexcel_ring%d", i); + mtx_init(&ring->mtx, buf, NULL, MTX_DEF); + STAILQ_INIT(&ring->free_requests); + STAILQ_INIT(&ring->ready_requests); + STAILQ_INIT(&ring->queued_requests); + + ring->cdr.read = ring->cdr.write = 0; + ring->rdr.read = ring->rdr.write = 0; + for (j = 0; j < SAFEXCEL_RING_SIZE; j++) { + cdesc = &ring->cdr.desc[j]; + atok = ring->dma_atok.paddr + + sc->sc_config.atok_offset * j; + cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok); + cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok); + } + } +} + +static void +safexcel_dma_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + struct safexcel_dma_mem *sdm; + + if (error != 0) + return; + + KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); + sdm = arg; + sdm->paddr = segs->ds_addr; +} + +static int +safexcel_dma_alloc_mem(struct safexcel_softc *sc, struct safexcel_dma_mem *sdm, + bus_size_t size) +{ + int error; + + KASSERT(sdm->vaddr == NULL, + ("%s: DMA memory descriptor in use.", __func__)); + + error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ + PAGE_SIZE, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + size, 1, /* maxsize, nsegments */ + size, BUS_DMA_COHERENT, /* maxsegsz, flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &sdm->tag); /* dmat */ + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate busdma tag, error %d\n", error); + goto err1; + } + + error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map); + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate DMA safe memory, error %d\n", error); + goto err2; + } + + error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size, + safexcel_dma_alloc_mem_cb, sdm, BUS_DMA_NOWAIT); + if (error != 0) { + device_printf(sc->sc_dev, + "cannot get address of the DMA memory, error %d\n", error); + goto err3; + } + + return (0); +err3: + bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); +err2: + bus_dma_tag_destroy(sdm->tag); +err1: + sdm->vaddr = NULL; + + return (error); +} + +static void +safexcel_dma_free_mem(struct safexcel_dma_mem *sdm) +{ + bus_dmamap_unload(sdm->tag, sdm->map); + bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); + bus_dma_tag_destroy(sdm->tag); +} + +static void +safexcel_dma_free_rings(struct safexcel_softc *sc) +{ + struct safexcel_ring *ring; + int i; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + safexcel_dma_free_mem(&ring->cdr.dma); + safexcel_dma_free_mem(&ring->dma_atok); + safexcel_dma_free_mem(&ring->rdr.dma); + bus_dma_tag_destroy(ring->data_dtag); + mtx_destroy(&ring->mtx); + } +} + +static int +safexcel_dma_init(struct safexcel_softc *sc) +{ + struct safexcel_ring *ring; + int error, i, size; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + + error = bus_dma_tag_create( + bus_get_dma_tag(sc->sc_dev),/* parent */ + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + SAFEXCEL_MAX_REQUEST_SIZE, /* maxsize */ + SAFEXCEL_MAX_FRAGMENTS, /* nsegments */ + SAFEXCEL_MAX_REQUEST_SIZE, /* maxsegsz */ + BUS_DMA_COHERENT, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->data_dtag); /* dmat */ + if (error != 0) { + device_printf(sc->sc_dev, + "bus_dma_tag_create main failed; error %d\n", error); + return (error); + } + + size = sizeof(uint32_t) * sc->sc_config.cd_offset * + SAFEXCEL_RING_SIZE; + error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size); + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate CDR DMA memory, error %d\n", + error); + goto err; + } + ring->cdr.desc = + (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr; + + /* Allocate additional CDR token memory. */ + error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, + sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE); + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate atoken DMA memory, error %d\n", + error); + goto err; + } + + size = sizeof(uint32_t) * sc->sc_config.rd_offset * + SAFEXCEL_RING_SIZE; + error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size); + if (error) { + device_printf(sc->sc_dev, + "failed to allocate RDR DMA memory, error %d\n", + error); + goto err; + } + ring->rdr.desc = + (struct safexcel_res_descr *)ring->rdr.dma.vaddr; + } + + return (0); +err: + safexcel_dma_free_rings(sc); + return (error); +} + +static void +safexcel_deinit_hw(struct safexcel_softc *sc) +{ + safexcel_hw_reset_rings(sc); + safexcel_dma_free_rings(sc); +} + +static int +safexcel_init_hw(struct safexcel_softc *sc) +{ + int pe; + + /* 23.3.7 Initialization */ + if (safexcel_configure(sc) != 0) + return (EINVAL); + + if (safexcel_dma_init(sc) != 0) + return (ENOMEM); + + safexcel_init_rings(sc); + + safexcel_init_hia_bus_access(sc); + + /* 23.3.7.2 Disable EIP-97 global Interrupts */ + safexcel_disable_global_interrupts(sc); + + for (pe = 0; pe < sc->sc_config.pes; pe++) { + /* 23.3.7.3 Configure Data Fetch Engine */ + safexcel_configure_dfe_engine(sc, pe); + + /* 23.3.7.4 Configure Data Store Engine */ + if (safexcel_configure_dse(sc, pe)) { + safexcel_deinit_hw(sc); + return (-1); + } + + /* 23.3.7.5 1. Protocol enables */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION_EN(pe), + 0xffffffff); + SAFEXCEL_WRITE(sc, + SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION2_EN(pe), + 0xffffffff); + } + + safexcel_hw_prepare_rings(sc); + + /* 23.3.7.5 Configure the Processing Engine(s). */ + for (pe = 0; pe < sc->sc_config.pes; pe++) + safexcel_enable_pe_engine(sc, pe); + + safexcel_hw_setup_rings(sc); + + return (0); +} + +static int +safexcel_setup_dev_interrupts(struct safexcel_softc *sc) +{ + int i, j; + + for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { + sc->sc_ih[i].sc = sc; + sc->sc_ih[i].ring = i; + + if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i], + INTR_TYPE_NET | INTR_MPSAFE, NULL, safexcel_ring_intr, + &sc->sc_ih[i], &sc->sc_ih[i].handle)) { + device_printf(sc->sc_dev, + "couldn't setup interrupt %d\n", i); + goto err; + } + } + + return (0); + +err: + for (j = 0; j < i; j++) + bus_teardown_intr(sc->sc_dev, sc->sc_intr[j], + sc->sc_ih[j].handle); + + return (ENXIO); +} + +static void +safexcel_teardown_dev_interrupts(struct safexcel_softc *sc) +{ + int i; + + for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) + bus_teardown_intr(sc->sc_dev, sc->sc_intr[i], + sc->sc_ih[i].handle); +} + +static int +safexcel_alloc_dev_resources(struct safexcel_softc *sc) +{ + char name[16]; + device_t dev; + phandle_t node; + int error, i, rid; + + dev = sc->sc_dev; + node = ofw_bus_get_node(dev); + + rid = 0; + sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->sc_res == NULL) { + device_printf(dev, "couldn't allocate memory resources\n"); + return (ENXIO); + } + + for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) { + (void)snprintf(name, sizeof(name), "ring%d", i); + error = ofw_bus_find_string_index(node, "interrupt-names", name, + &rid); + if (error != 0) + break; + + sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->sc_intr[i] == NULL) { + error = ENXIO; + goto out; + } + } + if (i == 0) { + device_printf(dev, "couldn't allocate interrupt resources\n"); + error = ENXIO; + goto out; + } + + mtx_init(&sc->sc_mtx, "safexcel softc", NULL, MTX_DEF); + + return (0); + +out: + for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) + bus_release_resource(dev, SYS_RES_IRQ, + rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); + bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), + sc->sc_res); + return (error); +} + +static void +safexcel_free_dev_resources(struct safexcel_softc *sc) +{ + int i; + + mtx_destroy(&sc->sc_mtx); + + for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) + bus_release_resource(sc->sc_dev, SYS_RES_IRQ, + rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); + if (sc->sc_res != NULL) + bus_release_resource(sc->sc_dev, SYS_RES_MEMORY, + rman_get_rid(sc->sc_res), sc->sc_res); +} + +static int +safexcel_probe(device_t dev) +{ + struct safexcel_softc *sc; + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + sc = device_get_softc(dev); + sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data; + if (sc->sc_type == 0) + return (ENXIO); + + device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator"); + + return (BUS_PROBE_DEFAULT); +} + +static int +safexcel_attach(device_t dev) +{ + struct sysctl_ctx_list *sctx; + struct safexcel_softc *sc; + struct safexcel_request *req; + struct safexcel_ring *ring; + int i, j, ringidx; + + sc = device_get_softc(dev); + sc->sc_dev = dev; + sc->sc_blocked = 0; + sc->sc_cid = -1; + + if (safexcel_alloc_dev_resources(sc)) + goto err; + + if (safexcel_setup_dev_interrupts(sc)) + goto err1; + + if (safexcel_init_hw(sc)) + goto err2; + + for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { + ring = &sc->sc_ring[ringidx]; + + ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); + ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); + + ring->requests = mallocarray(SAFEXCEL_REQUESTS_PER_RING, + sizeof(struct safexcel_request), M_SAFEXCEL, + M_WAITOK | M_ZERO); + + for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { + req = &ring->requests[i]; + req->sc = sc; + if (bus_dmamap_create(ring->data_dtag, + BUS_DMA_COHERENT, &req->dmap) != 0) { + for (j = 0; j < i; j++) + bus_dmamap_destroy(ring->data_dtag, + ring->requests[j].dmap); + goto err2; + } + if (safexcel_dma_alloc_mem(sc, &req->ctx, + sizeof(struct safexcel_context_record)) != 0) { + for (j = 0; j < i; j++) { + bus_dmamap_destroy(ring->data_dtag, + ring->requests[j].dmap); + safexcel_dma_free_mem( + &ring->requests[j].ctx); + } + goto err2; + } + STAILQ_INSERT_TAIL(&ring->free_requests, req, link); + } + } + + sctx = device_get_sysctl_ctx(dev); + SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0, + "Debug message verbosity"); + + sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session), + CRYPTOCAP_F_HARDWARE); + if (sc->sc_cid < 0) + goto err2; + + return (0); + +err2: + safexcel_teardown_dev_interrupts(sc); +err1: + safexcel_free_dev_resources(sc); +err: + return (ENXIO); +} + +static int +safexcel_detach(device_t dev) +{ + struct safexcel_ring *ring; + struct safexcel_softc *sc; + int i, ringidx; + + sc = device_get_softc(dev); + + if (sc->sc_cid >= 0) + crypto_unregister_all(sc->sc_cid); + for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { + ring = &sc->sc_ring[ringidx]; + for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { + bus_dmamap_destroy(ring->data_dtag, + ring->requests[i].dmap); + safexcel_dma_free_mem(&ring->requests[i].ctx); + } + free(ring->requests, M_SAFEXCEL); + sglist_free(ring->cmd_data); + sglist_free(ring->res_data); + } + safexcel_deinit_hw(sc); + safexcel_teardown_dev_interrupts(sc); + safexcel_free_dev_resources(sc); + + return (0); +} + +/* + * Populate the request's context record with pre-computed key material. + */ +static int +safexcel_set_context(struct safexcel_request *req) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_context_record *ctx; + struct safexcel_session *sess; + uint8_t *data; + int off; + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + sess = req->sess; + + ctx = (struct safexcel_context_record *)req->ctx.vaddr; + data = (uint8_t *)ctx->data; + if (csp->csp_cipher_alg != 0) { + if (crp->crp_cipher_key != NULL) + memcpy(data, crp->crp_cipher_key, sess->klen); + else + memcpy(data, csp->csp_cipher_key, sess->klen); + off = sess->klen; + } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { + if (crp->crp_auth_key != NULL) + memcpy(data, crp->crp_auth_key, sess->klen); + else + memcpy(data, csp->csp_auth_key, sess->klen); + off = sess->klen; + } else { + off = 0; + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + off += GMAC_BLOCK_LEN; + break; + case CRYPTO_AES_CCM_16: + memcpy(data + off, sess->xcbc_key, + AES_BLOCK_LEN * 2 + sess->klen); + off += AES_BLOCK_LEN * 2 + sess->klen; + break; + case CRYPTO_AES_XTS: + memcpy(data + off, sess->tweak_key, sess->klen); + off += sess->klen; + break; + } + + switch (csp->csp_auth_alg) { + case CRYPTO_AES_NIST_GMAC: + memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + off += GMAC_BLOCK_LEN; + break; + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + memcpy(data + off, sess->hmac_ipad, sess->statelen); + off += sess->statelen; + memcpy(data + off, sess->hmac_opad, sess->statelen); + off += sess->statelen; + break; + } + + return (off); +} + +/* + * Populate fields in the first command descriptor of the chain used to encode + * the specified request. These fields indicate the algorithms used, the size + * of the key material stored in the associated context record, the primitive + * operations to be performed on input data, and the location of the IV if any. + */ +static void +safexcel_set_command(struct safexcel_request *req, + struct safexcel_cmd_descr *cdesc) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_session *sess; + uint32_t ctrl0, ctrl1, ctxr_len; + int alg; + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + sess = req->sess; + + ctrl0 = sess->alg | sess->digest | sess->hash; + ctrl1 = sess->mode; + + ctxr_len = safexcel_set_context(req) / sizeof(uint32_t); + ctrl0 |= SAFEXCEL_CONTROL0_SIZE(ctxr_len); + + alg = csp->csp_cipher_alg; + if (alg == 0) + alg = csp->csp_auth_alg; + + switch (alg) { + case CRYPTO_AES_CCM_16: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT | + SAFEXCEL_CONTROL0_KEY_EN; + } else { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN | + SAFEXCEL_CONTROL0_KEY_EN; + } + ctrl1 |= SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | + SAFEXCEL_CONTROL1_IV2 | SAFEXCEL_CONTROL1_IV3; + break; + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + case CRYPTO_AES_XTS: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | + SAFEXCEL_CONTROL0_KEY_EN; + if (csp->csp_auth_alg != 0) + ctrl0 |= + SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT; + } else { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | + SAFEXCEL_CONTROL0_KEY_EN; + if (csp->csp_auth_alg != 0) + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; + } + break; + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_NIST_GMAC: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) || + csp->csp_auth_alg != 0) { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | + SAFEXCEL_CONTROL0_KEY_EN | + SAFEXCEL_CONTROL0_TYPE_HASH_OUT; + } else { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | + SAFEXCEL_CONTROL0_KEY_EN | + SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; + } + if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { + ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE | + SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | + SAFEXCEL_CONTROL1_IV2; + } + break; + case CRYPTO_SHA1: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_512: + ctrl0 |= SAFEXCEL_CONTROL0_RESTART_HASH; + /* FALLTHROUGH */ + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_OUT; + break; + } + + cdesc->control_data.control0 = ctrl0; + cdesc->control_data.control1 = ctrl1; +} + +/* + * Construct a no-op instruction, used to pad input tokens. + */ +static void +safexcel_instr_nop(struct safexcel_instr **instrp) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = (1 << 2); + instr->status = 0; + instr->instructions = 0; + + *instrp = instr + 1; +} + +/* + * Insert the digest of the input payload. This is typically the last + * instruction of a sequence. + */ +static void +safexcel_instr_insert_digest(struct safexcel_instr **instrp, int len) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = len; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_INSERT_HASH_DIGEST; + + *instrp = instr + 1; +} + +/* + * Retrieve and verify a digest. + */ +static void +safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_RETRIEVE; + instr->length = len; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST; + instr++; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS; + instr->length = len | SAFEXCEL_INSTR_VERIFY_HASH; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_VERIFY_PADDING; + + *instrp = instr + 1; +} + +static void +safexcel_instr_temp_aes_block(struct safexcel_instr **instrp) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT; + instr->length = 0; + instr->status = 0; + instr->instructions = AES_BLOCK_LEN; + instr++; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = AES_BLOCK_LEN; + instr->status = 0; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_DEST_CRYPTO; + + *instrp = instr + 1; +} + +/* + * Handle a request for an unauthenticated block cipher. + */ +static void +safexcel_instr_cipher(struct safexcel_request *req, + struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + + crp = req->crp; + + /* Insert the payload. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | + SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_OUTPUT; + + cdesc->additional_cdata_size = 1; +} + +static void +safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_instr *start; + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + start = instr; + + /* Insert the AAD. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_aad_length; + instr->status = crp->crp_payload_length == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_HASH; + instr++; + + /* Encrypt any data left in the request. */ + if (crp->crp_payload_length > 0) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_CRYPTO | + SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_DEST_OUTPUT; + instr++; + } + + /* + * Compute the digest, or extract it and place it in the output stream. + */ + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + else + safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_instr_sha_hash(struct safexcel_request *req, + struct safexcel_instr *instr) +{ + struct cryptop *crp; + struct safexcel_instr *start; + + crp = req->crp; + start = instr; + + /* Pass the input data to the hash engine. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + + /* Insert the hash result into the output stream. */ + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + + /* Pad the rest of the inline instruction space. */ + while (instr != start + SAFEXCEL_MAX_ITOKENS) + safexcel_instr_nop(&instr); +} + +static void +safexcel_instr_ccm(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + struct safexcel_instr *start; + uint8_t *a0, *b0, *alenp, L; + int aalign, blen; + + crp = req->crp; + start = instr; + + /* + * Construct two blocks, A0 and B0, used in encryption and + * authentication, respectively. A0 is embedded in the token + * descriptor, and B0 is inserted directly into the data stream using + * instructions below. + * + * OCF seems to assume a 12-byte IV, fixing L (the payload length size) + * at 3 bytes due to the layout of B0. This is fine since the driver + * has a maximum of 65535 bytes anyway. + */ + blen = AES_BLOCK_LEN; + L = 3; + + a0 = (uint8_t *)&cdesc->control_data.token[0]; + memset(a0, 0, blen); + a0[0] = L - 1; + memcpy(&a0[1], req->iv, AES_CCM_IV_LEN); + + /* + * Insert B0 and the AAD length into the input stream. + */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0); + instr->status = 0; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_INSERT_IMMEDIATE; + instr++; + + b0 = (uint8_t *)instr; + memset(b0, 0, blen); + b0[0] = + L - 1 | /* payload length size */ + ((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ | + (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; + memcpy(&b0[1], req->iv, AES_CCM_IV_LEN); + b0[14] = crp->crp_payload_length >> 8; + b0[15] = crp->crp_payload_length & 0xff; + instr += blen / sizeof(*instr); + + /* Insert the AAD length and data into the input stream. */ + if (crp->crp_aad_length > 0) { + alenp = (uint8_t *)instr; + alenp[0] = crp->crp_aad_length >> 8; + alenp[1] = crp->crp_aad_length & 0xff; + alenp[2] = 0; + alenp[3] = 0; + instr++; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_aad_length; + instr->status = 0; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + + /* Insert zero padding. */ + aalign = (crp->crp_aad_length + 2) & (blen - 1); + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = aalign == 0 ? 0 : + blen - ((crp->crp_aad_length + 2) & (blen - 1)); + instr->status = crp->crp_payload_length == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + } + + safexcel_instr_temp_aes_block(&instr); + + /* Insert the cipher payload into the input stream. */ + if (crp->crp_payload_length > 0) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_DEST_CRYPTO | + SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_INS_LAST; + instr++; + + /* Insert zero padding. */ + if (crp->crp_payload_length & (blen - 1)) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = blen - + (crp->crp_payload_length & (blen - 1)); + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + } + } + + /* + * Compute the digest, or extract it and place it in the output stream. + */ + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + else + safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_instr_gcm(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + struct safexcel_instr *start; + + memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); + cdesc->control_data.token[3] = htobe32(1); + + crp = req->crp; + start = instr; + + /* Insert the AAD into the input stream. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_aad_length; + instr->status = crp->crp_payload_length == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_HASH; + instr++; + + safexcel_instr_temp_aes_block(&instr); + + /* Insert the cipher payload into the input stream. */ + if (crp->crp_payload_length > 0) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_INS_LAST; + instr++; + } + + /* + * Compute the digest, or extract it and place it in the output stream. + */ + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + else + safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + struct safexcel_instr *start; + + memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); + cdesc->control_data.token[3] = htobe32(1); + + crp = req->crp; + start = instr; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_HASH; + instr++; + + safexcel_instr_temp_aes_block(&instr); + + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_set_token(struct safexcel_request *req) +{ + const struct crypto_session_params *csp; + struct safexcel_cmd_descr *cdesc; + struct safexcel_instr *instr; + struct safexcel_softc *sc; + int ringidx; + + csp = crypto_get_params(req->crp->crp_session); + cdesc = req->cdesc; + sc = req->sc; + ringidx = req->sess->ringidx; + + safexcel_set_command(req, cdesc); + + /* + * For keyless hash operations, the token instructions can be embedded + * in the token itself. Otherwise we use an additional token descriptor + * and the embedded instruction space is used to store the IV. + */ + if (csp->csp_cipher_alg == 0 && + csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { + instr = (void *)cdesc->control_data.token; + } else { + instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + + sc->sc_config.atok_offset * + (cdesc - sc->sc_ring[ringidx].cdr.desc)); + cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + safexcel_instr_gcm(req, instr, cdesc); + break; + case CRYPTO_AES_CCM_16: + safexcel_instr_ccm(req, instr, cdesc); + break; + case CRYPTO_AES_XTS: + memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); + memset(cdesc->control_data.token + + AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); + + safexcel_instr_cipher(req, instr, cdesc); + break; + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); + if (csp->csp_auth_alg != 0) + safexcel_instr_eta(req, instr, cdesc); + else + safexcel_instr_cipher(req, instr, cdesc); + break; + default: + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + safexcel_instr_sha_hash(req, instr); + break; + case CRYPTO_AES_NIST_GMAC: + safexcel_instr_gmac(req, instr, cdesc); + break; + default: + panic("unhandled auth request %d", csp->csp_auth_alg); + } + break; + } +} + +static struct safexcel_res_descr * +safexcel_res_descr_add(struct safexcel_ring *ring, bool first, bool last, + bus_addr_t data, uint32_t len) +{ + struct safexcel_res_descr *rdesc; + struct safexcel_res_descr_ring *rring; + + mtx_assert(&ring->mtx, MA_OWNED); + + rring = &ring->rdr; + if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read) + return (NULL); + + rdesc = &rring->desc[rring->write]; + rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE; + + rdesc->particle_size = len; + rdesc->rsvd0 = 0; + rdesc->descriptor_overflow = 0; + rdesc->buffer_overflow = 0; + rdesc->last_seg = last; + rdesc->first_seg = first; + rdesc->result_size = + sizeof(struct safexcel_res_data) / sizeof(uint32_t); + rdesc->rsvd1 = 0; + rdesc->data_lo = SAFEXCEL_ADDR_LO(data); + rdesc->data_hi = SAFEXCEL_ADDR_HI(data); + + if (first) { + rdesc->result_data.packet_length = 0; + rdesc->result_data.error_code = 0; + } + + return (rdesc); +} + +static struct safexcel_cmd_descr * +safexcel_cmd_descr_add(struct safexcel_ring *ring, bool first, bool last, + bus_addr_t data, uint32_t seglen, uint32_t reqlen, bus_addr_t context) +{ + struct safexcel_cmd_descr *cdesc; + struct safexcel_cmd_descr_ring *cring; + + KASSERT(full_data_len <= SAFEXCEL_MAX_REQUEST_SIZE, + ("%s: request length %u too long", __func__, full_data_len)); + mtx_assert(&ring->mtx, MA_OWNED); + + cring = &ring->cdr; + if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read) + return (NULL); + + cdesc = &cring->desc[cring->write]; + cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE; + + cdesc->particle_size = seglen; + cdesc->rsvd0 = 0; + cdesc->last_seg = last; + cdesc->first_seg = first; + cdesc->additional_cdata_size = 0; + cdesc->rsvd1 = 0; + cdesc->data_lo = SAFEXCEL_ADDR_LO(data); + cdesc->data_hi = SAFEXCEL_ADDR_HI(data); + if (first) { + cdesc->control_data.packet_length = reqlen; + cdesc->control_data.options = SAFEXCEL_OPTION_IP | + SAFEXCEL_OPTION_CP | SAFEXCEL_OPTION_CTX_CTRL_IN_CMD | + SAFEXCEL_OPTION_RC_AUTO; + cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS; + cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) | + SAFEXCEL_CONTEXT_SMALL; + cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context); + } + + return (cdesc); +} + +static void +safexcel_cmd_descr_rollback(struct safexcel_ring *ring, int count) +{ + struct safexcel_cmd_descr_ring *cring; + + mtx_assert(&ring->mtx, MA_OWNED); + + cring = &ring->cdr; + cring->write -= count; + if (cring->write < 0) + cring->write += SAFEXCEL_RING_SIZE; +} + +static void +safexcel_res_descr_rollback(struct safexcel_ring *ring, int count) +{ + struct safexcel_res_descr_ring *rring; + + mtx_assert(&ring->mtx, MA_OWNED); + + rring = &ring->rdr; + rring->write -= count; + if (rring->write < 0) + rring->write += SAFEXCEL_RING_SIZE; +} + +static void +safexcel_append_segs(bus_dma_segment_t *segs, int nseg, struct sglist *sg, + int start, int len) +{ + bus_dma_segment_t *seg; + size_t seglen; + int error, i; + + for (i = 0; i < nseg && len > 0; i++) { + seg = &segs[i]; + + if (seg->ds_len <= start) { + start -= seg->ds_len; + continue; + } + + seglen = MIN(len, seg->ds_len - start); + error = sglist_append_phys(sg, seg->ds_addr + start, seglen); + if (error != 0) + panic("%s: ran out of segments: %d", __func__, error); + len -= seglen; + start = 0; + } +} + +static void +safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_cmd_descr *cdesc; + struct safexcel_request *req; + struct safexcel_ring *ring; + struct safexcel_session *sess; + struct sglist *sg; + size_t inlen; + int i; + bool first, last; + + req = arg; + if (error != 0) { + req->error = error; + return; + } + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + sess = req->sess; + ring = &req->sc->sc_ring[sess->ringidx]; + + mtx_assert(&ring->mtx, MA_OWNED); + + /* + * Set up descriptors for input and output data. + * + * The processing engine programs require that any AAD comes first, + * followed by the cipher plaintext, followed by the digest. Some + * consumers place the digest first in the input buffer, in which case + * we have to create an extra descriptor. + * + * As an optimization, unmodified data is not passed to the output + * stream. + */ + sglist_reset(ring->cmd_data); + sglist_reset(ring->res_data); + if (crp->crp_aad_length != 0) { + safexcel_append_segs(segs, nseg, ring->cmd_data, + crp->crp_aad_start, crp->crp_aad_length); + } + safexcel_append_segs(segs, nseg, ring->cmd_data, + crp->crp_payload_start, crp->crp_payload_length); + if (csp->csp_cipher_alg != 0) { + safexcel_append_segs(segs, nseg, ring->res_data, + crp->crp_payload_start, crp->crp_payload_length); + } + if (sess->digestlen > 0) { + if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { + safexcel_append_segs(segs, nseg, ring->cmd_data, + crp->crp_digest_start, sess->digestlen); + } else { + safexcel_append_segs(segs, nseg, ring->res_data, + crp->crp_digest_start, sess->digestlen); + } + } + + sg = ring->cmd_data; + if (sg->sg_nseg == 0) { + /* + * Fake a segment for the command descriptor if the input has + * length zero. The EIP97 apparently does not handle + * zero-length packets properly since subsequent requests return + * bogus errors, so provide a dummy segment using the context + * descriptor. + */ + (void)sglist_append_phys(sg, req->ctx.paddr, 1); + } + for (i = 0, inlen = 0; i < sg->sg_nseg; i++) + inlen += sg->sg_segs[i].ss_len; + for (i = 0; i < sg->sg_nseg; i++) { + first = i == 0; + last = i == sg->sg_nseg - 1; + + cdesc = safexcel_cmd_descr_add(ring, first, last, + sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len, + (uint32_t)inlen, req->ctx.paddr); + if (cdesc == NULL) { + safexcel_cmd_descr_rollback(ring, i); + req->error = EAGAIN; + return; + } + if (i == 0) + req->cdesc = cdesc; + } + req->cdescs = sg->sg_nseg; + + sg = ring->res_data; + if (sg->sg_nseg == 0) { + /* + * We need a result descriptor even if the output stream will be + * empty, for example when verifying an AAD digest. + */ + sg->sg_segs[0].ss_paddr = 0; + sg->sg_segs[0].ss_len = 0; + sg->sg_nseg = 1; + } + for (i = 0; i < sg->sg_nseg; i++) { + first = i == 0; + last = i == sg->sg_nseg - 1; + + if (safexcel_res_descr_add(ring, first, last, + sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) { + safexcel_cmd_descr_rollback(ring, + ring->cmd_data->sg_nseg); + safexcel_res_descr_rollback(ring, i); + req->error = EAGAIN; + return; + } + } + req->rdescs = sg->sg_nseg; +} + +static int +safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req) +{ + int error; + + req->error = 0; + req->cdescs = req->rdescs = 0; + + error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp, + safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); + if (error == 0) + req->dmap_loaded = true; + + if (req->error != 0) + error = req->error; + + return (error); +} + +static bool +safexcel_probe_cipher(const struct crypto_session_params *csp) +{ + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return (false); + break; + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return (false); + break; + default: + return (false); + } + + return (true); +} + +/* + * Determine whether the driver can implement a session with the requested + * parameters. + */ +static int +safexcel_probesession(device_t dev, const struct crypto_session_params *csp) +{ + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + if (!safexcel_probe_cipher(csp)) + return (EINVAL); + break; + case CSP_MODE_DIGEST: + switch (csp->csp_auth_alg) { + case CRYPTO_AES_NIST_GMAC: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); + break; + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + break; + default: + return (EINVAL); + } + break; + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); + break; + case CRYPTO_AES_CCM_16: + if (csp->csp_ivlen != AES_CCM_IV_LEN) + return (EINVAL); + break; + default: + return (EINVAL); + } + break; + case CSP_MODE_ETA: + if (!safexcel_probe_cipher(csp)) + return (EINVAL); + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + /* + * The EIP-97 does not support combining AES-XTS with + * hash operations. + */ + if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) + return (EINVAL); + break; + default: + return (EINVAL); + } + break; + default: + return (EINVAL); + } + + return (CRYPTODEV_PROBE_HARDWARE); +} + +/* + * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted + * using the cipher key. + */ +static void +safexcel_setkey_ghash(struct safexcel_session *sess, const uint8_t *key, + int klen) +{ + uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; + uint8_t zeros[AES_BLOCK_LEN]; + int i, rounds; + + memset(zeros, 0, sizeof(zeros)); + + rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); + rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)sess->ghash_key); + for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) + sess->ghash_key[i] = htobe32(sess->ghash_key[i]); + + explicit_bzero(ks, sizeof(ks)); +} + +/* + * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 + * in the hardware implementation. K1 is the cipher key and comes last in the + * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC + * is not implemented so K2 and K3 are fixed. + */ +static void +safexcel_setkey_xcbcmac(struct safexcel_session *sess, const uint8_t *key, + int klen) +{ + int i, off; + + memset(sess->xcbc_key, 0, sizeof(sess->xcbc_key)); + off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); + for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) + sess->xcbc_key[i + off] = htobe32(le32dec(key)); +} + +static void +safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, + char *buf) +{ + int hashwords, i; + + switch (ahash->type) { + case CRYPTO_SHA1_HMAC: + hashwords = ahash->hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); + break; + case CRYPTO_SHA2_224_HMAC: + hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); + break; + case CRYPTO_SHA2_256_HMAC: + hashwords = ahash->hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); + break; + case CRYPTO_SHA2_384_HMAC: + hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); + for (i = 0; i < hashwords; i++) + ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); + break; + case CRYPTO_SHA2_512_HMAC: + hashwords = ahash->hashsize / sizeof(uint64_t); + for (i = 0; i < hashwords; i++) + ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); + break; + } +} + +/* + * Pre-compute the inner and outer digests used in the HMAC algorithm. + */ +static void +safexcel_setkey_hmac(const struct crypto_session_params *csp, + struct safexcel_session *sess, const uint8_t *key, int klen) +{ + union authctx ctx; + struct auth_hash *ahash; + + ahash = crypto_auth_hash(csp); + hmac_init_ipad(ahash, key, klen, &ctx); + safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_ipad); + hmac_init_opad(ahash, key, klen, &ctx); + safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_opad); + explicit_bzero(&ctx, ahash->ctxsize); +} + +static void +safexcel_setkey_xts(struct safexcel_session *sess, const uint8_t *key, int klen) +{ + memcpy(sess->tweak_key, key + klen / 2, klen / 2); +} + +static void +safexcel_setkey(struct safexcel_session *sess, + const struct crypto_session_params *csp, struct cryptop *crp) +{ + const uint8_t *akey, *ckey; + int aklen, cklen; + + aklen = csp->csp_auth_klen; + cklen = csp->csp_cipher_klen; + akey = ckey = NULL; + if (crp != NULL) { + akey = crp->crp_auth_key; + ckey = crp->crp_cipher_key; + } + if (akey == NULL) + akey = csp->csp_auth_key; + if (ckey == NULL) + ckey = csp->csp_cipher_key; + + sess->klen = cklen; + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + safexcel_setkey_ghash(sess, ckey, cklen); + break; + case CRYPTO_AES_CCM_16: + safexcel_setkey_xcbcmac(sess, ckey, cklen); + break; + case CRYPTO_AES_XTS: + safexcel_setkey_xts(sess, ckey, cklen); + sess->klen /= 2; + break; + } + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + safexcel_setkey_hmac(csp, sess, akey, aklen); + break; + case CRYPTO_AES_NIST_GMAC: + sess->klen = aklen; + safexcel_setkey_ghash(sess, akey, aklen); + break; + } +} + +static uint32_t +safexcel_aes_algid(int keylen) +{ + switch (keylen) { + case 16: + return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128); + case 24: + return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192); + case 32: + return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256); + default: + panic("invalid AES key length %d", keylen); + } +} + +static uint32_t +safexcel_aes_ccm_hashid(int keylen) +{ + switch (keylen) { + case 16: + return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC128); + case 24: + return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC192); + case 32: + return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC256); + default: + panic("invalid AES key length %d", keylen); + } +} + +static uint32_t +safexcel_sha_hashid(int alg) +{ + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA1); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA224); + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA256); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA384); + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA512); + default: + __assert_unreachable(); + } +} + +static int +safexcel_sha_hashlen(int alg) +{ + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (SHA1_HASH_LEN); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + return (SHA2_224_HASH_LEN); + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (SHA2_256_HASH_LEN); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + return (SHA2_384_HASH_LEN); + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (SHA2_512_HASH_LEN); + default: + __assert_unreachable(); + } +} + +static int +safexcel_sha_statelen(int alg) +{ + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (SHA1_HASH_LEN); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (SHA2_256_HASH_LEN); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (SHA2_512_HASH_LEN); + default: + __assert_unreachable(); + } +} + +static int +safexcel_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct safexcel_session *sess; + struct safexcel_softc *sc; + + sc = device_get_softc(dev); + sess = crypto_get_driver_session(cses); + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_512: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED; + sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); + sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); + sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); + break; + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_HMAC; + sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); + sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); + sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); + break; + case CRYPTO_AES_NIST_GMAC: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; + sess->digestlen = GMAC_DIGEST_LEN; + sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; + sess->alg = safexcel_aes_algid(csp->csp_auth_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; + break; + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; + sess->digestlen = GMAC_DIGEST_LEN; + sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; + break; + case CRYPTO_AES_CCM_16: + sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen); + sess->digest = SAFEXCEL_CONTROL0_DIGEST_CCM; + sess->digestlen = CCM_CBC_MAX_DIGEST_LEN; + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM; + break; + case CRYPTO_AES_CBC: + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC; + break; + case CRYPTO_AES_ICM: + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR; + break; + case CRYPTO_AES_XTS: + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS; + break; + } + + if (csp->csp_auth_mlen != 0) + sess->digestlen = csp->csp_auth_mlen; + + safexcel_setkey(sess, csp, NULL); + + /* Bind each session to a fixed ring to minimize lock contention. */ + sess->ringidx = atomic_fetchadd_int(&sc->sc_ringidx, 1); + sess->ringidx %= sc->sc_config.rings; + + return (0); +} + +static int +safexcel_process(device_t dev, struct cryptop *crp, int hint) +{ + const struct crypto_session_params *csp; + struct safexcel_request *req; + struct safexcel_ring *ring; + struct safexcel_session *sess; + struct safexcel_softc *sc; + int error; + + sc = device_get_softc(dev); + sess = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); + + if (__predict_false(crypto_buffer_len(&crp->crp_buf) > + SAFEXCEL_MAX_REQUEST_SIZE)) { + crp->crp_etype = E2BIG; + crypto_done(crp); + return (0); + } + + if (crp->crp_cipher_key != NULL || crp->crp_auth_key != NULL) + safexcel_setkey(sess, csp, crp); + + ring = &sc->sc_ring[sess->ringidx]; + mtx_lock(&ring->mtx); + req = safexcel_alloc_request(sc, ring); + if (__predict_false(req == NULL)) { + mtx_lock(&sc->sc_mtx); + mtx_unlock(&ring->mtx); + sc->sc_blocked = CRYPTO_SYMQ; + mtx_unlock(&sc->sc_mtx); + return (ERESTART); + } + + req->crp = crp; + req->sess = sess; + + crypto_read_iv(crp, req->iv); + + error = safexcel_create_chain(ring, req); + if (__predict_false(error != 0)) { + safexcel_free_request(ring, req); + mtx_unlock(&ring->mtx); + crp->crp_etype = error; + crypto_done(crp); + return (0); + } + + safexcel_set_token(req); + + bus_dmamap_sync(ring->data_dtag, req->dmap, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(req->ctx.tag, req->ctx.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + safexcel_enqueue_request(sc, ring, req); + + if ((hint & CRYPTO_HINT_MORE) == 0) + safexcel_execute(sc, ring, req); + mtx_unlock(&ring->mtx); + + return (0); +} + +static device_method_t safexcel_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, safexcel_probe), + DEVMETHOD(device_attach, safexcel_attach), + DEVMETHOD(device_detach, safexcel_detach), + + /* Cryptodev interface */ + DEVMETHOD(cryptodev_probesession, safexcel_probesession), + DEVMETHOD(cryptodev_newsession, safexcel_newsession), + DEVMETHOD(cryptodev_process, safexcel_process), + + DEVMETHOD_END +}; + +static devclass_t safexcel_devclass; + +static driver_t safexcel_driver = { + .name = "safexcel", + .methods = safexcel_methods, + .size = sizeof(struct safexcel_softc), +}; + +DRIVER_MODULE(safexcel, simplebus, safexcel_driver, safexcel_devclass, 0, 0); +MODULE_VERSION(safexcel, 1); +MODULE_DEPEND(safexcel, crypto, 1, 1, 1); diff --git a/sys/dev/safexcel/safexcel_reg.h b/sys/dev/safexcel/safexcel_reg.h new file mode 100644 index 000000000000..fb26a8f99a50 --- /dev/null +++ b/sys/dev/safexcel/safexcel_reg.h @@ -0,0 +1,457 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SAFEXCEL_REGS_H_ +#define _SAFEXCEL_REGS_H_ + +#define SAFEXCEL_HIA_VERSION_LE 0x35ca +#define SAFEXCEL_HIA_VERSION_BE 0xca35 +#define EIP201_VERSION_LE 0x36c9 +#define SAFEXCEL_REG_LO16(_reg) ((_reg) & 0xffff) +#define SAFEXCEL_REG_HI16(_reg) (((_reg) >> 16) & 0xffff) + + +/* HIA, Command Descriptor Ring Manager */ +#define CDR_BASE_ADDR_LO(x) (0x0 + ((x) << 12)) +#define CDR_BASE_ADDR_HI(x) (0x4 + ((x) << 12)) +#define CDR_DATA_BASE_ADDR_LO(x) (0x8 + ((x) << 12)) +#define CDR_DATA_BASE_ADDR_HI(x) (0xC + ((x) << 12)) +#define CDR_ACD_BASE_ADDR_LO(x) (0x10 + ((x) << 12)) +#define CDR_ACD_BASE_ADDR_HI(x) (0x14 + ((x) << 12)) +#define CDR_RING_SIZE(x) (0x18 + ((x) << 12)) +#define CDR_DESC_SIZE(x) (0x1C + ((x) << 12)) +#define CDR_CFG(x) (0x20 + ((x) << 12)) +#define CDR_DMA_CFG(x) (0x24 + ((x) << 12)) +#define CDR_THR(x) (0x28 + ((x) << 12)) +#define CDR_PREP_COUNT(x) (0x2C + ((x) << 12)) +#define CDR_PROC_COUNT(x) (0x30 + ((x) << 12)) +#define CDR_PREP_PNTR(x) (0x34 + ((x) << 12)) +#define CDR_PROC_PNTR(x) (0x38 + ((x) << 12)) +#define CDR_STAT(x) (0x3C + ((x) << 12)) + +/* HIA, Result Descriptor Ring Manager */ +#define RDR_BASE_ADDR_LO(x) (0x800 + ((x) << 12)) +#define RDR_BASE_ADDR_HI(x) (0x804 + ((x) << 12)) +#define RDR_DATA_BASE_ADDR_LO(x) (0x808 + ((x) << 12)) +#define RDR_DATA_BASE_ADDR_HI(x) (0x80C + ((x) << 12)) +#define RDR_ACD_BASE_ADDR_LO(x) (0x810 + ((x) << 12)) +#define RDR_ACD_BASE_ADDR_HI(x) (0x814 + ((x) << 12)) +#define RDR_RING_SIZE(x) (0x818 + ((x) << 12)) +#define RDR_DESC_SIZE(x) (0x81C + ((x) << 12)) +#define RDR_CFG(x) (0x820 + ((x) << 12)) +#define RDR_DMA_CFG(x) (0x824 + ((x) << 12)) +#define RDR_THR(x) (0x828 + ((x) << 12)) +#define RDR_PREP_COUNT(x) (0x82C + ((x) << 12)) +#define RDR_PROC_COUNT(x) (0x830 + ((x) << 12)) +#define RDR_PREP_PNTR(x) (0x834 + ((x) << 12)) +#define RDR_PROC_PNTR(x) (0x838 + ((x) << 12)) +#define RDR_STAT(x) (0x83C + ((x) << 12)) + +/* HIA, Ring AIC */ +#define AIC_POL_CTRL(x) (0xE000 - ((x) << 12)) +#define AIC_TYPE_CTRL(x) (0xE004 - ((x) << 12)) +#define AIC_ENABLE_CTRL(x) (0xE008 - ((x) << 12)) +#define AIC_RAW_STAL(x) (0xE00C - ((x) << 12)) +#define AIC_ENABLE_SET(x) (0xE00C - ((x) << 12)) +#define AIC_ENABLED_STAT(x) (0xE010 - ((x) << 12)) +#define AIC_ACK(x) (0xE010 - ((x) << 12)) +#define AIC_ENABLE_CLR(x) (0xE014 - ((x) << 12)) +#define AIC_OPTIONS(x) (0xE018 - ((x) << 12)) +#define AIC_VERSION(x) (0xE01C - ((x) << 12)) + +/* HIA, Global AIC */ +#define AIC_G_POL_CTRL 0xF800 +#define AIC_G_TYPE_CTRL 0xF804 +#define AIC_G_ENABLE_CTRL 0xF808 +#define AIC_G_RAW_STAT 0xF80C +#define AIC_G_ENABLE_SET 0xF80C +#define AIC_G_ENABLED_STAT 0xF810 +#define AIC_G_ACK 0xF810 +#define AIC_G_ENABLE_CLR 0xF814 +#define AIC_G_OPTIONS 0xF818 +#define AIC_G_VERSION 0xF81C + +/* HIA, Data Fetch Engine */ +#define DFE_CFG 0xF000 +#define DFE_PRIO_0 0xF010 +#define DFE_PRIO_1 0xF014 +#define DFE_PRIO_2 0xF018 +#define DFE_PRIO_3 0xF01C + +/* HIA, Data Fetch Engine access monitoring for CDR */ +#define DFE_RING_REGION_LO(x) (0xF080 + ((x) << 3)) +#define DFE_RING_REGION_HI(x) (0xF084 + ((x) << 3)) + +/* HIA, Data Fetch Engine thread control and status for thread */ +#define DFE_THR_CTRL 0xF200 +#define DFE_THR_STAT 0xF204 +#define DFE_THR_DESC_CTRL 0xF208 +#define DFE_THR_DESC_DPTR_LO 0xF210 +#define DFE_THR_DESC_DPTR_HI 0xF214 +#define DFE_THR_DESC_ACDPTR_LO 0xF218 +#define DFE_THR_DESC_ACDPTR_HI 0xF21C + +/* HIA, Data Store Engine */ +#define DSE_CFG 0xF400 +#define DSE_PRIO_0 0xF410 +#define DSE_PRIO_1 0xF414 +#define DSE_PRIO_2 0xF418 +#define DSE_PRIO_3 0xF41C + +/* HIA, Data Store Engine access monitoring for RDR */ +#define DSE_RING_REGION_LO(x) (0xF480 + ((x) << 3)) +#define DSE_RING_REGION_HI(x) (0xF484 + ((x) << 3)) + +/* HIA, Data Store Engine thread control and status for thread */ +#define DSE_THR_CTRL 0xF600 +#define DSE_THR_STAT 0xF604 +#define DSE_THR_DESC_CTRL 0xF608 +#define DSE_THR_DESC_DPTR_LO 0xF610 +#define DSE_THR_DESC_DPTR_HI 0xF614 +#define DSE_THR_DESC_S_DPTR_LO 0xF618 +#define DSE_THR_DESC_S_DPTR_HI 0xF61C +#define DSE_THR_ERROR_STAT 0xF620 + +/* HIA Global */ +#define HIA_MST_CTRL 0xFFF4 +#define HIA_OPTIONS 0xFFF8 +#define HIA_VERSION 0xFFFC + +/* Processing Engine Input Side, Processing Engine */ +#define PE_IN_DBUF_THRESH 0x10000 +#define PE_IN_TBUF_THRESH 0x10100 + +/* Packet Engine Configuration / Status Registers */ +#define PE_TOKEN_CTRL_STAT 0x11000 +#define PE_FUNCTION_EN 0x11004 +#define PE_CONTEXT_CTRL 0x11008 +#define PE_INTERRUPT_CTRL_STAT 0x11010 +#define PE_CONTEXT_STAT 0x1100C +#define PE_OUT_TRANS_CTRL_STAT 0x11018 +#define PE_OUT_BUF_CTRL 0x1101C + +/* Packet Engine AIC */ +#define PE_EIP96_AIC_POL_CTRL 0x113C0 +#define PE_EIP96_AIC_TYPE_CTRL 0x113C4 +#define PE_EIP96_AIC_ENABLE_CTRL 0x113C8 +#define PE_EIP96_AIC_RAW_STAT 0x113CC +#define PE_EIP96_AIC_ENABLE_SET 0x113CC +#define PE_EIP96_AIC_ENABLED_STAT 0x113D0 +#define PE_EIP96_AIC_ACK 0x113D0 +#define PE_EIP96_AIC_ENABLE_CLR 0x113D4 +#define PE_EIP96_AIC_OPTIONS 0x113D8 +#define PE_EIP96_AIC_VERSION 0x113DC + +/* Packet Engine Options & Version Registers */ +#define PE_EIP96_OPTIONS 0x113F8 +#define PE_EIP96_VERSION 0x113FC + +#define SAFEXCEL_OPT + +/* Processing Engine Output Side */ +#define PE_OUT_DBUF_THRESH 0x11C00 +#define PE_OUT_TBUF_THRESH 0x11D00 + +/* Processing Engine Local AIC */ +#define PE_AIC_POL_CTRL 0x11F00 +#define PE_AIC_TYPE_CTRL 0x11F04 +#define PE_AIC_ENABLE_CTRL 0x11F08 +#define PE_AIC_RAW_STAT 0x11F0C +#define PE_AIC_ENABLE_SET 0x11F0C +#define PE_AIC_ENABLED_STAT 0x11F10 +#define PE_AIC_ENABLE_CLR 0x11F14 +#define PE_AIC_OPTIONS 0x11F18 +#define PE_AIC_VERSION 0x11F1C + +/* Processing Engine General Configuration and Version */ +#define PE_IN_FLIGHT 0x11FF0 +#define PE_OPTIONS 0x11FF8 +#define PE_VERSION 0x11FFC + +/* EIP-97 - Global */ +#define EIP97_CLOCK_STATE 0x1FFE4 +#define EIP97_FORCE_CLOCK_ON 0x1FFE8 +#define EIP97_FORCE_CLOCK_OFF 0x1FFEC +#define EIP97_MST_CTRL 0x1FFF4 +#define EIP97_OPTIONS 0x1FFF8 +#define EIP97_VERSION 0x1FFFC + +/* Register base offsets */ +#define SAFEXCEL_HIA_AIC(_sc) ((_sc)->sc_offsets.hia_aic) +#define SAFEXCEL_HIA_AIC_G(_sc) ((_sc)->sc_offsets.hia_aic_g) +#define SAFEXCEL_HIA_AIC_R(_sc) ((_sc)->sc_offsets.hia_aic_r) +#define SAFEXCEL_HIA_AIC_xDR(_sc) ((_sc)->sc_offsets.hia_aic_xdr) +#define SAFEXCEL_HIA_DFE(_sc) ((_sc)->sc_offsets.hia_dfe) +#define SAFEXCEL_HIA_DFE_THR(_sc) ((_sc)->sc_offsets.hia_dfe_thr) +#define SAFEXCEL_HIA_DSE(_sc) ((_sc)->sc_offsets.hia_dse) +#define SAFEXCEL_HIA_DSE_THR(_sc) ((_sc)->sc_offsets.hia_dse_thr) +#define SAFEXCEL_HIA_GEN_CFG(_sc) ((_sc)->sc_offsets.hia_gen_cfg) +#define SAFEXCEL_PE(_sc) ((_sc)->sc_offsets.pe) + +/* EIP197 base offsets */ +#define SAFEXCEL_EIP197_HIA_AIC_BASE 0x90000 +#define SAFEXCEL_EIP197_HIA_AIC_G_BASE 0x90000 +#define SAFEXCEL_EIP197_HIA_AIC_R_BASE 0x90800 +#define SAFEXCEL_EIP197_HIA_AIC_xDR_BASE 0x80000 +#define SAFEXCEL_EIP197_HIA_DFE_BASE 0x8c000 +#define SAFEXCEL_EIP197_HIA_DFE_THR_BASE 0x8c040 +#define SAFEXCEL_EIP197_HIA_DSE_BASE 0x8d000 +#define SAFEXCEL_EIP197_HIA_DSE_THR_BASE 0x8d040 +#define SAFEXCEL_EIP197_HIA_GEN_CFG_BASE 0xf0000 +#define SAFEXCEL_EIP197_PE_BASE 0xa0000 + +/* EIP97 base offsets */ +#define SAFEXCEL_EIP97_HIA_AIC_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_AIC_G_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_AIC_R_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_AIC_xDR_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_DFE_BASE 0xf000 +#define SAFEXCEL_EIP97_HIA_DFE_THR_BASE 0xf200 +#define SAFEXCEL_EIP97_HIA_DSE_BASE 0xf400 +#define SAFEXCEL_EIP97_HIA_DSE_THR_BASE 0xf600 +#define SAFEXCEL_EIP97_HIA_GEN_CFG_BASE 0x10000 +#define SAFEXCEL_EIP97_PE_BASE 0x10000 + +/* CDR/RDR register offsets */ +#define SAFEXCEL_HIA_xDR_OFF(priv, r) (SAFEXCEL_HIA_AIC_xDR(priv) + (r) * 0x1000) +#define SAFEXCEL_HIA_CDR(priv, r) (SAFEXCEL_HIA_xDR_OFF(priv, r)) +#define SAFEXCEL_HIA_RDR(priv, r) (SAFEXCEL_HIA_xDR_OFF(priv, r) + 0x800) +#define SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO 0x0000 +#define SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI 0x0004 +#define SAFEXCEL_HIA_xDR_RING_SIZE 0x0018 +#define SAFEXCEL_HIA_xDR_DESC_SIZE 0x001c +#define SAFEXCEL_HIA_xDR_CFG 0x0020 +#define SAFEXCEL_HIA_xDR_DMA_CFG 0x0024 +#define SAFEXCEL_HIA_xDR_THRESH 0x0028 +#define SAFEXCEL_HIA_xDR_PREP_COUNT 0x002c +#define SAFEXCEL_HIA_xDR_PROC_COUNT 0x0030 +#define SAFEXCEL_HIA_xDR_PREP_PNTR 0x0034 +#define SAFEXCEL_HIA_xDR_PROC_PNTR 0x0038 +#define SAFEXCEL_HIA_xDR_STAT 0x003c + +/* register offsets */ +#define SAFEXCEL_HIA_DFE_CFG(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DFE_THR_CTRL(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DFE_THR_STAT(n) (0x004 + (128 * (n))) +#define SAFEXCEL_HIA_DSE_CFG(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DSE_THR_CTRL(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DSE_THR_STAT(n) (0x004 + (128 * (n))) +#define SAFEXCEL_HIA_RA_PE_CTRL(n) (0x010 + (8 * (n))) +#define SAFEXCEL_HIA_RA_PE_STAT 0x0014 +#define SAFEXCEL_HIA_AIC_R_OFF(r) ((r) * 0x1000) +#define SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(r) (0xe008 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_ENABLED_STAT(r) (0xe010 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_ACK(r) (0xe010 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_ENABLE_CLR(r) (0xe014 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_VERSION(r) (0xe01c - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_G_ENABLE_CTRL 0xf808 +#define SAFEXCEL_HIA_AIC_G_ENABLED_STAT 0xf810 +#define SAFEXCEL_HIA_AIC_G_ACK 0xf810 +#define SAFEXCEL_HIA_MST_CTRL 0xfff4 +#define SAFEXCEL_HIA_OPTIONS 0xfff8 +#define SAFEXCEL_HIA_VERSION 0xfffc +#define SAFEXCEL_PE_IN_DBUF_THRES(n) (0x0000 + (0x2000 * (n))) +#define SAFEXCEL_PE_IN_TBUF_THRES(n) (0x0100 + (0x2000 * (n))) +#define SAFEXCEL_PE_ICE_SCRATCH_RAM(x, n) ((0x800 + (x) * 4) + 0x2000 * (n)) +#define SAFEXCEL_PE_ICE_PUE_CTRL(n) (0xc80 + (0x2000 * (n))) +#define SAFEXCEL_PE_ICE_SCRATCH_CTRL 0x0d04 +#define SAFEXCEL_PE_ICE_FPP_CTRL(n) (0xd80 + (0x2000 * (n))) +#define SAFEXCEL_PE_ICE_RAM_CTRL(n) (0xff0 + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n))) +#define SAFEXCEL_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n))) +#define SAFEXCEL_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n))) + +/* EIP-197 Classification Engine */ + +/* Classification regs */ +#define SAFEXCEL_CS_RAM_CTRL 0xf7ff0 + +/* SAFEXCEL_HIA_xDR_DESC_SIZE */ +#define SAFEXCEL_xDR_DESC_MODE_64BIT (1U << 31) +#define SAFEXCEL_CDR_DESC_MODE_ADCP (1 << 30) +#define SAFEXCEL_xDR_DESC_xD_OFFSET 16 + +/* SAFEXCEL_DIA_xDR_CFG */ +#define SAFEXCEL_xDR_xD_FETCH_THRESH 16 + +/* SAFEXCEL_HIA_xDR_DMA_CFG */ +#define SAFEXCEL_HIA_xDR_WR_RES_BUF (1 << 22) +#define SAFEXCEL_HIA_xDR_WR_CTRL_BUF (1 << 23) +#define SAFEXCEL_HIA_xDR_WR_OWN_BUF (1 << 24) +#define SAFEXCEL_HIA_xDR_CFG_xD_PROT(n) (((n) & 0xf) << 4) +#define SAFEXCEL_HIA_xDR_CFG_DATA_PROT(n) (((n) & 0xf) << 12) +#define SAFEXCEL_HIA_xDR_CFG_ACD_PROT(n) (((n) & 0xf) << 20) +#define SAFEXCEL_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 25) +#define SAFEXCEL_HIA_xDR_CFG_RD_CACHE(n) (((n) & 0x7) << 29) + +/* SAFEXCEL_HIA_CDR_THRESH */ +#define SAFEXCEL_HIA_CDR_THRESH_PROC_PKT(n) ((n) & 0xffff) +#define SAFEXCEL_HIA_CDR_THRESH_PROC_MODE (1 << 22) +#define SAFEXCEL_HIA_CDR_THRESH_PKT_MODE (1 << 23) + /* x256 clk cycles */ +#define SAFEXCEL_HIA_CDR_THRESH_TIMEOUT(n) (((n) & 0xff) << 24) + +/* SAFEXCEL_HIA_RDR_THRESH */ +#define SAFEXCEL_HIA_RDR_THRESH_PROC_PKT(n) ((n) & 0xffff) +#define SAFEXCEL_HIA_RDR_THRESH_PKT_MODE (1 << 23) + /* x256 clk cycles */ +#define SAFEXCEL_HIA_RDR_THRESH_TIMEOUT(n) (((n) & 0xff) << 24) + +/* SAFEXCEL_HIA_xDR_PREP_COUNT */ +#define SAFEXCEL_xDR_PREP_CLR_COUNT (1U << 31) +#define SAFEXCEL_xDR_PREP_xD_COUNT_INCR_OFFSET 2 +#define SAFEXCEL_xDR_PREP_RD_COUNT_INCR_MASK 0x3fff + +/* SAFEXCEL_HIA_xDR_PROC_COUNT */ +#define SAFEXCEL_xDR_PROC_xD_PKT_OFFSET 24 +#define SAFEXCEL_xDR_PROC_xD_PKT_MASK 0x7f +#define SAFEXCEL_xDR_PROC_xD_COUNT(n) ((n) << 2) +#define SAFEXCEL_xDR_PROC_xD_PKT(n) \ + (((n) & SAFEXCEL_xDR_PROC_xD_PKT_MASK) << SAFEXCEL_xDR_PROC_xD_PKT_OFFSET) +#define SAFEXCEL_xDR_PROC_CLR_COUNT (1U << 31) + +/* SAFEXCEL_HIA_xDR_STAT */ +#define SAFEXCEL_xDR_DMA_ERR (1 << 0) +#define SAFEXCEL_xDR_PREP_CMD_THRES (1 << 1) +#define SAFEXCEL_xDR_ERR (1 << 2) +#define SAFEXCEL_xDR_THRESH (1 << 4) +#define SAFEXCEL_xDR_TIMEOUT (1 << 5) +#define SAFEXCEL_CDR_INTR_MASK 0x3f +#define SAFEXCEL_RDR_INTR_MASK 0xff + +#define SAFEXCEL_HIA_RA_PE_CTRL_RESET (1U << 31) +#define SAFEXCEL_HIA_RA_PE_CTRL_EN (1 << 30) + +/* Register offsets */ + +/* SAFEXCEL_HIA_DSE_THR_STAT */ +#define SAFEXCEL_DSE_THR_RDR_ID_MASK 0xf000 + +/* SAFEXCEL_HIA_OPTIONS */ +#define SAFEXCEL_OPT_ADDR_64 (1U << 31) +#define SAFEXCEL_OPT_TGT_ALIGN_OFFSET 28 +#define SAFEXCEL_OPT_TGT_ALIGN_MASK 0x70000000 +#define SAFEXCEL_xDR_HDW_OFFSET 25 +#define SAFEXCEL_xDR_HDW_MASK 0x6000000 +#define SAFEXCEL_N_RINGS_MASK 0xf +#define SAFEXCEL_N_PES_OFFSET 4 +#define SAFEXCEL_N_PES_MASK 0x1f0 +#define EIP97_N_PES_MASK 0x70 + +/* SAFEXCEL_HIA_AIC_R_ENABLE_CTRL */ +#define SAFEXCEL_CDR_IRQ(n) (1 << ((n) * 2)) +#define SAFEXCEL_RDR_IRQ(n) (1 << ((n) * 2 + 1)) + +/* SAFEXCEL_HIA_DFE/DSE_CFG */ +#define SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(n) ((n) << 0) +#define SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(n) (((n) & 0x7) << 4) +#define SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(n) ((n) << 8) +#define SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE 0xc000 +#define SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(n) ((n) << 16) +#define SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20) +#define SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24) +#define SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG 0xe0000000 +#define SAFEXCEL_HIA_DSE_CFG_EN_SINGLE_WR (1 << 29) +#define SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG 0xc0000000 + +/* SAFEXCEL_HIA_DFE/DSE_THR_CTRL */ +#define SAFEXCEL_DxE_THR_CTRL_EN (1 << 30) +#define SAFEXCEL_DxE_THR_CTRL_RESET_PE (1U << 31) + +/* SAFEXCEL_HIA_AIC_G_ENABLED_STAT */ +#define SAFEXCEL_G_IRQ_DFE(n) (1 << ((n) << 1)) +#define SAFEXCEL_G_IRQ_DSE(n) (1 << (((n) << 1) + 1)) +#define SAFEXCEL_G_IRQ_RING (1 << 16) +#define SAFEXCEL_G_IRQ_PE(n) (1 << ((n) + 20)) + +/* SAFEXCEL_HIA_MST_CTRL */ +#define RD_CACHE_3BITS 0x5U +#define WR_CACHE_3BITS 0x3U +#define RD_CACHE_4BITS (RD_CACHE_3BITS << 1 | (1 << 0)) +#define WR_CACHE_4BITS (WR_CACHE_3BITS << 1 | (1 << 0)) +#define SAFEXCEL_MST_CTRL_RD_CACHE(n) (((n) & 0xf) << 0) +#define SAFEXCEL_MST_CTRL_WD_CACHE(n) (((n) & 0xf) << 4) +#define MST_CTRL_SUPPORT_PROT(n) (((n) & 0xf) << 12) +#define SAFEXCEL_MST_CTRL_BYTE_SWAP (1 << 24) +#define SAFEXCEL_MST_CTRL_NO_BYTE_SWAP (1 << 25) + +/* SAFEXCEL_PE_IN_DBUF/TBUF_THRES */ +#define SAFEXCEL_PE_IN_xBUF_THRES_MIN(n) ((n) << 8) +#define SAFEXCEL_PE_IN_xBUF_THRES_MAX(n) ((n) << 12) + +/* SAFEXCEL_PE_OUT_DBUF_THRES */ +#define SAFEXCEL_PE_OUT_DBUF_THRES_MIN(n) ((n) << 0) +#define SAFEXCEL_PE_OUT_DBUF_THRES_MAX(n) ((n) << 4) + +/* SAFEXCEL_HIA_AIC_G_ACK */ +#define SAFEXCEL_AIC_G_ACK_ALL_MASK 0xffffffff +#define SAFEXCEL_AIC_G_ACK_HIA_MASK 0x7ff00000 + +/* SAFEXCEL_HIA_AIC_R_ENABLE_CLR */ +#define SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK 0xffffffff + +/* SAFEXCEL_PE_EIP96_CONTEXT_CTRL */ +#define SAFEXCEL_CONTEXT_SIZE(n) (n) +#define SAFEXCEL_ADDRESS_MODE (1 << 8) +#define SAFEXCEL_CONTROL_MODE (1 << 9) + +/* SAFEXCEL_PE_EIP96_FUNCTION_EN */ +#define SAFEXCEL_FUNCTION_RSVD ((1U << 6) | (1U << 15) | (1U << 20) | (1U << 23)) +#define SAFEXCEL_PROTOCOL_HASH_ONLY (1U << 0) +#define SAFEXCEL_PROTOCOL_ENCRYPT_ONLY (1U << 1) +#define SAFEXCEL_PROTOCOL_HASH_ENCRYPT (1U << 2) +#define SAFEXCEL_PROTOCOL_HASH_DECRYPT (1U << 3) +#define SAFEXCEL_PROTOCOL_ENCRYPT_HASH (1U << 4) +#define SAFEXCEL_PROTOCOL_DECRYPT_HASH (1U << 5) +#define SAFEXCEL_ALG_ARC4 (1U << 7) +#define SAFEXCEL_ALG_AES_ECB (1U << 8) +#define SAFEXCEL_ALG_AES_CBC (1U << 9) +#define SAFEXCEL_ALG_AES_CTR_ICM (1U << 10) +#define SAFEXCEL_ALG_AES_OFB (1U << 11) +#define SAFEXCEL_ALG_AES_CFB (1U << 12) +#define SAFEXCEL_ALG_DES_ECB (1U << 13) +#define SAFEXCEL_ALG_DES_CBC (1U << 14) +#define SAFEXCEL_ALG_DES_OFB (1U << 16) +#define SAFEXCEL_ALG_DES_CFB (1U << 17) +#define SAFEXCEL_ALG_3DES_ECB (1U << 18) +#define SAFEXCEL_ALG_3DES_CBC (1U << 19) +#define SAFEXCEL_ALG_3DES_OFB (1U << 21) +#define SAFEXCEL_ALG_3DES_CFB (1U << 22) +#define SAFEXCEL_ALG_MD5 (1U << 24) +#define SAFEXCEL_ALG_HMAC_MD5 (1U << 25) +#define SAFEXCEL_ALG_SHA1 (1U << 26) +#define SAFEXCEL_ALG_HMAC_SHA1 (1U << 27) +#define SAFEXCEL_ALG_SHA2 (1U << 28) +#define SAFEXCEL_ALG_HMAC_SHA2 (1U << 29) +#define SAFEXCEL_ALG_AES_XCBC_MAC (1U << 30) +#define SAFEXCEL_ALG_GCM_HASH (1U << 31) + +#endif /* _SAFEXCEL_REGS_H_ */ diff --git a/sys/dev/safexcel/safexcel_var.h b/sys/dev/safexcel/safexcel_var.h new file mode 100644 index 000000000000..619bcff00c57 --- /dev/null +++ b/sys/dev/safexcel/safexcel_var.h @@ -0,0 +1,420 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SAFEXCEL_VAR_H_ +#define _SAFEXCEL_VAR_H_ + +#define SAFEXCEL_MAX_RINGS 4 +#define SAFEXCEL_MAX_BATCH_SIZE 64 +#define SAFEXCEL_MAX_FRAGMENTS 64 +#define SAFEXCEL_MAX_IV_LEN 16 +#define SAFEXCEL_MAX_REQUEST_SIZE 65535 + +#define SAFEXCEL_RING_SIZE 512 +#define SAFEXCEL_REQUESTS_PER_RING 64 +#define SAFEXCEL_MAX_ITOKENS 4 +#define SAFEXCEL_MAX_ATOKENS 16 +#define SAFEXCEL_FETCH_COUNT 1 +#define SAFEXCEL_MAX_KEY_LEN 32 +#define SAFEXCEL_MAX_RING_AIC 14 + +/* + * Context Record format. + * + * In this driver the context control words are always set in the control data. + * This is configured by setting SAFEXCEL_OPTION_CTX_CTRL_IN_CMD. + */ +struct safexcel_context_record { + uint32_t control0; /* Unused. */ + uint32_t control1; /* Unused. */ + uint32_t data[40]; /* Key material. */ +} __packed; + +/* Processing Engine Control Data format. */ +struct safexcel_control_data { + uint32_t packet_length : 17; + uint32_t options : 13; + uint32_t type : 2; + + uint16_t application_id; + uint16_t rsvd; + + uint32_t context_lo; + uint32_t context_hi; + + uint32_t control0; + uint32_t control1; + + /* Inline instructions or IV. */ + uint32_t token[SAFEXCEL_MAX_ITOKENS]; +} __packed; + +/* + * Basic Command Descriptor. + * + * The Processing Engine and driver cooperate to maintain a set of command + * rings, representing outstanding crypto operation requests. Each descriptor + * corresponds to an input data segment, and thus a single crypto(9) request may + * span several contiguous command descriptors. + * + * The first command descriptor for a request stores the input token, which + * encodes data specific to the requested operation, such as the encryption + * mode. For some operations data is passed outside the descriptor, in a + * context record (e.g., encryption keys), or in an "additional token data" + * region (e.g., instructions). + */ +struct safexcel_cmd_descr { + uint32_t particle_size : 17; + uint32_t rsvd0 : 5; + uint32_t last_seg : 1; + uint32_t first_seg : 1; + uint32_t additional_cdata_size : 8; + uint32_t rsvd1; + + uint32_t data_lo; + uint32_t data_hi; + + uint32_t atok_lo; + uint32_t atok_hi; + + struct safexcel_control_data control_data; +} __packed; + +/* Context control word 0 fields. */ +#define SAFEXCEL_CONTROL0_TYPE_NULL_OUT 0x0 +#define SAFEXCEL_CONTROL0_TYPE_NULL_IN 0x1 +#define SAFEXCEL_CONTROL0_TYPE_HASH_OUT 0x2 +#define SAFEXCEL_CONTROL0_TYPE_HASH_IN 0x3 +#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT 0x4 +#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN 0x5 +#define SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT 0x6 +#define SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN 0x7 +#define SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT 0xe +#define SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN 0xf +#define SAFEXCEL_CONTROL0_RESTART_HASH (1 << 4) +#define SAFEXCEL_CONTROL0_NO_FINISH_HASH (1 << 5) +#define SAFEXCEL_CONTROL0_SIZE(n) (((n) & 0xff) << 8) +#define SAFEXCEL_CONTROL0_KEY_EN (1 << 16) +#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128 (0x5 << 17) +#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192 (0x6 << 17) +#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256 (0x7 << 17) +#define SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED (0x1 << 21) +#define SAFEXCEL_CONTROL0_DIGEST_CCM (0x2 << 21) +#define SAFEXCEL_CONTROL0_DIGEST_GMAC (0x2 << 21) +#define SAFEXCEL_CONTROL0_DIGEST_HMAC (0x3 << 21) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA1 (0x2 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA224 (0x4 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA256 (0x3 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA384 (0x6 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA512 (0x5 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC128 (0x1 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC192 (0x2 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC256 (0x3 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_GHASH (0x4 << 23) +#define SAFEXCEL_CONTROL0_INV_FR (0x5 << 24) +#define SAFEXCEL_CONTROL0_INV_TR (0x6 << 24) + +/* Context control word 1 fields. */ +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ECB 0x0 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC 0x1 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ICM 0x3 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_OFB 0x4 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CFB128 0x5 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR 0x6 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS 0x7 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM (0x6 | (1 << 17)) +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM (0x6 | (1 << 17)) +#define SAFEXCEL_CONTROL1_IV0 (1u << 5) +#define SAFEXCEL_CONTROL1_IV1 (1u << 6) +#define SAFEXCEL_CONTROL1_IV2 (1u << 7) +#define SAFEXCEL_CONTROL1_IV3 (1u << 8) +#define SAFEXCEL_CONTROL1_DIGEST_CNT (1u << 9) +#define SAFEXCEL_CONTROL1_COUNTER_MODE (1u << 10) +#define SAFEXCEL_CONTROL1_ENCRYPT_HASH_RES (1u << 17) +#define SAFEXCEL_CONTROL1_HASH_STORE (1u << 19) + +/* Control options. */ +#define SAFEXCEL_OPTION_IP (1u << 0) /* must be set */ +#define SAFEXCEL_OPTION_CP (1u << 1) /* 64-bit ctx addr */ +#define SAFEXCEL_OPTION_RC_AUTO (2u << 3) /* auto ctx reuse */ +#define SAFEXCEL_OPTION_CTX_CTRL_IN_CMD (1u << 8) /* ctx ctrl */ +#define SAFEXCEL_OPTION_4_TOKEN_IV_CMD 0xe00 /* IV in bypass */ + +struct safexcel_res_data { + uint32_t packet_length : 17; + uint32_t error_code : 15; + + uint32_t bypass_length : 4; + uint32_t e15 : 1; + uint32_t rsvd0 : 16; + uint32_t hash_bytes : 1; + uint32_t hash_length : 6; + uint32_t generic_bytes : 1; + uint32_t checksum : 1; + uint32_t next_header : 1; + uint32_t length : 1; + + uint16_t application_id; + uint16_t rsvd1; + + uint32_t rsvd2; +}; + +/* Basic Result Descriptor format */ +struct safexcel_res_descr { + uint32_t particle_size : 17; + uint32_t rsvd0 : 3; + uint32_t descriptor_overflow : 1; + uint32_t buffer_overflow : 1; + uint32_t last_seg : 1; + uint32_t first_seg : 1; + uint32_t result_size : 8; + + uint32_t rsvd1; + + uint32_t data_lo; + uint32_t data_hi; + + struct safexcel_res_data result_data; +} __packed; + +/* Result data error codes. */ +#define SAFEXCEL_RESULT_ERR_PACKET_LEN (1u << 0) +#define SAFEXCEL_RESULT_ERR_TOKEN_ERROR (1u << 1) +#define SAFEXCEL_RESULT_ERR_BYPASS (1u << 2) +#define SAFEXCEL_RESULT_ERR_CRYPTO_BLOCK_SIZE (1u << 3) +#define SAFEXCEL_RESULT_ERR_HASH_BLOCK_SIZE (1u << 4) +#define SAFEXCEL_RESULT_ERR_INVALID_CMD (1u << 5) +#define SAFEXCEL_RESULT_ERR_PROHIBITED_ALGO (1u << 6) +#define SAFEXCEL_RESULT_ERR_HASH_INPUT_OVERFLOW (1u << 7) +#define SAFEXCEL_RESULT_ERR_TTL_UNDERFLOW (1u << 8) +#define SAFEXCEL_RESULT_ERR_AUTH_FAILED (1u << 9) +#define SAFEXCEL_RESULT_ERR_SEQNO_CHECK_FAILED (1u << 10) +#define SAFEXCEL_RESULT_ERR_SPI_CHECK (1u << 11) +#define SAFEXCEL_RESULT_ERR_CHECKSUM (1u << 12) +#define SAFEXCEL_RESULT_ERR_PAD_VERIFICATION (1u << 13) +#define SAFEXCEL_RESULT_ERR_TIMEOUT (1u << 14) +#define SAFEXCEL_RESULT_ERR_OUTPUT_DMA (1u << 15) + +/* + * The EIP-96 (crypto transform engine) is programmed using a set of + * data processing instructions with the encodings defined below. + */ +struct safexcel_instr { + uint32_t length : 17; /* bytes to be processed */ + uint32_t status : 2; /* stream status */ + uint32_t instructions : 9; + uint32_t opcode : 4; +} __packed; + +/* Type 1, operational data instructions. */ +#define SAFEXCEL_INSTR_OPCODE_DIRECTION 0x0 +#define SAFEXCEL_INSTR_OPCODE_PRE_CHECKSUM 0x1 +#define SAFEXCEL_INSTR_OPCODE_INSERT 0x2 +#define SAFEXCEL_INSTR_OPCODE_INSERT_CTX 0x9 +#define SAFEXCEL_INSTR_OPCODE_REPLACE 0x3 +#define SAFEXCEL_INSTR_OPCODE_RETRIEVE 0x4 +#define SAFEXCEL_INSTR_OPCODE_MUTE 0x5 +/* Type 2, IP header instructions. */ +#define SAFEXCEL_INSTR_OPCODE_IPV4 0x7 +#define SAFEXCEL_INSTR_OPCODE_IPV4_CHECKSUM 0x6 +#define SAFEXCEL_INSTR_OPCODE_IPV6 0x8 +/* Type 3, postprocessing instructions. */ +#define SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT 0xa +#define SAFEXCEL_INSTR_OPCODE_REPLACE_BYTE 0xb +/* Type 4, result instructions. */ +#define SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS 0xd +/* Type 5, context control instructions. */ +#define SAFEXCEL_INSTR_OPCODE_CONTEXT_ACCESS 0xe +/* Type 6, context control instructions. */ +#define SAFEXCEL_INSTR_OPCODE_BYPASS_TOKEN_DATA 0xf + +/* Status bits for type 1 and 2 instructions. */ +#define SAFEXCEL_INSTR_STATUS_LAST_HASH (1u << 0) +#define SAFEXCEL_INSTR_STATUS_LAST_PACKET (1u << 1) +/* Status bits for type 3 instructions. */ +#define SAFEXCEL_INSTR_STATUS_NO_CKSUM_MOD (1u << 0) + +/* Instruction-dependent flags. */ +#define SAFEXCEL_INSTR_INSERT_HASH_DIGEST 0x1c +#define SAFEXCEL_INSTR_INSERT_IMMEDIATE 0x1b +#define SAFEXCEL_INSTR_DEST_OUTPUT (1u << 5) +#define SAFEXCEL_INSTR_DEST_HASH (1u << 6) +#define SAFEXCEL_INSTR_DEST_CRYPTO (1u << 7) +#define SAFEXCEL_INSTR_INS_LAST (1u << 8) + +#define SAFEXCEL_INSTR_VERIFY_HASH (1u << 16) +#define SAFEXCEL_INSTR_VERIFY_PADDING (1u << 5) + +#define SAFEXCEL_TOKEN_TYPE_BYPASS 0x0 +#define SAFEXCEL_TOKEN_TYPE_AUTONOMOUS 0x3 + +#define SAFEXCEL_CONTEXT_SMALL 0x2 +#define SAFEXCEL_CONTEXT_LARGE 0x3 + +struct safexcel_reg_offsets { + uint32_t hia_aic; + uint32_t hia_aic_g; + uint32_t hia_aic_r; + uint32_t hia_aic_xdr; + uint32_t hia_dfe; + uint32_t hia_dfe_thr; + uint32_t hia_dse; + uint32_t hia_dse_thr; + uint32_t hia_gen_cfg; + uint32_t pe; +}; + +struct safexcel_config { + uint32_t hdw; /* Host interface Data Width. */ + uint32_t aic_rings; /* Number of AIC rings. */ + uint32_t pes; /* Number of PEs. */ + uint32_t rings; /* Number of rings. */ + + uint32_t cd_size; /* CDR descriptor size. */ + uint32_t cd_offset; /* CDR offset (size + alignment). */ + + uint32_t rd_size; /* RDR descriptor size. */ + uint32_t rd_offset; /* RDR offset. */ + + uint32_t atok_offset; /* Additional token offset. */ + + uint32_t caps; /* Device capabilities. */ +}; + +#define SAFEXCEL_DPRINTF(sc, lvl, ...) do { \ + if ((sc)->sc_debug >= (lvl)) \ + device_printf((sc)->sc_dev, __VA_ARGS__); \ +} while (0) + +struct safexcel_dma_mem { + caddr_t vaddr; + bus_addr_t paddr; + bus_dma_tag_t tag; + bus_dmamap_t map; +}; + +struct safexcel_cmd_descr_ring { + struct safexcel_dma_mem dma; + struct safexcel_cmd_descr *desc; + int write; + int read; +}; + +struct safexcel_res_descr_ring { + struct safexcel_dma_mem dma; + struct safexcel_res_descr *desc; + int write; + int read; +}; + +struct safexcel_session { + int ringidx; + uint32_t alg; /* cipher algorithm */ + uint32_t digest; /* digest type */ + uint32_t hash; /* hash algorithm */ + uint32_t mode; /* cipher mode of operation */ + unsigned int digestlen; /* digest length */ + unsigned int statelen; /* HMAC hash state length */ + unsigned int klen; /* cipher key length */ + union { + uint32_t ghash_key[AES_BLOCK_LEN / sizeof(uint32_t)]; + uint32_t xcbc_key[(AES_BLOCK_LEN * 2 + AES_MAX_KEY) / + sizeof(uint32_t)]; + uint8_t tweak_key[AES_MAX_KEY]; + }; + struct { + uint8_t hmac_ipad[HMAC_MAX_BLOCK_LEN]; + uint8_t hmac_opad[HMAC_MAX_BLOCK_LEN]; + }; +}; + +struct safexcel_softc; + +struct safexcel_request { + STAILQ_ENTRY(safexcel_request) link; + bool dmap_loaded; + bus_dmamap_t dmap; + int error; + int cdescs, rdescs; + uint8_t iv[SAFEXCEL_MAX_IV_LEN]; + struct safexcel_cmd_descr *cdesc; + struct safexcel_dma_mem ctx; + struct safexcel_session *sess; + struct cryptop *crp; + struct safexcel_softc *sc; +}; + +struct safexcel_ring { + struct mtx mtx; + struct sglist *cmd_data; + struct safexcel_cmd_descr_ring cdr; + struct sglist *res_data; + struct safexcel_res_descr_ring rdr; + + struct safexcel_request *requests; + STAILQ_HEAD(, safexcel_request) ready_requests; + STAILQ_HEAD(, safexcel_request) queued_requests; + STAILQ_HEAD(, safexcel_request) free_requests; + + struct safexcel_dma_mem dma_atok; + bus_dma_tag_t data_dtag; +}; + +struct safexcel_intr_handle { + struct safexcel_softc *sc; + void *handle; + int ring; +}; + +struct safexcel_softc { + device_t sc_dev; + uint32_t sc_type; /* EIP-97 or 197 */ + int sc_debug; + + struct resource *sc_res; + struct resource *sc_intr[SAFEXCEL_MAX_RINGS]; + struct safexcel_intr_handle sc_ih[SAFEXCEL_MAX_RINGS]; + + struct safexcel_ring sc_ring[SAFEXCEL_MAX_RINGS]; + int sc_ringidx; + struct mtx sc_mtx; + + int sc_blocked; + int32_t sc_cid; + struct safexcel_reg_offsets sc_offsets; + struct safexcel_config sc_config; +}; + +#define SAFEXCEL_WRITE(sc, off, val) bus_write_4((sc)->sc_res, (off), (val)) +#define SAFEXCEL_READ(sc, off) bus_read_4((sc)->sc_res, (off)) + +#define SAFEXCEL_ADDR_LO(addr) ((uint64_t)(addr) & 0xffffffffu) +#define SAFEXCEL_ADDR_HI(addr) (((uint64_t)(addr) >> 32) & 0xffffffffu) + +#endif /* _SAFEXCEL_VAR_H_ */ diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 4244819427ec..230d875722e6 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -317,6 +317,7 @@ SUBDIR= \ ${_rtwnfw} \ ${_s3} \ ${_safe} \ + safexcel \ ${_sbni} \ scc \ ${_sctp} \ diff --git a/sys/modules/safexcel/Makefile b/sys/modules/safexcel/Makefile new file mode 100644 index 000000000000..985a3f40c282 --- /dev/null +++ b/sys/modules/safexcel/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ +.PATH: ${SRCTOP}/sys/dev/safexcel + +KMOD= safexcel +SRCS= safexcel.c + +SRCS+= bus_if.h cryptodev_if.h device_if.h ofw_bus_if.h + +.include