From b356ddf07671c7e0c960231344da7bddf5df42c4 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Tue, 14 Jul 2020 14:09:29 +0000 Subject: [PATCH] Add a driver for the SafeXcel EIP-97. The EIP-97 is a packet processing module found on the ESPRESSObin. This commit adds a crypto(9) driver for the crypto and hash engine in this device. An initial skeleton driver that could attach and submit requests was written by loos and others at Netgate, and the driver was finished by me. Support for separate AAD and output buffers will be added in a separate commit, to simplify merging to stable/12 (where those features don't exist). Reviewed by: gnn, jhb Feedback from: andrew, cem, manu MFC after: 1 week Sponsored by: Rubicon Communications, LLC (Netgate) Differential Revision: https://reviews.freebsd.org/D25417 --- share/man/man4/Makefile | 1 + share/man/man4/safexcel.4 | 84 + sys/arm64/conf/GENERIC | 3 + sys/conf/files.arm64 | 1 + sys/dev/safexcel/safexcel.c | 2637 +++++++++++++++++++++++++++++++ sys/dev/safexcel/safexcel_reg.h | 457 ++++++ sys/dev/safexcel/safexcel_var.h | 420 +++++ sys/modules/Makefile | 1 + sys/modules/safexcel/Makefile | 9 + 9 files changed, 3613 insertions(+) create mode 100644 share/man/man4/safexcel.4 create mode 100644 sys/dev/safexcel/safexcel.c create mode 100644 sys/dev/safexcel/safexcel_reg.h create mode 100644 sys/dev/safexcel/safexcel_var.h create mode 100644 sys/modules/safexcel/Makefile diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile index b3cdb95fb953..fe715f48288d 100644 --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -449,6 +449,7 @@ MAN= aac.4 \ rue.4 \ sa.4 \ safe.4 \ + safexcel.4 \ sbp.4 \ sbp_targ.4 \ scc.4 \ diff --git a/share/man/man4/safexcel.4 b/share/man/man4/safexcel.4 new file mode 100644 index 000000000000..6751570713f9 --- /dev/null +++ b/share/man/man4/safexcel.4 @@ -0,0 +1,84 @@ +.\"- +.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate) +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd June 23, 2020 +.Dt SAFEXCEL 4 +.Os +.Sh NAME +.Nm safexcel +.Nd Inside Secure SafeXcel-IP-97 security packet engine +.Sh SYNOPSIS +To compile this driver into the kernel, +place the following lines in your +kernel configuration file: +.Bd -ragged -offset indent +.Cd "device crypto" +.Cd "device cryptodev" +.Cd "device safexcel" +.Ed +.Pp +Alternatively, to load the driver as a +module at boot time, place the following line in +.Xr loader.conf 5 : +.Bd -literal -offset indent +safexcel_load="YES" +.Ed +.Sh DESCRIPTION +The +.Nm +driver implements +.Xr crypto 4 +support for the cryptographic acceleration functions of the EIP-97 device +found on some Marvell systems-on-chip. +The driver can accelerate the following AES modes: +.Pp +.Bl -bullet -compact +.It +AES-CBC +.It +AES-CTR +.It +AES-XTS +.It +AES-GCM +.It +AES-CCM +.El +.Pp +.Nm +also implements SHA1 and SHA2 transforms, and can combine AES-CBC and AES-CTR +with SHA1-HMAC and SHA2-HMAC for encrypt-then-authenticate operations. +.Sh SEE ALSO +.Xr crypto 4 , +.Xr ipsec 4 , +.Xr random 4 , +.Xr geli 8 , +.Xr crypto 9 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 13.0 . diff --git a/sys/arm64/conf/GENERIC b/sys/arm64/conf/GENERIC index 5e16c8f41990..30d1a8a0d9ad 100644 --- a/sys/arm64/conf/GENERIC +++ b/sys/arm64/conf/GENERIC @@ -283,6 +283,9 @@ device mv_ap806_sei # Marvell AP806 SEI device aw_rtc # Allwinner Real-time Clock device mv_rtc # Marvell Real-time Clock +# Crypto accelerators +device safexcel # Inside Secure EIP-97 + # Watchdog controllers device aw_wdog # Allwinner Watchdog diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index f17c53a78699..a538da81f1c4 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -324,6 +324,7 @@ dev/pci/pci_dw_if.m optional pci fdt dev/psci/psci.c standard dev/psci/smccc_arm64.S standard dev/psci/smccc.c standard +dev/safexcel/safexcel.c optional safexcel fdt dev/sdhci/sdhci_xenon.c optional sdhci_xenon sdhci fdt dev/uart/uart_cpu_arm64.c optional uart dev/uart/uart_dev_mu.c optional uart uart_mu diff --git a/sys/dev/safexcel/safexcel.c b/sys/dev/safexcel/safexcel.c new file mode 100644 index 000000000000..d7fee9b42a29 --- /dev/null +++ b/sys/dev/safexcel/safexcel.c @@ -0,0 +1,2637 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "cryptodev_if.h" + +#include "safexcel_reg.h" +#include "safexcel_var.h" + +static MALLOC_DEFINE(M_SAFEXCEL, "safexcel_req", "safexcel request buffers"); + +/* + * We only support the EIP97 for now. + */ +static struct ofw_compat_data safexcel_compat[] = { + { "inside-secure,safexcel-eip97ies", (uintptr_t)97 }, + { "inside-secure,safexcel-eip97", (uintptr_t)97 }, + { NULL, 0 } +}; + +const struct safexcel_reg_offsets eip97_regs_offset = { + .hia_aic = SAFEXCEL_EIP97_HIA_AIC_BASE, + .hia_aic_g = SAFEXCEL_EIP97_HIA_AIC_G_BASE, + .hia_aic_r = SAFEXCEL_EIP97_HIA_AIC_R_BASE, + .hia_aic_xdr = SAFEXCEL_EIP97_HIA_AIC_xDR_BASE, + .hia_dfe = SAFEXCEL_EIP97_HIA_DFE_BASE, + .hia_dfe_thr = SAFEXCEL_EIP97_HIA_DFE_THR_BASE, + .hia_dse = SAFEXCEL_EIP97_HIA_DSE_BASE, + .hia_dse_thr = SAFEXCEL_EIP97_HIA_DSE_THR_BASE, + .hia_gen_cfg = SAFEXCEL_EIP97_HIA_GEN_CFG_BASE, + .pe = SAFEXCEL_EIP97_PE_BASE, +}; + +const struct safexcel_reg_offsets eip197_regs_offset = { + .hia_aic = SAFEXCEL_EIP197_HIA_AIC_BASE, + .hia_aic_g = SAFEXCEL_EIP197_HIA_AIC_G_BASE, + .hia_aic_r = SAFEXCEL_EIP197_HIA_AIC_R_BASE, + .hia_aic_xdr = SAFEXCEL_EIP197_HIA_AIC_xDR_BASE, + .hia_dfe = SAFEXCEL_EIP197_HIA_DFE_BASE, + .hia_dfe_thr = SAFEXCEL_EIP197_HIA_DFE_THR_BASE, + .hia_dse = SAFEXCEL_EIP197_HIA_DSE_BASE, + .hia_dse_thr = SAFEXCEL_EIP197_HIA_DSE_THR_BASE, + .hia_gen_cfg = SAFEXCEL_EIP197_HIA_GEN_CFG_BASE, + .pe = SAFEXCEL_EIP197_PE_BASE, +}; + +static struct safexcel_cmd_descr * +safexcel_cmd_descr_next(struct safexcel_cmd_descr_ring *ring) +{ + struct safexcel_cmd_descr *cdesc; + + if (ring->write == ring->read) + return (NULL); + cdesc = &ring->desc[ring->read]; + ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; + return (cdesc); +} + +static struct safexcel_res_descr * +safexcel_res_descr_next(struct safexcel_res_descr_ring *ring) +{ + struct safexcel_res_descr *rdesc; + + if (ring->write == ring->read) + return (NULL); + rdesc = &ring->desc[ring->read]; + ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; + return (rdesc); +} + +static struct safexcel_request * +safexcel_alloc_request(struct safexcel_softc *sc, struct safexcel_ring *ring) +{ + struct safexcel_request *req; + + mtx_assert(&ring->mtx, MA_OWNED); + + if ((req = STAILQ_FIRST(&ring->free_requests)) != NULL) + STAILQ_REMOVE_HEAD(&ring->free_requests, link); + return (req); +} + +static void +safexcel_free_request(struct safexcel_ring *ring, struct safexcel_request *req) +{ + struct safexcel_context_record *ctx; + + mtx_assert(&ring->mtx, MA_OWNED); + + if (req->dmap_loaded) { + bus_dmamap_unload(ring->data_dtag, req->dmap); + req->dmap_loaded = false; + } + ctx = (struct safexcel_context_record *)req->ctx.vaddr; + explicit_bzero(ctx->data, sizeof(ctx->data)); + explicit_bzero(req->iv, sizeof(req->iv)); + STAILQ_INSERT_TAIL(&ring->free_requests, req, link); +} + +static void +safexcel_enqueue_request(struct safexcel_softc *sc, struct safexcel_ring *ring, + struct safexcel_request *req) +{ + mtx_assert(&ring->mtx, MA_OWNED); + + STAILQ_INSERT_TAIL(&ring->ready_requests, req, link); +} + +static void +safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx) +{ + struct safexcel_cmd_descr *cdesc; + struct safexcel_res_descr *rdesc; + struct safexcel_request *req; + struct safexcel_ring *ring; + uint32_t error, i, ncdescs, nrdescs, nreqs; + + ring = &sc->sc_ring[ringidx]; + + mtx_lock(&ring->mtx); + nreqs = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT); + nreqs >>= SAFEXCEL_xDR_PROC_xD_PKT_OFFSET; + nreqs &= SAFEXCEL_xDR_PROC_xD_PKT_MASK; + if (nreqs == 0) { + SAFEXCEL_DPRINTF(sc, 1, + "zero pending requests on ring %d\n", ringidx); + goto out; + } + + ring = &sc->sc_ring[ringidx]; + bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + ncdescs = nrdescs = 0; + for (i = 0; i < nreqs; i++) { + req = STAILQ_FIRST(&ring->queued_requests); + KASSERT(req != NULL, ("%s: expected %d pending requests", + __func__, nreqs)); + STAILQ_REMOVE_HEAD(&ring->queued_requests, link); + mtx_unlock(&ring->mtx); + + bus_dmamap_sync(req->ctx.tag, req->ctx.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(ring->data_dtag, req->dmap, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + ncdescs += req->cdescs; + while (req->cdescs-- > 0) { + cdesc = safexcel_cmd_descr_next(&ring->cdr); + KASSERT(cdesc != NULL, + ("%s: missing control descriptor", __func__)); + if (req->cdescs == 0) + KASSERT(cdesc->last_seg, + ("%s: chain is not terminated", __func__)); + } + nrdescs += req->rdescs; + while (req->rdescs-- > 0) { + rdesc = safexcel_res_descr_next(&ring->rdr); + error = rdesc->result_data.error_code; + if (error != 0) { + if (error == SAFEXCEL_RESULT_ERR_AUTH_FAILED && + req->crp->crp_etype == 0) { + req->crp->crp_etype = EBADMSG; + } else { + SAFEXCEL_DPRINTF(sc, 1, + "error code %#x\n", error); + req->crp->crp_etype = EIO; + } + } + } + + crypto_done(req->crp); + mtx_lock(&ring->mtx); + safexcel_free_request(ring, req); + } + + if (nreqs != 0) { + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_xD_PKT(nreqs) | + (sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t))); + } +out: + if (!STAILQ_EMPTY(&ring->queued_requests)) { + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, + SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | 1); + } + mtx_unlock(&ring->mtx); +} + +static void +safexcel_ring_intr(void *arg) +{ + struct safexcel_softc *sc; + struct safexcel_intr_handle *ih; + uint32_t status, stat; + int ring; + bool blocked, rdrpending; + + ih = arg; + sc = ih->sc; + ring = ih->ring; + + status = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + + SAFEXCEL_HIA_AIC_R_ENABLED_STAT(ring)); + /* CDR interrupts */ + if (status & SAFEXCEL_CDR_IRQ(ring)) { + stat = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, + stat & SAFEXCEL_CDR_INTR_MASK); + } + /* RDR interrupts */ + rdrpending = false; + if (status & SAFEXCEL_RDR_IRQ(ring)) { + stat = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); + if ((stat & SAFEXCEL_xDR_ERR) == 0) + rdrpending = true; + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, + stat & SAFEXCEL_RDR_INTR_MASK); + } + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ACK(ring), + status); + + if (rdrpending) + safexcel_rdr_intr(sc, ring); + + mtx_lock(&sc->sc_mtx); + blocked = sc->sc_blocked; + sc->sc_blocked = 0; + mtx_unlock(&sc->sc_mtx); + + if (blocked) + crypto_unblock(sc->sc_cid, blocked); +} + +static int +safexcel_configure(struct safexcel_softc *sc) +{ + uint32_t i, mask, pemask, reg; + device_t dev; + + if (sc->sc_type == 197) { + sc->sc_offsets = eip197_regs_offset; + pemask = SAFEXCEL_N_PES_MASK; + } else { + sc->sc_offsets = eip97_regs_offset; + pemask = EIP97_N_PES_MASK; + } + + dev = sc->sc_dev; + + /* Scan for valid ring interrupt controllers. */ + for (i = 0; i < SAFEXCEL_MAX_RING_AIC; i++) { + reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + + SAFEXCEL_HIA_AIC_R_VERSION(i)); + if (SAFEXCEL_REG_LO16(reg) != EIP201_VERSION_LE) + break; + } + sc->sc_config.aic_rings = i; + if (sc->sc_config.aic_rings == 0) + return (-1); + + reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_OPTIONS); + /* Check for 64bit addressing. */ + if ((reg & SAFEXCEL_OPT_ADDR_64) == 0) + return (-1); + /* Check alignment constraints (which we do not support). */ + if (((reg & SAFEXCEL_OPT_TGT_ALIGN_MASK) >> + SAFEXCEL_OPT_TGT_ALIGN_OFFSET) != 0) + return (-1); + + sc->sc_config.hdw = + (reg & SAFEXCEL_xDR_HDW_MASK) >> SAFEXCEL_xDR_HDW_OFFSET; + mask = (1 << sc->sc_config.hdw) - 1; + + sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK; + /* Limit the number of rings to the number of the AIC Rings. */ + sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings); + + sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET; + + sc->sc_config.cd_size = + sizeof(struct safexcel_cmd_descr) / sizeof(uint32_t); + sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask; + + sc->sc_config.rd_size = + sizeof(struct safexcel_res_descr) / sizeof(uint32_t); + sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask; + + sc->sc_config.atok_offset = + (SAFEXCEL_MAX_ATOKENS * sizeof(struct safexcel_instr) + mask) & + ~mask; + + return (0); +} + +static void +safexcel_init_hia_bus_access(struct safexcel_softc *sc) +{ + uint32_t version, val; + + /* Determine endianness and configure byte swap. */ + version = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_VERSION); + val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); + if (SAFEXCEL_REG_HI16(version) == SAFEXCEL_HIA_VERSION_BE) { + val = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); + val = val ^ (SAFEXCEL_MST_CTRL_NO_BYTE_SWAP >> 24); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL, + val); + } + + /* Configure wr/rd cache values. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_GEN_CFG(sc) + SAFEXCEL_HIA_MST_CTRL, + SAFEXCEL_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | + SAFEXCEL_MST_CTRL_WD_CACHE(WR_CACHE_4BITS)); +} + +static void +safexcel_disable_global_interrupts(struct safexcel_softc *sc) +{ + /* Disable and clear pending interrupts. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ENABLE_CTRL, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, + SAFEXCEL_AIC_G_ACK_ALL_MASK); +} + +/* + * Configure the data fetch engine. This component parses command descriptors + * and sets up DMA transfers from host memory to the corresponding processing + * engine. + */ +static void +safexcel_configure_dfe_engine(struct safexcel_softc *sc, int pe) +{ + /* Reset all DFE threads. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_RESET_PE); + + /* Deassert the DFE reset. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), 0); + + /* DMA transfer size to use. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE(sc) + SAFEXCEL_HIA_DFE_CFG(pe), + SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG | + SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(6) | + SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(9) | + SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | + SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(7) | + SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS) | + SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS)); + + /* Configure the PE DMA transfer thresholds. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_DBUF_THRES(pe), + SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | + SAFEXCEL_PE_IN_xBUF_THRES_MAX(9)); + SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_TBUF_THRES(pe), + SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | + SAFEXCEL_PE_IN_xBUF_THRES_MAX(7)); +} + +/* + * Configure the data store engine. This component parses result descriptors + * and sets up DMA transfers from the processing engine to host memory. + */ +static int +safexcel_configure_dse(struct safexcel_softc *sc, int pe) +{ + uint32_t val; + int count; + + /* Disable and reset all DSE threads. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_RESET_PE); + + /* Wait for a second for threads to go idle. */ + for (count = 0;;) { + val = SAFEXCEL_READ(sc, + SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_STAT(pe)); + if ((val & SAFEXCEL_DSE_THR_RDR_ID_MASK) == + SAFEXCEL_DSE_THR_RDR_ID_MASK) + break; + if (count++ > 10000) { + device_printf(sc->sc_dev, "DSE reset timeout\n"); + return (-1); + } + DELAY(100); + } + + /* Exit the reset state. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), 0); + + /* DMA transfer size to use */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE(sc) + SAFEXCEL_HIA_DSE_CFG(pe), + SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG | + SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(7) | + SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(8) | + SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS) | + SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE); + + /* Configure the procesing engine thresholds */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_PE(sc) + SAFEXCEL_PE_OUT_DBUF_THRES(pe), + SAFEXCEL_PE_OUT_DBUF_THRES_MIN(7) | + SAFEXCEL_PE_OUT_DBUF_THRES_MAX(8)); + + return (0); +} + +static void +safexcel_hw_prepare_rings(struct safexcel_softc *sc) +{ + int i; + + for (i = 0; i < sc->sc_config.rings; i++) { + /* + * Command descriptors. + */ + + /* Clear interrupts for this ring. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), + SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK); + + /* Disable external triggering. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, + SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset * + sizeof(uint32_t)); + + /* + * Result descriptors. + */ + + /* Disable external triggering. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + /* Ring size. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, + SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset * + sizeof(uint32_t)); + } +} + +static void +safexcel_hw_setup_rings(struct safexcel_softc *sc) +{ + struct safexcel_ring *ring; + uint32_t cd_size_rnd, mask, rd_size_rnd, val; + int i; + + mask = (1 << sc->sc_config.hdw) - 1; + cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw; + val = (sizeof(struct safexcel_res_descr) - + sizeof(struct safexcel_res_data)) / sizeof(uint32_t); + rd_size_rnd = (val + mask) >> sc->sc_config.hdw; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + + /* + * Command descriptors. + */ + + /* Ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, + SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr)); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, + SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr)); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, + SAFEXCEL_xDR_DESC_MODE_64BIT | SAFEXCEL_CDR_DESC_MODE_ADCP | + (sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | + sc->sc_config.cd_size); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, + ((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) << + SAFEXCEL_xDR_xD_FETCH_THRESH) | + (SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset)); + + /* Configure DMA tx control. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, + SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | + SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS)); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_CDR_INTR_MASK); + + /* + * Result descriptors. + */ + + /* Ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, + SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr)); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, + SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr)); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, + SAFEXCEL_xDR_DESC_MODE_64BIT | + (sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | + sc->sc_config.rd_size); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, + ((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) << + SAFEXCEL_xDR_xD_FETCH_THRESH) | + (SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset)); + + /* Configure DMA tx control. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, + SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | + SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS) | + SAFEXCEL_HIA_xDR_WR_RES_BUF | SAFEXCEL_HIA_xDR_WR_CTRL_BUF); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_RDR_INTR_MASK); + + /* Enable ring interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(i), + SAFEXCEL_RDR_IRQ(i)); + } +} + +/* Reset the command and result descriptor rings. */ +static void +safexcel_hw_reset_rings(struct safexcel_softc *sc) +{ + int i; + + for (i = 0; i < sc->sc_config.rings; i++) { + /* + * Result descriptor ring operations. + */ + + /* Reset ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_RDR_INTR_MASK); + + /* Disable ring interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), + SAFEXCEL_RDR_IRQ(i)); + + /* + * Command descriptor ring operations. + */ + + /* Reset ring base address. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); + + /* Clear the pending prepared counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, + SAFEXCEL_xDR_PREP_CLR_COUNT); + + /* Clear the pending processed counter. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, + SAFEXCEL_xDR_PROC_CLR_COUNT); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); + + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); + + /* Clear any pending interrupt. */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, + SAFEXCEL_CDR_INTR_MASK); + } +} + +static void +safexcel_enable_pe_engine(struct safexcel_softc *sc, int pe) +{ + int i, ring_mask; + + for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) { + ring_mask <<= 1; + ring_mask |= 1; + } + + /* Enable command descriptor rings. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); + + /* Enable result descriptor rings. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), + SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); + + /* Clear any HIA interrupt. */ + SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, + SAFEXCEL_AIC_G_ACK_HIA_MASK); +} + +static void +safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring, + struct safexcel_request *req) +{ + uint32_t ncdescs, nrdescs, nreqs; + int ringidx; + bool busy; + + mtx_assert(&ring->mtx, MA_OWNED); + + ringidx = req->sess->ringidx; + if (STAILQ_EMPTY(&ring->ready_requests)) + return; + busy = !STAILQ_EMPTY(&ring->queued_requests); + ncdescs = nrdescs = nreqs = 0; + while ((req = STAILQ_FIRST(&ring->ready_requests)) != NULL && + req->cdescs + ncdescs <= SAFEXCEL_MAX_BATCH_SIZE && + req->rdescs + nrdescs <= SAFEXCEL_MAX_BATCH_SIZE) { + STAILQ_REMOVE_HEAD(&ring->ready_requests, link); + STAILQ_INSERT_TAIL(&ring->queued_requests, req, link); + ncdescs += req->cdescs; + nrdescs += req->rdescs; + nreqs++; + } + + if (!busy) { + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, + SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | nreqs); + } + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, + nrdescs * sc->sc_config.rd_offset * sizeof(uint32_t)); + SAFEXCEL_WRITE(sc, + SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, + ncdescs * sc->sc_config.cd_offset * sizeof(uint32_t)); +} + +static void +safexcel_init_rings(struct safexcel_softc *sc) +{ + struct safexcel_cmd_descr *cdesc; + struct safexcel_ring *ring; + char buf[32]; + uint64_t atok; + int i, j; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + + snprintf(buf, sizeof(buf), "safexcel_ring%d", i); + mtx_init(&ring->mtx, buf, NULL, MTX_DEF); + STAILQ_INIT(&ring->free_requests); + STAILQ_INIT(&ring->ready_requests); + STAILQ_INIT(&ring->queued_requests); + + ring->cdr.read = ring->cdr.write = 0; + ring->rdr.read = ring->rdr.write = 0; + for (j = 0; j < SAFEXCEL_RING_SIZE; j++) { + cdesc = &ring->cdr.desc[j]; + atok = ring->dma_atok.paddr + + sc->sc_config.atok_offset * j; + cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok); + cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok); + } + } +} + +static void +safexcel_dma_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + struct safexcel_dma_mem *sdm; + + if (error != 0) + return; + + KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); + sdm = arg; + sdm->paddr = segs->ds_addr; +} + +static int +safexcel_dma_alloc_mem(struct safexcel_softc *sc, struct safexcel_dma_mem *sdm, + bus_size_t size) +{ + int error; + + KASSERT(sdm->vaddr == NULL, + ("%s: DMA memory descriptor in use.", __func__)); + + error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ + PAGE_SIZE, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + size, 1, /* maxsize, nsegments */ + size, BUS_DMA_COHERENT, /* maxsegsz, flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &sdm->tag); /* dmat */ + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate busdma tag, error %d\n", error); + goto err1; + } + + error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map); + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate DMA safe memory, error %d\n", error); + goto err2; + } + + error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size, + safexcel_dma_alloc_mem_cb, sdm, BUS_DMA_NOWAIT); + if (error != 0) { + device_printf(sc->sc_dev, + "cannot get address of the DMA memory, error %d\n", error); + goto err3; + } + + return (0); +err3: + bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); +err2: + bus_dma_tag_destroy(sdm->tag); +err1: + sdm->vaddr = NULL; + + return (error); +} + +static void +safexcel_dma_free_mem(struct safexcel_dma_mem *sdm) +{ + bus_dmamap_unload(sdm->tag, sdm->map); + bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); + bus_dma_tag_destroy(sdm->tag); +} + +static void +safexcel_dma_free_rings(struct safexcel_softc *sc) +{ + struct safexcel_ring *ring; + int i; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + safexcel_dma_free_mem(&ring->cdr.dma); + safexcel_dma_free_mem(&ring->dma_atok); + safexcel_dma_free_mem(&ring->rdr.dma); + bus_dma_tag_destroy(ring->data_dtag); + mtx_destroy(&ring->mtx); + } +} + +static int +safexcel_dma_init(struct safexcel_softc *sc) +{ + struct safexcel_ring *ring; + int error, i, size; + + for (i = 0; i < sc->sc_config.rings; i++) { + ring = &sc->sc_ring[i]; + + error = bus_dma_tag_create( + bus_get_dma_tag(sc->sc_dev),/* parent */ + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + SAFEXCEL_MAX_REQUEST_SIZE, /* maxsize */ + SAFEXCEL_MAX_FRAGMENTS, /* nsegments */ + SAFEXCEL_MAX_REQUEST_SIZE, /* maxsegsz */ + BUS_DMA_COHERENT, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->data_dtag); /* dmat */ + if (error != 0) { + device_printf(sc->sc_dev, + "bus_dma_tag_create main failed; error %d\n", error); + return (error); + } + + size = sizeof(uint32_t) * sc->sc_config.cd_offset * + SAFEXCEL_RING_SIZE; + error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size); + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate CDR DMA memory, error %d\n", + error); + goto err; + } + ring->cdr.desc = + (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr; + + /* Allocate additional CDR token memory. */ + error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, + sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE); + if (error != 0) { + device_printf(sc->sc_dev, + "failed to allocate atoken DMA memory, error %d\n", + error); + goto err; + } + + size = sizeof(uint32_t) * sc->sc_config.rd_offset * + SAFEXCEL_RING_SIZE; + error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size); + if (error) { + device_printf(sc->sc_dev, + "failed to allocate RDR DMA memory, error %d\n", + error); + goto err; + } + ring->rdr.desc = + (struct safexcel_res_descr *)ring->rdr.dma.vaddr; + } + + return (0); +err: + safexcel_dma_free_rings(sc); + return (error); +} + +static void +safexcel_deinit_hw(struct safexcel_softc *sc) +{ + safexcel_hw_reset_rings(sc); + safexcel_dma_free_rings(sc); +} + +static int +safexcel_init_hw(struct safexcel_softc *sc) +{ + int pe; + + /* 23.3.7 Initialization */ + if (safexcel_configure(sc) != 0) + return (EINVAL); + + if (safexcel_dma_init(sc) != 0) + return (ENOMEM); + + safexcel_init_rings(sc); + + safexcel_init_hia_bus_access(sc); + + /* 23.3.7.2 Disable EIP-97 global Interrupts */ + safexcel_disable_global_interrupts(sc); + + for (pe = 0; pe < sc->sc_config.pes; pe++) { + /* 23.3.7.3 Configure Data Fetch Engine */ + safexcel_configure_dfe_engine(sc, pe); + + /* 23.3.7.4 Configure Data Store Engine */ + if (safexcel_configure_dse(sc, pe)) { + safexcel_deinit_hw(sc); + return (-1); + } + + /* 23.3.7.5 1. Protocol enables */ + SAFEXCEL_WRITE(sc, + SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION_EN(pe), + 0xffffffff); + SAFEXCEL_WRITE(sc, + SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION2_EN(pe), + 0xffffffff); + } + + safexcel_hw_prepare_rings(sc); + + /* 23.3.7.5 Configure the Processing Engine(s). */ + for (pe = 0; pe < sc->sc_config.pes; pe++) + safexcel_enable_pe_engine(sc, pe); + + safexcel_hw_setup_rings(sc); + + return (0); +} + +static int +safexcel_setup_dev_interrupts(struct safexcel_softc *sc) +{ + int i, j; + + for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { + sc->sc_ih[i].sc = sc; + sc->sc_ih[i].ring = i; + + if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i], + INTR_TYPE_NET | INTR_MPSAFE, NULL, safexcel_ring_intr, + &sc->sc_ih[i], &sc->sc_ih[i].handle)) { + device_printf(sc->sc_dev, + "couldn't setup interrupt %d\n", i); + goto err; + } + } + + return (0); + +err: + for (j = 0; j < i; j++) + bus_teardown_intr(sc->sc_dev, sc->sc_intr[j], + sc->sc_ih[j].handle); + + return (ENXIO); +} + +static void +safexcel_teardown_dev_interrupts(struct safexcel_softc *sc) +{ + int i; + + for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) + bus_teardown_intr(sc->sc_dev, sc->sc_intr[i], + sc->sc_ih[i].handle); +} + +static int +safexcel_alloc_dev_resources(struct safexcel_softc *sc) +{ + char name[16]; + device_t dev; + phandle_t node; + int error, i, rid; + + dev = sc->sc_dev; + node = ofw_bus_get_node(dev); + + rid = 0; + sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->sc_res == NULL) { + device_printf(dev, "couldn't allocate memory resources\n"); + return (ENXIO); + } + + for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) { + (void)snprintf(name, sizeof(name), "ring%d", i); + error = ofw_bus_find_string_index(node, "interrupt-names", name, + &rid); + if (error != 0) + break; + + sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->sc_intr[i] == NULL) { + error = ENXIO; + goto out; + } + } + if (i == 0) { + device_printf(dev, "couldn't allocate interrupt resources\n"); + error = ENXIO; + goto out; + } + + mtx_init(&sc->sc_mtx, "safexcel softc", NULL, MTX_DEF); + + return (0); + +out: + for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) + bus_release_resource(dev, SYS_RES_IRQ, + rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); + bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), + sc->sc_res); + return (error); +} + +static void +safexcel_free_dev_resources(struct safexcel_softc *sc) +{ + int i; + + mtx_destroy(&sc->sc_mtx); + + for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) + bus_release_resource(sc->sc_dev, SYS_RES_IRQ, + rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); + if (sc->sc_res != NULL) + bus_release_resource(sc->sc_dev, SYS_RES_MEMORY, + rman_get_rid(sc->sc_res), sc->sc_res); +} + +static int +safexcel_probe(device_t dev) +{ + struct safexcel_softc *sc; + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + sc = device_get_softc(dev); + sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data; + if (sc->sc_type == 0) + return (ENXIO); + + device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator"); + + return (BUS_PROBE_DEFAULT); +} + +static int +safexcel_attach(device_t dev) +{ + struct sysctl_ctx_list *sctx; + struct safexcel_softc *sc; + struct safexcel_request *req; + struct safexcel_ring *ring; + int i, j, ringidx; + + sc = device_get_softc(dev); + sc->sc_dev = dev; + sc->sc_blocked = 0; + sc->sc_cid = -1; + + if (safexcel_alloc_dev_resources(sc)) + goto err; + + if (safexcel_setup_dev_interrupts(sc)) + goto err1; + + if (safexcel_init_hw(sc)) + goto err2; + + for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { + ring = &sc->sc_ring[ringidx]; + + ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); + ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); + + ring->requests = mallocarray(SAFEXCEL_REQUESTS_PER_RING, + sizeof(struct safexcel_request), M_SAFEXCEL, + M_WAITOK | M_ZERO); + + for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { + req = &ring->requests[i]; + req->sc = sc; + if (bus_dmamap_create(ring->data_dtag, + BUS_DMA_COHERENT, &req->dmap) != 0) { + for (j = 0; j < i; j++) + bus_dmamap_destroy(ring->data_dtag, + ring->requests[j].dmap); + goto err2; + } + if (safexcel_dma_alloc_mem(sc, &req->ctx, + sizeof(struct safexcel_context_record)) != 0) { + for (j = 0; j < i; j++) { + bus_dmamap_destroy(ring->data_dtag, + ring->requests[j].dmap); + safexcel_dma_free_mem( + &ring->requests[j].ctx); + } + goto err2; + } + STAILQ_INSERT_TAIL(&ring->free_requests, req, link); + } + } + + sctx = device_get_sysctl_ctx(dev); + SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0, + "Debug message verbosity"); + + sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session), + CRYPTOCAP_F_HARDWARE); + if (sc->sc_cid < 0) + goto err2; + + return (0); + +err2: + safexcel_teardown_dev_interrupts(sc); +err1: + safexcel_free_dev_resources(sc); +err: + return (ENXIO); +} + +static int +safexcel_detach(device_t dev) +{ + struct safexcel_ring *ring; + struct safexcel_softc *sc; + int i, ringidx; + + sc = device_get_softc(dev); + + if (sc->sc_cid >= 0) + crypto_unregister_all(sc->sc_cid); + for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { + ring = &sc->sc_ring[ringidx]; + for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { + bus_dmamap_destroy(ring->data_dtag, + ring->requests[i].dmap); + safexcel_dma_free_mem(&ring->requests[i].ctx); + } + free(ring->requests, M_SAFEXCEL); + sglist_free(ring->cmd_data); + sglist_free(ring->res_data); + } + safexcel_deinit_hw(sc); + safexcel_teardown_dev_interrupts(sc); + safexcel_free_dev_resources(sc); + + return (0); +} + +/* + * Populate the request's context record with pre-computed key material. + */ +static int +safexcel_set_context(struct safexcel_request *req) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_context_record *ctx; + struct safexcel_session *sess; + uint8_t *data; + int off; + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + sess = req->sess; + + ctx = (struct safexcel_context_record *)req->ctx.vaddr; + data = (uint8_t *)ctx->data; + if (csp->csp_cipher_alg != 0) { + if (crp->crp_cipher_key != NULL) + memcpy(data, crp->crp_cipher_key, sess->klen); + else + memcpy(data, csp->csp_cipher_key, sess->klen); + off = sess->klen; + } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { + if (crp->crp_auth_key != NULL) + memcpy(data, crp->crp_auth_key, sess->klen); + else + memcpy(data, csp->csp_auth_key, sess->klen); + off = sess->klen; + } else { + off = 0; + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + off += GMAC_BLOCK_LEN; + break; + case CRYPTO_AES_CCM_16: + memcpy(data + off, sess->xcbc_key, + AES_BLOCK_LEN * 2 + sess->klen); + off += AES_BLOCK_LEN * 2 + sess->klen; + break; + case CRYPTO_AES_XTS: + memcpy(data + off, sess->tweak_key, sess->klen); + off += sess->klen; + break; + } + + switch (csp->csp_auth_alg) { + case CRYPTO_AES_NIST_GMAC: + memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + off += GMAC_BLOCK_LEN; + break; + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + memcpy(data + off, sess->hmac_ipad, sess->statelen); + off += sess->statelen; + memcpy(data + off, sess->hmac_opad, sess->statelen); + off += sess->statelen; + break; + } + + return (off); +} + +/* + * Populate fields in the first command descriptor of the chain used to encode + * the specified request. These fields indicate the algorithms used, the size + * of the key material stored in the associated context record, the primitive + * operations to be performed on input data, and the location of the IV if any. + */ +static void +safexcel_set_command(struct safexcel_request *req, + struct safexcel_cmd_descr *cdesc) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_session *sess; + uint32_t ctrl0, ctrl1, ctxr_len; + int alg; + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + sess = req->sess; + + ctrl0 = sess->alg | sess->digest | sess->hash; + ctrl1 = sess->mode; + + ctxr_len = safexcel_set_context(req) / sizeof(uint32_t); + ctrl0 |= SAFEXCEL_CONTROL0_SIZE(ctxr_len); + + alg = csp->csp_cipher_alg; + if (alg == 0) + alg = csp->csp_auth_alg; + + switch (alg) { + case CRYPTO_AES_CCM_16: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT | + SAFEXCEL_CONTROL0_KEY_EN; + } else { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN | + SAFEXCEL_CONTROL0_KEY_EN; + } + ctrl1 |= SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | + SAFEXCEL_CONTROL1_IV2 | SAFEXCEL_CONTROL1_IV3; + break; + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + case CRYPTO_AES_XTS: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | + SAFEXCEL_CONTROL0_KEY_EN; + if (csp->csp_auth_alg != 0) + ctrl0 |= + SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT; + } else { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | + SAFEXCEL_CONTROL0_KEY_EN; + if (csp->csp_auth_alg != 0) + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; + } + break; + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_NIST_GMAC: + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) || + csp->csp_auth_alg != 0) { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | + SAFEXCEL_CONTROL0_KEY_EN | + SAFEXCEL_CONTROL0_TYPE_HASH_OUT; + } else { + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | + SAFEXCEL_CONTROL0_KEY_EN | + SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; + } + if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { + ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE | + SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | + SAFEXCEL_CONTROL1_IV2; + } + break; + case CRYPTO_SHA1: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_512: + ctrl0 |= SAFEXCEL_CONTROL0_RESTART_HASH; + /* FALLTHROUGH */ + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_OUT; + break; + } + + cdesc->control_data.control0 = ctrl0; + cdesc->control_data.control1 = ctrl1; +} + +/* + * Construct a no-op instruction, used to pad input tokens. + */ +static void +safexcel_instr_nop(struct safexcel_instr **instrp) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = (1 << 2); + instr->status = 0; + instr->instructions = 0; + + *instrp = instr + 1; +} + +/* + * Insert the digest of the input payload. This is typically the last + * instruction of a sequence. + */ +static void +safexcel_instr_insert_digest(struct safexcel_instr **instrp, int len) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = len; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_INSERT_HASH_DIGEST; + + *instrp = instr + 1; +} + +/* + * Retrieve and verify a digest. + */ +static void +safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_RETRIEVE; + instr->length = len; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST; + instr++; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS; + instr->length = len | SAFEXCEL_INSTR_VERIFY_HASH; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_VERIFY_PADDING; + + *instrp = instr + 1; +} + +static void +safexcel_instr_temp_aes_block(struct safexcel_instr **instrp) +{ + struct safexcel_instr *instr; + + instr = *instrp; + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT; + instr->length = 0; + instr->status = 0; + instr->instructions = AES_BLOCK_LEN; + instr++; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = AES_BLOCK_LEN; + instr->status = 0; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_DEST_CRYPTO; + + *instrp = instr + 1; +} + +/* + * Handle a request for an unauthenticated block cipher. + */ +static void +safexcel_instr_cipher(struct safexcel_request *req, + struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + + crp = req->crp; + + /* Insert the payload. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | + SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_OUTPUT; + + cdesc->additional_cdata_size = 1; +} + +static void +safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_instr *start; + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + start = instr; + + /* Insert the AAD. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_aad_length; + instr->status = crp->crp_payload_length == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_HASH; + instr++; + + /* Encrypt any data left in the request. */ + if (crp->crp_payload_length > 0) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_CRYPTO | + SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_DEST_OUTPUT; + instr++; + } + + /* + * Compute the digest, or extract it and place it in the output stream. + */ + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + else + safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_instr_sha_hash(struct safexcel_request *req, + struct safexcel_instr *instr) +{ + struct cryptop *crp; + struct safexcel_instr *start; + + crp = req->crp; + start = instr; + + /* Pass the input data to the hash engine. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + + /* Insert the hash result into the output stream. */ + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + + /* Pad the rest of the inline instruction space. */ + while (instr != start + SAFEXCEL_MAX_ITOKENS) + safexcel_instr_nop(&instr); +} + +static void +safexcel_instr_ccm(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + struct safexcel_instr *start; + uint8_t *a0, *b0, *alenp, L; + int aalign, blen; + + crp = req->crp; + start = instr; + + /* + * Construct two blocks, A0 and B0, used in encryption and + * authentication, respectively. A0 is embedded in the token + * descriptor, and B0 is inserted directly into the data stream using + * instructions below. + * + * OCF seems to assume a 12-byte IV, fixing L (the payload length size) + * at 3 bytes due to the layout of B0. This is fine since the driver + * has a maximum of 65535 bytes anyway. + */ + blen = AES_BLOCK_LEN; + L = 3; + + a0 = (uint8_t *)&cdesc->control_data.token[0]; + memset(a0, 0, blen); + a0[0] = L - 1; + memcpy(&a0[1], req->iv, AES_CCM_IV_LEN); + + /* + * Insert B0 and the AAD length into the input stream. + */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0); + instr->status = 0; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_INSERT_IMMEDIATE; + instr++; + + b0 = (uint8_t *)instr; + memset(b0, 0, blen); + b0[0] = + L - 1 | /* payload length size */ + ((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ | + (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; + memcpy(&b0[1], req->iv, AES_CCM_IV_LEN); + b0[14] = crp->crp_payload_length >> 8; + b0[15] = crp->crp_payload_length & 0xff; + instr += blen / sizeof(*instr); + + /* Insert the AAD length and data into the input stream. */ + if (crp->crp_aad_length > 0) { + alenp = (uint8_t *)instr; + alenp[0] = crp->crp_aad_length >> 8; + alenp[1] = crp->crp_aad_length & 0xff; + alenp[2] = 0; + alenp[3] = 0; + instr++; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_aad_length; + instr->status = 0; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + + /* Insert zero padding. */ + aalign = (crp->crp_aad_length + 2) & (blen - 1); + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = aalign == 0 ? 0 : + blen - ((crp->crp_aad_length + 2) & (blen - 1)); + instr->status = crp->crp_payload_length == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + } + + safexcel_instr_temp_aes_block(&instr); + + /* Insert the cipher payload into the input stream. */ + if (crp->crp_payload_length > 0) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_DEST_CRYPTO | + SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_INS_LAST; + instr++; + + /* Insert zero padding. */ + if (crp->crp_payload_length & (blen - 1)) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = blen - + (crp->crp_payload_length & (blen - 1)); + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_DEST_HASH; + instr++; + } + } + + /* + * Compute the digest, or extract it and place it in the output stream. + */ + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + else + safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_instr_gcm(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + struct safexcel_instr *start; + + memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); + cdesc->control_data.token[3] = htobe32(1); + + crp = req->crp; + start = instr; + + /* Insert the AAD into the input stream. */ + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_aad_length; + instr->status = crp->crp_payload_length == 0 ? + SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_HASH; + instr++; + + safexcel_instr_temp_aes_block(&instr); + + /* Insert the cipher payload into the input stream. */ + if (crp->crp_payload_length > 0) { + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | + SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_INS_LAST; + instr++; + } + + /* + * Compute the digest, or extract it and place it in the output stream. + */ + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + else + safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr, + struct safexcel_cmd_descr *cdesc) +{ + struct cryptop *crp; + struct safexcel_instr *start; + + memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); + cdesc->control_data.token[3] = htobe32(1); + + crp = req->crp; + start = instr; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; + instr->length = crp->crp_payload_length; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; + instr->instructions = SAFEXCEL_INSTR_INS_LAST | + SAFEXCEL_INSTR_DEST_HASH; + instr++; + + safexcel_instr_temp_aes_block(&instr); + + safexcel_instr_insert_digest(&instr, req->sess->digestlen); + + cdesc->additional_cdata_size = instr - start; +} + +static void +safexcel_set_token(struct safexcel_request *req) +{ + const struct crypto_session_params *csp; + struct safexcel_cmd_descr *cdesc; + struct safexcel_instr *instr; + struct safexcel_softc *sc; + int ringidx; + + csp = crypto_get_params(req->crp->crp_session); + cdesc = req->cdesc; + sc = req->sc; + ringidx = req->sess->ringidx; + + safexcel_set_command(req, cdesc); + + /* + * For keyless hash operations, the token instructions can be embedded + * in the token itself. Otherwise we use an additional token descriptor + * and the embedded instruction space is used to store the IV. + */ + if (csp->csp_cipher_alg == 0 && + csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { + instr = (void *)cdesc->control_data.token; + } else { + instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + + sc->sc_config.atok_offset * + (cdesc - sc->sc_ring[ringidx].cdr.desc)); + cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + safexcel_instr_gcm(req, instr, cdesc); + break; + case CRYPTO_AES_CCM_16: + safexcel_instr_ccm(req, instr, cdesc); + break; + case CRYPTO_AES_XTS: + memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); + memset(cdesc->control_data.token + + AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); + + safexcel_instr_cipher(req, instr, cdesc); + break; + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); + if (csp->csp_auth_alg != 0) + safexcel_instr_eta(req, instr, cdesc); + else + safexcel_instr_cipher(req, instr, cdesc); + break; + default: + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + safexcel_instr_sha_hash(req, instr); + break; + case CRYPTO_AES_NIST_GMAC: + safexcel_instr_gmac(req, instr, cdesc); + break; + default: + panic("unhandled auth request %d", csp->csp_auth_alg); + } + break; + } +} + +static struct safexcel_res_descr * +safexcel_res_descr_add(struct safexcel_ring *ring, bool first, bool last, + bus_addr_t data, uint32_t len) +{ + struct safexcel_res_descr *rdesc; + struct safexcel_res_descr_ring *rring; + + mtx_assert(&ring->mtx, MA_OWNED); + + rring = &ring->rdr; + if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read) + return (NULL); + + rdesc = &rring->desc[rring->write]; + rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE; + + rdesc->particle_size = len; + rdesc->rsvd0 = 0; + rdesc->descriptor_overflow = 0; + rdesc->buffer_overflow = 0; + rdesc->last_seg = last; + rdesc->first_seg = first; + rdesc->result_size = + sizeof(struct safexcel_res_data) / sizeof(uint32_t); + rdesc->rsvd1 = 0; + rdesc->data_lo = SAFEXCEL_ADDR_LO(data); + rdesc->data_hi = SAFEXCEL_ADDR_HI(data); + + if (first) { + rdesc->result_data.packet_length = 0; + rdesc->result_data.error_code = 0; + } + + return (rdesc); +} + +static struct safexcel_cmd_descr * +safexcel_cmd_descr_add(struct safexcel_ring *ring, bool first, bool last, + bus_addr_t data, uint32_t seglen, uint32_t reqlen, bus_addr_t context) +{ + struct safexcel_cmd_descr *cdesc; + struct safexcel_cmd_descr_ring *cring; + + KASSERT(full_data_len <= SAFEXCEL_MAX_REQUEST_SIZE, + ("%s: request length %u too long", __func__, full_data_len)); + mtx_assert(&ring->mtx, MA_OWNED); + + cring = &ring->cdr; + if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read) + return (NULL); + + cdesc = &cring->desc[cring->write]; + cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE; + + cdesc->particle_size = seglen; + cdesc->rsvd0 = 0; + cdesc->last_seg = last; + cdesc->first_seg = first; + cdesc->additional_cdata_size = 0; + cdesc->rsvd1 = 0; + cdesc->data_lo = SAFEXCEL_ADDR_LO(data); + cdesc->data_hi = SAFEXCEL_ADDR_HI(data); + if (first) { + cdesc->control_data.packet_length = reqlen; + cdesc->control_data.options = SAFEXCEL_OPTION_IP | + SAFEXCEL_OPTION_CP | SAFEXCEL_OPTION_CTX_CTRL_IN_CMD | + SAFEXCEL_OPTION_RC_AUTO; + cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS; + cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) | + SAFEXCEL_CONTEXT_SMALL; + cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context); + } + + return (cdesc); +} + +static void +safexcel_cmd_descr_rollback(struct safexcel_ring *ring, int count) +{ + struct safexcel_cmd_descr_ring *cring; + + mtx_assert(&ring->mtx, MA_OWNED); + + cring = &ring->cdr; + cring->write -= count; + if (cring->write < 0) + cring->write += SAFEXCEL_RING_SIZE; +} + +static void +safexcel_res_descr_rollback(struct safexcel_ring *ring, int count) +{ + struct safexcel_res_descr_ring *rring; + + mtx_assert(&ring->mtx, MA_OWNED); + + rring = &ring->rdr; + rring->write -= count; + if (rring->write < 0) + rring->write += SAFEXCEL_RING_SIZE; +} + +static void +safexcel_append_segs(bus_dma_segment_t *segs, int nseg, struct sglist *sg, + int start, int len) +{ + bus_dma_segment_t *seg; + size_t seglen; + int error, i; + + for (i = 0; i < nseg && len > 0; i++) { + seg = &segs[i]; + + if (seg->ds_len <= start) { + start -= seg->ds_len; + continue; + } + + seglen = MIN(len, seg->ds_len - start); + error = sglist_append_phys(sg, seg->ds_addr + start, seglen); + if (error != 0) + panic("%s: ran out of segments: %d", __func__, error); + len -= seglen; + start = 0; + } +} + +static void +safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + const struct crypto_session_params *csp; + struct cryptop *crp; + struct safexcel_cmd_descr *cdesc; + struct safexcel_request *req; + struct safexcel_ring *ring; + struct safexcel_session *sess; + struct sglist *sg; + size_t inlen; + int i; + bool first, last; + + req = arg; + if (error != 0) { + req->error = error; + return; + } + + crp = req->crp; + csp = crypto_get_params(crp->crp_session); + sess = req->sess; + ring = &req->sc->sc_ring[sess->ringidx]; + + mtx_assert(&ring->mtx, MA_OWNED); + + /* + * Set up descriptors for input and output data. + * + * The processing engine programs require that any AAD comes first, + * followed by the cipher plaintext, followed by the digest. Some + * consumers place the digest first in the input buffer, in which case + * we have to create an extra descriptor. + * + * As an optimization, unmodified data is not passed to the output + * stream. + */ + sglist_reset(ring->cmd_data); + sglist_reset(ring->res_data); + if (crp->crp_aad_length != 0) { + safexcel_append_segs(segs, nseg, ring->cmd_data, + crp->crp_aad_start, crp->crp_aad_length); + } + safexcel_append_segs(segs, nseg, ring->cmd_data, + crp->crp_payload_start, crp->crp_payload_length); + if (csp->csp_cipher_alg != 0) { + safexcel_append_segs(segs, nseg, ring->res_data, + crp->crp_payload_start, crp->crp_payload_length); + } + if (sess->digestlen > 0) { + if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { + safexcel_append_segs(segs, nseg, ring->cmd_data, + crp->crp_digest_start, sess->digestlen); + } else { + safexcel_append_segs(segs, nseg, ring->res_data, + crp->crp_digest_start, sess->digestlen); + } + } + + sg = ring->cmd_data; + if (sg->sg_nseg == 0) { + /* + * Fake a segment for the command descriptor if the input has + * length zero. The EIP97 apparently does not handle + * zero-length packets properly since subsequent requests return + * bogus errors, so provide a dummy segment using the context + * descriptor. + */ + (void)sglist_append_phys(sg, req->ctx.paddr, 1); + } + for (i = 0, inlen = 0; i < sg->sg_nseg; i++) + inlen += sg->sg_segs[i].ss_len; + for (i = 0; i < sg->sg_nseg; i++) { + first = i == 0; + last = i == sg->sg_nseg - 1; + + cdesc = safexcel_cmd_descr_add(ring, first, last, + sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len, + (uint32_t)inlen, req->ctx.paddr); + if (cdesc == NULL) { + safexcel_cmd_descr_rollback(ring, i); + req->error = EAGAIN; + return; + } + if (i == 0) + req->cdesc = cdesc; + } + req->cdescs = sg->sg_nseg; + + sg = ring->res_data; + if (sg->sg_nseg == 0) { + /* + * We need a result descriptor even if the output stream will be + * empty, for example when verifying an AAD digest. + */ + sg->sg_segs[0].ss_paddr = 0; + sg->sg_segs[0].ss_len = 0; + sg->sg_nseg = 1; + } + for (i = 0; i < sg->sg_nseg; i++) { + first = i == 0; + last = i == sg->sg_nseg - 1; + + if (safexcel_res_descr_add(ring, first, last, + sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) { + safexcel_cmd_descr_rollback(ring, + ring->cmd_data->sg_nseg); + safexcel_res_descr_rollback(ring, i); + req->error = EAGAIN; + return; + } + } + req->rdescs = sg->sg_nseg; +} + +static int +safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req) +{ + int error; + + req->error = 0; + req->cdescs = req->rdescs = 0; + + error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp, + safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); + if (error == 0) + req->dmap_loaded = true; + + if (req->error != 0) + error = req->error; + + return (error); +} + +static bool +safexcel_probe_cipher(const struct crypto_session_params *csp) +{ + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return (false); + break; + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return (false); + break; + default: + return (false); + } + + return (true); +} + +/* + * Determine whether the driver can implement a session with the requested + * parameters. + */ +static int +safexcel_probesession(device_t dev, const struct crypto_session_params *csp) +{ + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + if (!safexcel_probe_cipher(csp)) + return (EINVAL); + break; + case CSP_MODE_DIGEST: + switch (csp->csp_auth_alg) { + case CRYPTO_AES_NIST_GMAC: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); + break; + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + break; + default: + return (EINVAL); + } + break; + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); + break; + case CRYPTO_AES_CCM_16: + if (csp->csp_ivlen != AES_CCM_IV_LEN) + return (EINVAL); + break; + default: + return (EINVAL); + } + break; + case CSP_MODE_ETA: + if (!safexcel_probe_cipher(csp)) + return (EINVAL); + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + /* + * The EIP-97 does not support combining AES-XTS with + * hash operations. + */ + if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && + csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) + return (EINVAL); + break; + default: + return (EINVAL); + } + break; + default: + return (EINVAL); + } + + return (CRYPTODEV_PROBE_HARDWARE); +} + +/* + * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted + * using the cipher key. + */ +static void +safexcel_setkey_ghash(struct safexcel_session *sess, const uint8_t *key, + int klen) +{ + uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; + uint8_t zeros[AES_BLOCK_LEN]; + int i, rounds; + + memset(zeros, 0, sizeof(zeros)); + + rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); + rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)sess->ghash_key); + for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) + sess->ghash_key[i] = htobe32(sess->ghash_key[i]); + + explicit_bzero(ks, sizeof(ks)); +} + +/* + * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 + * in the hardware implementation. K1 is the cipher key and comes last in the + * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC + * is not implemented so K2 and K3 are fixed. + */ +static void +safexcel_setkey_xcbcmac(struct safexcel_session *sess, const uint8_t *key, + int klen) +{ + int i, off; + + memset(sess->xcbc_key, 0, sizeof(sess->xcbc_key)); + off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); + for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) + sess->xcbc_key[i + off] = htobe32(le32dec(key)); +} + +static void +safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, + char *buf) +{ + int hashwords, i; + + switch (ahash->type) { + case CRYPTO_SHA1_HMAC: + hashwords = ahash->hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); + break; + case CRYPTO_SHA2_224_HMAC: + hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); + break; + case CRYPTO_SHA2_256_HMAC: + hashwords = ahash->hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); + break; + case CRYPTO_SHA2_384_HMAC: + hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); + for (i = 0; i < hashwords; i++) + ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); + break; + case CRYPTO_SHA2_512_HMAC: + hashwords = ahash->hashsize / sizeof(uint64_t); + for (i = 0; i < hashwords; i++) + ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); + break; + } +} + +/* + * Pre-compute the inner and outer digests used in the HMAC algorithm. + */ +static void +safexcel_setkey_hmac(const struct crypto_session_params *csp, + struct safexcel_session *sess, const uint8_t *key, int klen) +{ + union authctx ctx; + struct auth_hash *ahash; + + ahash = crypto_auth_hash(csp); + hmac_init_ipad(ahash, key, klen, &ctx); + safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_ipad); + hmac_init_opad(ahash, key, klen, &ctx); + safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_opad); + explicit_bzero(&ctx, ahash->ctxsize); +} + +static void +safexcel_setkey_xts(struct safexcel_session *sess, const uint8_t *key, int klen) +{ + memcpy(sess->tweak_key, key + klen / 2, klen / 2); +} + +static void +safexcel_setkey(struct safexcel_session *sess, + const struct crypto_session_params *csp, struct cryptop *crp) +{ + const uint8_t *akey, *ckey; + int aklen, cklen; + + aklen = csp->csp_auth_klen; + cklen = csp->csp_cipher_klen; + akey = ckey = NULL; + if (crp != NULL) { + akey = crp->crp_auth_key; + ckey = crp->crp_cipher_key; + } + if (akey == NULL) + akey = csp->csp_auth_key; + if (ckey == NULL) + ckey = csp->csp_cipher_key; + + sess->klen = cklen; + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + safexcel_setkey_ghash(sess, ckey, cklen); + break; + case CRYPTO_AES_CCM_16: + safexcel_setkey_xcbcmac(sess, ckey, cklen); + break; + case CRYPTO_AES_XTS: + safexcel_setkey_xts(sess, ckey, cklen); + sess->klen /= 2; + break; + } + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + safexcel_setkey_hmac(csp, sess, akey, aklen); + break; + case CRYPTO_AES_NIST_GMAC: + sess->klen = aklen; + safexcel_setkey_ghash(sess, akey, aklen); + break; + } +} + +static uint32_t +safexcel_aes_algid(int keylen) +{ + switch (keylen) { + case 16: + return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128); + case 24: + return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192); + case 32: + return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256); + default: + panic("invalid AES key length %d", keylen); + } +} + +static uint32_t +safexcel_aes_ccm_hashid(int keylen) +{ + switch (keylen) { + case 16: + return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC128); + case 24: + return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC192); + case 32: + return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC256); + default: + panic("invalid AES key length %d", keylen); + } +} + +static uint32_t +safexcel_sha_hashid(int alg) +{ + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA1); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA224); + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA256); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA384); + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (SAFEXCEL_CONTROL0_HASH_ALG_SHA512); + default: + __assert_unreachable(); + } +} + +static int +safexcel_sha_hashlen(int alg) +{ + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (SHA1_HASH_LEN); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + return (SHA2_224_HASH_LEN); + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (SHA2_256_HASH_LEN); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + return (SHA2_384_HASH_LEN); + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (SHA2_512_HASH_LEN); + default: + __assert_unreachable(); + } +} + +static int +safexcel_sha_statelen(int alg) +{ + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (SHA1_HASH_LEN); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (SHA2_256_HASH_LEN); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (SHA2_512_HASH_LEN); + default: + __assert_unreachable(); + } +} + +static int +safexcel_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct safexcel_session *sess; + struct safexcel_softc *sc; + + sc = device_get_softc(dev); + sess = crypto_get_driver_session(cses); + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_512: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED; + sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); + sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); + sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); + break; + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_HMAC; + sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); + sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); + sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); + break; + case CRYPTO_AES_NIST_GMAC: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; + sess->digestlen = GMAC_DIGEST_LEN; + sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; + sess->alg = safexcel_aes_algid(csp->csp_auth_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; + break; + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; + sess->digestlen = GMAC_DIGEST_LEN; + sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; + break; + case CRYPTO_AES_CCM_16: + sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen); + sess->digest = SAFEXCEL_CONTROL0_DIGEST_CCM; + sess->digestlen = CCM_CBC_MAX_DIGEST_LEN; + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM; + break; + case CRYPTO_AES_CBC: + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC; + break; + case CRYPTO_AES_ICM: + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR; + break; + case CRYPTO_AES_XTS: + sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2); + sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS; + break; + } + + if (csp->csp_auth_mlen != 0) + sess->digestlen = csp->csp_auth_mlen; + + safexcel_setkey(sess, csp, NULL); + + /* Bind each session to a fixed ring to minimize lock contention. */ + sess->ringidx = atomic_fetchadd_int(&sc->sc_ringidx, 1); + sess->ringidx %= sc->sc_config.rings; + + return (0); +} + +static int +safexcel_process(device_t dev, struct cryptop *crp, int hint) +{ + const struct crypto_session_params *csp; + struct safexcel_request *req; + struct safexcel_ring *ring; + struct safexcel_session *sess; + struct safexcel_softc *sc; + int error; + + sc = device_get_softc(dev); + sess = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); + + if (__predict_false(crypto_buffer_len(&crp->crp_buf) > + SAFEXCEL_MAX_REQUEST_SIZE)) { + crp->crp_etype = E2BIG; + crypto_done(crp); + return (0); + } + + if (crp->crp_cipher_key != NULL || crp->crp_auth_key != NULL) + safexcel_setkey(sess, csp, crp); + + ring = &sc->sc_ring[sess->ringidx]; + mtx_lock(&ring->mtx); + req = safexcel_alloc_request(sc, ring); + if (__predict_false(req == NULL)) { + mtx_lock(&sc->sc_mtx); + mtx_unlock(&ring->mtx); + sc->sc_blocked = CRYPTO_SYMQ; + mtx_unlock(&sc->sc_mtx); + return (ERESTART); + } + + req->crp = crp; + req->sess = sess; + + crypto_read_iv(crp, req->iv); + + error = safexcel_create_chain(ring, req); + if (__predict_false(error != 0)) { + safexcel_free_request(ring, req); + mtx_unlock(&ring->mtx); + crp->crp_etype = error; + crypto_done(crp); + return (0); + } + + safexcel_set_token(req); + + bus_dmamap_sync(ring->data_dtag, req->dmap, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(req->ctx.tag, req->ctx.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + safexcel_enqueue_request(sc, ring, req); + + if ((hint & CRYPTO_HINT_MORE) == 0) + safexcel_execute(sc, ring, req); + mtx_unlock(&ring->mtx); + + return (0); +} + +static device_method_t safexcel_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, safexcel_probe), + DEVMETHOD(device_attach, safexcel_attach), + DEVMETHOD(device_detach, safexcel_detach), + + /* Cryptodev interface */ + DEVMETHOD(cryptodev_probesession, safexcel_probesession), + DEVMETHOD(cryptodev_newsession, safexcel_newsession), + DEVMETHOD(cryptodev_process, safexcel_process), + + DEVMETHOD_END +}; + +static devclass_t safexcel_devclass; + +static driver_t safexcel_driver = { + .name = "safexcel", + .methods = safexcel_methods, + .size = sizeof(struct safexcel_softc), +}; + +DRIVER_MODULE(safexcel, simplebus, safexcel_driver, safexcel_devclass, 0, 0); +MODULE_VERSION(safexcel, 1); +MODULE_DEPEND(safexcel, crypto, 1, 1, 1); diff --git a/sys/dev/safexcel/safexcel_reg.h b/sys/dev/safexcel/safexcel_reg.h new file mode 100644 index 000000000000..fb26a8f99a50 --- /dev/null +++ b/sys/dev/safexcel/safexcel_reg.h @@ -0,0 +1,457 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SAFEXCEL_REGS_H_ +#define _SAFEXCEL_REGS_H_ + +#define SAFEXCEL_HIA_VERSION_LE 0x35ca +#define SAFEXCEL_HIA_VERSION_BE 0xca35 +#define EIP201_VERSION_LE 0x36c9 +#define SAFEXCEL_REG_LO16(_reg) ((_reg) & 0xffff) +#define SAFEXCEL_REG_HI16(_reg) (((_reg) >> 16) & 0xffff) + + +/* HIA, Command Descriptor Ring Manager */ +#define CDR_BASE_ADDR_LO(x) (0x0 + ((x) << 12)) +#define CDR_BASE_ADDR_HI(x) (0x4 + ((x) << 12)) +#define CDR_DATA_BASE_ADDR_LO(x) (0x8 + ((x) << 12)) +#define CDR_DATA_BASE_ADDR_HI(x) (0xC + ((x) << 12)) +#define CDR_ACD_BASE_ADDR_LO(x) (0x10 + ((x) << 12)) +#define CDR_ACD_BASE_ADDR_HI(x) (0x14 + ((x) << 12)) +#define CDR_RING_SIZE(x) (0x18 + ((x) << 12)) +#define CDR_DESC_SIZE(x) (0x1C + ((x) << 12)) +#define CDR_CFG(x) (0x20 + ((x) << 12)) +#define CDR_DMA_CFG(x) (0x24 + ((x) << 12)) +#define CDR_THR(x) (0x28 + ((x) << 12)) +#define CDR_PREP_COUNT(x) (0x2C + ((x) << 12)) +#define CDR_PROC_COUNT(x) (0x30 + ((x) << 12)) +#define CDR_PREP_PNTR(x) (0x34 + ((x) << 12)) +#define CDR_PROC_PNTR(x) (0x38 + ((x) << 12)) +#define CDR_STAT(x) (0x3C + ((x) << 12)) + +/* HIA, Result Descriptor Ring Manager */ +#define RDR_BASE_ADDR_LO(x) (0x800 + ((x) << 12)) +#define RDR_BASE_ADDR_HI(x) (0x804 + ((x) << 12)) +#define RDR_DATA_BASE_ADDR_LO(x) (0x808 + ((x) << 12)) +#define RDR_DATA_BASE_ADDR_HI(x) (0x80C + ((x) << 12)) +#define RDR_ACD_BASE_ADDR_LO(x) (0x810 + ((x) << 12)) +#define RDR_ACD_BASE_ADDR_HI(x) (0x814 + ((x) << 12)) +#define RDR_RING_SIZE(x) (0x818 + ((x) << 12)) +#define RDR_DESC_SIZE(x) (0x81C + ((x) << 12)) +#define RDR_CFG(x) (0x820 + ((x) << 12)) +#define RDR_DMA_CFG(x) (0x824 + ((x) << 12)) +#define RDR_THR(x) (0x828 + ((x) << 12)) +#define RDR_PREP_COUNT(x) (0x82C + ((x) << 12)) +#define RDR_PROC_COUNT(x) (0x830 + ((x) << 12)) +#define RDR_PREP_PNTR(x) (0x834 + ((x) << 12)) +#define RDR_PROC_PNTR(x) (0x838 + ((x) << 12)) +#define RDR_STAT(x) (0x83C + ((x) << 12)) + +/* HIA, Ring AIC */ +#define AIC_POL_CTRL(x) (0xE000 - ((x) << 12)) +#define AIC_TYPE_CTRL(x) (0xE004 - ((x) << 12)) +#define AIC_ENABLE_CTRL(x) (0xE008 - ((x) << 12)) +#define AIC_RAW_STAL(x) (0xE00C - ((x) << 12)) +#define AIC_ENABLE_SET(x) (0xE00C - ((x) << 12)) +#define AIC_ENABLED_STAT(x) (0xE010 - ((x) << 12)) +#define AIC_ACK(x) (0xE010 - ((x) << 12)) +#define AIC_ENABLE_CLR(x) (0xE014 - ((x) << 12)) +#define AIC_OPTIONS(x) (0xE018 - ((x) << 12)) +#define AIC_VERSION(x) (0xE01C - ((x) << 12)) + +/* HIA, Global AIC */ +#define AIC_G_POL_CTRL 0xF800 +#define AIC_G_TYPE_CTRL 0xF804 +#define AIC_G_ENABLE_CTRL 0xF808 +#define AIC_G_RAW_STAT 0xF80C +#define AIC_G_ENABLE_SET 0xF80C +#define AIC_G_ENABLED_STAT 0xF810 +#define AIC_G_ACK 0xF810 +#define AIC_G_ENABLE_CLR 0xF814 +#define AIC_G_OPTIONS 0xF818 +#define AIC_G_VERSION 0xF81C + +/* HIA, Data Fetch Engine */ +#define DFE_CFG 0xF000 +#define DFE_PRIO_0 0xF010 +#define DFE_PRIO_1 0xF014 +#define DFE_PRIO_2 0xF018 +#define DFE_PRIO_3 0xF01C + +/* HIA, Data Fetch Engine access monitoring for CDR */ +#define DFE_RING_REGION_LO(x) (0xF080 + ((x) << 3)) +#define DFE_RING_REGION_HI(x) (0xF084 + ((x) << 3)) + +/* HIA, Data Fetch Engine thread control and status for thread */ +#define DFE_THR_CTRL 0xF200 +#define DFE_THR_STAT 0xF204 +#define DFE_THR_DESC_CTRL 0xF208 +#define DFE_THR_DESC_DPTR_LO 0xF210 +#define DFE_THR_DESC_DPTR_HI 0xF214 +#define DFE_THR_DESC_ACDPTR_LO 0xF218 +#define DFE_THR_DESC_ACDPTR_HI 0xF21C + +/* HIA, Data Store Engine */ +#define DSE_CFG 0xF400 +#define DSE_PRIO_0 0xF410 +#define DSE_PRIO_1 0xF414 +#define DSE_PRIO_2 0xF418 +#define DSE_PRIO_3 0xF41C + +/* HIA, Data Store Engine access monitoring for RDR */ +#define DSE_RING_REGION_LO(x) (0xF480 + ((x) << 3)) +#define DSE_RING_REGION_HI(x) (0xF484 + ((x) << 3)) + +/* HIA, Data Store Engine thread control and status for thread */ +#define DSE_THR_CTRL 0xF600 +#define DSE_THR_STAT 0xF604 +#define DSE_THR_DESC_CTRL 0xF608 +#define DSE_THR_DESC_DPTR_LO 0xF610 +#define DSE_THR_DESC_DPTR_HI 0xF614 +#define DSE_THR_DESC_S_DPTR_LO 0xF618 +#define DSE_THR_DESC_S_DPTR_HI 0xF61C +#define DSE_THR_ERROR_STAT 0xF620 + +/* HIA Global */ +#define HIA_MST_CTRL 0xFFF4 +#define HIA_OPTIONS 0xFFF8 +#define HIA_VERSION 0xFFFC + +/* Processing Engine Input Side, Processing Engine */ +#define PE_IN_DBUF_THRESH 0x10000 +#define PE_IN_TBUF_THRESH 0x10100 + +/* Packet Engine Configuration / Status Registers */ +#define PE_TOKEN_CTRL_STAT 0x11000 +#define PE_FUNCTION_EN 0x11004 +#define PE_CONTEXT_CTRL 0x11008 +#define PE_INTERRUPT_CTRL_STAT 0x11010 +#define PE_CONTEXT_STAT 0x1100C +#define PE_OUT_TRANS_CTRL_STAT 0x11018 +#define PE_OUT_BUF_CTRL 0x1101C + +/* Packet Engine AIC */ +#define PE_EIP96_AIC_POL_CTRL 0x113C0 +#define PE_EIP96_AIC_TYPE_CTRL 0x113C4 +#define PE_EIP96_AIC_ENABLE_CTRL 0x113C8 +#define PE_EIP96_AIC_RAW_STAT 0x113CC +#define PE_EIP96_AIC_ENABLE_SET 0x113CC +#define PE_EIP96_AIC_ENABLED_STAT 0x113D0 +#define PE_EIP96_AIC_ACK 0x113D0 +#define PE_EIP96_AIC_ENABLE_CLR 0x113D4 +#define PE_EIP96_AIC_OPTIONS 0x113D8 +#define PE_EIP96_AIC_VERSION 0x113DC + +/* Packet Engine Options & Version Registers */ +#define PE_EIP96_OPTIONS 0x113F8 +#define PE_EIP96_VERSION 0x113FC + +#define SAFEXCEL_OPT + +/* Processing Engine Output Side */ +#define PE_OUT_DBUF_THRESH 0x11C00 +#define PE_OUT_TBUF_THRESH 0x11D00 + +/* Processing Engine Local AIC */ +#define PE_AIC_POL_CTRL 0x11F00 +#define PE_AIC_TYPE_CTRL 0x11F04 +#define PE_AIC_ENABLE_CTRL 0x11F08 +#define PE_AIC_RAW_STAT 0x11F0C +#define PE_AIC_ENABLE_SET 0x11F0C +#define PE_AIC_ENABLED_STAT 0x11F10 +#define PE_AIC_ENABLE_CLR 0x11F14 +#define PE_AIC_OPTIONS 0x11F18 +#define PE_AIC_VERSION 0x11F1C + +/* Processing Engine General Configuration and Version */ +#define PE_IN_FLIGHT 0x11FF0 +#define PE_OPTIONS 0x11FF8 +#define PE_VERSION 0x11FFC + +/* EIP-97 - Global */ +#define EIP97_CLOCK_STATE 0x1FFE4 +#define EIP97_FORCE_CLOCK_ON 0x1FFE8 +#define EIP97_FORCE_CLOCK_OFF 0x1FFEC +#define EIP97_MST_CTRL 0x1FFF4 +#define EIP97_OPTIONS 0x1FFF8 +#define EIP97_VERSION 0x1FFFC + +/* Register base offsets */ +#define SAFEXCEL_HIA_AIC(_sc) ((_sc)->sc_offsets.hia_aic) +#define SAFEXCEL_HIA_AIC_G(_sc) ((_sc)->sc_offsets.hia_aic_g) +#define SAFEXCEL_HIA_AIC_R(_sc) ((_sc)->sc_offsets.hia_aic_r) +#define SAFEXCEL_HIA_AIC_xDR(_sc) ((_sc)->sc_offsets.hia_aic_xdr) +#define SAFEXCEL_HIA_DFE(_sc) ((_sc)->sc_offsets.hia_dfe) +#define SAFEXCEL_HIA_DFE_THR(_sc) ((_sc)->sc_offsets.hia_dfe_thr) +#define SAFEXCEL_HIA_DSE(_sc) ((_sc)->sc_offsets.hia_dse) +#define SAFEXCEL_HIA_DSE_THR(_sc) ((_sc)->sc_offsets.hia_dse_thr) +#define SAFEXCEL_HIA_GEN_CFG(_sc) ((_sc)->sc_offsets.hia_gen_cfg) +#define SAFEXCEL_PE(_sc) ((_sc)->sc_offsets.pe) + +/* EIP197 base offsets */ +#define SAFEXCEL_EIP197_HIA_AIC_BASE 0x90000 +#define SAFEXCEL_EIP197_HIA_AIC_G_BASE 0x90000 +#define SAFEXCEL_EIP197_HIA_AIC_R_BASE 0x90800 +#define SAFEXCEL_EIP197_HIA_AIC_xDR_BASE 0x80000 +#define SAFEXCEL_EIP197_HIA_DFE_BASE 0x8c000 +#define SAFEXCEL_EIP197_HIA_DFE_THR_BASE 0x8c040 +#define SAFEXCEL_EIP197_HIA_DSE_BASE 0x8d000 +#define SAFEXCEL_EIP197_HIA_DSE_THR_BASE 0x8d040 +#define SAFEXCEL_EIP197_HIA_GEN_CFG_BASE 0xf0000 +#define SAFEXCEL_EIP197_PE_BASE 0xa0000 + +/* EIP97 base offsets */ +#define SAFEXCEL_EIP97_HIA_AIC_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_AIC_G_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_AIC_R_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_AIC_xDR_BASE 0x0 +#define SAFEXCEL_EIP97_HIA_DFE_BASE 0xf000 +#define SAFEXCEL_EIP97_HIA_DFE_THR_BASE 0xf200 +#define SAFEXCEL_EIP97_HIA_DSE_BASE 0xf400 +#define SAFEXCEL_EIP97_HIA_DSE_THR_BASE 0xf600 +#define SAFEXCEL_EIP97_HIA_GEN_CFG_BASE 0x10000 +#define SAFEXCEL_EIP97_PE_BASE 0x10000 + +/* CDR/RDR register offsets */ +#define SAFEXCEL_HIA_xDR_OFF(priv, r) (SAFEXCEL_HIA_AIC_xDR(priv) + (r) * 0x1000) +#define SAFEXCEL_HIA_CDR(priv, r) (SAFEXCEL_HIA_xDR_OFF(priv, r)) +#define SAFEXCEL_HIA_RDR(priv, r) (SAFEXCEL_HIA_xDR_OFF(priv, r) + 0x800) +#define SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO 0x0000 +#define SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI 0x0004 +#define SAFEXCEL_HIA_xDR_RING_SIZE 0x0018 +#define SAFEXCEL_HIA_xDR_DESC_SIZE 0x001c +#define SAFEXCEL_HIA_xDR_CFG 0x0020 +#define SAFEXCEL_HIA_xDR_DMA_CFG 0x0024 +#define SAFEXCEL_HIA_xDR_THRESH 0x0028 +#define SAFEXCEL_HIA_xDR_PREP_COUNT 0x002c +#define SAFEXCEL_HIA_xDR_PROC_COUNT 0x0030 +#define SAFEXCEL_HIA_xDR_PREP_PNTR 0x0034 +#define SAFEXCEL_HIA_xDR_PROC_PNTR 0x0038 +#define SAFEXCEL_HIA_xDR_STAT 0x003c + +/* register offsets */ +#define SAFEXCEL_HIA_DFE_CFG(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DFE_THR_CTRL(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DFE_THR_STAT(n) (0x004 + (128 * (n))) +#define SAFEXCEL_HIA_DSE_CFG(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DSE_THR_CTRL(n) (0x000 + (128 * (n))) +#define SAFEXCEL_HIA_DSE_THR_STAT(n) (0x004 + (128 * (n))) +#define SAFEXCEL_HIA_RA_PE_CTRL(n) (0x010 + (8 * (n))) +#define SAFEXCEL_HIA_RA_PE_STAT 0x0014 +#define SAFEXCEL_HIA_AIC_R_OFF(r) ((r) * 0x1000) +#define SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(r) (0xe008 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_ENABLED_STAT(r) (0xe010 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_ACK(r) (0xe010 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_ENABLE_CLR(r) (0xe014 - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_R_VERSION(r) (0xe01c - SAFEXCEL_HIA_AIC_R_OFF(r)) +#define SAFEXCEL_HIA_AIC_G_ENABLE_CTRL 0xf808 +#define SAFEXCEL_HIA_AIC_G_ENABLED_STAT 0xf810 +#define SAFEXCEL_HIA_AIC_G_ACK 0xf810 +#define SAFEXCEL_HIA_MST_CTRL 0xfff4 +#define SAFEXCEL_HIA_OPTIONS 0xfff8 +#define SAFEXCEL_HIA_VERSION 0xfffc +#define SAFEXCEL_PE_IN_DBUF_THRES(n) (0x0000 + (0x2000 * (n))) +#define SAFEXCEL_PE_IN_TBUF_THRES(n) (0x0100 + (0x2000 * (n))) +#define SAFEXCEL_PE_ICE_SCRATCH_RAM(x, n) ((0x800 + (x) * 4) + 0x2000 * (n)) +#define SAFEXCEL_PE_ICE_PUE_CTRL(n) (0xc80 + (0x2000 * (n))) +#define SAFEXCEL_PE_ICE_SCRATCH_CTRL 0x0d04 +#define SAFEXCEL_PE_ICE_FPP_CTRL(n) (0xd80 + (0x2000 * (n))) +#define SAFEXCEL_PE_ICE_RAM_CTRL(n) (0xff0 + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n))) +#define SAFEXCEL_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n))) +#define SAFEXCEL_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n))) +#define SAFEXCEL_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n))) + +/* EIP-197 Classification Engine */ + +/* Classification regs */ +#define SAFEXCEL_CS_RAM_CTRL 0xf7ff0 + +/* SAFEXCEL_HIA_xDR_DESC_SIZE */ +#define SAFEXCEL_xDR_DESC_MODE_64BIT (1U << 31) +#define SAFEXCEL_CDR_DESC_MODE_ADCP (1 << 30) +#define SAFEXCEL_xDR_DESC_xD_OFFSET 16 + +/* SAFEXCEL_DIA_xDR_CFG */ +#define SAFEXCEL_xDR_xD_FETCH_THRESH 16 + +/* SAFEXCEL_HIA_xDR_DMA_CFG */ +#define SAFEXCEL_HIA_xDR_WR_RES_BUF (1 << 22) +#define SAFEXCEL_HIA_xDR_WR_CTRL_BUF (1 << 23) +#define SAFEXCEL_HIA_xDR_WR_OWN_BUF (1 << 24) +#define SAFEXCEL_HIA_xDR_CFG_xD_PROT(n) (((n) & 0xf) << 4) +#define SAFEXCEL_HIA_xDR_CFG_DATA_PROT(n) (((n) & 0xf) << 12) +#define SAFEXCEL_HIA_xDR_CFG_ACD_PROT(n) (((n) & 0xf) << 20) +#define SAFEXCEL_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 25) +#define SAFEXCEL_HIA_xDR_CFG_RD_CACHE(n) (((n) & 0x7) << 29) + +/* SAFEXCEL_HIA_CDR_THRESH */ +#define SAFEXCEL_HIA_CDR_THRESH_PROC_PKT(n) ((n) & 0xffff) +#define SAFEXCEL_HIA_CDR_THRESH_PROC_MODE (1 << 22) +#define SAFEXCEL_HIA_CDR_THRESH_PKT_MODE (1 << 23) + /* x256 clk cycles */ +#define SAFEXCEL_HIA_CDR_THRESH_TIMEOUT(n) (((n) & 0xff) << 24) + +/* SAFEXCEL_HIA_RDR_THRESH */ +#define SAFEXCEL_HIA_RDR_THRESH_PROC_PKT(n) ((n) & 0xffff) +#define SAFEXCEL_HIA_RDR_THRESH_PKT_MODE (1 << 23) + /* x256 clk cycles */ +#define SAFEXCEL_HIA_RDR_THRESH_TIMEOUT(n) (((n) & 0xff) << 24) + +/* SAFEXCEL_HIA_xDR_PREP_COUNT */ +#define SAFEXCEL_xDR_PREP_CLR_COUNT (1U << 31) +#define SAFEXCEL_xDR_PREP_xD_COUNT_INCR_OFFSET 2 +#define SAFEXCEL_xDR_PREP_RD_COUNT_INCR_MASK 0x3fff + +/* SAFEXCEL_HIA_xDR_PROC_COUNT */ +#define SAFEXCEL_xDR_PROC_xD_PKT_OFFSET 24 +#define SAFEXCEL_xDR_PROC_xD_PKT_MASK 0x7f +#define SAFEXCEL_xDR_PROC_xD_COUNT(n) ((n) << 2) +#define SAFEXCEL_xDR_PROC_xD_PKT(n) \ + (((n) & SAFEXCEL_xDR_PROC_xD_PKT_MASK) << SAFEXCEL_xDR_PROC_xD_PKT_OFFSET) +#define SAFEXCEL_xDR_PROC_CLR_COUNT (1U << 31) + +/* SAFEXCEL_HIA_xDR_STAT */ +#define SAFEXCEL_xDR_DMA_ERR (1 << 0) +#define SAFEXCEL_xDR_PREP_CMD_THRES (1 << 1) +#define SAFEXCEL_xDR_ERR (1 << 2) +#define SAFEXCEL_xDR_THRESH (1 << 4) +#define SAFEXCEL_xDR_TIMEOUT (1 << 5) +#define SAFEXCEL_CDR_INTR_MASK 0x3f +#define SAFEXCEL_RDR_INTR_MASK 0xff + +#define SAFEXCEL_HIA_RA_PE_CTRL_RESET (1U << 31) +#define SAFEXCEL_HIA_RA_PE_CTRL_EN (1 << 30) + +/* Register offsets */ + +/* SAFEXCEL_HIA_DSE_THR_STAT */ +#define SAFEXCEL_DSE_THR_RDR_ID_MASK 0xf000 + +/* SAFEXCEL_HIA_OPTIONS */ +#define SAFEXCEL_OPT_ADDR_64 (1U << 31) +#define SAFEXCEL_OPT_TGT_ALIGN_OFFSET 28 +#define SAFEXCEL_OPT_TGT_ALIGN_MASK 0x70000000 +#define SAFEXCEL_xDR_HDW_OFFSET 25 +#define SAFEXCEL_xDR_HDW_MASK 0x6000000 +#define SAFEXCEL_N_RINGS_MASK 0xf +#define SAFEXCEL_N_PES_OFFSET 4 +#define SAFEXCEL_N_PES_MASK 0x1f0 +#define EIP97_N_PES_MASK 0x70 + +/* SAFEXCEL_HIA_AIC_R_ENABLE_CTRL */ +#define SAFEXCEL_CDR_IRQ(n) (1 << ((n) * 2)) +#define SAFEXCEL_RDR_IRQ(n) (1 << ((n) * 2 + 1)) + +/* SAFEXCEL_HIA_DFE/DSE_CFG */ +#define SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(n) ((n) << 0) +#define SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(n) (((n) & 0x7) << 4) +#define SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(n) ((n) << 8) +#define SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE 0xc000 +#define SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(n) ((n) << 16) +#define SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20) +#define SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24) +#define SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG 0xe0000000 +#define SAFEXCEL_HIA_DSE_CFG_EN_SINGLE_WR (1 << 29) +#define SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG 0xc0000000 + +/* SAFEXCEL_HIA_DFE/DSE_THR_CTRL */ +#define SAFEXCEL_DxE_THR_CTRL_EN (1 << 30) +#define SAFEXCEL_DxE_THR_CTRL_RESET_PE (1U << 31) + +/* SAFEXCEL_HIA_AIC_G_ENABLED_STAT */ +#define SAFEXCEL_G_IRQ_DFE(n) (1 << ((n) << 1)) +#define SAFEXCEL_G_IRQ_DSE(n) (1 << (((n) << 1) + 1)) +#define SAFEXCEL_G_IRQ_RING (1 << 16) +#define SAFEXCEL_G_IRQ_PE(n) (1 << ((n) + 20)) + +/* SAFEXCEL_HIA_MST_CTRL */ +#define RD_CACHE_3BITS 0x5U +#define WR_CACHE_3BITS 0x3U +#define RD_CACHE_4BITS (RD_CACHE_3BITS << 1 | (1 << 0)) +#define WR_CACHE_4BITS (WR_CACHE_3BITS << 1 | (1 << 0)) +#define SAFEXCEL_MST_CTRL_RD_CACHE(n) (((n) & 0xf) << 0) +#define SAFEXCEL_MST_CTRL_WD_CACHE(n) (((n) & 0xf) << 4) +#define MST_CTRL_SUPPORT_PROT(n) (((n) & 0xf) << 12) +#define SAFEXCEL_MST_CTRL_BYTE_SWAP (1 << 24) +#define SAFEXCEL_MST_CTRL_NO_BYTE_SWAP (1 << 25) + +/* SAFEXCEL_PE_IN_DBUF/TBUF_THRES */ +#define SAFEXCEL_PE_IN_xBUF_THRES_MIN(n) ((n) << 8) +#define SAFEXCEL_PE_IN_xBUF_THRES_MAX(n) ((n) << 12) + +/* SAFEXCEL_PE_OUT_DBUF_THRES */ +#define SAFEXCEL_PE_OUT_DBUF_THRES_MIN(n) ((n) << 0) +#define SAFEXCEL_PE_OUT_DBUF_THRES_MAX(n) ((n) << 4) + +/* SAFEXCEL_HIA_AIC_G_ACK */ +#define SAFEXCEL_AIC_G_ACK_ALL_MASK 0xffffffff +#define SAFEXCEL_AIC_G_ACK_HIA_MASK 0x7ff00000 + +/* SAFEXCEL_HIA_AIC_R_ENABLE_CLR */ +#define SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK 0xffffffff + +/* SAFEXCEL_PE_EIP96_CONTEXT_CTRL */ +#define SAFEXCEL_CONTEXT_SIZE(n) (n) +#define SAFEXCEL_ADDRESS_MODE (1 << 8) +#define SAFEXCEL_CONTROL_MODE (1 << 9) + +/* SAFEXCEL_PE_EIP96_FUNCTION_EN */ +#define SAFEXCEL_FUNCTION_RSVD ((1U << 6) | (1U << 15) | (1U << 20) | (1U << 23)) +#define SAFEXCEL_PROTOCOL_HASH_ONLY (1U << 0) +#define SAFEXCEL_PROTOCOL_ENCRYPT_ONLY (1U << 1) +#define SAFEXCEL_PROTOCOL_HASH_ENCRYPT (1U << 2) +#define SAFEXCEL_PROTOCOL_HASH_DECRYPT (1U << 3) +#define SAFEXCEL_PROTOCOL_ENCRYPT_HASH (1U << 4) +#define SAFEXCEL_PROTOCOL_DECRYPT_HASH (1U << 5) +#define SAFEXCEL_ALG_ARC4 (1U << 7) +#define SAFEXCEL_ALG_AES_ECB (1U << 8) +#define SAFEXCEL_ALG_AES_CBC (1U << 9) +#define SAFEXCEL_ALG_AES_CTR_ICM (1U << 10) +#define SAFEXCEL_ALG_AES_OFB (1U << 11) +#define SAFEXCEL_ALG_AES_CFB (1U << 12) +#define SAFEXCEL_ALG_DES_ECB (1U << 13) +#define SAFEXCEL_ALG_DES_CBC (1U << 14) +#define SAFEXCEL_ALG_DES_OFB (1U << 16) +#define SAFEXCEL_ALG_DES_CFB (1U << 17) +#define SAFEXCEL_ALG_3DES_ECB (1U << 18) +#define SAFEXCEL_ALG_3DES_CBC (1U << 19) +#define SAFEXCEL_ALG_3DES_OFB (1U << 21) +#define SAFEXCEL_ALG_3DES_CFB (1U << 22) +#define SAFEXCEL_ALG_MD5 (1U << 24) +#define SAFEXCEL_ALG_HMAC_MD5 (1U << 25) +#define SAFEXCEL_ALG_SHA1 (1U << 26) +#define SAFEXCEL_ALG_HMAC_SHA1 (1U << 27) +#define SAFEXCEL_ALG_SHA2 (1U << 28) +#define SAFEXCEL_ALG_HMAC_SHA2 (1U << 29) +#define SAFEXCEL_ALG_AES_XCBC_MAC (1U << 30) +#define SAFEXCEL_ALG_GCM_HASH (1U << 31) + +#endif /* _SAFEXCEL_REGS_H_ */ diff --git a/sys/dev/safexcel/safexcel_var.h b/sys/dev/safexcel/safexcel_var.h new file mode 100644 index 000000000000..619bcff00c57 --- /dev/null +++ b/sys/dev/safexcel/safexcel_var.h @@ -0,0 +1,420 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SAFEXCEL_VAR_H_ +#define _SAFEXCEL_VAR_H_ + +#define SAFEXCEL_MAX_RINGS 4 +#define SAFEXCEL_MAX_BATCH_SIZE 64 +#define SAFEXCEL_MAX_FRAGMENTS 64 +#define SAFEXCEL_MAX_IV_LEN 16 +#define SAFEXCEL_MAX_REQUEST_SIZE 65535 + +#define SAFEXCEL_RING_SIZE 512 +#define SAFEXCEL_REQUESTS_PER_RING 64 +#define SAFEXCEL_MAX_ITOKENS 4 +#define SAFEXCEL_MAX_ATOKENS 16 +#define SAFEXCEL_FETCH_COUNT 1 +#define SAFEXCEL_MAX_KEY_LEN 32 +#define SAFEXCEL_MAX_RING_AIC 14 + +/* + * Context Record format. + * + * In this driver the context control words are always set in the control data. + * This is configured by setting SAFEXCEL_OPTION_CTX_CTRL_IN_CMD. + */ +struct safexcel_context_record { + uint32_t control0; /* Unused. */ + uint32_t control1; /* Unused. */ + uint32_t data[40]; /* Key material. */ +} __packed; + +/* Processing Engine Control Data format. */ +struct safexcel_control_data { + uint32_t packet_length : 17; + uint32_t options : 13; + uint32_t type : 2; + + uint16_t application_id; + uint16_t rsvd; + + uint32_t context_lo; + uint32_t context_hi; + + uint32_t control0; + uint32_t control1; + + /* Inline instructions or IV. */ + uint32_t token[SAFEXCEL_MAX_ITOKENS]; +} __packed; + +/* + * Basic Command Descriptor. + * + * The Processing Engine and driver cooperate to maintain a set of command + * rings, representing outstanding crypto operation requests. Each descriptor + * corresponds to an input data segment, and thus a single crypto(9) request may + * span several contiguous command descriptors. + * + * The first command descriptor for a request stores the input token, which + * encodes data specific to the requested operation, such as the encryption + * mode. For some operations data is passed outside the descriptor, in a + * context record (e.g., encryption keys), or in an "additional token data" + * region (e.g., instructions). + */ +struct safexcel_cmd_descr { + uint32_t particle_size : 17; + uint32_t rsvd0 : 5; + uint32_t last_seg : 1; + uint32_t first_seg : 1; + uint32_t additional_cdata_size : 8; + uint32_t rsvd1; + + uint32_t data_lo; + uint32_t data_hi; + + uint32_t atok_lo; + uint32_t atok_hi; + + struct safexcel_control_data control_data; +} __packed; + +/* Context control word 0 fields. */ +#define SAFEXCEL_CONTROL0_TYPE_NULL_OUT 0x0 +#define SAFEXCEL_CONTROL0_TYPE_NULL_IN 0x1 +#define SAFEXCEL_CONTROL0_TYPE_HASH_OUT 0x2 +#define SAFEXCEL_CONTROL0_TYPE_HASH_IN 0x3 +#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT 0x4 +#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN 0x5 +#define SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT 0x6 +#define SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN 0x7 +#define SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT 0xe +#define SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN 0xf +#define SAFEXCEL_CONTROL0_RESTART_HASH (1 << 4) +#define SAFEXCEL_CONTROL0_NO_FINISH_HASH (1 << 5) +#define SAFEXCEL_CONTROL0_SIZE(n) (((n) & 0xff) << 8) +#define SAFEXCEL_CONTROL0_KEY_EN (1 << 16) +#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128 (0x5 << 17) +#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192 (0x6 << 17) +#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256 (0x7 << 17) +#define SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED (0x1 << 21) +#define SAFEXCEL_CONTROL0_DIGEST_CCM (0x2 << 21) +#define SAFEXCEL_CONTROL0_DIGEST_GMAC (0x2 << 21) +#define SAFEXCEL_CONTROL0_DIGEST_HMAC (0x3 << 21) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA1 (0x2 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA224 (0x4 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA256 (0x3 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA384 (0x6 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_SHA512 (0x5 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC128 (0x1 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC192 (0x2 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC256 (0x3 << 23) +#define SAFEXCEL_CONTROL0_HASH_ALG_GHASH (0x4 << 23) +#define SAFEXCEL_CONTROL0_INV_FR (0x5 << 24) +#define SAFEXCEL_CONTROL0_INV_TR (0x6 << 24) + +/* Context control word 1 fields. */ +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ECB 0x0 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC 0x1 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ICM 0x3 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_OFB 0x4 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CFB128 0x5 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR 0x6 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS 0x7 +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM (0x6 | (1 << 17)) +#define SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM (0x6 | (1 << 17)) +#define SAFEXCEL_CONTROL1_IV0 (1u << 5) +#define SAFEXCEL_CONTROL1_IV1 (1u << 6) +#define SAFEXCEL_CONTROL1_IV2 (1u << 7) +#define SAFEXCEL_CONTROL1_IV3 (1u << 8) +#define SAFEXCEL_CONTROL1_DIGEST_CNT (1u << 9) +#define SAFEXCEL_CONTROL1_COUNTER_MODE (1u << 10) +#define SAFEXCEL_CONTROL1_ENCRYPT_HASH_RES (1u << 17) +#define SAFEXCEL_CONTROL1_HASH_STORE (1u << 19) + +/* Control options. */ +#define SAFEXCEL_OPTION_IP (1u << 0) /* must be set */ +#define SAFEXCEL_OPTION_CP (1u << 1) /* 64-bit ctx addr */ +#define SAFEXCEL_OPTION_RC_AUTO (2u << 3) /* auto ctx reuse */ +#define SAFEXCEL_OPTION_CTX_CTRL_IN_CMD (1u << 8) /* ctx ctrl */ +#define SAFEXCEL_OPTION_4_TOKEN_IV_CMD 0xe00 /* IV in bypass */ + +struct safexcel_res_data { + uint32_t packet_length : 17; + uint32_t error_code : 15; + + uint32_t bypass_length : 4; + uint32_t e15 : 1; + uint32_t rsvd0 : 16; + uint32_t hash_bytes : 1; + uint32_t hash_length : 6; + uint32_t generic_bytes : 1; + uint32_t checksum : 1; + uint32_t next_header : 1; + uint32_t length : 1; + + uint16_t application_id; + uint16_t rsvd1; + + uint32_t rsvd2; +}; + +/* Basic Result Descriptor format */ +struct safexcel_res_descr { + uint32_t particle_size : 17; + uint32_t rsvd0 : 3; + uint32_t descriptor_overflow : 1; + uint32_t buffer_overflow : 1; + uint32_t last_seg : 1; + uint32_t first_seg : 1; + uint32_t result_size : 8; + + uint32_t rsvd1; + + uint32_t data_lo; + uint32_t data_hi; + + struct safexcel_res_data result_data; +} __packed; + +/* Result data error codes. */ +#define SAFEXCEL_RESULT_ERR_PACKET_LEN (1u << 0) +#define SAFEXCEL_RESULT_ERR_TOKEN_ERROR (1u << 1) +#define SAFEXCEL_RESULT_ERR_BYPASS (1u << 2) +#define SAFEXCEL_RESULT_ERR_CRYPTO_BLOCK_SIZE (1u << 3) +#define SAFEXCEL_RESULT_ERR_HASH_BLOCK_SIZE (1u << 4) +#define SAFEXCEL_RESULT_ERR_INVALID_CMD (1u << 5) +#define SAFEXCEL_RESULT_ERR_PROHIBITED_ALGO (1u << 6) +#define SAFEXCEL_RESULT_ERR_HASH_INPUT_OVERFLOW (1u << 7) +#define SAFEXCEL_RESULT_ERR_TTL_UNDERFLOW (1u << 8) +#define SAFEXCEL_RESULT_ERR_AUTH_FAILED (1u << 9) +#define SAFEXCEL_RESULT_ERR_SEQNO_CHECK_FAILED (1u << 10) +#define SAFEXCEL_RESULT_ERR_SPI_CHECK (1u << 11) +#define SAFEXCEL_RESULT_ERR_CHECKSUM (1u << 12) +#define SAFEXCEL_RESULT_ERR_PAD_VERIFICATION (1u << 13) +#define SAFEXCEL_RESULT_ERR_TIMEOUT (1u << 14) +#define SAFEXCEL_RESULT_ERR_OUTPUT_DMA (1u << 15) + +/* + * The EIP-96 (crypto transform engine) is programmed using a set of + * data processing instructions with the encodings defined below. + */ +struct safexcel_instr { + uint32_t length : 17; /* bytes to be processed */ + uint32_t status : 2; /* stream status */ + uint32_t instructions : 9; + uint32_t opcode : 4; +} __packed; + +/* Type 1, operational data instructions. */ +#define SAFEXCEL_INSTR_OPCODE_DIRECTION 0x0 +#define SAFEXCEL_INSTR_OPCODE_PRE_CHECKSUM 0x1 +#define SAFEXCEL_INSTR_OPCODE_INSERT 0x2 +#define SAFEXCEL_INSTR_OPCODE_INSERT_CTX 0x9 +#define SAFEXCEL_INSTR_OPCODE_REPLACE 0x3 +#define SAFEXCEL_INSTR_OPCODE_RETRIEVE 0x4 +#define SAFEXCEL_INSTR_OPCODE_MUTE 0x5 +/* Type 2, IP header instructions. */ +#define SAFEXCEL_INSTR_OPCODE_IPV4 0x7 +#define SAFEXCEL_INSTR_OPCODE_IPV4_CHECKSUM 0x6 +#define SAFEXCEL_INSTR_OPCODE_IPV6 0x8 +/* Type 3, postprocessing instructions. */ +#define SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT 0xa +#define SAFEXCEL_INSTR_OPCODE_REPLACE_BYTE 0xb +/* Type 4, result instructions. */ +#define SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS 0xd +/* Type 5, context control instructions. */ +#define SAFEXCEL_INSTR_OPCODE_CONTEXT_ACCESS 0xe +/* Type 6, context control instructions. */ +#define SAFEXCEL_INSTR_OPCODE_BYPASS_TOKEN_DATA 0xf + +/* Status bits for type 1 and 2 instructions. */ +#define SAFEXCEL_INSTR_STATUS_LAST_HASH (1u << 0) +#define SAFEXCEL_INSTR_STATUS_LAST_PACKET (1u << 1) +/* Status bits for type 3 instructions. */ +#define SAFEXCEL_INSTR_STATUS_NO_CKSUM_MOD (1u << 0) + +/* Instruction-dependent flags. */ +#define SAFEXCEL_INSTR_INSERT_HASH_DIGEST 0x1c +#define SAFEXCEL_INSTR_INSERT_IMMEDIATE 0x1b +#define SAFEXCEL_INSTR_DEST_OUTPUT (1u << 5) +#define SAFEXCEL_INSTR_DEST_HASH (1u << 6) +#define SAFEXCEL_INSTR_DEST_CRYPTO (1u << 7) +#define SAFEXCEL_INSTR_INS_LAST (1u << 8) + +#define SAFEXCEL_INSTR_VERIFY_HASH (1u << 16) +#define SAFEXCEL_INSTR_VERIFY_PADDING (1u << 5) + +#define SAFEXCEL_TOKEN_TYPE_BYPASS 0x0 +#define SAFEXCEL_TOKEN_TYPE_AUTONOMOUS 0x3 + +#define SAFEXCEL_CONTEXT_SMALL 0x2 +#define SAFEXCEL_CONTEXT_LARGE 0x3 + +struct safexcel_reg_offsets { + uint32_t hia_aic; + uint32_t hia_aic_g; + uint32_t hia_aic_r; + uint32_t hia_aic_xdr; + uint32_t hia_dfe; + uint32_t hia_dfe_thr; + uint32_t hia_dse; + uint32_t hia_dse_thr; + uint32_t hia_gen_cfg; + uint32_t pe; +}; + +struct safexcel_config { + uint32_t hdw; /* Host interface Data Width. */ + uint32_t aic_rings; /* Number of AIC rings. */ + uint32_t pes; /* Number of PEs. */ + uint32_t rings; /* Number of rings. */ + + uint32_t cd_size; /* CDR descriptor size. */ + uint32_t cd_offset; /* CDR offset (size + alignment). */ + + uint32_t rd_size; /* RDR descriptor size. */ + uint32_t rd_offset; /* RDR offset. */ + + uint32_t atok_offset; /* Additional token offset. */ + + uint32_t caps; /* Device capabilities. */ +}; + +#define SAFEXCEL_DPRINTF(sc, lvl, ...) do { \ + if ((sc)->sc_debug >= (lvl)) \ + device_printf((sc)->sc_dev, __VA_ARGS__); \ +} while (0) + +struct safexcel_dma_mem { + caddr_t vaddr; + bus_addr_t paddr; + bus_dma_tag_t tag; + bus_dmamap_t map; +}; + +struct safexcel_cmd_descr_ring { + struct safexcel_dma_mem dma; + struct safexcel_cmd_descr *desc; + int write; + int read; +}; + +struct safexcel_res_descr_ring { + struct safexcel_dma_mem dma; + struct safexcel_res_descr *desc; + int write; + int read; +}; + +struct safexcel_session { + int ringidx; + uint32_t alg; /* cipher algorithm */ + uint32_t digest; /* digest type */ + uint32_t hash; /* hash algorithm */ + uint32_t mode; /* cipher mode of operation */ + unsigned int digestlen; /* digest length */ + unsigned int statelen; /* HMAC hash state length */ + unsigned int klen; /* cipher key length */ + union { + uint32_t ghash_key[AES_BLOCK_LEN / sizeof(uint32_t)]; + uint32_t xcbc_key[(AES_BLOCK_LEN * 2 + AES_MAX_KEY) / + sizeof(uint32_t)]; + uint8_t tweak_key[AES_MAX_KEY]; + }; + struct { + uint8_t hmac_ipad[HMAC_MAX_BLOCK_LEN]; + uint8_t hmac_opad[HMAC_MAX_BLOCK_LEN]; + }; +}; + +struct safexcel_softc; + +struct safexcel_request { + STAILQ_ENTRY(safexcel_request) link; + bool dmap_loaded; + bus_dmamap_t dmap; + int error; + int cdescs, rdescs; + uint8_t iv[SAFEXCEL_MAX_IV_LEN]; + struct safexcel_cmd_descr *cdesc; + struct safexcel_dma_mem ctx; + struct safexcel_session *sess; + struct cryptop *crp; + struct safexcel_softc *sc; +}; + +struct safexcel_ring { + struct mtx mtx; + struct sglist *cmd_data; + struct safexcel_cmd_descr_ring cdr; + struct sglist *res_data; + struct safexcel_res_descr_ring rdr; + + struct safexcel_request *requests; + STAILQ_HEAD(, safexcel_request) ready_requests; + STAILQ_HEAD(, safexcel_request) queued_requests; + STAILQ_HEAD(, safexcel_request) free_requests; + + struct safexcel_dma_mem dma_atok; + bus_dma_tag_t data_dtag; +}; + +struct safexcel_intr_handle { + struct safexcel_softc *sc; + void *handle; + int ring; +}; + +struct safexcel_softc { + device_t sc_dev; + uint32_t sc_type; /* EIP-97 or 197 */ + int sc_debug; + + struct resource *sc_res; + struct resource *sc_intr[SAFEXCEL_MAX_RINGS]; + struct safexcel_intr_handle sc_ih[SAFEXCEL_MAX_RINGS]; + + struct safexcel_ring sc_ring[SAFEXCEL_MAX_RINGS]; + int sc_ringidx; + struct mtx sc_mtx; + + int sc_blocked; + int32_t sc_cid; + struct safexcel_reg_offsets sc_offsets; + struct safexcel_config sc_config; +}; + +#define SAFEXCEL_WRITE(sc, off, val) bus_write_4((sc)->sc_res, (off), (val)) +#define SAFEXCEL_READ(sc, off) bus_read_4((sc)->sc_res, (off)) + +#define SAFEXCEL_ADDR_LO(addr) ((uint64_t)(addr) & 0xffffffffu) +#define SAFEXCEL_ADDR_HI(addr) (((uint64_t)(addr) >> 32) & 0xffffffffu) + +#endif /* _SAFEXCEL_VAR_H_ */ diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 4244819427ec..230d875722e6 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -317,6 +317,7 @@ SUBDIR= \ ${_rtwnfw} \ ${_s3} \ ${_safe} \ + safexcel \ ${_sbni} \ scc \ ${_sctp} \ diff --git a/sys/modules/safexcel/Makefile b/sys/modules/safexcel/Makefile new file mode 100644 index 000000000000..985a3f40c282 --- /dev/null +++ b/sys/modules/safexcel/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ +.PATH: ${SRCTOP}/sys/dev/safexcel + +KMOD= safexcel +SRCS= safexcel.c + +SRCS+= bus_if.h cryptodev_if.h device_if.h ofw_bus_if.h + +.include