b356ddf076
The EIP-97 is a packet processing module found on the ESPRESSObin. This commit adds a crypto(9) driver for the crypto and hash engine in this device. An initial skeleton driver that could attach and submit requests was written by loos and others at Netgate, and the driver was finished by me. Support for separate AAD and output buffers will be added in a separate commit, to simplify merging to stable/12 (where those features don't exist). Reviewed by: gnn, jhb Feedback from: andrew, cem, manu MFC after: 1 week Sponsored by: Rubicon Communications, LLC (Netgate) Differential Revision: https://reviews.freebsd.org/D25417
421 lines
13 KiB
C
421 lines
13 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
*
|
|
* Copyright (c) 2020 Rubicon Communications, LLC (Netgate)
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* $FreeBSD$
|
|
*/
|
|
|
|
#ifndef _SAFEXCEL_VAR_H_
|
|
#define _SAFEXCEL_VAR_H_
|
|
|
|
#define SAFEXCEL_MAX_RINGS 4
|
|
#define SAFEXCEL_MAX_BATCH_SIZE 64
|
|
#define SAFEXCEL_MAX_FRAGMENTS 64
|
|
#define SAFEXCEL_MAX_IV_LEN 16
|
|
#define SAFEXCEL_MAX_REQUEST_SIZE 65535
|
|
|
|
#define SAFEXCEL_RING_SIZE 512
|
|
#define SAFEXCEL_REQUESTS_PER_RING 64
|
|
#define SAFEXCEL_MAX_ITOKENS 4
|
|
#define SAFEXCEL_MAX_ATOKENS 16
|
|
#define SAFEXCEL_FETCH_COUNT 1
|
|
#define SAFEXCEL_MAX_KEY_LEN 32
|
|
#define SAFEXCEL_MAX_RING_AIC 14
|
|
|
|
/*
|
|
* Context Record format.
|
|
*
|
|
* In this driver the context control words are always set in the control data.
|
|
* This is configured by setting SAFEXCEL_OPTION_CTX_CTRL_IN_CMD.
|
|
*/
|
|
struct safexcel_context_record {
|
|
uint32_t control0; /* Unused. */
|
|
uint32_t control1; /* Unused. */
|
|
uint32_t data[40]; /* Key material. */
|
|
} __packed;
|
|
|
|
/* Processing Engine Control Data format. */
|
|
struct safexcel_control_data {
|
|
uint32_t packet_length : 17;
|
|
uint32_t options : 13;
|
|
uint32_t type : 2;
|
|
|
|
uint16_t application_id;
|
|
uint16_t rsvd;
|
|
|
|
uint32_t context_lo;
|
|
uint32_t context_hi;
|
|
|
|
uint32_t control0;
|
|
uint32_t control1;
|
|
|
|
/* Inline instructions or IV. */
|
|
uint32_t token[SAFEXCEL_MAX_ITOKENS];
|
|
} __packed;
|
|
|
|
/*
|
|
* Basic Command Descriptor.
|
|
*
|
|
* The Processing Engine and driver cooperate to maintain a set of command
|
|
* rings, representing outstanding crypto operation requests. Each descriptor
|
|
* corresponds to an input data segment, and thus a single crypto(9) request may
|
|
* span several contiguous command descriptors.
|
|
*
|
|
* The first command descriptor for a request stores the input token, which
|
|
* encodes data specific to the requested operation, such as the encryption
|
|
* mode. For some operations data is passed outside the descriptor, in a
|
|
* context record (e.g., encryption keys), or in an "additional token data"
|
|
* region (e.g., instructions).
|
|
*/
|
|
struct safexcel_cmd_descr {
|
|
uint32_t particle_size : 17;
|
|
uint32_t rsvd0 : 5;
|
|
uint32_t last_seg : 1;
|
|
uint32_t first_seg : 1;
|
|
uint32_t additional_cdata_size : 8;
|
|
uint32_t rsvd1;
|
|
|
|
uint32_t data_lo;
|
|
uint32_t data_hi;
|
|
|
|
uint32_t atok_lo;
|
|
uint32_t atok_hi;
|
|
|
|
struct safexcel_control_data control_data;
|
|
} __packed;
|
|
|
|
/* Context control word 0 fields. */
|
|
#define SAFEXCEL_CONTROL0_TYPE_NULL_OUT 0x0
|
|
#define SAFEXCEL_CONTROL0_TYPE_NULL_IN 0x1
|
|
#define SAFEXCEL_CONTROL0_TYPE_HASH_OUT 0x2
|
|
#define SAFEXCEL_CONTROL0_TYPE_HASH_IN 0x3
|
|
#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT 0x4
|
|
#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN 0x5
|
|
#define SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT 0x6
|
|
#define SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN 0x7
|
|
#define SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT 0xe
|
|
#define SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN 0xf
|
|
#define SAFEXCEL_CONTROL0_RESTART_HASH (1 << 4)
|
|
#define SAFEXCEL_CONTROL0_NO_FINISH_HASH (1 << 5)
|
|
#define SAFEXCEL_CONTROL0_SIZE(n) (((n) & 0xff) << 8)
|
|
#define SAFEXCEL_CONTROL0_KEY_EN (1 << 16)
|
|
#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128 (0x5 << 17)
|
|
#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192 (0x6 << 17)
|
|
#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256 (0x7 << 17)
|
|
#define SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED (0x1 << 21)
|
|
#define SAFEXCEL_CONTROL0_DIGEST_CCM (0x2 << 21)
|
|
#define SAFEXCEL_CONTROL0_DIGEST_GMAC (0x2 << 21)
|
|
#define SAFEXCEL_CONTROL0_DIGEST_HMAC (0x3 << 21)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_SHA1 (0x2 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_SHA224 (0x4 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_SHA256 (0x3 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_SHA384 (0x6 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_SHA512 (0x5 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC128 (0x1 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC192 (0x2 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC256 (0x3 << 23)
|
|
#define SAFEXCEL_CONTROL0_HASH_ALG_GHASH (0x4 << 23)
|
|
#define SAFEXCEL_CONTROL0_INV_FR (0x5 << 24)
|
|
#define SAFEXCEL_CONTROL0_INV_TR (0x6 << 24)
|
|
|
|
/* Context control word 1 fields. */
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ECB 0x0
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC 0x1
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ICM 0x3
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_OFB 0x4
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CFB128 0x5
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR 0x6
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS 0x7
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM (0x6 | (1 << 17))
|
|
#define SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM (0x6 | (1 << 17))
|
|
#define SAFEXCEL_CONTROL1_IV0 (1u << 5)
|
|
#define SAFEXCEL_CONTROL1_IV1 (1u << 6)
|
|
#define SAFEXCEL_CONTROL1_IV2 (1u << 7)
|
|
#define SAFEXCEL_CONTROL1_IV3 (1u << 8)
|
|
#define SAFEXCEL_CONTROL1_DIGEST_CNT (1u << 9)
|
|
#define SAFEXCEL_CONTROL1_COUNTER_MODE (1u << 10)
|
|
#define SAFEXCEL_CONTROL1_ENCRYPT_HASH_RES (1u << 17)
|
|
#define SAFEXCEL_CONTROL1_HASH_STORE (1u << 19)
|
|
|
|
/* Control options. */
|
|
#define SAFEXCEL_OPTION_IP (1u << 0) /* must be set */
|
|
#define SAFEXCEL_OPTION_CP (1u << 1) /* 64-bit ctx addr */
|
|
#define SAFEXCEL_OPTION_RC_AUTO (2u << 3) /* auto ctx reuse */
|
|
#define SAFEXCEL_OPTION_CTX_CTRL_IN_CMD (1u << 8) /* ctx ctrl */
|
|
#define SAFEXCEL_OPTION_4_TOKEN_IV_CMD 0xe00 /* IV in bypass */
|
|
|
|
struct safexcel_res_data {
|
|
uint32_t packet_length : 17;
|
|
uint32_t error_code : 15;
|
|
|
|
uint32_t bypass_length : 4;
|
|
uint32_t e15 : 1;
|
|
uint32_t rsvd0 : 16;
|
|
uint32_t hash_bytes : 1;
|
|
uint32_t hash_length : 6;
|
|
uint32_t generic_bytes : 1;
|
|
uint32_t checksum : 1;
|
|
uint32_t next_header : 1;
|
|
uint32_t length : 1;
|
|
|
|
uint16_t application_id;
|
|
uint16_t rsvd1;
|
|
|
|
uint32_t rsvd2;
|
|
};
|
|
|
|
/* Basic Result Descriptor format */
|
|
struct safexcel_res_descr {
|
|
uint32_t particle_size : 17;
|
|
uint32_t rsvd0 : 3;
|
|
uint32_t descriptor_overflow : 1;
|
|
uint32_t buffer_overflow : 1;
|
|
uint32_t last_seg : 1;
|
|
uint32_t first_seg : 1;
|
|
uint32_t result_size : 8;
|
|
|
|
uint32_t rsvd1;
|
|
|
|
uint32_t data_lo;
|
|
uint32_t data_hi;
|
|
|
|
struct safexcel_res_data result_data;
|
|
} __packed;
|
|
|
|
/* Result data error codes. */
|
|
#define SAFEXCEL_RESULT_ERR_PACKET_LEN (1u << 0)
|
|
#define SAFEXCEL_RESULT_ERR_TOKEN_ERROR (1u << 1)
|
|
#define SAFEXCEL_RESULT_ERR_BYPASS (1u << 2)
|
|
#define SAFEXCEL_RESULT_ERR_CRYPTO_BLOCK_SIZE (1u << 3)
|
|
#define SAFEXCEL_RESULT_ERR_HASH_BLOCK_SIZE (1u << 4)
|
|
#define SAFEXCEL_RESULT_ERR_INVALID_CMD (1u << 5)
|
|
#define SAFEXCEL_RESULT_ERR_PROHIBITED_ALGO (1u << 6)
|
|
#define SAFEXCEL_RESULT_ERR_HASH_INPUT_OVERFLOW (1u << 7)
|
|
#define SAFEXCEL_RESULT_ERR_TTL_UNDERFLOW (1u << 8)
|
|
#define SAFEXCEL_RESULT_ERR_AUTH_FAILED (1u << 9)
|
|
#define SAFEXCEL_RESULT_ERR_SEQNO_CHECK_FAILED (1u << 10)
|
|
#define SAFEXCEL_RESULT_ERR_SPI_CHECK (1u << 11)
|
|
#define SAFEXCEL_RESULT_ERR_CHECKSUM (1u << 12)
|
|
#define SAFEXCEL_RESULT_ERR_PAD_VERIFICATION (1u << 13)
|
|
#define SAFEXCEL_RESULT_ERR_TIMEOUT (1u << 14)
|
|
#define SAFEXCEL_RESULT_ERR_OUTPUT_DMA (1u << 15)
|
|
|
|
/*
|
|
* The EIP-96 (crypto transform engine) is programmed using a set of
|
|
* data processing instructions with the encodings defined below.
|
|
*/
|
|
struct safexcel_instr {
|
|
uint32_t length : 17; /* bytes to be processed */
|
|
uint32_t status : 2; /* stream status */
|
|
uint32_t instructions : 9;
|
|
uint32_t opcode : 4;
|
|
} __packed;
|
|
|
|
/* Type 1, operational data instructions. */
|
|
#define SAFEXCEL_INSTR_OPCODE_DIRECTION 0x0
|
|
#define SAFEXCEL_INSTR_OPCODE_PRE_CHECKSUM 0x1
|
|
#define SAFEXCEL_INSTR_OPCODE_INSERT 0x2
|
|
#define SAFEXCEL_INSTR_OPCODE_INSERT_CTX 0x9
|
|
#define SAFEXCEL_INSTR_OPCODE_REPLACE 0x3
|
|
#define SAFEXCEL_INSTR_OPCODE_RETRIEVE 0x4
|
|
#define SAFEXCEL_INSTR_OPCODE_MUTE 0x5
|
|
/* Type 2, IP header instructions. */
|
|
#define SAFEXCEL_INSTR_OPCODE_IPV4 0x7
|
|
#define SAFEXCEL_INSTR_OPCODE_IPV4_CHECKSUM 0x6
|
|
#define SAFEXCEL_INSTR_OPCODE_IPV6 0x8
|
|
/* Type 3, postprocessing instructions. */
|
|
#define SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT 0xa
|
|
#define SAFEXCEL_INSTR_OPCODE_REPLACE_BYTE 0xb
|
|
/* Type 4, result instructions. */
|
|
#define SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS 0xd
|
|
/* Type 5, context control instructions. */
|
|
#define SAFEXCEL_INSTR_OPCODE_CONTEXT_ACCESS 0xe
|
|
/* Type 6, context control instructions. */
|
|
#define SAFEXCEL_INSTR_OPCODE_BYPASS_TOKEN_DATA 0xf
|
|
|
|
/* Status bits for type 1 and 2 instructions. */
|
|
#define SAFEXCEL_INSTR_STATUS_LAST_HASH (1u << 0)
|
|
#define SAFEXCEL_INSTR_STATUS_LAST_PACKET (1u << 1)
|
|
/* Status bits for type 3 instructions. */
|
|
#define SAFEXCEL_INSTR_STATUS_NO_CKSUM_MOD (1u << 0)
|
|
|
|
/* Instruction-dependent flags. */
|
|
#define SAFEXCEL_INSTR_INSERT_HASH_DIGEST 0x1c
|
|
#define SAFEXCEL_INSTR_INSERT_IMMEDIATE 0x1b
|
|
#define SAFEXCEL_INSTR_DEST_OUTPUT (1u << 5)
|
|
#define SAFEXCEL_INSTR_DEST_HASH (1u << 6)
|
|
#define SAFEXCEL_INSTR_DEST_CRYPTO (1u << 7)
|
|
#define SAFEXCEL_INSTR_INS_LAST (1u << 8)
|
|
|
|
#define SAFEXCEL_INSTR_VERIFY_HASH (1u << 16)
|
|
#define SAFEXCEL_INSTR_VERIFY_PADDING (1u << 5)
|
|
|
|
#define SAFEXCEL_TOKEN_TYPE_BYPASS 0x0
|
|
#define SAFEXCEL_TOKEN_TYPE_AUTONOMOUS 0x3
|
|
|
|
#define SAFEXCEL_CONTEXT_SMALL 0x2
|
|
#define SAFEXCEL_CONTEXT_LARGE 0x3
|
|
|
|
struct safexcel_reg_offsets {
|
|
uint32_t hia_aic;
|
|
uint32_t hia_aic_g;
|
|
uint32_t hia_aic_r;
|
|
uint32_t hia_aic_xdr;
|
|
uint32_t hia_dfe;
|
|
uint32_t hia_dfe_thr;
|
|
uint32_t hia_dse;
|
|
uint32_t hia_dse_thr;
|
|
uint32_t hia_gen_cfg;
|
|
uint32_t pe;
|
|
};
|
|
|
|
struct safexcel_config {
|
|
uint32_t hdw; /* Host interface Data Width. */
|
|
uint32_t aic_rings; /* Number of AIC rings. */
|
|
uint32_t pes; /* Number of PEs. */
|
|
uint32_t rings; /* Number of rings. */
|
|
|
|
uint32_t cd_size; /* CDR descriptor size. */
|
|
uint32_t cd_offset; /* CDR offset (size + alignment). */
|
|
|
|
uint32_t rd_size; /* RDR descriptor size. */
|
|
uint32_t rd_offset; /* RDR offset. */
|
|
|
|
uint32_t atok_offset; /* Additional token offset. */
|
|
|
|
uint32_t caps; /* Device capabilities. */
|
|
};
|
|
|
|
#define SAFEXCEL_DPRINTF(sc, lvl, ...) do { \
|
|
if ((sc)->sc_debug >= (lvl)) \
|
|
device_printf((sc)->sc_dev, __VA_ARGS__); \
|
|
} while (0)
|
|
|
|
struct safexcel_dma_mem {
|
|
caddr_t vaddr;
|
|
bus_addr_t paddr;
|
|
bus_dma_tag_t tag;
|
|
bus_dmamap_t map;
|
|
};
|
|
|
|
struct safexcel_cmd_descr_ring {
|
|
struct safexcel_dma_mem dma;
|
|
struct safexcel_cmd_descr *desc;
|
|
int write;
|
|
int read;
|
|
};
|
|
|
|
struct safexcel_res_descr_ring {
|
|
struct safexcel_dma_mem dma;
|
|
struct safexcel_res_descr *desc;
|
|
int write;
|
|
int read;
|
|
};
|
|
|
|
struct safexcel_session {
|
|
int ringidx;
|
|
uint32_t alg; /* cipher algorithm */
|
|
uint32_t digest; /* digest type */
|
|
uint32_t hash; /* hash algorithm */
|
|
uint32_t mode; /* cipher mode of operation */
|
|
unsigned int digestlen; /* digest length */
|
|
unsigned int statelen; /* HMAC hash state length */
|
|
unsigned int klen; /* cipher key length */
|
|
union {
|
|
uint32_t ghash_key[AES_BLOCK_LEN / sizeof(uint32_t)];
|
|
uint32_t xcbc_key[(AES_BLOCK_LEN * 2 + AES_MAX_KEY) /
|
|
sizeof(uint32_t)];
|
|
uint8_t tweak_key[AES_MAX_KEY];
|
|
};
|
|
struct {
|
|
uint8_t hmac_ipad[HMAC_MAX_BLOCK_LEN];
|
|
uint8_t hmac_opad[HMAC_MAX_BLOCK_LEN];
|
|
};
|
|
};
|
|
|
|
struct safexcel_softc;
|
|
|
|
struct safexcel_request {
|
|
STAILQ_ENTRY(safexcel_request) link;
|
|
bool dmap_loaded;
|
|
bus_dmamap_t dmap;
|
|
int error;
|
|
int cdescs, rdescs;
|
|
uint8_t iv[SAFEXCEL_MAX_IV_LEN];
|
|
struct safexcel_cmd_descr *cdesc;
|
|
struct safexcel_dma_mem ctx;
|
|
struct safexcel_session *sess;
|
|
struct cryptop *crp;
|
|
struct safexcel_softc *sc;
|
|
};
|
|
|
|
struct safexcel_ring {
|
|
struct mtx mtx;
|
|
struct sglist *cmd_data;
|
|
struct safexcel_cmd_descr_ring cdr;
|
|
struct sglist *res_data;
|
|
struct safexcel_res_descr_ring rdr;
|
|
|
|
struct safexcel_request *requests;
|
|
STAILQ_HEAD(, safexcel_request) ready_requests;
|
|
STAILQ_HEAD(, safexcel_request) queued_requests;
|
|
STAILQ_HEAD(, safexcel_request) free_requests;
|
|
|
|
struct safexcel_dma_mem dma_atok;
|
|
bus_dma_tag_t data_dtag;
|
|
};
|
|
|
|
struct safexcel_intr_handle {
|
|
struct safexcel_softc *sc;
|
|
void *handle;
|
|
int ring;
|
|
};
|
|
|
|
struct safexcel_softc {
|
|
device_t sc_dev;
|
|
uint32_t sc_type; /* EIP-97 or 197 */
|
|
int sc_debug;
|
|
|
|
struct resource *sc_res;
|
|
struct resource *sc_intr[SAFEXCEL_MAX_RINGS];
|
|
struct safexcel_intr_handle sc_ih[SAFEXCEL_MAX_RINGS];
|
|
|
|
struct safexcel_ring sc_ring[SAFEXCEL_MAX_RINGS];
|
|
int sc_ringidx;
|
|
struct mtx sc_mtx;
|
|
|
|
int sc_blocked;
|
|
int32_t sc_cid;
|
|
struct safexcel_reg_offsets sc_offsets;
|
|
struct safexcel_config sc_config;
|
|
};
|
|
|
|
#define SAFEXCEL_WRITE(sc, off, val) bus_write_4((sc)->sc_res, (off), (val))
|
|
#define SAFEXCEL_READ(sc, off) bus_read_4((sc)->sc_res, (off))
|
|
|
|
#define SAFEXCEL_ADDR_LO(addr) ((uint64_t)(addr) & 0xffffffffu)
|
|
#define SAFEXCEL_ADDR_HI(addr) (((uint64_t)(addr) >> 32) & 0xffffffffu)
|
|
|
|
#endif /* _SAFEXCEL_VAR_H_ */
|