d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/cnxk: support stream cipher chained operations

Browse Source 
Adding support for zuc, snow3g and aes-ctr-cmac
chained operations on cn9k using key and IV scheme
in microcode.

Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>
This commit is contained in:
Tejasree Kondoj 2022-06-20 17:56:53 +05:30 committed by Akhil Goyal
parent 57fe58d0a5
commit 0fbb3e6cf2
4 changed files with 535 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

271
drivers/common/cnxk/roc_se.c
View File

@ -88,21 +88,24 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
fc_type = ROC_SE_FC_GEN;
break;
case ROC_SE_ZUC_EEA3:
/* No support for chained operations */
if (unlikely(ctx->hash_type))
return -1;
fc_type = ROC_SE_PDCP;
if (ctx->hash_type)
fc_type = ROC_SE_PDCP_CHAIN;
else
fc_type = ROC_SE_PDCP;
break;
case ROC_SE_SNOW3G_UEA2:
if (unlikely(key_len != 16))
return -1;
/* No support for AEAD yet */
if (unlikely(ctx->hash_type))
return -1;
fc_type = ROC_SE_PDCP;
if (ctx->hash_type)
fc_type = ROC_SE_PDCP_CHAIN;
else
fc_type = ROC_SE_PDCP;
break;
case ROC_SE_AES_CTR_EEA2:
fc_type = ROC_SE_PDCP;
if (ctx->hash_type)
fc_type = ROC_SE_PDCP_CHAIN;
else
fc_type = ROC_SE_PDCP;
break;
case ROC_SE_KASUMI_F8_CBC:
case ROC_SE_KASUMI_F8_ECB:
@ -171,6 +174,29 @@ cpt_pdcp_key_type_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t key_len)
return 0;
}
static int
cpt_pdcp_chain_key_type_get(uint16_t key_len)
{
roc_se_aes_type key_type;
switch (key_len) {
case 16:
key_type = ROC_SE_AES_128_BIT;
break;
case 24:
key_type = ROC_SE_AES_192_BIT;
break;
case 32:
key_type = ROC_SE_AES_256_BIT;
break;
default:
plt_err("Invalid key len");
return -ENOTSUP;
}
return key_type;
}
static int
cpt_pdcp_mac_len_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t mac_len)
{
@ -202,7 +228,7 @@ cpt_pdcp_mac_len_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t mac_len)
}
static void
cpt_pdcp_update_zuc_const(uint8_t *zuc_const, int key_len, int mac_len)
cpt_zuc_const_update(uint8_t *zuc_const, int key_len, int mac_len)
{
if (key_len == 16) {
memcpy(zuc_const, zuc_key128, 32);
@ -227,15 +253,19 @@ int
roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
const uint8_t *key, uint16_t key_len, uint16_t mac_len)
{
struct roc_se_zuc_snow3g_chain_ctx *zs_ch_ctx;
struct roc_se_zuc_snow3g_ctx *zs_ctx;
struct roc_se_kasumi_ctx *k_ctx;
struct roc_se_context *fctx;
uint8_t opcode_minor;
uint8_t pdcp_alg;
int ret;
if (se_ctx == NULL)
return -1;
zs_ctx = &se_ctx->se_ctx.zs_ctx;
zs_ch_ctx = &se_ctx->se_ctx.zs_ch_ctx;
k_ctx = &se_ctx->se_ctx.k_ctx;
fctx = &se_ctx->se_ctx.fctx;
@ -243,14 +273,12 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
uint8_t *zuc_const;
uint32_t keyx[4];
uint8_t *ci_key;
bool chained_op =
se_ctx->ciph_then_auth || se_ctx->auth_then_ciph;
if (!key_len)
return -1;
/* No support for chained operations yet */
if (se_ctx->enc_cipher)
return -1;
if (roc_model_is_cn9k()) {
ci_key = zs_ctx->zuc.onk_ctx.ci_key;
zuc_const = zs_ctx->zuc.onk_ctx.zuc_const;
@ -262,41 +290,88 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
/* For ZUC/SNOW3G/Kasumi */
switch (type) {
case ROC_SE_SNOW3G_UIA2:
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_SNOW3G;
zs_ctx->zuc.otk_ctx.w0.s.mac_len =
ROC_SE_PDCP_MAC_LEN_32_BIT;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
cpt_snow3g_key_gen(key, keyx);
memcpy(ci_key, keyx, key_len);
se_ctx->fc_type = ROC_SE_PDCP;
if (chained_op) {
struct roc_se_onk_zuc_chain_ctx *ctx =
&zs_ch_ctx->zuc.onk_ctx;
zs_ch_ctx->zuc.onk_ctx.w0.s.state_conf =
ROC_SE_PDCP_CHAIN_CTX_KEY_IV;
ctx->w0.s.auth_type =
ROC_SE_PDCP_CHAIN_ALG_TYPE_SNOW3G;
ctx->w0.s.mac_len = mac_len;
ctx->w0.s.auth_key_len = key_len;
se_ctx->fc_type = ROC_SE_PDCP_CHAIN;
cpt_snow3g_key_gen(key, keyx);
memcpy(ctx->st.auth_key, keyx, key_len);
} else {
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_SNOW3G;
zs_ctx->zuc.otk_ctx.w0.s.mac_len =
ROC_SE_PDCP_MAC_LEN_32_BIT;
cpt_snow3g_key_gen(key, keyx);
memcpy(ci_key, keyx, key_len);
se_ctx->fc_type = ROC_SE_PDCP;
}
se_ctx->pdcp_auth_alg = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
se_ctx->zsk_flags = 0x1;
break;
case ROC_SE_ZUC_EIA3:
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_ZUC;
ret = cpt_pdcp_key_type_set(zs_ctx, key_len);
if (ret)
return ret;
ret = cpt_pdcp_mac_len_set(zs_ctx, mac_len);
if (ret)
return ret;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
memcpy(ci_key, key, key_len);
if (key_len == 32)
roc_se_zuc_bytes_swap(ci_key, key_len);
cpt_pdcp_update_zuc_const(zuc_const, key_len, mac_len);
se_ctx->fc_type = ROC_SE_PDCP;
if (chained_op) {
struct roc_se_onk_zuc_chain_ctx *ctx =
&zs_ch_ctx->zuc.onk_ctx;
ctx->w0.s.state_conf =
ROC_SE_PDCP_CHAIN_CTX_KEY_IV;
ctx->w0.s.auth_type =
ROC_SE_PDCP_CHAIN_ALG_TYPE_ZUC;
ctx->w0.s.mac_len = mac_len;
ctx->w0.s.auth_key_len = key_len;
memcpy(ctx->st.auth_key, key, key_len);
cpt_zuc_const_update(ctx->st.auth_zuc_const,
key_len, mac_len);
se_ctx->fc_type = ROC_SE_PDCP_CHAIN;
} else {
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_ZUC;
ret = cpt_pdcp_key_type_set(zs_ctx, key_len);
if (ret)
return ret;
ret = cpt_pdcp_mac_len_set(zs_ctx, mac_len);
if (ret)
return ret;
memcpy(ci_key, key, key_len);
if (key_len == 32)
roc_se_zuc_bytes_swap(ci_key, key_len);
cpt_zuc_const_update(zuc_const, key_len,
mac_len);
se_ctx->fc_type = ROC_SE_PDCP;
}
se_ctx->pdcp_auth_alg = ROC_SE_PDCP_ALG_TYPE_ZUC;
se_ctx->zsk_flags = 0x1;
break;
case ROC_SE_AES_CMAC_EIA2:
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_AES_CTR;
zs_ctx->zuc.otk_ctx.w0.s.mac_len =
ROC_SE_PDCP_MAC_LEN_32_BIT;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
memcpy(ci_key, key, key_len);
se_ctx->fc_type = ROC_SE_PDCP;
if (chained_op) {
struct roc_se_onk_zuc_chain_ctx *ctx =
&zs_ch_ctx->zuc.onk_ctx;
int key_type;
key_type = cpt_pdcp_chain_key_type_get(key_len);
if (key_type < 0)
return key_type;
ctx->w0.s.auth_key_len = key_type;
ctx->w0.s.state_conf =
ROC_SE_PDCP_CHAIN_CTX_KEY_IV;
ctx->w0.s.auth_type =
ROC_SE_PDCP_ALG_TYPE_AES_CTR;
ctx->w0.s.mac_len = mac_len;
memcpy(ctx->st.auth_key, key, key_len);
se_ctx->fc_type = ROC_SE_PDCP_CHAIN;
} else {
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_AES_CTR;
zs_ctx->zuc.otk_ctx.w0.s.mac_len =
ROC_SE_PDCP_MAC_LEN_32_BIT;
memcpy(ci_key, key, key_len);
se_ctx->fc_type = ROC_SE_PDCP;
}
se_ctx->pdcp_auth_alg = ROC_SE_PDCP_ALG_TYPE_AES_CMAC;
se_ctx->zsk_flags = 0x1;
break;
case ROC_SE_KASUMI_F9_ECB:
@ -316,11 +391,16 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
}
se_ctx->mac_len = mac_len;
se_ctx->hash_type = type;
pdcp_alg = zs_ctx->zuc.otk_ctx.w0.s.alg_type;
if (roc_model_is_cn9k())
se_ctx->template_w4.s.opcode_minor =
((1 << 7) | (se_ctx->pdcp_alg_type << 5) | 1);
if (chained_op == true)
opcode_minor = se_ctx->ciph_then_auth ? 2 : 3;
else
opcode_minor = ((1 << 7) | (pdcp_alg << 5) | 1);
else
se_ctx->template_w4.s.opcode_minor = ((1 << 4) | 1);
opcode_minor = ((1 << 4) | 1);
se_ctx->template_w4.s.opcode_minor = opcode_minor;
return 0;
}
@ -363,13 +443,18 @@ int
roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
const uint8_t *key, uint16_t key_len, uint8_t *salt)
{
bool chained_op = se_ctx->ciph_then_auth || se_ctx->auth_then_ciph;
struct roc_se_zuc_snow3g_ctx *zs_ctx = &se_ctx->se_ctx.zs_ctx;
struct roc_se_context *fctx = &se_ctx->se_ctx.fctx;
struct roc_se_zuc_snow3g_chain_ctx *zs_ch_ctx;
uint8_t opcode_minor;
uint8_t *zuc_const;
uint32_t keyx[4];
uint8_t *ci_key;
int ret;
zs_ch_ctx = &se_ctx->se_ctx.zs_ch_ctx;
if (roc_model_is_cn9k()) {
ci_key = zs_ctx->zuc.onk_ctx.ci_key;
zuc_const = zs_ctx->zuc.onk_ctx.zuc_const;
@ -447,34 +532,73 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
memcpy(fctx->hmac.ipad, &key[key_len], key_len);
break;
case ROC_SE_SNOW3G_UEA2:
zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
zs_ctx->zuc.otk_ctx.w0.s.alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
cpt_snow3g_key_gen(key, keyx);
memcpy(ci_key, keyx, key_len);
if (chained_op == true) {
struct roc_se_onk_zuc_chain_ctx *ctx =
&zs_ch_ctx->zuc.onk_ctx;
zs_ch_ctx->zuc.onk_ctx.w0.s.state_conf =
ROC_SE_PDCP_CHAIN_CTX_KEY_IV;
zs_ch_ctx->zuc.onk_ctx.w0.s.cipher_type =
ROC_SE_PDCP_CHAIN_ALG_TYPE_SNOW3G;
zs_ch_ctx->zuc.onk_ctx.w0.s.ci_key_len = key_len;
cpt_snow3g_key_gen(key, keyx);
memcpy(ctx->st.ci_key, keyx, key_len);
} else {
zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_SNOW3G;
cpt_snow3g_key_gen(key, keyx);
memcpy(ci_key, keyx, key_len);
}
se_ctx->pdcp_ci_alg = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
se_ctx->zsk_flags = 0;
goto success;
case ROC_SE_ZUC_EEA3:
ret = cpt_pdcp_key_type_set(zs_ctx, key_len);
if (ret)
return ret;
zs_ctx->zuc.otk_ctx.w0.s.alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
memcpy(ci_key, key, key_len);
if (key_len == 32) {
roc_se_zuc_bytes_swap(ci_key, key_len);
memcpy(zuc_const, zuc_key256, 16);
} else
memcpy(zuc_const, zuc_key128, 32);
if (chained_op == true) {
struct roc_se_onk_zuc_chain_ctx *ctx =
&zs_ch_ctx->zuc.onk_ctx;
zs_ch_ctx->zuc.onk_ctx.w0.s.state_conf =
ROC_SE_PDCP_CHAIN_CTX_KEY_IV;
zs_ch_ctx->zuc.onk_ctx.w0.s.cipher_type =
ROC_SE_PDCP_CHAIN_ALG_TYPE_ZUC;
memcpy(ctx->st.ci_key, key, key_len);
memcpy(ctx->st.ci_zuc_const, zuc_key128, 32);
zs_ch_ctx->zuc.onk_ctx.w0.s.ci_key_len = key_len;
} else {
ret = cpt_pdcp_key_type_set(zs_ctx, key_len);
if (ret)
return ret;
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_ZUC;
memcpy(ci_key, key, key_len);
if (key_len == 32) {
roc_se_zuc_bytes_swap(ci_key, key_len);
memcpy(zuc_const, zuc_key256, 16);
} else
memcpy(zuc_const, zuc_key128, 32);
}
se_ctx->pdcp_ci_alg = ROC_SE_PDCP_ALG_TYPE_ZUC;
se_ctx->zsk_flags = 0;
goto success;
case ROC_SE_AES_CTR_EEA2:
zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_AES_CTR;
se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
memcpy(ci_key, key, key_len);
if (chained_op == true) {
struct roc_se_onk_zuc_chain_ctx *ctx =
&zs_ch_ctx->zuc.onk_ctx;
int key_type;
key_type = cpt_pdcp_chain_key_type_get(key_len);
if (key_type < 0)
return key_type;
ctx->w0.s.ci_key_len = key_type;
ctx->w0.s.state_conf = ROC_SE_PDCP_CHAIN_CTX_KEY_IV;
ctx->w0.s.cipher_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
memcpy(ctx->st.ci_key, key, key_len);
} else {
zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
zs_ctx->zuc.otk_ctx.w0.s.alg_type =
ROC_SE_PDCP_ALG_TYPE_AES_CTR;
memcpy(ci_key, key, key_len);
}
se_ctx->pdcp_ci_alg = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
se_ctx->zsk_flags = 0;
goto success;
case ROC_SE_KASUMI_F8_ECB:
@ -502,11 +626,16 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
se_ctx->enc_cipher = type;
if (se_ctx->fc_type == ROC_SE_PDCP) {
if (roc_model_is_cn9k())
se_ctx->template_w4.s.opcode_minor =
((1 << 7) | (se_ctx->pdcp_alg_type << 5) |
(se_ctx->zsk_flags & 0x7));
if (chained_op == true)
opcode_minor = se_ctx->ciph_then_auth ? 2 : 3;
else
opcode_minor =
((1 << 7) | (se_ctx->pdcp_ci_alg << 5) |
(se_ctx->zsk_flags & 0x7));
else
se_ctx->template_w4.s.opcode_minor = ((1 << 4));
opcode_minor = ((1 << 4));
se_ctx->template_w4.s.opcode_minor = opcode_minor;
}
return 0;
}

74
drivers/common/cnxk/roc_se.h
View File

@ -11,10 +11,11 @@
#define ROC_SE_FC_MINOR_OP_DECRYPT 0x1
#define ROC_SE_FC_MINOR_OP_HMAC_FIRST 0x10
#define ROC_SE_MAJOR_OP_HASH 0x34
#define ROC_SE_MAJOR_OP_HMAC 0x35
#define ROC_SE_MAJOR_OP_PDCP 0x37
#define ROC_SE_MAJOR_OP_KASUMI 0x38
#define ROC_SE_MAJOR_OP_HASH 0x34
#define ROC_SE_MAJOR_OP_HMAC 0x35
#define ROC_SE_MAJOR_OP_PDCP 0x37
#define ROC_SE_MAJOR_OP_KASUMI 0x38
#define ROC_SE_MAJOR_OP_PDCP_CHAIN 0x3C
#define ROC_SE_MAJOR_OP_MISC 0x01
#define ROC_SE_MISC_MINOR_OP_PASSTHROUGH 0x03
@ -38,10 +39,11 @@
#define ROC_SE_K_F8 0x4
#define ROC_SE_K_F9 0x8
#define ROC_SE_FC_GEN 0x1
#define ROC_SE_PDCP 0x2
#define ROC_SE_KASUMI 0x3
#define ROC_SE_HASH_HMAC 0x4
#define ROC_SE_FC_GEN 0x1
#define ROC_SE_PDCP 0x2
#define ROC_SE_KASUMI 0x3
#define ROC_SE_HASH_HMAC 0x4
#define ROC_SE_PDCP_CHAIN 0x5
#define ROC_SE_OP_CIPHER_ENCRYPT 0x1
#define ROC_SE_OP_CIPHER_DECRYPT 0x2
@ -224,6 +226,42 @@ struct roc_se_onk_zuc_ctx {
uint8_t zuc_const[32];
};
struct roc_se_onk_zuc_chain_ctx {
union {
uint64_t u64;
struct {
uint64_t cipher_type : 2;
uint64_t rsvd58_59 : 2;
uint64_t auth_type : 2;
uint64_t rsvd62_63 : 2;
uint64_t mac_len : 4;
uint64_t ci_key_len : 2;
uint64_t auth_key_len : 2;
uint64_t rsvd42_47 : 6;
uint64_t state_conf : 2;
uint64_t rsvd0_39 : 40;
} s;
} w0;
union {
struct {
uint8_t encr_lfsr_state[64];
uint8_t auth_lfsr_state[64];
};
struct {
uint8_t ci_key[32];
uint8_t ci_zuc_const[32];
uint8_t auth_key[32];
uint8_t auth_zuc_const[32];
};
} st;
};
struct roc_se_zuc_snow3g_chain_ctx {
union {
struct roc_se_onk_zuc_chain_ctx onk_ctx;
} zuc;
};
struct roc_se_zuc_snow3g_ctx {
union {
struct roc_se_onk_zuc_ctx onk_ctx;
@ -275,9 +313,15 @@ struct roc_se_fc_params {
PLT_STATIC_ASSERT((offsetof(struct roc_se_fc_params, aad_buf) % 128) == 0);
#define ROC_SE_PDCP_ALG_TYPE_ZUC 0
#define ROC_SE_PDCP_ALG_TYPE_SNOW3G 1
#define ROC_SE_PDCP_ALG_TYPE_AES_CTR 2
#define ROC_SE_PDCP_ALG_TYPE_ZUC 0
#define ROC_SE_PDCP_ALG_TYPE_SNOW3G 1
#define ROC_SE_PDCP_ALG_TYPE_AES_CTR 2
#define ROC_SE_PDCP_ALG_TYPE_AES_CMAC 3
#define ROC_SE_PDCP_CHAIN_ALG_TYPE_SNOW3G 1
#define ROC_SE_PDCP_CHAIN_ALG_TYPE_ZUC 3
#define ROC_SE_PDCP_CHAIN_CTX_LFSR 0
#define ROC_SE_PDCP_CHAIN_CTX_KEY_IV 1
struct roc_se_ctx {
/* Below fields are accessed by sw */
@ -289,13 +333,17 @@ struct roc_se_ctx {
uint64_t hmac : 1;
uint64_t zsk_flags : 3;
uint64_t k_ecb : 1;
uint64_t pdcp_alg_type : 2;
uint64_t rsvd : 21;
uint64_t pdcp_ci_alg : 2;
uint64_t pdcp_auth_alg : 2;
uint16_t ciph_then_auth : 1;
uint16_t auth_then_ciph : 1;
uint64_t rsvd : 17;
union cpt_inst_w4 template_w4;
/* Below fields are accessed by hardware */
union {
struct roc_se_context fctx;
struct roc_se_zuc_snow3g_ctx zs_ctx;
struct roc_se_zuc_snow3g_chain_ctx zs_ch_ctx;
struct roc_se_kasumi_ctx k_ctx;
} se_ctx;
uint8_t *auth_key;

69
drivers/crypto/cnxk/cnxk_cryptodev_ops.c
View File

@ -421,14 +421,39 @@ cnxk_cpt_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
return sizeof(struct cnxk_se_sess);
}
static bool
is_valid_pdcp_cipher_alg(struct rte_crypto_sym_xform *c_xfrm,
struct cnxk_se_sess *sess)
{
switch (c_xfrm->cipher.algo) {
case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
case RTE_CRYPTO_CIPHER_ZUC_EEA3:
break;
case RTE_CRYPTO_CIPHER_AES_CTR:
sess->aes_ctr_eea2 = 1;
break;
default:
return false;
}
return true;
}
static int
cnxk_sess_fill(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
cnxk_sess_fill(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xform,
struct cnxk_se_sess *sess)
{
struct rte_crypto_sym_xform *aead_xfrm = NULL;
struct rte_crypto_sym_xform *c_xfrm = NULL;
struct rte_crypto_sym_xform *a_xfrm = NULL;
bool pdcp_chain_supported = false;
bool ciph_then_auth = false;
if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_96XX_B0 ||
roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_96XX_C0 ||
roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_98XX)
pdcp_chain_supported = true;
if (xform == NULL)
return -EINVAL;
@ -506,6 +531,32 @@ cnxk_sess_fill(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
/* Cipher then auth */
if (ciph_then_auth) {
if (c_xfrm->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
if (a_xfrm->auth.op != RTE_CRYPTO_AUTH_OP_VERIFY)
return -EINVAL;
sess->auth_first = 1;
switch (a_xfrm->auth.algo) {
case RTE_CRYPTO_AUTH_SHA1_HMAC:
switch (c_xfrm->cipher.algo) {
case RTE_CRYPTO_CIPHER_AES_CBC:
break;
default:
return -ENOTSUP;
}
break;
case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
case RTE_CRYPTO_AUTH_ZUC_EIA3:
case RTE_CRYPTO_AUTH_AES_CMAC:
if (!pdcp_chain_supported ||
!is_valid_pdcp_cipher_alg(c_xfrm, sess))
return -ENOTSUP;
break;
default:
return -ENOTSUP;
}
}
sess->roc_se_ctx.ciph_then_auth = 1;
sess->chained_op = 1;
if (fill_sess_cipher(c_xfrm, sess))
return -ENOTSUP;
if (fill_sess_auth(a_xfrm, sess))
@ -517,6 +568,9 @@ cnxk_sess_fill(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
/* else */
if (c_xfrm->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
if (a_xfrm->auth.op != RTE_CRYPTO_AUTH_OP_GENERATE)
return -EINVAL;
sess->auth_first = 1;
switch (a_xfrm->auth.algo) {
case RTE_CRYPTO_AUTH_SHA1_HMAC:
switch (c_xfrm->cipher.algo) {
@ -526,11 +580,20 @@ cnxk_sess_fill(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
return -ENOTSUP;
}
break;
case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
case RTE_CRYPTO_AUTH_ZUC_EIA3:
case RTE_CRYPTO_AUTH_AES_CMAC:
if (!pdcp_chain_supported ||
!is_valid_pdcp_cipher_alg(c_xfrm, sess))
return -ENOTSUP;
break;
default:
return -ENOTSUP;
}
}
sess->roc_se_ctx.auth_then_ciph = 1;
sess->chained_op = 1;
if (fill_sess_auth(a_xfrm, sess))
return -ENOTSUP;
if (fill_sess_cipher(c_xfrm, sess))
@ -547,7 +610,7 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
/* Set the engine group */
if (sess->zsk_flag || sess->chacha_poly)
if (sess->zsk_flag || sess->chacha_poly || sess->aes_ctr_eea2)
inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE];
else
inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
@ -574,7 +637,7 @@ sym_session_configure(struct roc_cpt *roc_cpt, int driver_id,
sess_priv = priv;
ret = cnxk_sess_fill(xform, sess_priv);
ret = cnxk_sess_fill(roc_cpt, xform, sess_priv);
if (ret)
goto priv_put;

233
drivers/crypto/cnxk/cnxk_se.h
View File

@ -24,7 +24,12 @@ struct cnxk_se_sess {
uint16_t chacha_poly : 1;
uint16_t is_null : 1;
uint16_t is_gmac : 1;
uint16_t rsvd1 : 3;
uint16_t chained_op : 1;
uint16_t auth_first : 1;
uint16_t aes_ctr_eea2 : 1;
uint16_t zs_cipher : 4;
uint16_t zs_auth : 4;
uint16_t rsvd2 : 8;
uint16_t aad_length;
uint8_t mac_len;
uint8_t iv_length;
@ -63,6 +68,11 @@ pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type,
uint32_t *iv_s_temp, iv_temp[4];
int j;
if (unlikely(iv_s == NULL)) {
memset(iv_d, 0, 16);
return;
}
if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
/*
* DPDK seems to provide it in form of IV3 IV2 IV1 IV0
@ -74,7 +84,8 @@ pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type,
for (j = 0; j < 4; j++)
iv_temp[j] = iv_s_temp[3 - j];
memcpy(iv_d, iv_temp, 16);
} else if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_ZUC) {
} else if ((pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_ZUC) ||
pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_AES_CTR) {
if (pack_iv) {
cpt_pack_iv(iv_s, iv_d);
memcpy(iv_d + 6, iv_s + 8, 17);
@ -997,6 +1008,110 @@ cpt_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
return 0;
}
static __rte_always_inline int
cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
struct roc_se_fc_params *params,
struct cpt_inst_s *inst)
{
uint32_t encr_offset, auth_offset, iv_offset = 0;
uint8_t *auth_iv = NULL, *cipher_iv = NULL;
uint32_t encr_data_len, auth_data_len;
uint8_t pdcp_ci_alg, pdcp_auth_alg;
union cpt_inst_w4 cpt_inst_w4;
struct roc_se_ctx *se_ctx;
const int iv_len = 32;
uint32_t mac_len = 0;
uint8_t pack_iv = 0;
void *offset_vaddr;
int32_t inputlen;
void *dm_vaddr;
uint8_t *iv_d;
if (unlikely((!(req_flags & ROC_SE_SINGLE_BUF_INPLACE)) ||
(!(req_flags & ROC_SE_SINGLE_BUF_HEADROOM)))) {
plt_dp_err("Scatter gather mode is not supported");
return -1;
}
encr_offset = ROC_SE_ENCR_OFFSET(d_offs);
auth_offset = ROC_SE_AUTH_OFFSET(d_offs);
if (auth_offset != encr_offset) {
plt_dp_err("encr_offset and auth_offset are not same");
plt_dp_err("enc_offset: %d", encr_offset);
plt_dp_err("auth_offset: %d", auth_offset);
return -1;
}
if (unlikely(encr_offset >> 16)) {
plt_dp_err("Offset not supported");
plt_dp_err("enc_offset: %d", encr_offset);
return -1;
}
se_ctx = params->ctx_buf.vaddr;
mac_len = se_ctx->mac_len;
pdcp_ci_alg = se_ctx->pdcp_ci_alg;
pdcp_auth_alg = se_ctx->pdcp_auth_alg;
encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
auth_data_len = ROC_SE_AUTH_DLEN(d_lens);
if ((auth_data_len + mac_len) != encr_data_len) {
plt_dp_err("(auth_data_len + mac_len) != encr_data_len");
plt_dp_err("auth_data_len: %d", auth_data_len);
plt_dp_err("encr_data_len: %d", encr_data_len);
plt_dp_err("mac_len: %d", mac_len);
return -1;
}
cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_PDCP_CHAIN;
cpt_inst_w4.s.opcode_minor = se_ctx->template_w4.s.opcode_minor;
cpt_inst_w4.s.param1 = auth_data_len;
cpt_inst_w4.s.param2 = 0;
if (likely(params->auth_iv_len))
auth_iv = params->auth_iv_buf;
if (likely(params->cipher_iv_len))
cipher_iv = params->iv_buf;
encr_offset += iv_len;
if (se_ctx->auth_then_ciph)
inputlen = encr_offset + auth_data_len;
else
inputlen = encr_offset + encr_data_len;
dm_vaddr = params->bufs[0].vaddr;
/* Use Direct mode */
offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - ROC_SE_OFF_CTRL_LEN -
iv_len);
/* DPTR */
inst->dptr = (uint64_t)offset_vaddr;
/* RPTR should just exclude offset control word */
inst->rptr = (uint64_t)dm_vaddr - iv_len;
cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN;
*(uint64_t *)offset_vaddr = rte_cpu_to_be_64(
((uint64_t)(iv_offset) << 16) | ((uint64_t)(encr_offset)));
iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN);
pdcp_iv_copy(iv_d, cipher_iv, pdcp_ci_alg, pack_iv);
iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN + 16);
pdcp_iv_copy(iv_d, auth_iv, pdcp_auth_alg, pack_iv);
inst->w4.u64 = cpt_inst_w4.u64;
return 0;
}
static __rte_always_inline int
cpt_pdcp_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
struct roc_se_fc_params *params, struct cpt_inst_s *inst)
@ -1018,7 +1133,6 @@ cpt_pdcp_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
se_ctx = params->ctx_buf.vaddr;
flags = se_ctx->zsk_flags;
mac_len = se_ctx->mac_len;
pdcp_alg_type = se_ctx->pdcp_alg_type;
cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_PDCP;
cpt_inst_w4.s.opcode_minor = se_ctx->template_w4.s.opcode_minor;
@ -1032,8 +1146,9 @@ cpt_pdcp_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
*/
auth_data_len = ROC_SE_AUTH_DLEN(d_lens);
auth_offset = ROC_SE_AUTH_OFFSET(d_offs);
pdcp_alg_type = se_ctx->pdcp_auth_alg;
if (se_ctx->pdcp_alg_type != ROC_SE_PDCP_ALG_TYPE_AES_CTR) {
if (pdcp_alg_type != ROC_SE_PDCP_ALG_TYPE_AES_CMAC) {
iv_len = params->auth_iv_len;
if (iv_len == 25) {
@ -1067,6 +1182,7 @@ cpt_pdcp_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
} else {
iv_s = params->iv_buf;
iv_len = params->cipher_iv_len;
pdcp_alg_type = se_ctx->pdcp_ci_alg;
if (iv_len == 25) {
roc_se_zuc_bytes_swap(iv_s, iv_len);
@ -1609,6 +1725,9 @@ cpt_fc_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
ret = cpt_pdcp_alg_prep(flags, d_offs, d_lens, fc_params, inst);
} else if (fc_type == ROC_SE_KASUMI) {
ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, inst);
} else if (fc_type == ROC_SE_PDCP_CHAIN) {
ret = cpt_pdcp_chain_alg_prep(flags, d_offs, d_lens, fc_params,
inst);
}
/*
@ -1640,6 +1759,9 @@ cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
inst);
} else if (fc_type == ROC_SE_HASH_HMAC) {
ret = cpt_digest_gen_prep(flags, d_lens, fc_params, inst);
} else if (fc_type == ROC_SE_PDCP_CHAIN) {
ret = cpt_pdcp_chain_alg_prep(flags, d_offs, d_lens, fc_params,
inst);
}
return ret;
@ -1713,10 +1835,10 @@ fill_sess_aead(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
static __rte_always_inline int
fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
{
uint8_t zsk_flag = 0, zs_cipher = 0, aes_ctr = 0, is_null = 0;
struct rte_crypto_cipher_xform *c_form;
roc_se_cipher_type enc_type = 0; /* NULL Cipher type */
uint32_t cipher_key_len = 0;
uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0;
c_form = &xform->cipher;
@ -1750,28 +1872,37 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
cipher_key_len = 8;
break;
case RTE_CRYPTO_CIPHER_AES_CTR:
enc_type = ROC_SE_AES_CTR;
if (sess->aes_ctr_eea2) {
enc_type = ROC_SE_AES_CTR_EEA2;
} else {
enc_type = ROC_SE_AES_CTR;
aes_ctr = 1;
}
cipher_key_len = 16;
aes_ctr = 1;
break;
case RTE_CRYPTO_CIPHER_NULL:
enc_type = 0;
is_null = 1;
break;
case RTE_CRYPTO_CIPHER_KASUMI_F8:
if (sess->chained_op)
return -ENOTSUP;
enc_type = ROC_SE_KASUMI_F8_ECB;
cipher_key_len = 16;
zsk_flag = ROC_SE_K_F8;
zs_cipher = ROC_SE_K_F8;
break;
case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
enc_type = ROC_SE_SNOW3G_UEA2;
cipher_key_len = 16;
zsk_flag = ROC_SE_ZS_EA;
zs_cipher = ROC_SE_ZS_EA;
break;
case RTE_CRYPTO_CIPHER_ZUC_EEA3:
enc_type = ROC_SE_ZUC_EEA3;
cipher_key_len = c_form->key.length;
zsk_flag = ROC_SE_ZS_EA;
zs_cipher = ROC_SE_ZS_EA;
break;
case RTE_CRYPTO_CIPHER_AES_XTS:
enc_type = ROC_SE_AES_XTS;
@ -1802,7 +1933,19 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
return -1;
}
if (zsk_flag && sess->roc_se_ctx.ciph_then_auth) {
struct rte_crypto_auth_xform *a_form;
a_form = &xform->next->auth;
if (c_form->op != RTE_CRYPTO_CIPHER_OP_DECRYPT &&
a_form->op != RTE_CRYPTO_AUTH_OP_VERIFY) {
plt_dp_err("Crypto: PDCP cipher then auth must use"
" options: decrypt and verify");
return -EINVAL;
}
}
sess->zsk_flag = zsk_flag;
sess->zs_cipher = zs_cipher;
sess->aes_gcm = 0;
sess->aes_ctr = aes_ctr;
sess->iv_offset = c_form->iv.offset;
@ -1822,9 +1965,9 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
static __rte_always_inline int
fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
{
uint8_t zsk_flag = 0, zs_auth = 0, aes_gcm = 0, is_null = 0;
struct rte_crypto_auth_xform *a_form;
roc_se_auth_type auth_type = 0; /* NULL Auth type */
uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC)
return fill_sess_gmac(xform, sess);
@ -1879,20 +2022,25 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
auth_type = ROC_SE_MD5_TYPE;
break;
case RTE_CRYPTO_AUTH_KASUMI_F9:
if (sess->chained_op)
return -ENOTSUP;
auth_type = ROC_SE_KASUMI_F9_ECB;
/*
* Indicate that direction needs to be taken out
* from end of src
*/
zsk_flag = ROC_SE_K_F9;
zs_auth = ROC_SE_K_F9;
break;
case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
auth_type = ROC_SE_SNOW3G_UIA2;
zsk_flag = ROC_SE_ZS_IA;
zs_auth = ROC_SE_ZS_IA;
break;
case RTE_CRYPTO_AUTH_ZUC_EIA3:
auth_type = ROC_SE_ZUC_EIA3;
zsk_flag = ROC_SE_ZS_IA;
zs_auth = ROC_SE_ZS_IA;
break;
case RTE_CRYPTO_AUTH_NULL:
auth_type = 0;
@ -1912,7 +2060,19 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
return -1;
}
if (zsk_flag && sess->roc_se_ctx.auth_then_ciph) {
struct rte_crypto_cipher_xform *c_form;
c_form = &xform->next->cipher;
if (c_form->op != RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
a_form->op != RTE_CRYPTO_AUTH_OP_GENERATE) {
plt_dp_err("Crypto: PDCP auth then cipher must use"
" options: encrypt and generate");
return -EINVAL;
}
}
sess->zsk_flag = zsk_flag;
sess->zs_auth = zs_auth;
sess->aes_gcm = aes_gcm;
sess->mac_len = a_form->digest_length;
sess->is_null = is_null;
@ -2121,11 +2281,15 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
uint8_t inplace = 1;
#endif
struct roc_se_fc_params fc_params;
bool chain = sess->chained_op;
char src[SRC_IOV_SIZE];
char dst[SRC_IOV_SIZE];
uint32_t iv_buf[4];
bool pdcp_chain;
int ret;
pdcp_chain = chain && (sess->zs_auth || sess->zs_cipher);
fc_params.cipher_iv_len = sess->iv_length;
fc_params.auth_iv_len = sess->auth_iv_length;
@ -2143,10 +2307,11 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
}
}
if (sess->zsk_flag) {
fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(
cop, uint8_t *, sess->auth_iv_offset);
if (sess->zsk_flag != ROC_SE_ZS_EA)
if (sess->zsk_flag || sess->zs_auth) {
if (sess->auth_iv_length)
fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(
cop, uint8_t *, sess->auth_iv_offset);
if ((!chain) && (sess->zsk_flag != ROC_SE_ZS_EA))
inplace = 0;
}
m_src = sym_op->m_src;
@ -2203,17 +2368,35 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
}
}
} else {
d_offs = sym_op->cipher.data.offset;
d_lens = sym_op->cipher.data.length;
mc_hash_off =
sym_op->cipher.data.offset + sym_op->cipher.data.length;
d_offs = (d_offs << 16) | sym_op->auth.data.offset;
d_lens = (d_lens << 32) | sym_op->auth.data.length;
uint32_t ci_data_length = sym_op->cipher.data.length;
uint32_t ci_data_offset = sym_op->cipher.data.offset;
uint32_t a_data_length = sym_op->auth.data.length;
uint32_t a_data_offset = sym_op->auth.data.offset;
if (mc_hash_off <
(sym_op->auth.data.offset + sym_op->auth.data.length)) {
mc_hash_off = (sym_op->auth.data.offset +
sym_op->auth.data.length);
if (pdcp_chain) {
if (sess->zs_cipher) {
ci_data_length /= 8;
ci_data_offset /= 8;
}
if (sess->zs_auth) {
a_data_length /= 8;
a_data_offset /= 8;
}
}
d_offs = ci_data_offset;
d_offs = (d_offs << 16) | a_data_offset;
d_lens = ci_data_length;
d_lens = (d_lens << 32) | a_data_length;
if (sess->auth_first)
mc_hash_off = a_data_offset + a_data_length;
else
mc_hash_off = ci_data_offset + ci_data_length;
if (mc_hash_off < (a_data_offset + a_data_length)) {
mc_hash_off = (a_data_offset + a_data_length);
}
/* for gmac, salt should be updated like in gcm */
if (unlikely(sess->is_gmac)) {
@ -2247,7 +2430,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
}
fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
if (!(op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST) &&
if (!(sess->auth_first) && (!pdcp_chain) &&
unlikely(sess->is_null || sess->cpt_op == ROC_SE_OP_DECODE))
inplace = 0;
@ -2304,8 +2487,8 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
if (unlikely(!((flags & ROC_SE_SINGLE_BUF_INPLACE) &&
(flags & ROC_SE_SINGLE_BUF_HEADROOM) &&
((ctx->fc_type == ROC_SE_FC_GEN) ||
(ctx->fc_type == ROC_SE_PDCP))))) {
((ctx->fc_type != ROC_SE_KASUMI) &&
(ctx->fc_type != ROC_SE_HASH_HMAC))))) {
mdata = alloc_op_meta(&fc_params.meta_buf, m_info->mlen,
m_info->pool, infl_req);
if (mdata == NULL) {