d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/qat: make response process function inline

Browse Source 
Optimize the dequeue function by inlining the response
processing function, assuming only symmetric
operations are supported.

Signed-off-by: Tomasz Jozwiak <tomaszx.jozwiak@intel.com>
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
This commit is contained in:
Fiona Trahe 2018-06-13 14:14:19 +02:00 committed by Pablo de Lara
parent 3ed20d8b6e
commit 91614c73b6
5 changed files with 131 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/crypto/qat/qat_qp.c
View File

@ -14,6 +14,7 @@
#include "qat_logs.h"
#include "qat_device.h"
#include "qat_qp.h"
#include "qat_sym.h"
#include "adf_transport_access_macros.h"
@ -238,7 +239,6 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
qp->qat_dev_gen = qat_dev->qat_dev_gen;
qp->build_request = qat_qp_conf->build_request;
qp->process_response = qat_qp_conf->process_response;
qp->qat_dev = qat_dev;
PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
@ -612,7 +612,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
resp_counter != nb_ops) {
tmp_qp->process_response(ops, resp_msg);
qat_sym_process_response(ops, resp_msg);
head = adf_modulo(head + rx_queue->msg_size,
rx_queue->modulo_mask);

5
drivers/crypto/qat/qat_qp.h
View File

@ -21,9 +21,6 @@ typedef int (*build_request_t)(void *op,
enum qat_device_gen qat_dev_gen);
/**< Build a request from an op. */
typedef int (*process_response_t)(void **ops, uint8_t *resp);
/**< Process a response descriptor and return the associated op. */
/**
* Structure with data needed for creation of queue pair.
*/
@ -44,7 +41,6 @@ struct qat_qp_config {
uint32_t cookie_size;
int socket_id;
build_request_t build_request;
process_response_t process_response;
const char *service_str;
};
@ -83,7 +79,6 @@ struct qat_qp {
uint32_t nb_descriptors;
enum qat_device_gen qat_dev_gen;
build_request_t build_request;
process_response_t process_response;
struct qat_pci_device *qat_dev;
/**< qat device this qp is on */
} __rte_cache_aligned;

127
drivers/crypto/qat/qat_sym.c
View File

@ -12,44 +12,7 @@
#include <rte_byteorder.h>
#include "qat_logs.h"
#include "qat_sym_session.h"
#include "qat_sym.h"
#include "qat_sym_pmd.h"
#define BYTE_LENGTH 8
/* bpi is only used for partial blocks of DES and AES
* so AES block len can be assumed as max len for iv, src and dst
*/
#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
/** Encrypt a single partial block
* Depends on openssl libcrypto
* Uses ECB+XOR to do CFB encryption, same result, more performant
*/
static inline int
bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
uint8_t *iv, int ivlen, int srclen,
void *bpi_ctx)
{
EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
int encrypted_ivlen;
uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
uint8_t *encr = encrypted_iv;
/* ECB method: encrypt the IV, then XOR this with plaintext */
if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
<= 0)
goto cipher_encrypt_err;
for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
*dst = *src ^ *encr;
return 0;
cipher_encrypt_err:
PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
return -EINVAL;
}
/** Decrypt a single partial block
* Depends on openssl libcrypto
@ -136,62 +99,6 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx,
return sym_op->cipher.data.length - last_block_len;
}
static inline uint32_t
qat_bpicipher_postprocess(struct qat_sym_session *ctx,
struct rte_crypto_op *op)
{
int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
struct rte_crypto_sym_op *sym_op = op->sym;
uint8_t last_block_len = block_len > 0 ?
sym_op->cipher.data.length % block_len : 0;
if (last_block_len > 0 &&
ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
/* Encrypt last block */
uint8_t *last_block, *dst, *iv;
uint32_t last_block_offset;
last_block_offset = sym_op->cipher.data.offset +
sym_op->cipher.data.length - last_block_len;
last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
uint8_t *, last_block_offset);
if (unlikely(sym_op->m_dst != NULL))
/* out-of-place operation (OOP) */
dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
uint8_t *, last_block_offset);
else
dst = last_block;
if (last_block_len < sym_op->cipher.data.length)
/* use previous block ciphertext as IV */
iv = dst - block_len;
else
/* runt block, i.e. less than one full block */
iv = rte_crypto_op_ctod_offset(op, uint8_t *,
ctx->cipher_iv.offset);
#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
rte_hexdump(stdout, "BPI: src before post-process:", last_block,
last_block_len);
if (sym_op->m_dst != NULL)
rte_hexdump(stdout, "BPI: dst before post-process:",
dst, last_block_len);
#endif
bpi_cipher_encrypt(last_block, dst, iv, block_len,
last_block_len, ctx->bpi_ctx);
#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
rte_hexdump(stdout, "BPI: src after post-process:", last_block,
last_block_len);
if (sym_op->m_dst != NULL)
rte_hexdump(stdout, "BPI: dst after post-process:", dst,
last_block_len);
#endif
}
return sym_op->cipher.data.length - last_block_len;
}
static inline void
set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
struct icp_qat_fw_la_cipher_req_params *cipher_param,
@ -659,37 +566,3 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
#endif
return 0;
}
int
qat_sym_process_response(void **op, uint8_t *resp)
{
struct icp_qat_fw_comn_resp *resp_msg =
(struct icp_qat_fw_comn_resp *)resp;
struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
(resp_msg->opaque_data);
#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
sizeof(struct icp_qat_fw_comn_resp));
#endif
if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
resp_msg->comn_hdr.comn_status)) {
rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
} else {
struct qat_sym_session *sess = (struct qat_sym_session *)
get_session_private_data(
rx_op->sym->session,
cryptodev_qat_driver_id);
if (sess->bpi_ctx)
qat_bpicipher_postprocess(sess, rx_op);
rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
}
*op = (void *)rx_op;
return 0;
}

131
drivers/crypto/qat/qat_sym.h
View File

@ -6,8 +6,19 @@
#define _QAT_SYM_H_
#include <rte_cryptodev_pmd.h>
#include <rte_hexdump.h>
#include <openssl/evp.h>
#include "qat_common.h"
#include "qat_sym_session.h"
#include "qat_sym_pmd.h"
#define BYTE_LENGTH 8
/* bpi is only used for partial blocks of DES and AES
* so AES block len can be assumed as max len for iv, src and dst
*/
#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
struct qat_sym_session;
@ -21,7 +32,123 @@ struct qat_sym_op_cookie {
int
qat_sym_build_request(void *in_op, uint8_t *out_msg,
void *op_cookie, enum qat_device_gen qat_dev_gen);
int
qat_sym_process_response(void **op, uint8_t *resp);
/** Encrypt a single partial block
* Depends on openssl libcrypto
* Uses ECB+XOR to do CFB encryption, same result, more performant
*/
static inline int
bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
uint8_t *iv, int ivlen, int srclen,
void *bpi_ctx)
{
EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
int encrypted_ivlen;
uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
uint8_t *encr = encrypted_iv;
/* ECB method: encrypt the IV, then XOR this with plaintext */
if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
<= 0)
goto cipher_encrypt_err;
for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
*dst = *src ^ *encr;
return 0;
cipher_encrypt_err:
PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
return -EINVAL;
}
static inline uint32_t
qat_bpicipher_postprocess(struct qat_sym_session *ctx,
struct rte_crypto_op *op)
{
int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
struct rte_crypto_sym_op *sym_op = op->sym;
uint8_t last_block_len = block_len > 0 ?
sym_op->cipher.data.length % block_len : 0;
if (last_block_len > 0 &&
ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
/* Encrypt last block */
uint8_t *last_block, *dst, *iv;
uint32_t last_block_offset;
last_block_offset = sym_op->cipher.data.offset +
sym_op->cipher.data.length - last_block_len;
last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
uint8_t *, last_block_offset);
if (unlikely(sym_op->m_dst != NULL))
/* out-of-place operation (OOP) */
dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
uint8_t *, last_block_offset);
else
dst = last_block;
if (last_block_len < sym_op->cipher.data.length)
/* use previous block ciphertext as IV */
iv = dst - block_len;
else
/* runt block, i.e. less than one full block */
iv = rte_crypto_op_ctod_offset(op, uint8_t *,
ctx->cipher_iv.offset);
#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
rte_hexdump(stdout, "BPI: src before post-process:", last_block,
last_block_len);
if (sym_op->m_dst != NULL)
rte_hexdump(stdout, "BPI: dst before post-process:",
dst, last_block_len);
#endif
bpi_cipher_encrypt(last_block, dst, iv, block_len,
last_block_len, ctx->bpi_ctx);
#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
rte_hexdump(stdout, "BPI: src after post-process:", last_block,
last_block_len);
if (sym_op->m_dst != NULL)
rte_hexdump(stdout, "BPI: dst after post-process:", dst,
last_block_len);
#endif
}
return sym_op->cipher.data.length - last_block_len;
}
static inline void
qat_sym_process_response(void **op, uint8_t *resp)
{
struct icp_qat_fw_comn_resp *resp_msg =
(struct icp_qat_fw_comn_resp *)resp;
struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
(resp_msg->opaque_data);
#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
sizeof(struct icp_qat_fw_comn_resp));
#endif
if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
resp_msg->comn_hdr.comn_status)) {
rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
} else {
struct qat_sym_session *sess = (struct qat_sym_session *)
get_session_private_data(
rx_op->sym->session,
cryptodev_qat_driver_id);
if (sess->bpi_ctx)
qat_bpicipher_postprocess(sess, rx_op);
rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
}
*op = (void *)rx_op;
}
#endif /* _QAT_SYM_H_ */

1
drivers/crypto/qat/qat_sym_pmd.c
View File

@ -160,7 +160,6 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
qat_qp_conf.hw = qp_hw_data;
qat_qp_conf.build_request = qat_sym_build_request;
qat_qp_conf.process_response = qat_sym_process_response;
qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
qat_qp_conf.socket_id = socket_id;