freebsd-nq/module/zfs/qat_crypt.c
Tom Caputi 17dd88352e Allow QAT to handle non page-aligned buffers
This patch adds some handling to the QAT acceleration functions
that allows them to handle buffers that are not aligned with the
page cache. At the moment this never happens since callers only
happen to work with page-aligned buffers, but this code should
prevent headaches if this isn't always true in the future. This
patch also adds some cleanups to align the QAT compression code
with the encryption and checksumming code.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Weigang Li <weigang.li@intel.com>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #7305
2018-03-16 16:02:49 -07:00

559 lines
16 KiB
C

/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* This file represents the QAT implementation of checksums and encryption.
* Internally, QAT shares the same cryptographic instances for both of these
* operations, so the code has been combined here. QAT data compression uses
* compression instances, so that code is separated into qat_compress.c
*/
#if defined(_KERNEL) && defined(HAVE_QAT)
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <sys/zfs_context.h>
#include <sys/zio_crypt.h>
#include "lac/cpa_cy_im.h"
#include "lac/cpa_cy_common.h"
#include "qat.h"
/*
* Max instances in QAT device, each instance is a channel to submit
* jobs to QAT hardware, this is only for pre-allocating instance,
* and session arrays, the actual number of instances are defined in
* the QAT driver's configure file.
*/
#define QAT_CRYPT_MAX_INSTANCES 48
#define MAX_PAGE_NUM 1024
static Cpa16U inst_num = 0;
static Cpa16U num_inst = 0;
static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
static boolean_t qat_crypt_init_done = B_FALSE;
int zfs_qat_encrypt_disable = 0;
int zfs_qat_checksum_disable = 0;
typedef struct cy_callback {
CpaBoolean verify_result;
struct completion complete;
} cy_callback_t;
static void
symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
{
cy_callback_t *cb = p_callback;
if (cb != NULL) {
/* indicate that the function has been called */
cb->verify_result = verify;
complete(&cb->complete);
}
}
boolean_t
qat_crypt_use_accel(size_t s_len)
{
return (!zfs_qat_encrypt_disable &&
qat_crypt_init_done &&
s_len >= QAT_MIN_BUF_SIZE &&
s_len <= QAT_MAX_BUF_SIZE);
}
boolean_t
qat_checksum_use_accel(size_t s_len)
{
return (!zfs_qat_checksum_disable &&
qat_crypt_init_done &&
s_len >= QAT_MIN_BUF_SIZE &&
s_len <= QAT_MAX_BUF_SIZE);
}
void
qat_crypt_clean(void)
{
for (Cpa32U i = 0; i < num_inst; i++)
cpaCyStopInstance(cy_inst_handles[i]);
num_inst = 0;
qat_crypt_init_done = B_FALSE;
}
int
qat_crypt_init(void)
{
Cpa32U i;
CpaStatus status = CPA_STATUS_FAIL;
status = cpaCyGetNumInstances(&num_inst);
if (status != CPA_STATUS_SUCCESS)
return (-1);
/* if the user has configured no QAT encryption units just return */
if (num_inst == 0)
return (0);
if (num_inst > QAT_CRYPT_MAX_INSTANCES)
num_inst = QAT_CRYPT_MAX_INSTANCES;
status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
if (status != CPA_STATUS_SUCCESS)
return (-1);
for (i = 0; i < num_inst; i++) {
status = cpaCySetAddressTranslation(cy_inst_handles[i],
(void *)virt_to_phys);
if (status != CPA_STATUS_SUCCESS)
goto error;
status = cpaCyStartInstance(cy_inst_handles[i]);
if (status != CPA_STATUS_SUCCESS)
goto error;
}
qat_crypt_init_done = B_TRUE;
return (0);
error:
qat_crypt_clean();
return (-1);
}
void
qat_crypt_fini(void)
{
if (!qat_crypt_init_done)
return;
qat_crypt_clean();
}
static CpaStatus
qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
Cpa64U crypt, Cpa32U aad_len)
{
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa32U ctx_size;
Cpa32U ciper_algorithm;
Cpa32U hash_algorithm;
CpaCySymSessionSetupData sd = { 0 };
if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
return (CPA_STATUS_FAIL);
} else {
ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
}
sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
sd.cipherSetupData.pCipherKey = key->ck_data;
sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
sd.hashSetupData.hashAlgorithm = hash_algorithm;
sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
sd.digestIsAppended = CPA_FALSE;
sd.verifyDigest = CPA_FALSE;
if (dir == QAT_ENCRYPT) {
sd.cipherSetupData.cipherDirection =
CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
sd.algChainOrder =
CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
} else {
ASSERT3U(dir, ==, QAT_DECRYPT);
sd.cipherSetupData.cipherDirection =
CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
sd.algChainOrder =
CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
}
status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
if (status != CPA_STATUS_SUCCESS)
return (status);
status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
if (status != CPA_STATUS_SUCCESS)
return (status);
status = cpaCySymInitSession(inst_handle, symcallback, &sd,
*cy_session_ctx);
if (status != CPA_STATUS_SUCCESS) {
QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
return (status);
}
return (CPA_STATUS_SUCCESS);
}
static CpaStatus
qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
{
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa32U ctx_size;
Cpa32U hash_algorithm;
CpaCySymSessionSetupData sd = { 0 };
/*
* ZFS's SHA512 checksum is actually SHA512/256, which uses
* a different IV from standard SHA512. QAT does not support
* SHA512/256, so we can only support SHA256.
*/
if (cksum == ZIO_CHECKSUM_SHA256)
hash_algorithm = CPA_CY_SYM_HASH_SHA256;
else
return (CPA_STATUS_FAIL);
sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
sd.symOperation = CPA_CY_SYM_OP_HASH;
sd.hashSetupData.hashAlgorithm = hash_algorithm;
sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
sd.digestIsAppended = CPA_FALSE;
sd.verifyDigest = CPA_FALSE;
status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
if (status != CPA_STATUS_SUCCESS)
return (status);
status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
if (status != CPA_STATUS_SUCCESS)
return (status);
status = cpaCySymInitSession(inst_handle, symcallback, &sd,
*cy_session_ctx);
if (status != CPA_STATUS_SUCCESS) {
QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
return (status);
}
return (CPA_STATUS_SUCCESS);
}
static CpaStatus
qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
CpaBufferList *src, CpaBufferList *dst)
{
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa32U meta_size = 0;
status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
if (status != CPA_STATUS_SUCCESS)
return (status);
status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
if (status != CPA_STATUS_SUCCESS)
goto error;
if (src != dst) {
status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
meta_size);
if (status != CPA_STATUS_SUCCESS)
goto error;
}
return (CPA_STATUS_SUCCESS);
error:
QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
if (src != dst)
QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
return (status);
}
int
qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
{
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa16U i;
CpaInstanceHandle cy_inst_handle;
Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left = 0;
Cpa8S *data = NULL;
CpaCySymSessionCtx *cy_session_ctx = NULL;
cy_callback_t cb;
CpaCySymOpData op_data = { 0 };
CpaBufferList src_buffer_list = { 0 };
CpaBufferList dst_buffer_list = { 0 };
CpaFlatBuffer *flat_src_buf_array = NULL;
CpaFlatBuffer *flat_src_buf = NULL;
CpaFlatBuffer *flat_dst_buf_array = NULL;
CpaFlatBuffer *flat_dst_buf = NULL;
struct page *in_pages[MAX_PAGE_NUM];
struct page *out_pages[MAX_PAGE_NUM];
Cpa32U in_page_num = 0;
Cpa32U out_page_num = 0;
Cpa32U in_page_off = 0;
Cpa32U out_page_off = 0;
if (dir == QAT_ENCRYPT) {
QAT_STAT_BUMP(encrypt_requests);
QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
} else {
QAT_STAT_BUMP(decrypt_requests);
QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
}
i = atomic_inc_32_nv(&inst_num) % num_inst;
cy_inst_handle = cy_inst_handles[i];
status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
&cy_session_ctx, key, crypt, aad_len);
if (status != CPA_STATUS_SUCCESS) {
/* don't count CCM as a failure since it's not supported */
if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
QAT_STAT_BUMP(crypt_fails);
return (status);
}
/*
* We increment nr_bufs by 2 to allow us to handle non
* page-aligned buffer addresses and buffers whose sizes
* are not divisible by PAGE_SIZE.
*/
status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
&src_buffer_list, &dst_buffer_list);
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
nr_bufs * sizeof (CpaFlatBuffer));
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
nr_bufs * sizeof (CpaFlatBuffer));
if (status != CPA_STATUS_SUCCESS)
goto fail;
bytes_left = enc_len;
data = src_buf;
flat_src_buf = flat_src_buf_array;
while (bytes_left > 0) {
in_page_off = ((long)data & ~PAGE_MASK);
in_pages[in_page_num] = qat_mem_to_page(data);
flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
flat_src_buf->dataLenInBytes =
min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
data += flat_src_buf->dataLenInBytes;
bytes_left -= flat_src_buf->dataLenInBytes;
flat_src_buf++;
in_page_num++;
}
src_buffer_list.pBuffers = flat_src_buf_array;
src_buffer_list.numBuffers = in_page_num;
bytes_left = enc_len;
data = dst_buf;
flat_dst_buf = flat_dst_buf_array;
while (bytes_left > 0) {
out_page_off = ((long)data & ~PAGE_MASK);
out_pages[out_page_num] = qat_mem_to_page(data);
flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
out_page_off;
flat_dst_buf->dataLenInBytes =
min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
data += flat_dst_buf->dataLenInBytes;
bytes_left -= flat_dst_buf->dataLenInBytes;
flat_dst_buf++;
out_page_num++;
}
dst_buffer_list.pBuffers = flat_dst_buf_array;
dst_buffer_list.numBuffers = out_page_num;
op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
op_data.pIv = NULL; /* set this later as the J0 block */
op_data.ivLenInBytes = 0;
op_data.cryptoStartSrcOffsetInBytes = 0;
op_data.messageLenToCipherInBytes = 0;
op_data.hashStartSrcOffsetInBytes = 0;
op_data.messageLenToHashInBytes = 0;
op_data.pDigestResult = 0;
op_data.messageLenToCipherInBytes = enc_len;
op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
op_data.pDigestResult = digest_buf;
op_data.pAdditionalAuthData = aad_buf;
op_data.pIv = iv_buf;
cb.verify_result = CPA_FALSE;
init_completion(&cb.complete);
status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
&src_buffer_list, &dst_buffer_list, NULL);
if (status != CPA_STATUS_SUCCESS)
goto fail;
if (!wait_for_completion_interruptible_timeout(&cb.complete,
QAT_TIMEOUT_MS)) {
status = CPA_STATUS_FAIL;
goto fail;
}
if (cb.verify_result == CPA_FALSE) {
status = CPA_STATUS_FAIL;
goto fail;
}
if (dir == QAT_ENCRYPT)
QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
else
QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
fail:
if (status != CPA_STATUS_SUCCESS)
QAT_STAT_BUMP(crypt_fails);
for (i = 0; i < in_page_num; i++)
kunmap(in_pages[i]);
for (i = 0; i < out_page_num; i++)
kunmap(out_pages[i]);
cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(cy_session_ctx);
QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
return (status);
}
int
qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
{
CpaStatus status;
Cpa16U i;
CpaInstanceHandle cy_inst_handle;
Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
Cpa32U bytes_left = 0;
Cpa8S *data = NULL;
CpaCySymSessionCtx *cy_session_ctx = NULL;
cy_callback_t cb;
Cpa8U digest_buffer[sizeof (zio_cksum_t)];
CpaCySymOpData op_data = { 0 };
CpaBufferList src_buffer_list = { 0 };
CpaFlatBuffer *flat_src_buf_array = NULL;
CpaFlatBuffer *flat_src_buf = NULL;
struct page *in_pages[MAX_PAGE_NUM];
Cpa32U page_num = 0;
Cpa32U page_off = 0;
QAT_STAT_BUMP(cksum_requests);
QAT_STAT_INCR(cksum_total_in_bytes, size);
i = atomic_inc_32_nv(&inst_num) % num_inst;
cy_inst_handle = cy_inst_handles[i];
status = qat_init_checksum_session_ctx(cy_inst_handle,
&cy_session_ctx, cksum);
if (status != CPA_STATUS_SUCCESS) {
/* don't count unsupported checksums as a failure */
if (cksum == ZIO_CHECKSUM_SHA256 ||
cksum == ZIO_CHECKSUM_SHA512)
QAT_STAT_BUMP(cksum_fails);
return (status);
}
/*
* We increment nr_bufs by 2 to allow us to handle non
* page-aligned buffer addresses and buffers whose sizes
* are not divisible by PAGE_SIZE.
*/
status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
&src_buffer_list, &src_buffer_list);
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
nr_bufs * sizeof (CpaFlatBuffer));
if (status != CPA_STATUS_SUCCESS)
goto fail;
bytes_left = size;
data = buf;
flat_src_buf = flat_src_buf_array;
while (bytes_left > 0) {
page_off = ((long)data & ~PAGE_MASK);
in_pages[page_num] = qat_mem_to_page(data);
flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
flat_src_buf->dataLenInBytes =
min((long)PAGE_SIZE - page_off, (long)bytes_left);
data += flat_src_buf->dataLenInBytes;
bytes_left -= flat_src_buf->dataLenInBytes;
flat_src_buf++;
page_num++;
}
src_buffer_list.pBuffers = flat_src_buf_array;
src_buffer_list.numBuffers = page_num;
op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
op_data.hashStartSrcOffsetInBytes = 0;
op_data.messageLenToHashInBytes = size;
op_data.pDigestResult = digest_buffer;
cb.verify_result = CPA_FALSE;
init_completion(&cb.complete);
status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
&src_buffer_list, &src_buffer_list, NULL);
if (status != CPA_STATUS_SUCCESS)
goto fail;
if (!wait_for_completion_interruptible_timeout(&cb.complete,
QAT_TIMEOUT_MS)) {
status = CPA_STATUS_FAIL;
goto fail;
}
bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
fail:
if (status != CPA_STATUS_SUCCESS)
QAT_STAT_BUMP(cksum_fails);
for (i = 0; i < page_num; i++)
kunmap(in_pages[i]);
cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(cy_session_ctx);
QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
return (status);
}
module_param(zfs_qat_encrypt_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
module_param(zfs_qat_checksum_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
#endif