diff --git a/config/common_base b/config/common_base index 38beaabb3d..b68278b47b 100644 --- a/config/common_base +++ b/config/common_base @@ -537,6 +537,7 @@ CONFIG_RTE_LIBRTE_PMD_QAT_SYM=n # CONFIG_RTE_PMD_QAT_MAX_PCI_DEVICES=48 CONFIG_RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS=16 +CONFIG_RTE_PMD_QAT_COMP_IM_BUFFER_SIZE=65536 # # Compile PMD for virtio crypto devices diff --git a/config/rte_config.h b/config/rte_config.h index 816e6f879e..333fb0be95 100644 --- a/config/rte_config.h +++ b/config/rte_config.h @@ -95,6 +95,7 @@ /* Max. number of QuickAssist devices which can be attached */ #define RTE_PMD_QAT_MAX_PCI_DEVICES 48 #define RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS 16 +#define RTE_PMD_QAT_COMP_IM_BUFFER_SIZE 65536 /* virtio crypto defines */ #define RTE_MAX_VIRTIO_CRYPTO 32 diff --git a/doc/guides/compressdevs/features/qat.ini b/doc/guides/compressdevs/features/qat.ini index 5cd4524ba9..6b1e7f9352 100644 --- a/doc/guides/compressdevs/features/qat.ini +++ b/doc/guides/compressdevs/features/qat.ini @@ -13,3 +13,4 @@ Adler32 = Y Crc32 = Y Adler32&Crc32 = Y Fixed = Y +Dynamic = Y diff --git a/doc/guides/compressdevs/qat_comp.rst b/doc/guides/compressdevs/qat_comp.rst index 7bffbe6ffa..aee3b99b04 100644 --- a/doc/guides/compressdevs/qat_comp.rst +++ b/doc/guides/compressdevs/qat_comp.rst @@ -18,11 +18,7 @@ QAT compression PMD has support for: Compression/Decompression algorithm: - * DEFLATE - -Huffman code type: - - * FIXED + * DEFLATE - using Fixed and Dynamic Huffman encoding Window size support: @@ -36,7 +32,6 @@ Limitations ----------- * Compressdev level 0, no compression, is not supported. -* Dynamic Huffman encoding is not yet supported. * Queue pairs are not thread-safe (that is, within a single queue pair, RX and TX from different lcores is not supported). * No BSD support as BSD QAT kernel driver not available. diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst index c6256939b0..07e1edf9fb 100644 --- a/doc/guides/rel_notes/release_18_11.rst +++ b/doc/guides/rel_notes/release_18_11.rst @@ -201,6 +201,11 @@ New Features Added the new caam job ring driver for NXP platforms. See the "NXP CAAM JOB RING (caam_jr)" document for more details on this new driver. +* **Added support for Dynamic Huffman Encoding to Intel QAT comp PMD.** + + The Intel QuickAssist (QAT) compression PMD has been updated with support + for Dynamic Huffman Encoding for the Deflate algorithm. + * **Added Event Ethernet Tx Adapter.** Added event ethernet Tx adapter library that provides configuration and diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c index 4ff8b88485..2a1cf3e179 100644 --- a/drivers/common/qat/qat_device.c +++ b/drivers/common/qat/qat_device.c @@ -7,6 +7,7 @@ #include "qat_device.h" #include "adf_transport_access_macros.h" #include "qat_sym_pmd.h" +#include "qat_comp_pmd.h" /* Hardware device information per generation */ __extension__ @@ -14,15 +15,18 @@ struct qat_gen_hw_data qat_gen_config[] = { [QAT_GEN1] = { .dev_gen = QAT_GEN1, .qp_hw_data = qat_gen1_qps, + .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1 }, [QAT_GEN2] = { .dev_gen = QAT_GEN2, .qp_hw_data = qat_gen1_qps, /* gen2 has same ring layout as gen1 */ + .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2 }, [QAT_GEN3] = { .dev_gen = QAT_GEN3, .qp_hw_data = qat_gen3_qps, + .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3 }, }; diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h index 3a71cd439a..eb81c78f9c 100644 --- a/drivers/common/qat/qat_device.h +++ b/drivers/common/qat/qat_device.h @@ -16,6 +16,12 @@ #define QAT_DEV_NAME_MAX_LEN 64 +enum qat_comp_num_im_buffers { + QAT_NUM_INTERM_BUFS_GEN1 = 12, + QAT_NUM_INTERM_BUFS_GEN2 = 20, + QAT_NUM_INTERM_BUFS_GEN3 = 20 +}; + /* * This struct holds all the data about a QAT pci device * including data about all services it supports. @@ -72,6 +78,7 @@ struct qat_pci_device { struct qat_gen_hw_data { enum qat_device_gen dev_gen; const struct qat_qp_hw_data (*qp_hw_data)[ADF_MAX_QPS_ON_ANY_SERVICE]; + enum qat_comp_num_im_buffers comp_num_im_bufs_required; }; extern struct qat_gen_hw_data qat_gen_config[]; diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c index b9336f356c..d70c594928 100644 --- a/drivers/compress/qat/qat_comp.c +++ b/drivers/compress/qat/qat_comp.c @@ -192,7 +192,7 @@ static void qat_comp_create_req_hdr(struct icp_qat_fw_comn_req_hdr *header, } static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, - const struct rte_memzone *interm_buff_mz __rte_unused, + const struct rte_memzone *interm_buff_mz, const struct rte_comp_xform *xform) { struct icp_qat_fw_comp_req *comp_req; @@ -280,10 +280,20 @@ static int qat_comp_create_templates(struct qat_comp_xform *qat_xform, ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl, ICP_QAT_FW_SLICE_COMP); } else if (qat_xform->qat_comp_request_type == - QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) { + QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) { - QAT_LOG(ERR, "Dynamic huffman encoding not supported"); - return -EINVAL; + ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->comp_cd_ctrl, + ICP_QAT_FW_SLICE_XLAT); + ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl, + ICP_QAT_FW_SLICE_COMP); + + ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->u2.xlt_cd_ctrl, + ICP_QAT_FW_SLICE_DRAM_WR); + ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->u2.xlt_cd_ctrl, + ICP_QAT_FW_SLICE_XLAT); + + comp_req->u1.xlt_pars.inter_buff_ptr = + interm_buff_mz->phys_addr; } #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG @@ -334,12 +344,6 @@ qat_comp_private_xform_create(struct rte_compressdev *dev, (struct qat_comp_xform *)*private_xform; if (xform->type == RTE_COMP_COMPRESS) { - if (xform->compress.deflate.huffman == - RTE_COMP_HUFFMAN_DYNAMIC) { - QAT_LOG(ERR, - "QAT device doesn't support dynamic compression"); - return -ENOTSUP; - } if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED || ((xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_DEFAULT) @@ -347,6 +351,21 @@ qat_comp_private_xform_create(struct rte_compressdev *dev, qat_xform->qat_comp_request_type = QAT_COMP_REQUEST_FIXED_COMP_STATELESS; + else if ((xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_DYNAMIC || + xform->compress.deflate.huffman == + RTE_COMP_HUFFMAN_DEFAULT) && + qat->interm_buff_mz != NULL) + + qat_xform->qat_comp_request_type = + QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS; + + else { + QAT_LOG(ERR, + "IM buffers needed for dynamic deflate. Set size in config file"); + return -EINVAL; + } + qat_xform->checksum_type = xform->compress.chksum; } else { diff --git a/drivers/compress/qat/qat_comp.h b/drivers/compress/qat/qat_comp.h index 8d315efb03..99a4462eb8 100644 --- a/drivers/compress/qat/qat_comp.h +++ b/drivers/compress/qat/qat_comp.h @@ -15,6 +15,10 @@ #include "icp_qat_fw_comp.h" #include "icp_qat_fw_la.h" +#define QAT_64_BYTE_ALIGN_MASK (~0x3f) +#define QAT_64_BYTE_ALIGN (64) +#define QAT_NUM_BUFS_IN_IM_SGL 1 + #define ERR_CODE_QAT_COMP_WRONG_FW -99 enum qat_comp_request_type { @@ -24,6 +28,15 @@ enum qat_comp_request_type { REQ_COMP_END }; +struct array_of_ptrs { + phys_addr_t pointer[0]; +}; + +struct qat_inter_sgl { + qat_sgl_hdr; + struct qat_flat_buf buffers[QAT_NUM_BUFS_IN_IM_SGL]; +} __rte_packed __rte_cache_aligned; + struct qat_comp_sgl { qat_sgl_hdr; struct qat_flat_buf buffers[RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS]; diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c index 63af23a333..01dd736133 100644 --- a/drivers/compress/qat/qat_comp_pmd.c +++ b/drivers/compress/qat/qat_comp_pmd.c @@ -14,6 +14,7 @@ static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = { RTE_COMP_FF_CRC32_ADLER32_CHECKSUM | RTE_COMP_FF_SHAREABLE_PRIV_XFORM | RTE_COMP_FF_HUFFMAN_FIXED | + RTE_COMP_FF_HUFFMAN_DYNAMIC | RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_SGL_IN_LB_OUT | RTE_COMP_FF_OOP_LB_IN_SGL_OUT, @@ -112,7 +113,7 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, /* store a link to the qp in the qat_pci_device */ qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][qp_id] - = *qp_addr; + = *qp_addr; qp = (struct qat_qp *)*qp_addr; @@ -135,6 +136,103 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, return ret; } + +#define QAT_IM_BUFFER_DEBUG 0 +static const struct rte_memzone * +qat_comp_setup_inter_buffers(struct qat_comp_dev_private *comp_dev, + uint32_t buff_size) +{ + char inter_buff_mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *memzone; + uint8_t *mz_start = NULL; + rte_iova_t mz_start_phys = 0; + struct array_of_ptrs *array_of_pointers; + int size_of_ptr_array; + uint32_t full_size; + uint32_t offset_of_sgls, offset_of_flat_buffs = 0; + int i; + int num_im_sgls = qat_gen_config[ + comp_dev->qat_dev->qat_dev_gen].comp_num_im_bufs_required; + + QAT_LOG(DEBUG, "QAT COMP device %s needs %d sgls", + comp_dev->qat_dev->name, num_im_sgls); + snprintf(inter_buff_mz_name, RTE_MEMZONE_NAMESIZE, + "%s_inter_buff", comp_dev->qat_dev->name); + memzone = rte_memzone_lookup(inter_buff_mz_name); + if (memzone != NULL) { + QAT_LOG(DEBUG, "QAT COMP im buffer memzone created already"); + return memzone; + } + + /* Create a memzone to hold intermediate buffers and associated + * meta-data needed by the firmware. The memzone contains: + * - a list of num_im_sgls physical pointers to sgls + * - the num_im_sgl sgl structures, each pointing to 2 flat buffers + * - the flat buffers: num_im_sgl * 2 + * where num_im_sgls depends on the hardware generation of the device + */ + + size_of_ptr_array = num_im_sgls * sizeof(phys_addr_t); + offset_of_sgls = (size_of_ptr_array + (~QAT_64_BYTE_ALIGN_MASK)) + & QAT_64_BYTE_ALIGN_MASK; + offset_of_flat_buffs = + offset_of_sgls + num_im_sgls * sizeof(struct qat_inter_sgl); + full_size = offset_of_flat_buffs + + num_im_sgls * buff_size * QAT_NUM_BUFS_IN_IM_SGL; + + memzone = rte_memzone_reserve_aligned(inter_buff_mz_name, full_size, + comp_dev->compressdev->data->socket_id, + RTE_MEMZONE_2MB, QAT_64_BYTE_ALIGN); + if (memzone == NULL) { + QAT_LOG(ERR, "Can't allocate intermediate buffers" + " for device %s", comp_dev->qat_dev->name); + return NULL; + } + + mz_start = (uint8_t *)memzone->addr; + mz_start_phys = memzone->phys_addr; + QAT_LOG(DEBUG, "Memzone %s: addr = %p, phys = 0x%"PRIx64 + ", size required %d, size created %zu", + inter_buff_mz_name, mz_start, mz_start_phys, + full_size, memzone->len); + + array_of_pointers = (struct array_of_ptrs *)mz_start; + for (i = 0; i < num_im_sgls; i++) { + uint32_t curr_sgl_offset = + offset_of_sgls + i * sizeof(struct qat_inter_sgl); + struct qat_inter_sgl *sgl = + (struct qat_inter_sgl *)(mz_start + curr_sgl_offset); + array_of_pointers->pointer[i] = mz_start_phys + curr_sgl_offset; + + sgl->num_bufs = QAT_NUM_BUFS_IN_IM_SGL; + sgl->num_mapped_bufs = 0; + sgl->resrvd = 0; + sgl->buffers[0].addr = mz_start_phys + offset_of_flat_buffs + + ((i * QAT_NUM_BUFS_IN_IM_SGL) * buff_size); + sgl->buffers[0].len = buff_size; + sgl->buffers[0].resrvd = 0; + sgl->buffers[1].addr = mz_start_phys + offset_of_flat_buffs + + (((i * QAT_NUM_BUFS_IN_IM_SGL) + 1) * buff_size); + sgl->buffers[1].len = buff_size; + sgl->buffers[1].resrvd = 0; + +#if QAT_IM_BUFFER_DEBUG + QAT_LOG(DEBUG, " : phys addr of sgl[%i] in array_of_pointers" + "= 0x%"PRIx64, i, array_of_pointers->pointer[i]); + QAT_LOG(DEBUG, " : virt address of sgl[%i] = %p", i, sgl); + QAT_LOG(DEBUG, " : sgl->buffers[0].addr = 0x%"PRIx64", len=%d", + sgl->buffers[0].addr, sgl->buffers[0].len); + QAT_LOG(DEBUG, " : sgl->buffers[1].addr = 0x%"PRIx64", len=%d", + sgl->buffers[1].addr, sgl->buffers[1].len); +#endif + } +#if QAT_IM_BUFFER_DEBUG + QAT_DP_HEXDUMP_LOG(DEBUG, "IM buffer memzone start:", + mz_start, offset_of_flat_buffs + 32); +#endif + return memzone; +} + static struct rte_mempool * qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev, uint32_t num_elements) @@ -176,6 +274,12 @@ qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev, static void _qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev) { + /* Free intermediate buffers */ + if (comp_dev->interm_buff_mz) { + rte_memzone_free(comp_dev->interm_buff_mz); + comp_dev->interm_buff_mz = NULL; + } + /* Free private_xform pool */ if (comp_dev->xformpool) { /* Free internal mempool for private xforms */ @@ -197,6 +301,21 @@ qat_comp_dev_config(struct rte_compressdev *dev, return -EINVAL; } + if (RTE_PMD_QAT_COMP_IM_BUFFER_SIZE == 0) { + QAT_LOG(WARNING, + "RTE_PMD_QAT_COMP_IM_BUFFER_SIZE = 0 in config file, so" + " QAT device can't be used for Dynamic Deflate. " + "Did you really intend to do this?"); + } else { + comp_dev->interm_buff_mz = + qat_comp_setup_inter_buffers(comp_dev, + RTE_PMD_QAT_COMP_IM_BUFFER_SIZE); + if (comp_dev->interm_buff_mz == NULL) { + ret = -ENOMEM; + goto error_out; + } + } + comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev, config->max_nb_priv_xforms); if (comp_dev->xformpool == NULL) { @@ -365,6 +484,10 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev) QAT_LOG(ERR, "Compression PMD not supported on QAT dh895xcc"); return 0; } + if (qat_pci_dev->qat_dev_gen == QAT_GEN3) { + QAT_LOG(ERR, "Compression PMD not supported on QAT c4xxx"); + return 0; + } struct rte_compressdev_pmd_init_params init_params = { .name = "",