d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

common/qat: support GEN4 devices

Browse Source 
This commit adds support for fourth generation (GEN4) of
Intel QuickAssist (QAT) Technology devices.

Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>
This commit is contained in:
Arek Kusztal 2021-06-28 17:34:20 +01:00 committed by Akhil Goyal
parent 7b976dd079
commit 8f393c4ffd
13 changed files with 384 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/guides/cryptodevs/qat.rst
View File

@ -25,6 +25,7 @@ poll mode crypto driver support for the following hardware accelerator devices:
* ``Intel QuickAssist Technology 200xx``
* ``Intel QuickAssist Technology D15xx``
* ``Intel QuickAssist Technology C4xxx``
* ``Intel QuickAssist Technology 4xxx``
Features
@ -94,15 +95,16 @@ All the usual chains are supported and also some mixed chains:
+==================+===========+=============+==========+==========+
| NULL CIPHER | Y | 2&3 | 2&3 | Y |
+------------------+-----------+-------------+----------+----------+
| SNOW3G UEA2 | 2&3 | Y | 2&3 | 2&3 |
| SNOW3G UEA2 | 2&3 | 1&2&3 | 2&3 | 2&3 |
+------------------+-----------+-------------+----------+----------+
| ZUC EEA3 | 2&3 | 2&3 | 2&3 | 2&3 |
+------------------+-----------+-------------+----------+----------+
| AES CTR | Y | 2&3 | 2&3 | Y |
| AES CTR | 1&2&3 | 2&3 | 2&3 | Y |
+------------------+-----------+-------------+----------+----------+
* The combinations marked as "Y" are supported on all QAT hardware versions.
* The combinations marked as "2&3" are supported on GEN2/GEN3 QAT hardware only.
* The combinations marked as "2&3" are supported on GEN2 and GEN3 QAT hardware only.
* The combinations marked as "1&2&3" are supported on GEN1, GEN2 and GEN3 QAT hardware only.
Limitations
@ -373,6 +375,8 @@ to see the full table)
+-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+
| Yes | No | No | 3 | C4xxx | p | qat_c4xxx | c4xxx | 18a0 | 1 | 18a1 | 128 |
+-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+
| Yes | No | No | 4 | 4xxx | N/A | qat_4xxx | 4xxx | 4940 | 4 | 4941 | 16 |
+-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+
* Note: Symmetric mixed crypto algorithms feature on Gen 2 works only with 01.org driver version 4.9.0+

6
doc/guides/rel_notes/release_21_08.rst
View File

@ -105,6 +105,12 @@ New Features
* Added COUNT action support for SN1000 NICs
* **Updated Intel QuickAssist crypto PMD.**
Added fourth generation of QuickAssist Technology(QAT) devices support.
Only symmetric crypto has been currently enabled, compression and asymmetric
crypto PMD will fail to create.
* **Added support for Marvell CNXK crypto driver.**
* Added cnxk crypto PMD which provides support for an integrated

52
drivers/common/qat/qat_adf/adf_transport_access_macros_gen4.h Normal file
View File

@ -0,0 +1,52 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
* Copyright(c) 2021 Intel Corporation
*/
#ifndef ADF_TRANSPORT_ACCESS_MACROS_GEN4_H
#define ADF_TRANSPORT_ACCESS_MACROS_GEN4_H
#include "adf_transport_access_macros.h"
#define ADF_RINGS_PER_INT_SRCSEL_GEN4 2
#define ADF_BANK_INT_SRC_SEL_MASK_GEN4 0x44UL
#define ADF_BANK_INT_FLAG_CLEAR_MASK_GEN4 0x3
#define ADF_RING_BUNDLE_SIZE_GEN4 0x2000
#define ADF_RING_CSR_ADDR_OFFSET_GEN4 0x100000
#define ADF_RING_CSR_RING_CONFIG_GEN4 0x1000
#define ADF_RING_CSR_RING_LBASE_GEN4 0x1040
#define ADF_RING_CSR_RING_UBASE_GEN4 0x1080
#define BUILD_RING_BASE_ADDR_GEN4(addr, size) \
((((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) << 6)
#define WRITE_CSR_RING_BASE_GEN4(csr_base_addr, bank, ring, value) \
do { \
uint32_t l_base = 0, u_base = 0; \
l_base = (uint32_t)(value & 0xFFFFFFFF); \
u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \
ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
(ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
ADF_RING_CSR_RING_LBASE_GEN4 + (ring << 2), \
l_base); \
ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
(ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
ADF_RING_CSR_RING_UBASE_GEN4 + (ring << 2), \
u_base); \
} while (0)
#define WRITE_CSR_RING_CONFIG_GEN4(csr_base_addr, bank, ring, value) \
ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
(ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
ADF_RING_CSR_RING_CONFIG_GEN4 + (ring << 2), value)
#define WRITE_CSR_RING_TAIL_GEN4(csr_base_addr, bank, ring, value) \
ADF_CSR_WR((u8 *)(csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
(ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
ADF_RING_CSR_RING_TAIL + ((ring) << 2), value)
#define WRITE_CSR_RING_HEAD_GEN4(csr_base_addr, bank, ring, value) \
ADF_CSR_WR((u8 *)(csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
(ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
ADF_RING_CSR_RING_HEAD + ((ring) << 2), value)
#endif

48
drivers/common/qat/qat_adf/adf_transport_access_macros_gen4vf.h Normal file
View File

@ -0,0 +1,48 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
* Copyright(c) 2021 Intel Corporation
*/
#ifndef ADF_TRANSPORT_ACCESS_MACROS_GEN4VF_H
#define ADF_TRANSPORT_ACCESS_MACROS_GEN4VF_H
#include "adf_transport_access_macros.h"
#include "adf_transport_access_macros_gen4.h"
#define ADF_RING_CSR_ADDR_OFFSET_GEN4VF 0x0
#define WRITE_CSR_RING_BASE_GEN4VF(csr_base_addr, bank, ring, value) \
do { \
uint32_t l_base = 0, u_base = 0; \
l_base = (uint32_t)(value & 0xFFFFFFFF); \
u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \
ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
(ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
ADF_RING_CSR_RING_LBASE_GEN4 + (ring << 2), \
l_base); \
ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
(ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
ADF_RING_CSR_RING_UBASE_GEN4 + (ring << 2), \
u_base); \
} while (0)
#define WRITE_CSR_RING_CONFIG_GEN4VF(csr_base_addr, bank, ring, value) \
ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
(ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
ADF_RING_CSR_RING_CONFIG_GEN4 + (ring << 2), value)
#define WRITE_CSR_RING_TAIL_GEN4VF(csr_base_addr, bank, ring, value) \
ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
(ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
ADF_RING_CSR_RING_TAIL + ((ring) << 2), (value))
#define WRITE_CSR_RING_HEAD_GEN4VF(csr_base_addr, bank, ring, value) \
ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
(ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
ADF_RING_CSR_RING_HEAD + ((ring) << 2), (value))
#define WRITE_CSR_RING_SRV_ARB_EN_GEN4VF(csr_base_addr, bank, value) \
ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
(ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
ADF_RING_CSR_RING_SRV_ARB_EN, (value))
#endif

3
drivers/common/qat/qat_common.h
View File

@ -18,7 +18,8 @@
enum qat_device_gen {
QAT_GEN1 = 1,
QAT_GEN2,
QAT_GEN3
QAT_GEN3,
QAT_GEN4
};
enum qat_service_type {

22
drivers/common/qat/qat_device.c
View File

@ -30,6 +30,11 @@ struct qat_gen_hw_data qat_gen_config[] = {
.qp_hw_data = qat_gen3_qps,
.comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
},
[QAT_GEN4] = {
.dev_gen = QAT_GEN4,
.qp_hw_data = NULL,
.comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
},
};
/* per-process array of device data */
@ -59,6 +64,9 @@ static const struct rte_pci_id pci_id_qat_map[] = {
{
RTE_PCI_DEVICE(0x8086, 0x18a1),
},
{
RTE_PCI_DEVICE(0x8086, 0x4941),
},
{.device_id = 0},
};
@ -232,6 +240,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev,
case 0x18a1:
qat_dev->qat_dev_gen = QAT_GEN3;
break;
case 0x4941:
qat_dev->qat_dev_gen = QAT_GEN4;
break;
default:
QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
@ -241,6 +252,17 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev,
if (devargs && devargs->drv_str)
qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
if (qat_dev->qat_dev_gen >= QAT_GEN4) {
int ret = qat_read_qp_config(qat_dev, qat_dev->qat_dev_gen);
if (ret) {
QAT_LOG(ERR,
"Cannot acquire ring configuration for QAT_%d",
qat_dev_id);
return NULL;
}
}
rte_spinlock_init(&qat_dev->arb_csr_lock);
qat_nb_pci_devices++;

3
drivers/common/qat/qat_device.h
View File

@ -105,6 +105,9 @@ struct qat_pci_device {
/* Data relating to compression service */
struct qat_comp_dev_private *comp_dev;
/**< link back to compressdev private data */
struct qat_qp_hw_data qp_gen4_data[QAT_GEN4_BUNDLE_NUM]
[QAT_GEN4_QPS_PER_BUNDLE_NUM];
/**< Data of ring configuration on gen4 */
};
struct qat_gen_hw_data {

239
drivers/common/qat/qat_qp.c
View File

@ -19,6 +19,7 @@
#include "qat_asym.h"
#include "qat_comp.h"
#include "adf_transport_access_macros.h"
#include "adf_transport_access_macros_gen4vf.h"
#define QAT_CQ_MAX_DEQ_RETRIES 10
@ -138,25 +139,33 @@ static int qat_queue_create(struct qat_pci_device *qat_dev,
struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
uint32_t *queue_size_for_csr);
static void adf_configure_queues(struct qat_qp *queue);
static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
rte_spinlock_t *lock);
static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
rte_spinlock_t *lock);
static void adf_configure_queues(struct qat_qp *queue,
enum qat_device_gen qat_dev_gen);
static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen,
struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
int qat_qps_per_service(struct qat_pci_device *qat_dev,
enum qat_service_type service)
{
int i = 0, count = 0, max_ops_per_srv = 0;
const struct qat_qp_hw_data*
sym_hw_qps = qat_gen_config[qat_dev->qat_dev_gen]
.qp_hw_data[service];
max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
for (; i < max_ops_per_srv; i++)
if (sym_hw_qps[i].service_type == service)
count++;
if (qat_dev->qat_dev_gen == QAT_GEN4) {
max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
for (i = 0, count = 0; i < max_ops_per_srv; i++)
if (qat_dev->qp_gen4_data[i][0].service_type == service)
count++;
} else {
const struct qat_qp_hw_data *sym_hw_qps =
qat_gen_config[qat_dev->qat_dev_gen]
.qp_hw_data[service];
max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
for (i = 0, count = 0; i < max_ops_per_srv; i++)
if (sym_hw_qps[i].service_type == service)
count++;
}
return count;
}
@ -195,12 +204,12 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
struct qat_qp **qp_addr,
uint16_t queue_pair_id,
struct qat_qp_config *qat_qp_conf)
{
struct qat_qp *qp;
struct rte_pci_device *pci_dev =
qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
char op_cookie_pool_name[RTE_RING_NAMESIZE];
enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
uint32_t i;
QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
@ -264,8 +273,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
goto create_err;
}
adf_configure_queues(qp);
adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
adf_configure_queues(qp, qat_dev_gen);
adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr,
&qat_dev->arb_csr_lock);
snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
@ -314,7 +323,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
return -EFAULT;
}
int qat_qp_release(struct qat_qp **qp_addr)
int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr)
{
struct qat_qp *qp = *qp_addr;
uint32_t i;
@ -335,8 +345,8 @@ int qat_qp_release(struct qat_qp **qp_addr)
return -EAGAIN;
}
adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
&qp->qat_dev->arb_csr_lock);
adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr,
&qp->qat_dev->arb_csr_lock);
for (i = 0; i < qp->nb_descriptors; i++)
rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
@ -386,6 +396,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
const struct rte_memzone *qp_mz;
struct rte_pci_device *pci_dev =
qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
int ret = 0;
uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
@ -445,14 +456,19 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
* Write an unused pattern to the queue memory.
*/
memset(queue->base_addr, 0x7F, queue_size_bytes);
queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
queue->queue_size);
io_addr = pci_dev->mem_resource[0].addr;
WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
if (qat_dev_gen == QAT_GEN4) {
queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
queue->queue_size);
WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
queue->hw_queue_number, queue_base);
} else {
queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
queue->queue_size);
WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
queue->hw_queue_number, queue_base);
}
QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
" nb msgs %u, msg_size %u, modulo mask %u",
@ -468,6 +484,61 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
return ret;
}
int
qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
enum qat_service_type service_type)
{
if (qat_dev->qat_dev_gen == QAT_GEN4) {
int i = 0, valid_qps = 0;
for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
if (qat_dev->qp_gen4_data[i][0].service_type ==
service_type) {
if (valid_qps == qp_id)
return i;
++valid_qps;
}
}
}
return -1;
}
int
qat_read_qp_config(struct qat_pci_device *qat_dev,
enum qat_device_gen qat_dev_gen)
{
if (qat_dev_gen == QAT_GEN4) {
/* Read default configuration,
* until some probe of it can be done
*/
int i = 0;
for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
struct qat_qp_hw_data *hw_data =
&qat_dev->qp_gen4_data[i][0];
enum qat_service_type service_type =
(QAT_GEN4_QP_DEFCON >> (8 * i)) & 0xFF;
memset(hw_data, 0, sizeof(*hw_data));
hw_data->service_type = service_type;
if (service_type == QAT_SERVICE_ASYMMETRIC) {
hw_data->tx_msg_size = 64;
hw_data->rx_msg_size = 32;
} else if (service_type == QAT_SERVICE_SYMMETRIC ||
service_type ==
QAT_SERVICE_COMPRESSION) {
hw_data->tx_msg_size = 128;
hw_data->rx_msg_size = 32;
}
hw_data->tx_ring_num = 0;
hw_data->rx_ring_num = 1;
hw_data->hw_bundle_num = i;
}
}
/* With default config will always return success */
return 0;
}
static int qat_qp_check_queue_alignment(uint64_t phys_addr,
uint32_t queue_size_bytes)
{
@ -491,54 +562,81 @@ static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
return -EINVAL;
}
static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
rte_spinlock_t *lock)
static void
adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq,
void *base_addr, rte_spinlock_t *lock)
{
uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
(ADF_ARB_REG_SLOT *
txq->hw_bundle_number);
uint32_t value;
uint32_t arb_csr_offset = 0, value;
rte_spinlock_lock(lock);
value = ADF_CSR_RD(base_addr, arb_csr_offset);
if (qat_dev_gen == QAT_GEN4) {
arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
(ADF_RING_BUNDLE_SIZE_GEN4 *
txq->hw_bundle_number);
value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
arb_csr_offset);
} else {
arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
(ADF_ARB_REG_SLOT *
txq->hw_bundle_number);
value = ADF_CSR_RD(base_addr,
arb_csr_offset);
}
value |= (0x01 << txq->hw_queue_number);
ADF_CSR_WR(base_addr, arb_csr_offset, value);
rte_spinlock_unlock(lock);
}
static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
rte_spinlock_t *lock)
static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock)
{
uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
(ADF_ARB_REG_SLOT *
txq->hw_bundle_number);
uint32_t value;
uint32_t arb_csr_offset = 0, value;
rte_spinlock_lock(lock);
value = ADF_CSR_RD(base_addr, arb_csr_offset);
if (qat_dev_gen == QAT_GEN4) {
arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
(ADF_RING_BUNDLE_SIZE_GEN4 *
txq->hw_bundle_number);
value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
arb_csr_offset);
} else {
arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
(ADF_ARB_REG_SLOT *
txq->hw_bundle_number);
value = ADF_CSR_RD(base_addr,
arb_csr_offset);
}
value &= ~(0x01 << txq->hw_queue_number);
ADF_CSR_WR(base_addr, arb_csr_offset, value);
rte_spinlock_unlock(lock);
}
static void adf_configure_queues(struct qat_qp *qp)
static void adf_configure_queues(struct qat_qp *qp,
enum qat_device_gen qat_dev_gen)
{
uint32_t queue_config;
struct qat_queue *queue = &qp->tx_q;
uint32_t q_tx_config, q_resp_config;
struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
queue_config = BUILD_RING_CONFIG(queue->queue_size);
q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
ADF_RING_NEAR_WATERMARK_512,
ADF_RING_NEAR_WATERMARK_0);
WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
queue->hw_queue_number, queue_config);
queue = &qp->rx_q;
queue_config =
BUILD_RESP_RING_CONFIG(queue->queue_size,
ADF_RING_NEAR_WATERMARK_512,
ADF_RING_NEAR_WATERMARK_0);
WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
queue->hw_queue_number, queue_config);
if (qat_dev_gen == QAT_GEN4) {
WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
q_tx->hw_bundle_number, q_tx->hw_queue_number,
q_tx_config);
WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
q_rx->hw_bundle_number, q_rx->hw_queue_number,
q_resp_config);
} else {
WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
q_tx->hw_bundle_number, q_tx->hw_queue_number,
q_tx_config);
WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
q_rx->hw_bundle_number, q_rx->hw_queue_number,
q_resp_config);
}
}
static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
@ -547,14 +645,21 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
}
static inline void
txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
txq_write_tail(enum qat_device_gen qat_dev_gen,
struct qat_qp *qp, struct qat_queue *q) {
if (qat_dev_gen == QAT_GEN4) {
WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
q->hw_bundle_number, q->hw_queue_number, q->tail);
} else {
WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
q->hw_queue_number, q->tail);
q->csr_tail = q->tail;
}
}
static inline
void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp,
struct qat_queue *q)
{
uint32_t old_head, new_head;
uint32_t max_head;
@ -576,8 +681,14 @@ void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
q->csr_head = new_head;
/* write current head to CSR */
WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
q->hw_queue_number, new_head);
if (qat_dev_gen == QAT_GEN4) {
WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
q->hw_bundle_number, q->hw_queue_number, new_head);
} else {
WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
q->hw_queue_number, new_head);
}
}
uint16_t
@ -670,7 +781,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
queue->tail = tail;
tmp_qp->enqueued += nb_ops_sent;
tmp_qp->stats.enqueued_count += nb_ops_sent;
txq_write_tail(tmp_qp, queue);
txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
return nb_ops_sent;
}
@ -843,7 +954,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
queue->tail = tail;
tmp_qp->enqueued += total_descriptors_built;
tmp_qp->stats.enqueued_count += nb_ops_sent;
txq_write_tail(tmp_qp, queue);
txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
return nb_ops_sent;
}
@ -909,7 +1020,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
rx_queue->head = head;
if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
rxq_free_desc(tmp_qp, rx_queue);
rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue);
QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
op_resp_counter, fw_resp_counter);
@ -951,7 +1062,7 @@ qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
queue->head = adf_modulo(queue->head + queue->msg_size,
queue->modulo_mask);
rxq_free_desc(qp, queue);
rxq_free_desc(qp->qat_dev_gen, qp, queue);
}
return result;
@ -986,7 +1097,7 @@ qat_cq_get_fw_version(struct qat_qp *qp)
memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
queue->tail = adf_modulo(queue->tail + queue->msg_size,
queue->modulo_mask);
txq_write_tail(qp, queue);
txq_write_tail(qp->qat_dev_gen, qp, queue);
/* receive a response */
if (qat_cq_dequeue_response(qp, &response)) {

29
drivers/common/qat/qat_qp.h
View File

@ -14,6 +14,16 @@ struct qat_pci_device;
#define QAT_QP_MIN_INFL_THRESHOLD 256
/* Default qp configuration for GEN4 devices */
#define QAT_GEN4_QP_DEFCON (QAT_SERVICE_SYMMETRIC | \
QAT_SERVICE_SYMMETRIC << 8 | \
QAT_SERVICE_SYMMETRIC << 16 | \
QAT_SERVICE_SYMMETRIC << 24)
/* QAT GEN 4 specific macros */
#define QAT_GEN4_BUNDLE_NUM 4
#define QAT_GEN4_QPS_PER_BUNDLE_NUM 1
/**
* Structure with data needed for creation of queue pair.
*/
@ -26,6 +36,15 @@ struct qat_qp_hw_data {
uint16_t rx_msg_size;
};
/**
* Structure with data needed for creation of queue pair on gen4.
*/
struct qat_qp_gen4_data {
struct qat_qp_hw_data qat_qp_hw_data;
uint8_t reserved;
uint8_t valid;
};
/**
* Structure with data needed for creation of queue pair.
*/
@ -90,7 +109,7 @@ uint16_t
qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops);
int
qat_qp_release(struct qat_qp **qp_addr);
qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr);
int
qat_qp_setup(struct qat_pci_device *qat_dev,
@ -110,4 +129,12 @@ qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
void *op_cookie __rte_unused,
uint64_t *dequeue_err_count __rte_unused);
int
qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
enum qat_service_type service_type);
int
qat_read_qp_config(struct qat_pci_device *qat_dev,
enum qat_device_gen qat_dev_gen);
#endif /* _QAT_QP_H_ */

7
drivers/compress/qat/qat_comp_pmd.c
View File

@ -74,6 +74,7 @@ qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id)
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[queue_pair_id]);
struct qat_qp *qp = (struct qat_qp *)*qp_addr;
enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
uint32_t i;
QAT_LOG(DEBUG, "Release comp qp %u on device %d",
@ -90,7 +91,7 @@ qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id)
rte_free(cookie->qat_sgl_dst_d);
}
return qat_qp_release((struct qat_qp **)
return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
@ -710,6 +711,10 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev,
const struct rte_compressdev_capabilities *capabilities;
uint64_t capa_size;
if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
QAT_LOG(ERR, "Compression PMD not supported on QAT 4xxx");
return 0;
}
snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "comp");
QAT_LOG(DEBUG, "Creating QAT COMP device %s", name);

7
drivers/crypto/qat/qat_asym_pmd.c
View File

@ -103,6 +103,7 @@ static int qat_asym_qp_release(struct rte_cryptodev *dev,
uint16_t queue_pair_id)
{
struct qat_asym_dev_private *qat_private = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
QAT_LOG(DEBUG, "Release asym qp %u on device %d",
queue_pair_id, dev->data->dev_id);
@ -110,7 +111,7 @@ static int qat_asym_qp_release(struct rte_cryptodev *dev,
qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id]
= NULL;
return qat_qp_release((struct qat_qp **)
return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
@ -250,6 +251,10 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
struct rte_cryptodev *cryptodev;
struct qat_asym_dev_private *internals;
if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
QAT_LOG(ERR, "Asymmetric crypto PMD not supported on QAT 4xxx");
return 0;
}
snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "asym");
QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);

33
drivers/crypto/qat/qat_sym_pmd.c
View File

@ -139,6 +139,7 @@ static void qat_sym_stats_reset(struct rte_cryptodev *dev)
static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
{
struct qat_sym_dev_private *qat_private = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
QAT_LOG(DEBUG, "Release sym qp %u on device %d",
queue_pair_id, dev->data->dev_id);
@ -146,7 +147,7 @@ static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id]
= NULL;
return qat_qp_release((struct qat_qp **)
return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
@ -158,15 +159,33 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
int ret = 0;
uint32_t i;
struct qat_qp_config qat_qp_conf;
const struct qat_qp_hw_data *sym_hw_qps = NULL;
const struct qat_qp_hw_data *qp_hw_data = NULL;
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
struct qat_sym_dev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
const struct qat_qp_hw_data *sym_hw_qps =
qat_gen_config[qat_private->qat_dev->qat_dev_gen]
.qp_hw_data[QAT_SERVICE_SYMMETRIC];
const struct qat_qp_hw_data *qp_hw_data = sym_hw_qps + qp_id;
if (qat_dev->qat_dev_gen == QAT_GEN4) {
int ring_pair =
qat_select_valid_queue(qat_dev, qp_id,
QAT_SERVICE_SYMMETRIC);
sym_hw_qps =
&qat_dev->qp_gen4_data[0][0];
qp_hw_data =
&qat_dev->qp_gen4_data[ring_pair][0];
if (ring_pair < 0) {
QAT_LOG(ERR,
"qp_id %u invalid for this device, no enough services allocated for GEN4 device",
qp_id);
return -EINVAL;
}
} else {
sym_hw_qps = qat_gen_config[qat_dev->qat_dev_gen]
.qp_hw_data[QAT_SERVICE_SYMMETRIC];
qp_hw_data = sym_hw_qps + qp_id;
}
/* If qp is already in use free ring memory and qp metadata. */
if (*qp_addr != NULL) {
@ -430,6 +449,10 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
capabilities = qat_gen3_sym_capabilities;
capa_size = sizeof(qat_gen3_sym_capabilities);
break;
case QAT_GEN4:
capabilities = NULL;
capa_size = 0;
break;
default:
QAT_LOG(DEBUG,
"QAT gen %d capabilities unknown",

1
drivers/crypto/qat/qat_sym_session.c
View File

@ -550,6 +550,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
return -EINVAL;
}
memset(session, 0, sizeof(*session));
/* Set context descriptor physical address */
session->cd_paddr = session_paddr +
offsetof(struct qat_sym_session, cd);