d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/nfp: add flower ctrl VNIC

Browse Source 
Adds the setup/start logic for the ctrl vNIC. This vNIC is used by
the PMD and flower firmware application as a communication channel
between driver and firmware. In the case of OVS it is also used to
communicate flow statistics from hardware to the driver.

A rte_eth device is not exposed to DPDK for this vNIC as it is strictly
used internally by flower logic.

Because of the add of ctrl vNIC, a new PCItoCPPBar is needed. Modify the
related logics.

Signed-off-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Niklas Söderlund <niklas.soderlund@corigine.com>
This commit is contained in:
Chaoyong He 2022-09-26 14:59:50 +08:00 committed by Ferruh Yigit
parent 5e13debf44
commit 945441ebdb
3 changed files with 401 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

376
drivers/net/nfp/flower/nfp_flower.c
View File

@ -19,6 +19,7 @@
#include "../nfpcore/nfp_nsp.h"
#include "nfp_flower.h"
#define CTRL_VNIC_NB_DESC 512
#define DEFAULT_FLBUF_SIZE 9216
static const struct eth_dev_ops nfp_flower_pf_vnic_ops = {
@ -97,12 +98,353 @@ nfp_flower_init_vnic_common(struct nfp_net_hw *hw, const char *vnic_type)
return 0;
}
static int
nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw)
{
uint32_t i;
int ret = 0;
uint16_t n_txq;
uint16_t n_rxq;
unsigned int numa_node;
struct rte_mempool *mp;
struct nfp_net_rxq *rxq;
struct nfp_net_txq *txq;
struct nfp_pf_dev *pf_dev;
struct rte_eth_dev *eth_dev;
const struct rte_memzone *tz;
struct nfp_app_fw_flower *app_fw_flower;
/* Set up some pointers here for ease of use */
pf_dev = hw->pf_dev;
app_fw_flower = NFP_PRIV_TO_APP_FW_FLOWER(pf_dev->app_fw_priv);
ret = nfp_flower_init_vnic_common(hw, "ctrl_vnic");
if (ret != 0) {
PMD_INIT_LOG(ERR, "Could not init pf vnic");
return -EINVAL;
}
/* Allocate memory for the eth_dev of the vNIC */
hw->eth_dev = rte_zmalloc("nfp_ctrl_vnic",
sizeof(struct rte_eth_dev), RTE_CACHE_LINE_SIZE);
if (hw->eth_dev == NULL) {
PMD_INIT_LOG(ERR, "Could not allocate ctrl vnic");
return -ENOMEM;
}
/* Grab the pointer to the newly created rte_eth_dev here */
eth_dev = hw->eth_dev;
/* Also allocate memory for the data part of the eth_dev */
eth_dev->data = rte_zmalloc("nfp_ctrl_vnic_data",
sizeof(struct rte_eth_dev_data), RTE_CACHE_LINE_SIZE);
if (eth_dev->data == NULL) {
PMD_INIT_LOG(ERR, "Could not allocate ctrl vnic data");
ret = -ENOMEM;
goto eth_dev_cleanup;
}
/* Create a mbuf pool for the ctrl vNIC */
numa_node = rte_socket_id();
app_fw_flower->ctrl_pktmbuf_pool = rte_pktmbuf_pool_create("ctrl_mbuf_pool",
4 * CTRL_VNIC_NB_DESC, 64, 0, 9216, numa_node);
if (app_fw_flower->ctrl_pktmbuf_pool == NULL) {
PMD_INIT_LOG(ERR, "Create mbuf pool for ctrl vnic failed");
ret = -ENOMEM;
goto dev_data_cleanup;
}
mp = app_fw_flower->ctrl_pktmbuf_pool;
/* Configure the ctrl vNIC device */
n_rxq = hw->max_rx_queues;
n_txq = hw->max_tx_queues;
eth_dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
sizeof(eth_dev->data->rx_queues[0]) * n_rxq,
RTE_CACHE_LINE_SIZE);
if (eth_dev->data->rx_queues == NULL) {
PMD_INIT_LOG(ERR, "rte_zmalloc failed for ctrl vNIC rx queues");
ret = -ENOMEM;
goto mempool_cleanup;
}
eth_dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
sizeof(eth_dev->data->tx_queues[0]) * n_txq,
RTE_CACHE_LINE_SIZE);
if (eth_dev->data->tx_queues == NULL) {
PMD_INIT_LOG(ERR, "rte_zmalloc failed for ctrl vNIC tx queues");
ret = -ENOMEM;
goto rx_queue_free;
}
/* Fill in some of the eth_dev fields */
eth_dev->device = &pf_dev->pci_dev->device;
eth_dev->data->nb_tx_queues = n_rxq;
eth_dev->data->nb_rx_queues = n_txq;
eth_dev->data->dev_private = hw;
/* Set up the Rx queues */
for (i = 0; i < n_rxq; i++) {
rxq = rte_zmalloc_socket("ethdev RX queue",
sizeof(struct nfp_net_rxq), RTE_CACHE_LINE_SIZE,
numa_node);
if (rxq == NULL) {
PMD_DRV_LOG(ERR, "Error allocating rxq");
ret = -ENOMEM;
goto rx_queue_setup_cleanup;
}
eth_dev->data->rx_queues[i] = rxq;
/* Hw queues mapping based on firmware configuration */
rxq->qidx = i;
rxq->fl_qcidx = i * hw->stride_rx;
rxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1);
rxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx);
rxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx);
/*
* Tracking mbuf size for detecting a potential mbuf overflow due to
* RX offset
*/
rxq->mem_pool = mp;
rxq->mbuf_size = rxq->mem_pool->elt_size;
rxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
hw->flbufsz = rxq->mbuf_size;
rxq->rx_count = CTRL_VNIC_NB_DESC;
rxq->rx_free_thresh = DEFAULT_RX_FREE_THRESH;
rxq->drop_en = 1;
/*
* Allocate RX ring hardware descriptors. A memzone large enough to
* handle the maximum ring size is allocated in order to allow for
* resizing in later calls to the queue setup function.
*/
tz = rte_eth_dma_zone_reserve(eth_dev, "ctrl_rx_ring", i,
sizeof(struct nfp_net_rx_desc) * NFP_NET_MAX_RX_DESC,
NFP_MEMZONE_ALIGN, numa_node);
if (tz == NULL) {
PMD_DRV_LOG(ERR, "Error allocating rx dma");
rte_free(rxq);
ret = -ENOMEM;
goto rx_queue_setup_cleanup;
}
/* Saving physical and virtual addresses for the RX ring */
rxq->dma = (uint64_t)tz->iova;
rxq->rxds = (struct nfp_net_rx_desc *)tz->addr;
/* Mbuf pointers array for referencing mbufs linked to RX descriptors */
rxq->rxbufs = rte_zmalloc_socket("rxq->rxbufs",
sizeof(*rxq->rxbufs) * CTRL_VNIC_NB_DESC,
RTE_CACHE_LINE_SIZE, numa_node);
if (rxq->rxbufs == NULL) {
rte_eth_dma_zone_free(eth_dev, "ctrl_rx_ring", i);
rte_free(rxq);
ret = -ENOMEM;
goto rx_queue_setup_cleanup;
}
nfp_net_reset_rx_queue(rxq);
rxq->hw = hw;
/*
* Telling the HW about the physical address of the RX ring and number
* of descriptors in log2 format
*/
nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(i), rxq->dma);
nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(i), rte_log2_u32(CTRL_VNIC_NB_DESC));
}
/* Set up the Tx queues */
for (i = 0; i < n_txq; i++) {
txq = rte_zmalloc_socket("ethdev TX queue",
sizeof(struct nfp_net_txq), RTE_CACHE_LINE_SIZE,
numa_node);
if (txq == NULL) {
PMD_DRV_LOG(ERR, "Error allocating txq");
ret = -ENOMEM;
goto tx_queue_setup_cleanup;
}
eth_dev->data->tx_queues[i] = txq;
/*
* Allocate TX ring hardware descriptors. A memzone large enough to
* handle the maximum ring size is allocated in order to allow for
* resizing in later calls to the queue setup function.
*/
tz = rte_eth_dma_zone_reserve(eth_dev, "ctrl_tx_ring", i,
sizeof(struct nfp_net_nfd3_tx_desc) * NFP_NET_MAX_TX_DESC,
NFP_MEMZONE_ALIGN, numa_node);
if (tz == NULL) {
PMD_DRV_LOG(ERR, "Error allocating tx dma");
rte_free(txq);
ret = -ENOMEM;
goto tx_queue_setup_cleanup;
}
txq->tx_count = CTRL_VNIC_NB_DESC;
txq->tx_free_thresh = DEFAULT_RX_FREE_THRESH;
txq->tx_pthresh = DEFAULT_TX_PTHRESH;
txq->tx_hthresh = DEFAULT_TX_HTHRESH;
txq->tx_wthresh = DEFAULT_TX_WTHRESH;
/* Queue mapping based on firmware configuration */
txq->qidx = i;
txq->tx_qcidx = i * hw->stride_tx;
txq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx);
/* Saving physical and virtual addresses for the TX ring */
txq->dma = (uint64_t)tz->iova;
txq->txds = (struct nfp_net_nfd3_tx_desc *)tz->addr;
/* Mbuf pointers array for referencing mbufs linked to TX descriptors */
txq->txbufs = rte_zmalloc_socket("txq->txbufs",
sizeof(*txq->txbufs) * CTRL_VNIC_NB_DESC,
RTE_CACHE_LINE_SIZE, numa_node);
if (txq->txbufs == NULL) {
rte_eth_dma_zone_free(eth_dev, "ctrl_tx_ring", i);
rte_free(txq);
ret = -ENOMEM;
goto tx_queue_setup_cleanup;
}
nfp_net_reset_tx_queue(txq);
txq->hw = hw;
/*
* Telling the HW about the physical address of the TX ring and number
* of descriptors in log2 format
*/
nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(i), txq->dma);
nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(i), rte_log2_u32(CTRL_VNIC_NB_DESC));
}
return 0;
tx_queue_setup_cleanup:
for (i = 0; i < hw->max_tx_queues; i++) {
txq = eth_dev->data->tx_queues[i];
if (txq != NULL) {
rte_free(txq->txbufs);
rte_eth_dma_zone_free(eth_dev, "ctrl_tx_ring", i);
rte_free(txq);
}
}
rx_queue_setup_cleanup:
for (i = 0; i < hw->max_rx_queues; i++) {
rxq = eth_dev->data->rx_queues[i];
if (rxq != NULL) {
rte_free(rxq->rxbufs);
rte_eth_dma_zone_free(eth_dev, "ctrl_rx_ring", i);
rte_free(rxq);
}
}
rte_free(eth_dev->data->tx_queues);
rx_queue_free:
rte_free(eth_dev->data->rx_queues);
mempool_cleanup:
rte_mempool_free(mp);
dev_data_cleanup:
rte_free(eth_dev->data);
eth_dev_cleanup:
rte_free(eth_dev);
return ret;
}
static void
nfp_flower_cleanup_ctrl_vnic(struct nfp_net_hw *hw)
{
uint32_t i;
struct nfp_net_rxq *rxq;
struct nfp_net_txq *txq;
struct rte_eth_dev *eth_dev;
struct nfp_app_fw_flower *app_fw_flower;
eth_dev = hw->eth_dev;
app_fw_flower = NFP_PRIV_TO_APP_FW_FLOWER(hw->pf_dev->app_fw_priv);
for (i = 0; i < hw->max_tx_queues; i++) {
txq = eth_dev->data->tx_queues[i];
if (txq != NULL) {
rte_free(txq->txbufs);
rte_eth_dma_zone_free(eth_dev, "ctrl_tx_ring", i);
rte_free(txq);
}
}
for (i = 0; i < hw->max_rx_queues; i++) {
rxq = eth_dev->data->rx_queues[i];
if (rxq != NULL) {
rte_free(rxq->rxbufs);
rte_eth_dma_zone_free(eth_dev, "ctrl_rx_ring", i);
rte_free(rxq);
}
}
rte_free(eth_dev->data->tx_queues);
rte_free(eth_dev->data->rx_queues);
rte_mempool_free(app_fw_flower->ctrl_pktmbuf_pool);
rte_free(eth_dev->data);
rte_free(eth_dev);
}
static int
nfp_flower_start_ctrl_vnic(struct nfp_net_hw *hw)
{
int ret;
uint32_t update;
uint32_t new_ctrl;
struct rte_eth_dev *dev;
dev = hw->eth_dev;
/* Disabling queues just in case... */
nfp_net_disable_queues(dev);
/* Enabling the required queues in the device */
nfp_net_enable_queues(dev);
/* Writing configuration parameters in the device */
nfp_net_params_setup(hw);
new_ctrl = NFP_NET_CFG_CTRL_ENABLE;
update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING |
NFP_NET_CFG_UPDATE_MSIX;
rte_wmb();
/* If an error when reconfig we avoid to change hw state */
ret = nfp_net_reconfig(hw, new_ctrl, update);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Failed to reconfig ctrl vnic");
return -EIO;
}
hw->ctrl = new_ctrl;
/* Setup the freelist ring */
ret = nfp_net_rx_freelist_setup(dev);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Error with flower ctrl vNIC freelist setup");
return -EIO;
}
return 0;
}
int
nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev)
{
int ret;
unsigned int numa_node;
struct nfp_net_hw *pf_hw;
struct nfp_net_hw *ctrl_hw;
struct nfp_app_fw_flower *app_fw_flower;
numa_node = rte_socket_id();
@ -147,8 +489,42 @@ nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev)
goto pf_cpp_area_cleanup;
}
/* The ctrl vNIC struct comes directly after the PF one */
app_fw_flower->ctrl_hw = pf_hw + 1;
ctrl_hw = app_fw_flower->ctrl_hw;
/* Map the ctrl vNIC ctrl bar */
ctrl_hw->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_ctrl_bar",
32768, &ctrl_hw->ctrl_area);
if (ctrl_hw->ctrl_bar == NULL) {
PMD_INIT_LOG(ERR, "Cloud not map the ctrl vNIC ctrl bar");
ret = -ENODEV;
goto pf_cpp_area_cleanup;
}
/* Now populate the ctrl vNIC */
ctrl_hw->pf_dev = pf_dev;
ctrl_hw->cpp = pf_dev->cpp;
ret = nfp_flower_init_ctrl_vnic(app_fw_flower->ctrl_hw);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Could not initialize flower ctrl vNIC");
goto ctrl_cpp_area_cleanup;
}
/* Start the ctrl vNIC */
ret = nfp_flower_start_ctrl_vnic(app_fw_flower->ctrl_hw);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Could not start flower ctrl vNIC");
goto ctrl_vnic_cleanup;
}
return 0;
ctrl_vnic_cleanup:
nfp_flower_cleanup_ctrl_vnic(app_fw_flower->ctrl_hw);
ctrl_cpp_area_cleanup:
nfp_cpp_area_free(ctrl_hw->ctrl_area);
pf_cpp_area_cleanup:
nfp_cpp_area_free(pf_dev->ctrl_area);
vnic_cleanup:

6
drivers/net/nfp/flower/nfp_flower.h
View File

@ -10,6 +10,12 @@
struct nfp_app_fw_flower {
/* Pointer to the PF vNIC */
struct nfp_net_hw *pf_hw;
/* Pointer to a mempool for the ctrlvNIC */
struct rte_mempool *ctrl_pktmbuf_pool;
/* Pointer to the ctrl vNIC */
struct nfp_net_hw *ctrl_hw;
};
int nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev);

31
drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c
View File

@ -91,7 +91,10 @@ struct nfp6000_area_priv;
* @refcnt: number of current users
* @iomem: mapped IO memory
*/
#define NFP_BAR_MIN 1
#define NFP_BAR_MID 5
#define NFP_BAR_MAX 7
struct nfp_bar {
struct nfp_pcie_user *nfp;
uint32_t barcfg;
@ -292,6 +295,7 @@ nfp_reconfigure_bar(struct nfp_pcie_user *nfp, struct nfp_bar *bar, int tgt,
* BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
*
* Halving PCItoCPPBars for primary and secondary processes.
* For CoreNIC firmware:
* NFP PMD just requires two fixed slots, one for configuration BAR,
* and another for accessing the hw queues. Another slot is needed
* for setting the link up or down. Secondary processes do not need
@ -301,6 +305,9 @@ nfp_reconfigure_bar(struct nfp_pcie_user *nfp, struct nfp_bar *bar, int tgt,
* supported. Due to this requirement and future extensions requiring
* new slots per process, only one secondary process is supported by
* now.
* For Flower firmware:
* NFP PMD need another fixed slots, used as the configureation BAR
* for ctrl vNIC.
*/
static int
nfp_enable_bars(struct nfp_pcie_user *nfp)
@ -309,11 +316,11 @@ nfp_enable_bars(struct nfp_pcie_user *nfp)
int x, start, end;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
start = 4;
end = 1;
start = NFP_BAR_MID;
end = NFP_BAR_MIN;
} else {
start = 7;
end = 4;
start = NFP_BAR_MAX;
end = NFP_BAR_MID;
}
for (x = start; x > end; x--) {
bar = &nfp->bar[x - 1];
@ -341,11 +348,11 @@ nfp_alloc_bar(struct nfp_pcie_user *nfp)
int x, start, end;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
start = 4;
end = 1;
start = NFP_BAR_MID;
end = NFP_BAR_MIN;
} else {
start = 7;
end = 4;
start = NFP_BAR_MAX;
end = NFP_BAR_MID;
}
for (x = start; x > end; x--) {
bar = &nfp->bar[x - 1];
@ -364,11 +371,11 @@ nfp_disable_bars(struct nfp_pcie_user *nfp)
int x, start, end;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
start = 4;
end = 1;
start = NFP_BAR_MID;
end = NFP_BAR_MIN;
} else {
start = 7;
end = 4;
start = NFP_BAR_MAX;
end = NFP_BAR_MID;
}
for (x = start; x > end; x--) {