91c1667da0
strerror() input parameter should be > 0.
Coverity issue: 302864
Fixes: 3810ae4357 ("eventdev: add interrupt driven queues to Rx adapter")
Cc: stable@dpdk.org
Signed-off-by: Nikhil Rao <nikhil.rao@intel.com>
Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
2429 lines
60 KiB
C
2429 lines
60 KiB
C
/* SPDX-License-Identifier: BSD-3-Clause
|
|
* Copyright(c) 2017 Intel Corporation.
|
|
* All rights reserved.
|
|
*/
|
|
#if defined(LINUX)
|
|
#include <sys/epoll.h>
|
|
#endif
|
|
#include <unistd.h>
|
|
|
|
#include <rte_cycles.h>
|
|
#include <rte_common.h>
|
|
#include <rte_dev.h>
|
|
#include <rte_errno.h>
|
|
#include <rte_ethdev.h>
|
|
#include <rte_log.h>
|
|
#include <rte_malloc.h>
|
|
#include <rte_service_component.h>
|
|
#include <rte_thash.h>
|
|
#include <rte_interrupts.h>
|
|
|
|
#include "rte_eventdev.h"
|
|
#include "rte_eventdev_pmd.h"
|
|
#include "rte_event_eth_rx_adapter.h"
|
|
|
|
#define BATCH_SIZE 32
|
|
#define BLOCK_CNT_THRESHOLD 10
|
|
#define ETH_EVENT_BUFFER_SIZE (4*BATCH_SIZE)
|
|
|
|
#define ETH_RX_ADAPTER_SERVICE_NAME_LEN 32
|
|
#define ETH_RX_ADAPTER_MEM_NAME_LEN 32
|
|
|
|
#define RSS_KEY_SIZE 40
|
|
/* value written to intr thread pipe to signal thread exit */
|
|
#define ETH_BRIDGE_INTR_THREAD_EXIT 1
|
|
/* Sentinel value to detect initialized file handle */
|
|
#define INIT_FD -1
|
|
|
|
/*
|
|
* Used to store port and queue ID of interrupting Rx queue
|
|
*/
|
|
union queue_data {
|
|
RTE_STD_C11
|
|
void *ptr;
|
|
struct {
|
|
uint16_t port;
|
|
uint16_t queue;
|
|
};
|
|
};
|
|
|
|
/*
|
|
* There is an instance of this struct per polled Rx queue added to the
|
|
* adapter
|
|
*/
|
|
struct eth_rx_poll_entry {
|
|
/* Eth port to poll */
|
|
uint16_t eth_dev_id;
|
|
/* Eth rx queue to poll */
|
|
uint16_t eth_rx_qid;
|
|
};
|
|
|
|
/* Instance per adapter */
|
|
struct rte_eth_event_enqueue_buffer {
|
|
/* Count of events in this buffer */
|
|
uint16_t count;
|
|
/* Array of events in this buffer */
|
|
struct rte_event events[ETH_EVENT_BUFFER_SIZE];
|
|
};
|
|
|
|
struct rte_event_eth_rx_adapter {
|
|
/* RSS key */
|
|
uint8_t rss_key_be[RSS_KEY_SIZE];
|
|
/* Event device identifier */
|
|
uint8_t eventdev_id;
|
|
/* Per ethernet device structure */
|
|
struct eth_device_info *eth_devices;
|
|
/* Event port identifier */
|
|
uint8_t event_port_id;
|
|
/* Lock to serialize config updates with service function */
|
|
rte_spinlock_t rx_lock;
|
|
/* Max mbufs processed in any service function invocation */
|
|
uint32_t max_nb_rx;
|
|
/* Receive queues that need to be polled */
|
|
struct eth_rx_poll_entry *eth_rx_poll;
|
|
/* Size of the eth_rx_poll array */
|
|
uint16_t num_rx_polled;
|
|
/* Weighted round robin schedule */
|
|
uint32_t *wrr_sched;
|
|
/* wrr_sched[] size */
|
|
uint32_t wrr_len;
|
|
/* Next entry in wrr[] to begin polling */
|
|
uint32_t wrr_pos;
|
|
/* Event burst buffer */
|
|
struct rte_eth_event_enqueue_buffer event_enqueue_buffer;
|
|
/* Per adapter stats */
|
|
struct rte_event_eth_rx_adapter_stats stats;
|
|
/* Block count, counts up to BLOCK_CNT_THRESHOLD */
|
|
uint16_t enq_block_count;
|
|
/* Block start ts */
|
|
uint64_t rx_enq_block_start_ts;
|
|
/* epoll fd used to wait for Rx interrupts */
|
|
int epd;
|
|
/* Num of interrupt driven interrupt queues */
|
|
uint32_t num_rx_intr;
|
|
/* Used to send <dev id, queue id> of interrupting Rx queues from
|
|
* the interrupt thread to the Rx thread
|
|
*/
|
|
struct rte_ring *intr_ring;
|
|
/* Rx Queue data (dev id, queue id) for the last non-empty
|
|
* queue polled
|
|
*/
|
|
union queue_data qd;
|
|
/* queue_data is valid */
|
|
int qd_valid;
|
|
/* Interrupt ring lock, synchronizes Rx thread
|
|
* and interrupt thread
|
|
*/
|
|
rte_spinlock_t intr_ring_lock;
|
|
/* event array passed to rte_poll_wait */
|
|
struct rte_epoll_event *epoll_events;
|
|
/* Count of interrupt vectors in use */
|
|
uint32_t num_intr_vec;
|
|
/* Thread blocked on Rx interrupts */
|
|
pthread_t rx_intr_thread;
|
|
/* Configuration callback for rte_service configuration */
|
|
rte_event_eth_rx_adapter_conf_cb conf_cb;
|
|
/* Configuration callback argument */
|
|
void *conf_arg;
|
|
/* Set if default_cb is being used */
|
|
int default_cb_arg;
|
|
/* Service initialization state */
|
|
uint8_t service_inited;
|
|
/* Total count of Rx queues in adapter */
|
|
uint32_t nb_queues;
|
|
/* Memory allocation name */
|
|
char mem_name[ETH_RX_ADAPTER_MEM_NAME_LEN];
|
|
/* Socket identifier cached from eventdev */
|
|
int socket_id;
|
|
/* Per adapter EAL service */
|
|
uint32_t service_id;
|
|
/* Adapter started flag */
|
|
uint8_t rxa_started;
|
|
/* Adapter ID */
|
|
uint8_t id;
|
|
} __rte_cache_aligned;
|
|
|
|
/* Per eth device */
|
|
struct eth_device_info {
|
|
struct rte_eth_dev *dev;
|
|
struct eth_rx_queue_info *rx_queue;
|
|
/* Rx callback */
|
|
rte_event_eth_rx_adapter_cb_fn cb_fn;
|
|
/* Rx callback argument */
|
|
void *cb_arg;
|
|
/* Set if ethdev->eventdev packet transfer uses a
|
|
* hardware mechanism
|
|
*/
|
|
uint8_t internal_event_port;
|
|
/* Set if the adapter is processing rx queues for
|
|
* this eth device and packet processing has been
|
|
* started, allows for the code to know if the PMD
|
|
* rx_adapter_stop callback needs to be invoked
|
|
*/
|
|
uint8_t dev_rx_started;
|
|
/* Number of queues added for this device */
|
|
uint16_t nb_dev_queues;
|
|
/* Number of poll based queues
|
|
* If nb_rx_poll > 0, the start callback will
|
|
* be invoked if not already invoked
|
|
*/
|
|
uint16_t nb_rx_poll;
|
|
/* Number of interrupt based queues
|
|
* If nb_rx_intr > 0, the start callback will
|
|
* be invoked if not already invoked.
|
|
*/
|
|
uint16_t nb_rx_intr;
|
|
/* Number of queues that use the shared interrupt */
|
|
uint16_t nb_shared_intr;
|
|
/* sum(wrr(q)) for all queues within the device
|
|
* useful when deleting all device queues
|
|
*/
|
|
uint32_t wrr_len;
|
|
/* Intr based queue index to start polling from, this is used
|
|
* if the number of shared interrupts is non-zero
|
|
*/
|
|
uint16_t next_q_idx;
|
|
/* Intr based queue indices */
|
|
uint16_t *intr_queue;
|
|
/* device generates per Rx queue interrupt for queue index
|
|
* for queue indices < RTE_MAX_RXTX_INTR_VEC_ID - 1
|
|
*/
|
|
int multi_intr_cap;
|
|
/* shared interrupt enabled */
|
|
int shared_intr_enabled;
|
|
};
|
|
|
|
/* Per Rx queue */
|
|
struct eth_rx_queue_info {
|
|
int queue_enabled; /* True if added */
|
|
int intr_enabled;
|
|
uint16_t wt; /* Polling weight */
|
|
uint8_t event_queue_id; /* Event queue to enqueue packets to */
|
|
uint8_t sched_type; /* Sched type for events */
|
|
uint8_t priority; /* Event priority */
|
|
uint32_t flow_id; /* App provided flow identifier */
|
|
uint32_t flow_id_mask; /* Set to ~0 if app provides flow id else 0 */
|
|
};
|
|
|
|
static struct rte_event_eth_rx_adapter **event_eth_rx_adapter;
|
|
|
|
static inline int
|
|
rxa_validate_id(uint8_t id)
|
|
{
|
|
return id < RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
|
|
}
|
|
|
|
#define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
|
|
if (!rxa_validate_id(id)) { \
|
|
RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \
|
|
return retval; \
|
|
} \
|
|
} while (0)
|
|
|
|
static inline int
|
|
rxa_sw_adapter_queue_count(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
return rx_adapter->num_rx_polled + rx_adapter->num_rx_intr;
|
|
}
|
|
|
|
/* Greatest common divisor */
|
|
static uint16_t rxa_gcd_u16(uint16_t a, uint16_t b)
|
|
{
|
|
uint16_t r = a % b;
|
|
|
|
return r ? rxa_gcd_u16(b, r) : b;
|
|
}
|
|
|
|
/* Returns the next queue in the polling sequence
|
|
*
|
|
* http://kb.linuxvirtualserver.org/wiki/Weighted_Round-Robin_Scheduling
|
|
*/
|
|
static int
|
|
rxa_wrr_next(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
unsigned int n, int *cw,
|
|
struct eth_rx_poll_entry *eth_rx_poll, uint16_t max_wt,
|
|
uint16_t gcd, int prev)
|
|
{
|
|
int i = prev;
|
|
uint16_t w;
|
|
|
|
while (1) {
|
|
uint16_t q;
|
|
uint16_t d;
|
|
|
|
i = (i + 1) % n;
|
|
if (i == 0) {
|
|
*cw = *cw - gcd;
|
|
if (*cw <= 0)
|
|
*cw = max_wt;
|
|
}
|
|
|
|
q = eth_rx_poll[i].eth_rx_qid;
|
|
d = eth_rx_poll[i].eth_dev_id;
|
|
w = rx_adapter->eth_devices[d].rx_queue[q].wt;
|
|
|
|
if ((int)w >= *cw)
|
|
return i;
|
|
}
|
|
}
|
|
|
|
static inline int
|
|
rxa_shared_intr(struct eth_device_info *dev_info,
|
|
int rx_queue_id)
|
|
{
|
|
int multi_intr_cap;
|
|
|
|
if (dev_info->dev->intr_handle == NULL)
|
|
return 0;
|
|
|
|
multi_intr_cap = rte_intr_cap_multiple(dev_info->dev->intr_handle);
|
|
return !multi_intr_cap ||
|
|
rx_queue_id >= RTE_MAX_RXTX_INTR_VEC_ID - 1;
|
|
}
|
|
|
|
static inline int
|
|
rxa_intr_queue(struct eth_device_info *dev_info,
|
|
int rx_queue_id)
|
|
{
|
|
struct eth_rx_queue_info *queue_info;
|
|
|
|
queue_info = &dev_info->rx_queue[rx_queue_id];
|
|
return dev_info->rx_queue &&
|
|
!dev_info->internal_event_port &&
|
|
queue_info->queue_enabled && queue_info->wt == 0;
|
|
}
|
|
|
|
static inline int
|
|
rxa_polled_queue(struct eth_device_info *dev_info,
|
|
int rx_queue_id)
|
|
{
|
|
struct eth_rx_queue_info *queue_info;
|
|
|
|
queue_info = &dev_info->rx_queue[rx_queue_id];
|
|
return !dev_info->internal_event_port &&
|
|
dev_info->rx_queue &&
|
|
queue_info->queue_enabled && queue_info->wt != 0;
|
|
}
|
|
|
|
/* Calculate change in number of vectors after Rx queue ID is add/deleted */
|
|
static int
|
|
rxa_nb_intr_vect(struct eth_device_info *dev_info, int rx_queue_id, int add)
|
|
{
|
|
uint16_t i;
|
|
int n, s;
|
|
uint16_t nbq;
|
|
|
|
nbq = dev_info->dev->data->nb_rx_queues;
|
|
n = 0; /* non shared count */
|
|
s = 0; /* shared count */
|
|
|
|
if (rx_queue_id == -1) {
|
|
for (i = 0; i < nbq; i++) {
|
|
if (!rxa_shared_intr(dev_info, i))
|
|
n += add ? !rxa_intr_queue(dev_info, i) :
|
|
rxa_intr_queue(dev_info, i);
|
|
else
|
|
s += add ? !rxa_intr_queue(dev_info, i) :
|
|
rxa_intr_queue(dev_info, i);
|
|
}
|
|
|
|
if (s > 0) {
|
|
if ((add && dev_info->nb_shared_intr == 0) ||
|
|
(!add && dev_info->nb_shared_intr))
|
|
n += 1;
|
|
}
|
|
} else {
|
|
if (!rxa_shared_intr(dev_info, rx_queue_id))
|
|
n = add ? !rxa_intr_queue(dev_info, rx_queue_id) :
|
|
rxa_intr_queue(dev_info, rx_queue_id);
|
|
else
|
|
n = add ? !dev_info->nb_shared_intr :
|
|
dev_info->nb_shared_intr == 1;
|
|
}
|
|
|
|
return add ? n : -n;
|
|
}
|
|
|
|
/* Calculate nb_rx_intr after deleting interrupt mode rx queues
|
|
*/
|
|
static void
|
|
rxa_calc_nb_post_intr_del(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id,
|
|
uint32_t *nb_rx_intr)
|
|
{
|
|
uint32_t intr_diff;
|
|
|
|
if (rx_queue_id == -1)
|
|
intr_diff = dev_info->nb_rx_intr;
|
|
else
|
|
intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
|
|
|
|
*nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
|
|
}
|
|
|
|
/* Calculate nb_rx_* after adding interrupt mode rx queues, newly added
|
|
* interrupt queues could currently be poll mode Rx queues
|
|
*/
|
|
static void
|
|
rxa_calc_nb_post_add_intr(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id,
|
|
uint32_t *nb_rx_poll,
|
|
uint32_t *nb_rx_intr,
|
|
uint32_t *nb_wrr)
|
|
{
|
|
uint32_t intr_diff;
|
|
uint32_t poll_diff;
|
|
uint32_t wrr_len_diff;
|
|
|
|
if (rx_queue_id == -1) {
|
|
intr_diff = dev_info->dev->data->nb_rx_queues -
|
|
dev_info->nb_rx_intr;
|
|
poll_diff = dev_info->nb_rx_poll;
|
|
wrr_len_diff = dev_info->wrr_len;
|
|
} else {
|
|
intr_diff = !rxa_intr_queue(dev_info, rx_queue_id);
|
|
poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
|
|
wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
|
|
0;
|
|
}
|
|
|
|
*nb_rx_intr = rx_adapter->num_rx_intr + intr_diff;
|
|
*nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
|
|
*nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
|
|
}
|
|
|
|
/* Calculate size of the eth_rx_poll and wrr_sched arrays
|
|
* after deleting poll mode rx queues
|
|
*/
|
|
static void
|
|
rxa_calc_nb_post_poll_del(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id,
|
|
uint32_t *nb_rx_poll,
|
|
uint32_t *nb_wrr)
|
|
{
|
|
uint32_t poll_diff;
|
|
uint32_t wrr_len_diff;
|
|
|
|
if (rx_queue_id == -1) {
|
|
poll_diff = dev_info->nb_rx_poll;
|
|
wrr_len_diff = dev_info->wrr_len;
|
|
} else {
|
|
poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
|
|
wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
|
|
0;
|
|
}
|
|
|
|
*nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
|
|
*nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
|
|
}
|
|
|
|
/* Calculate nb_rx_* after adding poll mode rx queues
|
|
*/
|
|
static void
|
|
rxa_calc_nb_post_add_poll(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id,
|
|
uint16_t wt,
|
|
uint32_t *nb_rx_poll,
|
|
uint32_t *nb_rx_intr,
|
|
uint32_t *nb_wrr)
|
|
{
|
|
uint32_t intr_diff;
|
|
uint32_t poll_diff;
|
|
uint32_t wrr_len_diff;
|
|
|
|
if (rx_queue_id == -1) {
|
|
intr_diff = dev_info->nb_rx_intr;
|
|
poll_diff = dev_info->dev->data->nb_rx_queues -
|
|
dev_info->nb_rx_poll;
|
|
wrr_len_diff = wt*dev_info->dev->data->nb_rx_queues
|
|
- dev_info->wrr_len;
|
|
} else {
|
|
intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
|
|
poll_diff = !rxa_polled_queue(dev_info, rx_queue_id);
|
|
wrr_len_diff = rxa_polled_queue(dev_info, rx_queue_id) ?
|
|
wt - dev_info->rx_queue[rx_queue_id].wt :
|
|
wt;
|
|
}
|
|
|
|
*nb_rx_poll = rx_adapter->num_rx_polled + poll_diff;
|
|
*nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
|
|
*nb_wrr = rx_adapter->wrr_len + wrr_len_diff;
|
|
}
|
|
|
|
/* Calculate nb_rx_* after adding rx_queue_id */
|
|
static void
|
|
rxa_calc_nb_post_add(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id,
|
|
uint16_t wt,
|
|
uint32_t *nb_rx_poll,
|
|
uint32_t *nb_rx_intr,
|
|
uint32_t *nb_wrr)
|
|
{
|
|
if (wt != 0)
|
|
rxa_calc_nb_post_add_poll(rx_adapter, dev_info, rx_queue_id,
|
|
wt, nb_rx_poll, nb_rx_intr, nb_wrr);
|
|
else
|
|
rxa_calc_nb_post_add_intr(rx_adapter, dev_info, rx_queue_id,
|
|
nb_rx_poll, nb_rx_intr, nb_wrr);
|
|
}
|
|
|
|
/* Calculate nb_rx_* after deleting rx_queue_id */
|
|
static void
|
|
rxa_calc_nb_post_del(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id,
|
|
uint32_t *nb_rx_poll,
|
|
uint32_t *nb_rx_intr,
|
|
uint32_t *nb_wrr)
|
|
{
|
|
rxa_calc_nb_post_poll_del(rx_adapter, dev_info, rx_queue_id, nb_rx_poll,
|
|
nb_wrr);
|
|
rxa_calc_nb_post_intr_del(rx_adapter, dev_info, rx_queue_id,
|
|
nb_rx_intr);
|
|
}
|
|
|
|
/*
|
|
* Allocate the rx_poll array
|
|
*/
|
|
static struct eth_rx_poll_entry *
|
|
rxa_alloc_poll(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
uint32_t num_rx_polled)
|
|
{
|
|
size_t len;
|
|
|
|
len = RTE_ALIGN(num_rx_polled * sizeof(*rx_adapter->eth_rx_poll),
|
|
RTE_CACHE_LINE_SIZE);
|
|
return rte_zmalloc_socket(rx_adapter->mem_name,
|
|
len,
|
|
RTE_CACHE_LINE_SIZE,
|
|
rx_adapter->socket_id);
|
|
}
|
|
|
|
/*
|
|
* Allocate the WRR array
|
|
*/
|
|
static uint32_t *
|
|
rxa_alloc_wrr(struct rte_event_eth_rx_adapter *rx_adapter, int nb_wrr)
|
|
{
|
|
size_t len;
|
|
|
|
len = RTE_ALIGN(nb_wrr * sizeof(*rx_adapter->wrr_sched),
|
|
RTE_CACHE_LINE_SIZE);
|
|
return rte_zmalloc_socket(rx_adapter->mem_name,
|
|
len,
|
|
RTE_CACHE_LINE_SIZE,
|
|
rx_adapter->socket_id);
|
|
}
|
|
|
|
static int
|
|
rxa_alloc_poll_arrays(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
uint32_t nb_poll,
|
|
uint32_t nb_wrr,
|
|
struct eth_rx_poll_entry **rx_poll,
|
|
uint32_t **wrr_sched)
|
|
{
|
|
|
|
if (nb_poll == 0) {
|
|
*rx_poll = NULL;
|
|
*wrr_sched = NULL;
|
|
return 0;
|
|
}
|
|
|
|
*rx_poll = rxa_alloc_poll(rx_adapter, nb_poll);
|
|
if (*rx_poll == NULL) {
|
|
*wrr_sched = NULL;
|
|
return -ENOMEM;
|
|
}
|
|
|
|
*wrr_sched = rxa_alloc_wrr(rx_adapter, nb_wrr);
|
|
if (*wrr_sched == NULL) {
|
|
rte_free(*rx_poll);
|
|
return -ENOMEM;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* Precalculate WRR polling sequence for all queues in rx_adapter */
|
|
static void
|
|
rxa_calc_wrr_sequence(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_rx_poll_entry *rx_poll,
|
|
uint32_t *rx_wrr)
|
|
{
|
|
uint16_t d;
|
|
uint16_t q;
|
|
unsigned int i;
|
|
int prev = -1;
|
|
int cw = -1;
|
|
|
|
/* Initialize variables for calculation of wrr schedule */
|
|
uint16_t max_wrr_pos = 0;
|
|
unsigned int poll_q = 0;
|
|
uint16_t max_wt = 0;
|
|
uint16_t gcd = 0;
|
|
|
|
if (rx_poll == NULL)
|
|
return;
|
|
|
|
/* Generate array of all queues to poll, the size of this
|
|
* array is poll_q
|
|
*/
|
|
RTE_ETH_FOREACH_DEV(d) {
|
|
uint16_t nb_rx_queues;
|
|
struct eth_device_info *dev_info =
|
|
&rx_adapter->eth_devices[d];
|
|
nb_rx_queues = dev_info->dev->data->nb_rx_queues;
|
|
if (dev_info->rx_queue == NULL)
|
|
continue;
|
|
if (dev_info->internal_event_port)
|
|
continue;
|
|
dev_info->wrr_len = 0;
|
|
for (q = 0; q < nb_rx_queues; q++) {
|
|
struct eth_rx_queue_info *queue_info =
|
|
&dev_info->rx_queue[q];
|
|
uint16_t wt;
|
|
|
|
if (!rxa_polled_queue(dev_info, q))
|
|
continue;
|
|
wt = queue_info->wt;
|
|
rx_poll[poll_q].eth_dev_id = d;
|
|
rx_poll[poll_q].eth_rx_qid = q;
|
|
max_wrr_pos += wt;
|
|
dev_info->wrr_len += wt;
|
|
max_wt = RTE_MAX(max_wt, wt);
|
|
gcd = (gcd) ? rxa_gcd_u16(gcd, wt) : wt;
|
|
poll_q++;
|
|
}
|
|
}
|
|
|
|
/* Generate polling sequence based on weights */
|
|
prev = -1;
|
|
cw = -1;
|
|
for (i = 0; i < max_wrr_pos; i++) {
|
|
rx_wrr[i] = rxa_wrr_next(rx_adapter, poll_q, &cw,
|
|
rx_poll, max_wt, gcd, prev);
|
|
prev = rx_wrr[i];
|
|
}
|
|
}
|
|
|
|
static inline void
|
|
rxa_mtoip(struct rte_mbuf *m, struct ipv4_hdr **ipv4_hdr,
|
|
struct ipv6_hdr **ipv6_hdr)
|
|
{
|
|
struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
|
|
struct vlan_hdr *vlan_hdr;
|
|
|
|
*ipv4_hdr = NULL;
|
|
*ipv6_hdr = NULL;
|
|
|
|
switch (eth_hdr->ether_type) {
|
|
case RTE_BE16(ETHER_TYPE_IPv4):
|
|
*ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
|
|
break;
|
|
|
|
case RTE_BE16(ETHER_TYPE_IPv6):
|
|
*ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1);
|
|
break;
|
|
|
|
case RTE_BE16(ETHER_TYPE_VLAN):
|
|
vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
|
|
switch (vlan_hdr->eth_proto) {
|
|
case RTE_BE16(ETHER_TYPE_IPv4):
|
|
*ipv4_hdr = (struct ipv4_hdr *)(vlan_hdr + 1);
|
|
break;
|
|
case RTE_BE16(ETHER_TYPE_IPv6):
|
|
*ipv6_hdr = (struct ipv6_hdr *)(vlan_hdr + 1);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
break;
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* Calculate RSS hash for IPv4/6 */
|
|
static inline uint32_t
|
|
rxa_do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
|
|
{
|
|
uint32_t input_len;
|
|
void *tuple;
|
|
struct rte_ipv4_tuple ipv4_tuple;
|
|
struct rte_ipv6_tuple ipv6_tuple;
|
|
struct ipv4_hdr *ipv4_hdr;
|
|
struct ipv6_hdr *ipv6_hdr;
|
|
|
|
rxa_mtoip(m, &ipv4_hdr, &ipv6_hdr);
|
|
|
|
if (ipv4_hdr) {
|
|
ipv4_tuple.src_addr = rte_be_to_cpu_32(ipv4_hdr->src_addr);
|
|
ipv4_tuple.dst_addr = rte_be_to_cpu_32(ipv4_hdr->dst_addr);
|
|
tuple = &ipv4_tuple;
|
|
input_len = RTE_THASH_V4_L3_LEN;
|
|
} else if (ipv6_hdr) {
|
|
rte_thash_load_v6_addrs(ipv6_hdr,
|
|
(union rte_thash_tuple *)&ipv6_tuple);
|
|
tuple = &ipv6_tuple;
|
|
input_len = RTE_THASH_V6_L3_LEN;
|
|
} else
|
|
return 0;
|
|
|
|
return rte_softrss_be(tuple, input_len, rss_key_be);
|
|
}
|
|
|
|
static inline int
|
|
rxa_enq_blocked(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
return !!rx_adapter->enq_block_count;
|
|
}
|
|
|
|
static inline void
|
|
rxa_enq_block_start_ts(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
if (rx_adapter->rx_enq_block_start_ts)
|
|
return;
|
|
|
|
rx_adapter->enq_block_count++;
|
|
if (rx_adapter->enq_block_count < BLOCK_CNT_THRESHOLD)
|
|
return;
|
|
|
|
rx_adapter->rx_enq_block_start_ts = rte_get_tsc_cycles();
|
|
}
|
|
|
|
static inline void
|
|
rxa_enq_block_end_ts(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct rte_event_eth_rx_adapter_stats *stats)
|
|
{
|
|
if (unlikely(!stats->rx_enq_start_ts))
|
|
stats->rx_enq_start_ts = rte_get_tsc_cycles();
|
|
|
|
if (likely(!rxa_enq_blocked(rx_adapter)))
|
|
return;
|
|
|
|
rx_adapter->enq_block_count = 0;
|
|
if (rx_adapter->rx_enq_block_start_ts) {
|
|
stats->rx_enq_end_ts = rte_get_tsc_cycles();
|
|
stats->rx_enq_block_cycles += stats->rx_enq_end_ts -
|
|
rx_adapter->rx_enq_block_start_ts;
|
|
rx_adapter->rx_enq_block_start_ts = 0;
|
|
}
|
|
}
|
|
|
|
/* Add event to buffer, free space check is done prior to calling
|
|
* this function
|
|
*/
|
|
static inline void
|
|
rxa_buffer_event(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct rte_event *ev)
|
|
{
|
|
struct rte_eth_event_enqueue_buffer *buf =
|
|
&rx_adapter->event_enqueue_buffer;
|
|
rte_memcpy(&buf->events[buf->count++], ev, sizeof(struct rte_event));
|
|
}
|
|
|
|
/* Enqueue buffered events to event device */
|
|
static inline uint16_t
|
|
rxa_flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
struct rte_eth_event_enqueue_buffer *buf =
|
|
&rx_adapter->event_enqueue_buffer;
|
|
struct rte_event_eth_rx_adapter_stats *stats = &rx_adapter->stats;
|
|
|
|
uint16_t n = rte_event_enqueue_new_burst(rx_adapter->eventdev_id,
|
|
rx_adapter->event_port_id,
|
|
buf->events,
|
|
buf->count);
|
|
if (n != buf->count) {
|
|
memmove(buf->events,
|
|
&buf->events[n],
|
|
(buf->count - n) * sizeof(struct rte_event));
|
|
stats->rx_enq_retry++;
|
|
}
|
|
|
|
n ? rxa_enq_block_end_ts(rx_adapter, stats) :
|
|
rxa_enq_block_start_ts(rx_adapter);
|
|
|
|
buf->count -= n;
|
|
stats->rx_enq_count += n;
|
|
|
|
return n;
|
|
}
|
|
|
|
static inline void
|
|
rxa_buffer_mbufs(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
uint16_t eth_dev_id,
|
|
uint16_t rx_queue_id,
|
|
struct rte_mbuf **mbufs,
|
|
uint16_t num)
|
|
{
|
|
uint32_t i;
|
|
struct eth_device_info *dev_info =
|
|
&rx_adapter->eth_devices[eth_dev_id];
|
|
struct eth_rx_queue_info *eth_rx_queue_info =
|
|
&dev_info->rx_queue[rx_queue_id];
|
|
struct rte_eth_event_enqueue_buffer *buf =
|
|
&rx_adapter->event_enqueue_buffer;
|
|
int32_t qid = eth_rx_queue_info->event_queue_id;
|
|
uint8_t sched_type = eth_rx_queue_info->sched_type;
|
|
uint8_t priority = eth_rx_queue_info->priority;
|
|
uint32_t flow_id;
|
|
struct rte_event events[BATCH_SIZE];
|
|
struct rte_mbuf *m = mbufs[0];
|
|
uint32_t rss_mask;
|
|
uint32_t rss;
|
|
int do_rss;
|
|
uint64_t ts;
|
|
struct rte_mbuf *cb_mbufs[BATCH_SIZE];
|
|
uint16_t nb_cb;
|
|
|
|
/* 0xffff ffff if PKT_RX_RSS_HASH is set, otherwise 0 */
|
|
rss_mask = ~(((m->ol_flags & PKT_RX_RSS_HASH) != 0) - 1);
|
|
do_rss = !rss_mask && !eth_rx_queue_info->flow_id_mask;
|
|
|
|
if ((m->ol_flags & PKT_RX_TIMESTAMP) == 0) {
|
|
ts = rte_get_tsc_cycles();
|
|
for (i = 0; i < num; i++) {
|
|
m = mbufs[i];
|
|
|
|
m->timestamp = ts;
|
|
m->ol_flags |= PKT_RX_TIMESTAMP;
|
|
}
|
|
}
|
|
|
|
|
|
nb_cb = dev_info->cb_fn ? dev_info->cb_fn(eth_dev_id, rx_queue_id,
|
|
ETH_EVENT_BUFFER_SIZE,
|
|
buf->count, mbufs,
|
|
num,
|
|
dev_info->cb_arg,
|
|
cb_mbufs) :
|
|
num;
|
|
if (nb_cb < num) {
|
|
mbufs = cb_mbufs;
|
|
num = nb_cb;
|
|
}
|
|
|
|
for (i = 0; i < num; i++) {
|
|
m = mbufs[i];
|
|
struct rte_event *ev = &events[i];
|
|
|
|
rss = do_rss ?
|
|
rxa_do_softrss(m, rx_adapter->rss_key_be) :
|
|
m->hash.rss;
|
|
flow_id =
|
|
eth_rx_queue_info->flow_id &
|
|
eth_rx_queue_info->flow_id_mask;
|
|
flow_id |= rss & ~eth_rx_queue_info->flow_id_mask;
|
|
ev->flow_id = flow_id;
|
|
ev->op = RTE_EVENT_OP_NEW;
|
|
ev->sched_type = sched_type;
|
|
ev->queue_id = qid;
|
|
ev->event_type = RTE_EVENT_TYPE_ETH_RX_ADAPTER;
|
|
ev->sub_event_type = 0;
|
|
ev->priority = priority;
|
|
ev->mbuf = m;
|
|
|
|
rxa_buffer_event(rx_adapter, ev);
|
|
}
|
|
}
|
|
|
|
/* Enqueue packets from <port, q> to event buffer */
|
|
static inline uint32_t
|
|
rxa_eth_rx(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
uint16_t port_id,
|
|
uint16_t queue_id,
|
|
uint32_t rx_count,
|
|
uint32_t max_rx,
|
|
int *rxq_empty)
|
|
{
|
|
struct rte_mbuf *mbufs[BATCH_SIZE];
|
|
struct rte_eth_event_enqueue_buffer *buf =
|
|
&rx_adapter->event_enqueue_buffer;
|
|
struct rte_event_eth_rx_adapter_stats *stats =
|
|
&rx_adapter->stats;
|
|
uint16_t n;
|
|
uint32_t nb_rx = 0;
|
|
|
|
if (rxq_empty)
|
|
*rxq_empty = 0;
|
|
/* Don't do a batch dequeue from the rx queue if there isn't
|
|
* enough space in the enqueue buffer.
|
|
*/
|
|
while (BATCH_SIZE <= (RTE_DIM(buf->events) - buf->count)) {
|
|
if (buf->count >= BATCH_SIZE)
|
|
rxa_flush_event_buffer(rx_adapter);
|
|
|
|
stats->rx_poll_count++;
|
|
n = rte_eth_rx_burst(port_id, queue_id, mbufs, BATCH_SIZE);
|
|
if (unlikely(!n)) {
|
|
if (rxq_empty)
|
|
*rxq_empty = 1;
|
|
break;
|
|
}
|
|
rxa_buffer_mbufs(rx_adapter, port_id, queue_id, mbufs, n);
|
|
nb_rx += n;
|
|
if (rx_count + nb_rx > max_rx)
|
|
break;
|
|
}
|
|
|
|
if (buf->count >= BATCH_SIZE)
|
|
rxa_flush_event_buffer(rx_adapter);
|
|
|
|
return nb_rx;
|
|
}
|
|
|
|
static inline void
|
|
rxa_intr_ring_enqueue(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
void *data)
|
|
{
|
|
uint16_t port_id;
|
|
uint16_t queue;
|
|
int err;
|
|
union queue_data qd;
|
|
struct eth_device_info *dev_info;
|
|
struct eth_rx_queue_info *queue_info;
|
|
int *intr_enabled;
|
|
|
|
qd.ptr = data;
|
|
port_id = qd.port;
|
|
queue = qd.queue;
|
|
|
|
dev_info = &rx_adapter->eth_devices[port_id];
|
|
queue_info = &dev_info->rx_queue[queue];
|
|
rte_spinlock_lock(&rx_adapter->intr_ring_lock);
|
|
if (rxa_shared_intr(dev_info, queue))
|
|
intr_enabled = &dev_info->shared_intr_enabled;
|
|
else
|
|
intr_enabled = &queue_info->intr_enabled;
|
|
|
|
if (*intr_enabled) {
|
|
*intr_enabled = 0;
|
|
err = rte_ring_enqueue(rx_adapter->intr_ring, data);
|
|
/* Entry should always be available.
|
|
* The ring size equals the maximum number of interrupt
|
|
* vectors supported (an interrupt vector is shared in
|
|
* case of shared interrupts)
|
|
*/
|
|
if (err)
|
|
RTE_EDEV_LOG_ERR("Failed to enqueue interrupt"
|
|
" to ring: %s", strerror(-err));
|
|
else
|
|
rte_eth_dev_rx_intr_disable(port_id, queue);
|
|
}
|
|
rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
|
|
}
|
|
|
|
static int
|
|
rxa_intr_ring_check_avail(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
uint32_t num_intr_vec)
|
|
{
|
|
if (rx_adapter->num_intr_vec + num_intr_vec >
|
|
RTE_EVENT_ETH_INTR_RING_SIZE) {
|
|
RTE_EDEV_LOG_ERR("Exceeded intr ring slots current"
|
|
" %d needed %d limit %d", rx_adapter->num_intr_vec,
|
|
num_intr_vec, RTE_EVENT_ETH_INTR_RING_SIZE);
|
|
return -ENOSPC;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Delete entries for (dev, queue) from the interrupt ring */
|
|
static void
|
|
rxa_intr_ring_del_entries(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
uint16_t rx_queue_id)
|
|
{
|
|
int i, n;
|
|
union queue_data qd;
|
|
|
|
rte_spinlock_lock(&rx_adapter->intr_ring_lock);
|
|
|
|
n = rte_ring_count(rx_adapter->intr_ring);
|
|
for (i = 0; i < n; i++) {
|
|
rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
|
|
if (!rxa_shared_intr(dev_info, rx_queue_id)) {
|
|
if (qd.port == dev_info->dev->data->port_id &&
|
|
qd.queue == rx_queue_id)
|
|
continue;
|
|
} else {
|
|
if (qd.port == dev_info->dev->data->port_id)
|
|
continue;
|
|
}
|
|
rte_ring_enqueue(rx_adapter->intr_ring, qd.ptr);
|
|
}
|
|
|
|
rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
|
|
}
|
|
|
|
/* pthread callback handling interrupt mode receive queues
|
|
* After receiving an Rx interrupt, it enqueues the port id and queue id of the
|
|
* interrupting queue to the adapter's ring buffer for interrupt events.
|
|
* These events are picked up by rxa_intr_ring_dequeue() which is invoked from
|
|
* the adapter service function.
|
|
*/
|
|
static void *
|
|
rxa_intr_thread(void *arg)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter = arg;
|
|
struct rte_epoll_event *epoll_events = rx_adapter->epoll_events;
|
|
int n, i;
|
|
|
|
while (1) {
|
|
n = rte_epoll_wait(rx_adapter->epd, epoll_events,
|
|
RTE_EVENT_ETH_INTR_RING_SIZE, -1);
|
|
if (unlikely(n < 0))
|
|
RTE_EDEV_LOG_ERR("rte_epoll_wait returned error %d",
|
|
n);
|
|
for (i = 0; i < n; i++) {
|
|
rxa_intr_ring_enqueue(rx_adapter,
|
|
epoll_events[i].epdata.data);
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* Dequeue <port, q> from interrupt ring and enqueue received
|
|
* mbufs to eventdev
|
|
*/
|
|
static inline uint32_t
|
|
rxa_intr_ring_dequeue(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
uint32_t n;
|
|
uint32_t nb_rx = 0;
|
|
int rxq_empty;
|
|
struct rte_eth_event_enqueue_buffer *buf;
|
|
rte_spinlock_t *ring_lock;
|
|
uint8_t max_done = 0;
|
|
|
|
if (rx_adapter->num_rx_intr == 0)
|
|
return 0;
|
|
|
|
if (rte_ring_count(rx_adapter->intr_ring) == 0
|
|
&& !rx_adapter->qd_valid)
|
|
return 0;
|
|
|
|
buf = &rx_adapter->event_enqueue_buffer;
|
|
ring_lock = &rx_adapter->intr_ring_lock;
|
|
|
|
if (buf->count >= BATCH_SIZE)
|
|
rxa_flush_event_buffer(rx_adapter);
|
|
|
|
while (BATCH_SIZE <= (RTE_DIM(buf->events) - buf->count)) {
|
|
struct eth_device_info *dev_info;
|
|
uint16_t port;
|
|
uint16_t queue;
|
|
union queue_data qd = rx_adapter->qd;
|
|
int err;
|
|
|
|
if (!rx_adapter->qd_valid) {
|
|
struct eth_rx_queue_info *queue_info;
|
|
|
|
rte_spinlock_lock(ring_lock);
|
|
err = rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
|
|
if (err) {
|
|
rte_spinlock_unlock(ring_lock);
|
|
break;
|
|
}
|
|
|
|
port = qd.port;
|
|
queue = qd.queue;
|
|
rx_adapter->qd = qd;
|
|
rx_adapter->qd_valid = 1;
|
|
dev_info = &rx_adapter->eth_devices[port];
|
|
if (rxa_shared_intr(dev_info, queue))
|
|
dev_info->shared_intr_enabled = 1;
|
|
else {
|
|
queue_info = &dev_info->rx_queue[queue];
|
|
queue_info->intr_enabled = 1;
|
|
}
|
|
rte_eth_dev_rx_intr_enable(port, queue);
|
|
rte_spinlock_unlock(ring_lock);
|
|
} else {
|
|
port = qd.port;
|
|
queue = qd.queue;
|
|
|
|
dev_info = &rx_adapter->eth_devices[port];
|
|
}
|
|
|
|
if (rxa_shared_intr(dev_info, queue)) {
|
|
uint16_t i;
|
|
uint16_t nb_queues;
|
|
|
|
nb_queues = dev_info->dev->data->nb_rx_queues;
|
|
n = 0;
|
|
for (i = dev_info->next_q_idx; i < nb_queues; i++) {
|
|
uint8_t enq_buffer_full;
|
|
|
|
if (!rxa_intr_queue(dev_info, i))
|
|
continue;
|
|
n = rxa_eth_rx(rx_adapter, port, i, nb_rx,
|
|
rx_adapter->max_nb_rx,
|
|
&rxq_empty);
|
|
nb_rx += n;
|
|
|
|
enq_buffer_full = !rxq_empty && n == 0;
|
|
max_done = nb_rx > rx_adapter->max_nb_rx;
|
|
|
|
if (enq_buffer_full || max_done) {
|
|
dev_info->next_q_idx = i;
|
|
goto done;
|
|
}
|
|
}
|
|
|
|
rx_adapter->qd_valid = 0;
|
|
|
|
/* Reinitialize for next interrupt */
|
|
dev_info->next_q_idx = dev_info->multi_intr_cap ?
|
|
RTE_MAX_RXTX_INTR_VEC_ID - 1 :
|
|
0;
|
|
} else {
|
|
n = rxa_eth_rx(rx_adapter, port, queue, nb_rx,
|
|
rx_adapter->max_nb_rx,
|
|
&rxq_empty);
|
|
rx_adapter->qd_valid = !rxq_empty;
|
|
nb_rx += n;
|
|
if (nb_rx > rx_adapter->max_nb_rx)
|
|
break;
|
|
}
|
|
}
|
|
|
|
done:
|
|
rx_adapter->stats.rx_intr_packets += nb_rx;
|
|
return nb_rx;
|
|
}
|
|
|
|
/*
|
|
* Polls receive queues added to the event adapter and enqueues received
|
|
* packets to the event device.
|
|
*
|
|
* The receive code enqueues initially to a temporary buffer, the
|
|
* temporary buffer is drained anytime it holds >= BATCH_SIZE packets
|
|
*
|
|
* If there isn't space available in the temporary buffer, packets from the
|
|
* Rx queue aren't dequeued from the eth device, this back pressures the
|
|
* eth device, in virtual device environments this back pressure is relayed to
|
|
* the hypervisor's switching layer where adjustments can be made to deal with
|
|
* it.
|
|
*/
|
|
static inline uint32_t
|
|
rxa_poll(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
uint32_t num_queue;
|
|
uint32_t nb_rx = 0;
|
|
struct rte_eth_event_enqueue_buffer *buf;
|
|
uint32_t wrr_pos;
|
|
uint32_t max_nb_rx;
|
|
|
|
wrr_pos = rx_adapter->wrr_pos;
|
|
max_nb_rx = rx_adapter->max_nb_rx;
|
|
buf = &rx_adapter->event_enqueue_buffer;
|
|
|
|
/* Iterate through a WRR sequence */
|
|
for (num_queue = 0; num_queue < rx_adapter->wrr_len; num_queue++) {
|
|
unsigned int poll_idx = rx_adapter->wrr_sched[wrr_pos];
|
|
uint16_t qid = rx_adapter->eth_rx_poll[poll_idx].eth_rx_qid;
|
|
uint16_t d = rx_adapter->eth_rx_poll[poll_idx].eth_dev_id;
|
|
|
|
/* Don't do a batch dequeue from the rx queue if there isn't
|
|
* enough space in the enqueue buffer.
|
|
*/
|
|
if (buf->count >= BATCH_SIZE)
|
|
rxa_flush_event_buffer(rx_adapter);
|
|
if (BATCH_SIZE > (ETH_EVENT_BUFFER_SIZE - buf->count)) {
|
|
rx_adapter->wrr_pos = wrr_pos;
|
|
return nb_rx;
|
|
}
|
|
|
|
nb_rx += rxa_eth_rx(rx_adapter, d, qid, nb_rx, max_nb_rx,
|
|
NULL);
|
|
if (nb_rx > max_nb_rx) {
|
|
rx_adapter->wrr_pos =
|
|
(wrr_pos + 1) % rx_adapter->wrr_len;
|
|
break;
|
|
}
|
|
|
|
if (++wrr_pos == rx_adapter->wrr_len)
|
|
wrr_pos = 0;
|
|
}
|
|
return nb_rx;
|
|
}
|
|
|
|
static int
|
|
rxa_service_func(void *args)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter = args;
|
|
struct rte_event_eth_rx_adapter_stats *stats;
|
|
|
|
if (rte_spinlock_trylock(&rx_adapter->rx_lock) == 0)
|
|
return 0;
|
|
if (!rx_adapter->rxa_started) {
|
|
rte_spinlock_unlock(&rx_adapter->rx_lock);
|
|
return 0;
|
|
}
|
|
|
|
stats = &rx_adapter->stats;
|
|
stats->rx_packets += rxa_intr_ring_dequeue(rx_adapter);
|
|
stats->rx_packets += rxa_poll(rx_adapter);
|
|
rte_spinlock_unlock(&rx_adapter->rx_lock);
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
rte_event_eth_rx_adapter_init(void)
|
|
{
|
|
const char *name = "rte_event_eth_rx_adapter_array";
|
|
const struct rte_memzone *mz;
|
|
unsigned int sz;
|
|
|
|
sz = sizeof(*event_eth_rx_adapter) *
|
|
RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
|
|
sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
|
|
|
|
mz = rte_memzone_lookup(name);
|
|
if (mz == NULL) {
|
|
mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
|
|
RTE_CACHE_LINE_SIZE);
|
|
if (mz == NULL) {
|
|
RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
|
|
PRId32, rte_errno);
|
|
return -rte_errno;
|
|
}
|
|
}
|
|
|
|
event_eth_rx_adapter = mz->addr;
|
|
return 0;
|
|
}
|
|
|
|
static inline struct rte_event_eth_rx_adapter *
|
|
rxa_id_to_adapter(uint8_t id)
|
|
{
|
|
return event_eth_rx_adapter ?
|
|
event_eth_rx_adapter[id] : NULL;
|
|
}
|
|
|
|
static int
|
|
rxa_default_conf_cb(uint8_t id, uint8_t dev_id,
|
|
struct rte_event_eth_rx_adapter_conf *conf, void *arg)
|
|
{
|
|
int ret;
|
|
struct rte_eventdev *dev;
|
|
struct rte_event_dev_config dev_conf;
|
|
int started;
|
|
uint8_t port_id;
|
|
struct rte_event_port_conf *port_conf = arg;
|
|
struct rte_event_eth_rx_adapter *rx_adapter = rxa_id_to_adapter(id);
|
|
|
|
dev = &rte_eventdevs[rx_adapter->eventdev_id];
|
|
dev_conf = dev->data->dev_conf;
|
|
|
|
started = dev->data->dev_started;
|
|
if (started)
|
|
rte_event_dev_stop(dev_id);
|
|
port_id = dev_conf.nb_event_ports;
|
|
dev_conf.nb_event_ports += 1;
|
|
ret = rte_event_dev_configure(dev_id, &dev_conf);
|
|
if (ret) {
|
|
RTE_EDEV_LOG_ERR("failed to configure event dev %u\n",
|
|
dev_id);
|
|
if (started) {
|
|
if (rte_event_dev_start(dev_id))
|
|
return -EIO;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
ret = rte_event_port_setup(dev_id, port_id, port_conf);
|
|
if (ret) {
|
|
RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
|
|
port_id);
|
|
return ret;
|
|
}
|
|
|
|
conf->event_port_id = port_id;
|
|
conf->max_nb_rx = 128;
|
|
if (started)
|
|
ret = rte_event_dev_start(dev_id);
|
|
rx_adapter->default_cb_arg = 1;
|
|
return ret;
|
|
}
|
|
|
|
static int
|
|
rxa_epoll_create1(void)
|
|
{
|
|
#if defined(LINUX)
|
|
int fd;
|
|
fd = epoll_create1(EPOLL_CLOEXEC);
|
|
return fd < 0 ? -errno : fd;
|
|
#elif defined(BSD)
|
|
return -ENOTSUP;
|
|
#endif
|
|
}
|
|
|
|
static int
|
|
rxa_init_epd(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
if (rx_adapter->epd != INIT_FD)
|
|
return 0;
|
|
|
|
rx_adapter->epd = rxa_epoll_create1();
|
|
if (rx_adapter->epd < 0) {
|
|
int err = rx_adapter->epd;
|
|
rx_adapter->epd = INIT_FD;
|
|
RTE_EDEV_LOG_ERR("epoll_create1() failed, err %d", err);
|
|
return err;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
rxa_create_intr_thread(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
int err;
|
|
char thread_name[RTE_MAX_THREAD_NAME_LEN];
|
|
|
|
if (rx_adapter->intr_ring)
|
|
return 0;
|
|
|
|
rx_adapter->intr_ring = rte_ring_create("intr_ring",
|
|
RTE_EVENT_ETH_INTR_RING_SIZE,
|
|
rte_socket_id(), 0);
|
|
if (!rx_adapter->intr_ring)
|
|
return -ENOMEM;
|
|
|
|
rx_adapter->epoll_events = rte_zmalloc_socket(rx_adapter->mem_name,
|
|
RTE_EVENT_ETH_INTR_RING_SIZE *
|
|
sizeof(struct rte_epoll_event),
|
|
RTE_CACHE_LINE_SIZE,
|
|
rx_adapter->socket_id);
|
|
if (!rx_adapter->epoll_events) {
|
|
err = -ENOMEM;
|
|
goto error;
|
|
}
|
|
|
|
rte_spinlock_init(&rx_adapter->intr_ring_lock);
|
|
|
|
snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN,
|
|
"rx-intr-thread-%d", rx_adapter->id);
|
|
|
|
err = rte_ctrl_thread_create(&rx_adapter->rx_intr_thread, thread_name,
|
|
NULL, rxa_intr_thread, rx_adapter);
|
|
if (!err) {
|
|
rte_thread_setname(rx_adapter->rx_intr_thread, thread_name);
|
|
return 0;
|
|
}
|
|
|
|
RTE_EDEV_LOG_ERR("Failed to create interrupt thread err = %d\n", err);
|
|
error:
|
|
rte_ring_free(rx_adapter->intr_ring);
|
|
rx_adapter->intr_ring = NULL;
|
|
rx_adapter->epoll_events = NULL;
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
rxa_destroy_intr_thread(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
int err;
|
|
|
|
err = pthread_cancel(rx_adapter->rx_intr_thread);
|
|
if (err)
|
|
RTE_EDEV_LOG_ERR("Can't cancel interrupt thread err = %d\n",
|
|
err);
|
|
|
|
err = pthread_join(rx_adapter->rx_intr_thread, NULL);
|
|
if (err)
|
|
RTE_EDEV_LOG_ERR("Can't join interrupt thread err = %d\n", err);
|
|
|
|
rte_free(rx_adapter->epoll_events);
|
|
rte_ring_free(rx_adapter->intr_ring);
|
|
rx_adapter->intr_ring = NULL;
|
|
rx_adapter->epoll_events = NULL;
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
rxa_free_intr_resources(struct rte_event_eth_rx_adapter *rx_adapter)
|
|
{
|
|
int ret;
|
|
|
|
if (rx_adapter->num_rx_intr == 0)
|
|
return 0;
|
|
|
|
ret = rxa_destroy_intr_thread(rx_adapter);
|
|
if (ret)
|
|
return ret;
|
|
|
|
close(rx_adapter->epd);
|
|
rx_adapter->epd = INIT_FD;
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int
|
|
rxa_disable_intr(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
uint16_t rx_queue_id)
|
|
{
|
|
int err;
|
|
uint16_t eth_dev_id = dev_info->dev->data->port_id;
|
|
int sintr = rxa_shared_intr(dev_info, rx_queue_id);
|
|
|
|
err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
|
|
if (err) {
|
|
RTE_EDEV_LOG_ERR("Could not disable interrupt for Rx queue %u",
|
|
rx_queue_id);
|
|
return err;
|
|
}
|
|
|
|
err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
|
|
rx_adapter->epd,
|
|
RTE_INTR_EVENT_DEL,
|
|
0);
|
|
if (err)
|
|
RTE_EDEV_LOG_ERR("Interrupt event deletion failed %d", err);
|
|
|
|
if (sintr)
|
|
dev_info->rx_queue[rx_queue_id].intr_enabled = 0;
|
|
else
|
|
dev_info->shared_intr_enabled = 0;
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
rxa_del_intr_queue(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id)
|
|
{
|
|
int err;
|
|
int i;
|
|
int s;
|
|
|
|
if (dev_info->nb_rx_intr == 0)
|
|
return 0;
|
|
|
|
err = 0;
|
|
if (rx_queue_id == -1) {
|
|
s = dev_info->nb_shared_intr;
|
|
for (i = 0; i < dev_info->nb_rx_intr; i++) {
|
|
int sintr;
|
|
uint16_t q;
|
|
|
|
q = dev_info->intr_queue[i];
|
|
sintr = rxa_shared_intr(dev_info, q);
|
|
s -= sintr;
|
|
|
|
if (!sintr || s == 0) {
|
|
|
|
err = rxa_disable_intr(rx_adapter, dev_info,
|
|
q);
|
|
if (err)
|
|
return err;
|
|
rxa_intr_ring_del_entries(rx_adapter, dev_info,
|
|
q);
|
|
}
|
|
}
|
|
} else {
|
|
if (!rxa_intr_queue(dev_info, rx_queue_id))
|
|
return 0;
|
|
if (!rxa_shared_intr(dev_info, rx_queue_id) ||
|
|
dev_info->nb_shared_intr == 1) {
|
|
err = rxa_disable_intr(rx_adapter, dev_info,
|
|
rx_queue_id);
|
|
if (err)
|
|
return err;
|
|
rxa_intr_ring_del_entries(rx_adapter, dev_info,
|
|
rx_queue_id);
|
|
}
|
|
|
|
for (i = 0; i < dev_info->nb_rx_intr; i++) {
|
|
if (dev_info->intr_queue[i] == rx_queue_id) {
|
|
for (; i < dev_info->nb_rx_intr - 1; i++)
|
|
dev_info->intr_queue[i] =
|
|
dev_info->intr_queue[i + 1];
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
rxa_config_intr(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
uint16_t rx_queue_id)
|
|
{
|
|
int err, err1;
|
|
uint16_t eth_dev_id = dev_info->dev->data->port_id;
|
|
union queue_data qd;
|
|
int init_fd;
|
|
uint16_t *intr_queue;
|
|
int sintr = rxa_shared_intr(dev_info, rx_queue_id);
|
|
|
|
if (rxa_intr_queue(dev_info, rx_queue_id))
|
|
return 0;
|
|
|
|
intr_queue = dev_info->intr_queue;
|
|
if (dev_info->intr_queue == NULL) {
|
|
size_t len =
|
|
dev_info->dev->data->nb_rx_queues * sizeof(uint16_t);
|
|
dev_info->intr_queue =
|
|
rte_zmalloc_socket(
|
|
rx_adapter->mem_name,
|
|
len,
|
|
0,
|
|
rx_adapter->socket_id);
|
|
if (dev_info->intr_queue == NULL)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
init_fd = rx_adapter->epd;
|
|
err = rxa_init_epd(rx_adapter);
|
|
if (err)
|
|
goto err_free_queue;
|
|
|
|
qd.port = eth_dev_id;
|
|
qd.queue = rx_queue_id;
|
|
|
|
err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
|
|
rx_adapter->epd,
|
|
RTE_INTR_EVENT_ADD,
|
|
qd.ptr);
|
|
if (err) {
|
|
RTE_EDEV_LOG_ERR("Failed to add interrupt event for"
|
|
" Rx Queue %u err %d", rx_queue_id, err);
|
|
goto err_del_fd;
|
|
}
|
|
|
|
err = rte_eth_dev_rx_intr_enable(eth_dev_id, rx_queue_id);
|
|
if (err) {
|
|
RTE_EDEV_LOG_ERR("Could not enable interrupt for"
|
|
" Rx Queue %u err %d", rx_queue_id, err);
|
|
|
|
goto err_del_event;
|
|
}
|
|
|
|
err = rxa_create_intr_thread(rx_adapter);
|
|
if (!err) {
|
|
if (sintr)
|
|
dev_info->shared_intr_enabled = 1;
|
|
else
|
|
dev_info->rx_queue[rx_queue_id].intr_enabled = 1;
|
|
return 0;
|
|
}
|
|
|
|
|
|
err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
|
|
if (err)
|
|
RTE_EDEV_LOG_ERR("Could not disable interrupt for"
|
|
" Rx Queue %u err %d", rx_queue_id, err);
|
|
err_del_event:
|
|
err1 = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
|
|
rx_adapter->epd,
|
|
RTE_INTR_EVENT_DEL,
|
|
0);
|
|
if (err1) {
|
|
RTE_EDEV_LOG_ERR("Could not delete event for"
|
|
" Rx Queue %u err %d", rx_queue_id, err1);
|
|
}
|
|
err_del_fd:
|
|
if (init_fd == INIT_FD) {
|
|
close(rx_adapter->epd);
|
|
rx_adapter->epd = -1;
|
|
}
|
|
err_free_queue:
|
|
if (intr_queue == NULL)
|
|
rte_free(dev_info->intr_queue);
|
|
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
rxa_add_intr_queue(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int rx_queue_id)
|
|
|
|
{
|
|
int i, j, err;
|
|
int si = -1;
|
|
int shared_done = (dev_info->nb_shared_intr > 0);
|
|
|
|
if (rx_queue_id != -1) {
|
|
if (rxa_shared_intr(dev_info, rx_queue_id) && shared_done)
|
|
return 0;
|
|
return rxa_config_intr(rx_adapter, dev_info, rx_queue_id);
|
|
}
|
|
|
|
err = 0;
|
|
for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) {
|
|
|
|
if (rxa_shared_intr(dev_info, i) && shared_done)
|
|
continue;
|
|
|
|
err = rxa_config_intr(rx_adapter, dev_info, i);
|
|
|
|
shared_done = err == 0 && rxa_shared_intr(dev_info, i);
|
|
if (shared_done) {
|
|
si = i;
|
|
dev_info->shared_intr_enabled = 1;
|
|
}
|
|
if (err)
|
|
break;
|
|
}
|
|
|
|
if (err == 0)
|
|
return 0;
|
|
|
|
shared_done = (dev_info->nb_shared_intr > 0);
|
|
for (j = 0; j < i; j++) {
|
|
if (rxa_intr_queue(dev_info, j))
|
|
continue;
|
|
if (rxa_shared_intr(dev_info, j) && si != j)
|
|
continue;
|
|
err = rxa_disable_intr(rx_adapter, dev_info, j);
|
|
if (err)
|
|
break;
|
|
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
|
|
static int
|
|
rxa_init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id)
|
|
{
|
|
int ret;
|
|
struct rte_service_spec service;
|
|
struct rte_event_eth_rx_adapter_conf rx_adapter_conf;
|
|
|
|
if (rx_adapter->service_inited)
|
|
return 0;
|
|
|
|
memset(&service, 0, sizeof(service));
|
|
snprintf(service.name, ETH_RX_ADAPTER_SERVICE_NAME_LEN,
|
|
"rte_event_eth_rx_adapter_%d", id);
|
|
service.socket_id = rx_adapter->socket_id;
|
|
service.callback = rxa_service_func;
|
|
service.callback_userdata = rx_adapter;
|
|
/* Service function handles locking for queue add/del updates */
|
|
service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
|
|
ret = rte_service_component_register(&service, &rx_adapter->service_id);
|
|
if (ret) {
|
|
RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
|
|
service.name, ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = rx_adapter->conf_cb(id, rx_adapter->eventdev_id,
|
|
&rx_adapter_conf, rx_adapter->conf_arg);
|
|
if (ret) {
|
|
RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
|
|
ret);
|
|
goto err_done;
|
|
}
|
|
rx_adapter->event_port_id = rx_adapter_conf.event_port_id;
|
|
rx_adapter->max_nb_rx = rx_adapter_conf.max_nb_rx;
|
|
rx_adapter->service_inited = 1;
|
|
rx_adapter->epd = INIT_FD;
|
|
return 0;
|
|
|
|
err_done:
|
|
rte_service_component_unregister(rx_adapter->service_id);
|
|
return ret;
|
|
}
|
|
|
|
static void
|
|
rxa_update_queue(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int32_t rx_queue_id,
|
|
uint8_t add)
|
|
{
|
|
struct eth_rx_queue_info *queue_info;
|
|
int enabled;
|
|
uint16_t i;
|
|
|
|
if (dev_info->rx_queue == NULL)
|
|
return;
|
|
|
|
if (rx_queue_id == -1) {
|
|
for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
|
|
rxa_update_queue(rx_adapter, dev_info, i, add);
|
|
} else {
|
|
queue_info = &dev_info->rx_queue[rx_queue_id];
|
|
enabled = queue_info->queue_enabled;
|
|
if (add) {
|
|
rx_adapter->nb_queues += !enabled;
|
|
dev_info->nb_dev_queues += !enabled;
|
|
} else {
|
|
rx_adapter->nb_queues -= enabled;
|
|
dev_info->nb_dev_queues -= enabled;
|
|
}
|
|
queue_info->queue_enabled = !!add;
|
|
}
|
|
}
|
|
|
|
static void
|
|
rxa_sw_del(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int32_t rx_queue_id)
|
|
{
|
|
int pollq;
|
|
int intrq;
|
|
int sintrq;
|
|
|
|
|
|
if (rx_adapter->nb_queues == 0)
|
|
return;
|
|
|
|
if (rx_queue_id == -1) {
|
|
uint16_t nb_rx_queues;
|
|
uint16_t i;
|
|
|
|
nb_rx_queues = dev_info->dev->data->nb_rx_queues;
|
|
for (i = 0; i < nb_rx_queues; i++)
|
|
rxa_sw_del(rx_adapter, dev_info, i);
|
|
return;
|
|
}
|
|
|
|
pollq = rxa_polled_queue(dev_info, rx_queue_id);
|
|
intrq = rxa_intr_queue(dev_info, rx_queue_id);
|
|
sintrq = rxa_shared_intr(dev_info, rx_queue_id);
|
|
rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 0);
|
|
rx_adapter->num_rx_polled -= pollq;
|
|
dev_info->nb_rx_poll -= pollq;
|
|
rx_adapter->num_rx_intr -= intrq;
|
|
dev_info->nb_rx_intr -= intrq;
|
|
dev_info->nb_shared_intr -= intrq && sintrq;
|
|
}
|
|
|
|
static void
|
|
rxa_add_queue(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
struct eth_device_info *dev_info,
|
|
int32_t rx_queue_id,
|
|
const struct rte_event_eth_rx_adapter_queue_conf *conf)
|
|
{
|
|
struct eth_rx_queue_info *queue_info;
|
|
const struct rte_event *ev = &conf->ev;
|
|
int pollq;
|
|
int intrq;
|
|
int sintrq;
|
|
|
|
if (rx_queue_id == -1) {
|
|
uint16_t nb_rx_queues;
|
|
uint16_t i;
|
|
|
|
nb_rx_queues = dev_info->dev->data->nb_rx_queues;
|
|
for (i = 0; i < nb_rx_queues; i++)
|
|
rxa_add_queue(rx_adapter, dev_info, i, conf);
|
|
return;
|
|
}
|
|
|
|
pollq = rxa_polled_queue(dev_info, rx_queue_id);
|
|
intrq = rxa_intr_queue(dev_info, rx_queue_id);
|
|
sintrq = rxa_shared_intr(dev_info, rx_queue_id);
|
|
|
|
queue_info = &dev_info->rx_queue[rx_queue_id];
|
|
queue_info->event_queue_id = ev->queue_id;
|
|
queue_info->sched_type = ev->sched_type;
|
|
queue_info->priority = ev->priority;
|
|
queue_info->wt = conf->servicing_weight;
|
|
|
|
if (conf->rx_queue_flags &
|
|
RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID) {
|
|
queue_info->flow_id = ev->flow_id;
|
|
queue_info->flow_id_mask = ~0;
|
|
}
|
|
|
|
rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 1);
|
|
if (rxa_polled_queue(dev_info, rx_queue_id)) {
|
|
rx_adapter->num_rx_polled += !pollq;
|
|
dev_info->nb_rx_poll += !pollq;
|
|
rx_adapter->num_rx_intr -= intrq;
|
|
dev_info->nb_rx_intr -= intrq;
|
|
dev_info->nb_shared_intr -= intrq && sintrq;
|
|
}
|
|
|
|
if (rxa_intr_queue(dev_info, rx_queue_id)) {
|
|
rx_adapter->num_rx_polled -= pollq;
|
|
dev_info->nb_rx_poll -= pollq;
|
|
rx_adapter->num_rx_intr += !intrq;
|
|
dev_info->nb_rx_intr += !intrq;
|
|
dev_info->nb_shared_intr += !intrq && sintrq;
|
|
if (dev_info->nb_shared_intr == 1) {
|
|
if (dev_info->multi_intr_cap)
|
|
dev_info->next_q_idx =
|
|
RTE_MAX_RXTX_INTR_VEC_ID - 1;
|
|
else
|
|
dev_info->next_q_idx = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
static int rxa_sw_add(struct rte_event_eth_rx_adapter *rx_adapter,
|
|
uint16_t eth_dev_id,
|
|
int rx_queue_id,
|
|
const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
|
|
{
|
|
struct eth_device_info *dev_info = &rx_adapter->eth_devices[eth_dev_id];
|
|
struct rte_event_eth_rx_adapter_queue_conf temp_conf;
|
|
int ret;
|
|
struct eth_rx_poll_entry *rx_poll;
|
|
struct eth_rx_queue_info *rx_queue;
|
|
uint32_t *rx_wrr;
|
|
uint16_t nb_rx_queues;
|
|
uint32_t nb_rx_poll, nb_wrr;
|
|
uint32_t nb_rx_intr;
|
|
int num_intr_vec;
|
|
uint16_t wt;
|
|
|
|
if (queue_conf->servicing_weight == 0) {
|
|
struct rte_eth_dev_data *data = dev_info->dev->data;
|
|
|
|
temp_conf = *queue_conf;
|
|
if (!data->dev_conf.intr_conf.rxq) {
|
|
/* If Rx interrupts are disabled set wt = 1 */
|
|
temp_conf.servicing_weight = 1;
|
|
}
|
|
queue_conf = &temp_conf;
|
|
}
|
|
|
|
nb_rx_queues = dev_info->dev->data->nb_rx_queues;
|
|
rx_queue = dev_info->rx_queue;
|
|
wt = queue_conf->servicing_weight;
|
|
|
|
if (dev_info->rx_queue == NULL) {
|
|
dev_info->rx_queue =
|
|
rte_zmalloc_socket(rx_adapter->mem_name,
|
|
nb_rx_queues *
|
|
sizeof(struct eth_rx_queue_info), 0,
|
|
rx_adapter->socket_id);
|
|
if (dev_info->rx_queue == NULL)
|
|
return -ENOMEM;
|
|
}
|
|
rx_wrr = NULL;
|
|
rx_poll = NULL;
|
|
|
|
rxa_calc_nb_post_add(rx_adapter, dev_info, rx_queue_id,
|
|
queue_conf->servicing_weight,
|
|
&nb_rx_poll, &nb_rx_intr, &nb_wrr);
|
|
|
|
if (dev_info->dev->intr_handle)
|
|
dev_info->multi_intr_cap =
|
|
rte_intr_cap_multiple(dev_info->dev->intr_handle);
|
|
|
|
ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
|
|
&rx_poll, &rx_wrr);
|
|
if (ret)
|
|
goto err_free_rxqueue;
|
|
|
|
if (wt == 0) {
|
|
num_intr_vec = rxa_nb_intr_vect(dev_info, rx_queue_id, 1);
|
|
|
|
ret = rxa_intr_ring_check_avail(rx_adapter, num_intr_vec);
|
|
if (ret)
|
|
goto err_free_rxqueue;
|
|
|
|
ret = rxa_add_intr_queue(rx_adapter, dev_info, rx_queue_id);
|
|
if (ret)
|
|
goto err_free_rxqueue;
|
|
} else {
|
|
|
|
num_intr_vec = 0;
|
|
if (rx_adapter->num_rx_intr > nb_rx_intr) {
|
|
num_intr_vec = rxa_nb_intr_vect(dev_info,
|
|
rx_queue_id, 0);
|
|
/* interrupt based queues are being converted to
|
|
* poll mode queues, delete the interrupt configuration
|
|
* for those.
|
|
*/
|
|
ret = rxa_del_intr_queue(rx_adapter,
|
|
dev_info, rx_queue_id);
|
|
if (ret)
|
|
goto err_free_rxqueue;
|
|
}
|
|
}
|
|
|
|
if (nb_rx_intr == 0) {
|
|
ret = rxa_free_intr_resources(rx_adapter);
|
|
if (ret)
|
|
goto err_free_rxqueue;
|
|
}
|
|
|
|
if (wt == 0) {
|
|
uint16_t i;
|
|
|
|
if (rx_queue_id == -1) {
|
|
for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
|
|
dev_info->intr_queue[i] = i;
|
|
} else {
|
|
if (!rxa_intr_queue(dev_info, rx_queue_id))
|
|
dev_info->intr_queue[nb_rx_intr - 1] =
|
|
rx_queue_id;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
rxa_add_queue(rx_adapter, dev_info, rx_queue_id, queue_conf);
|
|
rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
|
|
|
|
rte_free(rx_adapter->eth_rx_poll);
|
|
rte_free(rx_adapter->wrr_sched);
|
|
|
|
rx_adapter->eth_rx_poll = rx_poll;
|
|
rx_adapter->wrr_sched = rx_wrr;
|
|
rx_adapter->wrr_len = nb_wrr;
|
|
rx_adapter->num_intr_vec += num_intr_vec;
|
|
return 0;
|
|
|
|
err_free_rxqueue:
|
|
if (rx_queue == NULL) {
|
|
rte_free(dev_info->rx_queue);
|
|
dev_info->rx_queue = NULL;
|
|
}
|
|
|
|
rte_free(rx_poll);
|
|
rte_free(rx_wrr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
rxa_ctrl(uint8_t id, int start)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
struct rte_eventdev *dev;
|
|
struct eth_device_info *dev_info;
|
|
uint32_t i;
|
|
int use_service = 0;
|
|
int stop = !start;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL)
|
|
return -EINVAL;
|
|
|
|
dev = &rte_eventdevs[rx_adapter->eventdev_id];
|
|
|
|
RTE_ETH_FOREACH_DEV(i) {
|
|
dev_info = &rx_adapter->eth_devices[i];
|
|
/* if start check for num dev queues */
|
|
if (start && !dev_info->nb_dev_queues)
|
|
continue;
|
|
/* if stop check if dev has been started */
|
|
if (stop && !dev_info->dev_rx_started)
|
|
continue;
|
|
use_service |= !dev_info->internal_event_port;
|
|
dev_info->dev_rx_started = start;
|
|
if (dev_info->internal_event_port == 0)
|
|
continue;
|
|
start ? (*dev->dev_ops->eth_rx_adapter_start)(dev,
|
|
&rte_eth_devices[i]) :
|
|
(*dev->dev_ops->eth_rx_adapter_stop)(dev,
|
|
&rte_eth_devices[i]);
|
|
}
|
|
|
|
if (use_service) {
|
|
rte_spinlock_lock(&rx_adapter->rx_lock);
|
|
rx_adapter->rxa_started = start;
|
|
rte_service_runstate_set(rx_adapter->service_id, start);
|
|
rte_spinlock_unlock(&rx_adapter->rx_lock);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id,
|
|
rte_event_eth_rx_adapter_conf_cb conf_cb,
|
|
void *conf_arg)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
int ret;
|
|
int socket_id;
|
|
uint16_t i;
|
|
char mem_name[ETH_RX_ADAPTER_SERVICE_NAME_LEN];
|
|
const uint8_t default_rss_key[] = {
|
|
0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
|
|
0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
|
|
0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
|
|
0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
|
|
0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
|
|
};
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
|
|
if (conf_cb == NULL)
|
|
return -EINVAL;
|
|
|
|
if (event_eth_rx_adapter == NULL) {
|
|
ret = rte_event_eth_rx_adapter_init();
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter != NULL) {
|
|
RTE_EDEV_LOG_ERR("Eth Rx adapter exists id = %" PRIu8, id);
|
|
return -EEXIST;
|
|
}
|
|
|
|
socket_id = rte_event_dev_socket_id(dev_id);
|
|
snprintf(mem_name, ETH_RX_ADAPTER_MEM_NAME_LEN,
|
|
"rte_event_eth_rx_adapter_%d",
|
|
id);
|
|
|
|
rx_adapter = rte_zmalloc_socket(mem_name, sizeof(*rx_adapter),
|
|
RTE_CACHE_LINE_SIZE, socket_id);
|
|
if (rx_adapter == NULL) {
|
|
RTE_EDEV_LOG_ERR("failed to get mem for rx adapter");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
rx_adapter->eventdev_id = dev_id;
|
|
rx_adapter->socket_id = socket_id;
|
|
rx_adapter->conf_cb = conf_cb;
|
|
rx_adapter->conf_arg = conf_arg;
|
|
rx_adapter->id = id;
|
|
strcpy(rx_adapter->mem_name, mem_name);
|
|
rx_adapter->eth_devices = rte_zmalloc_socket(rx_adapter->mem_name,
|
|
RTE_MAX_ETHPORTS *
|
|
sizeof(struct eth_device_info), 0,
|
|
socket_id);
|
|
rte_convert_rss_key((const uint32_t *)default_rss_key,
|
|
(uint32_t *)rx_adapter->rss_key_be,
|
|
RTE_DIM(default_rss_key));
|
|
|
|
if (rx_adapter->eth_devices == NULL) {
|
|
RTE_EDEV_LOG_ERR("failed to get mem for eth devices\n");
|
|
rte_free(rx_adapter);
|
|
return -ENOMEM;
|
|
}
|
|
rte_spinlock_init(&rx_adapter->rx_lock);
|
|
for (i = 0; i < RTE_MAX_ETHPORTS; i++)
|
|
rx_adapter->eth_devices[i].dev = &rte_eth_devices[i];
|
|
|
|
event_eth_rx_adapter[id] = rx_adapter;
|
|
if (conf_cb == rxa_default_conf_cb)
|
|
rx_adapter->default_cb_arg = 1;
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_create(uint8_t id, uint8_t dev_id,
|
|
struct rte_event_port_conf *port_config)
|
|
{
|
|
struct rte_event_port_conf *pc;
|
|
int ret;
|
|
|
|
if (port_config == NULL)
|
|
return -EINVAL;
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
|
|
pc = rte_malloc(NULL, sizeof(*pc), 0);
|
|
if (pc == NULL)
|
|
return -ENOMEM;
|
|
*pc = *port_config;
|
|
ret = rte_event_eth_rx_adapter_create_ext(id, dev_id,
|
|
rxa_default_conf_cb,
|
|
pc);
|
|
if (ret)
|
|
rte_free(pc);
|
|
return ret;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_free(uint8_t id)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL)
|
|
return -EINVAL;
|
|
|
|
if (rx_adapter->nb_queues) {
|
|
RTE_EDEV_LOG_ERR("%" PRIu16 " Rx queues not deleted",
|
|
rx_adapter->nb_queues);
|
|
return -EBUSY;
|
|
}
|
|
|
|
if (rx_adapter->default_cb_arg)
|
|
rte_free(rx_adapter->conf_arg);
|
|
rte_free(rx_adapter->eth_devices);
|
|
rte_free(rx_adapter);
|
|
event_eth_rx_adapter[id] = NULL;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_queue_add(uint8_t id,
|
|
uint16_t eth_dev_id,
|
|
int32_t rx_queue_id,
|
|
const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
|
|
{
|
|
int ret;
|
|
uint32_t cap;
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
struct rte_eventdev *dev;
|
|
struct eth_device_info *dev_info;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if ((rx_adapter == NULL) || (queue_conf == NULL))
|
|
return -EINVAL;
|
|
|
|
dev = &rte_eventdevs[rx_adapter->eventdev_id];
|
|
ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
|
|
eth_dev_id,
|
|
&cap);
|
|
if (ret) {
|
|
RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
|
|
"eth port %" PRIu16, id, eth_dev_id);
|
|
return ret;
|
|
}
|
|
|
|
if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) == 0
|
|
&& (queue_conf->rx_queue_flags &
|
|
RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID)) {
|
|
RTE_EDEV_LOG_ERR("Flow ID override is not supported,"
|
|
" eth port: %" PRIu16 " adapter id: %" PRIu8,
|
|
eth_dev_id, id);
|
|
return -EINVAL;
|
|
}
|
|
|
|
if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ) == 0 &&
|
|
(rx_queue_id != -1)) {
|
|
RTE_EDEV_LOG_ERR("Rx queues can only be connected to single "
|
|
"event queue, eth port: %" PRIu16 " adapter id: %"
|
|
PRIu8, eth_dev_id, id);
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (rx_queue_id != -1 && (uint16_t)rx_queue_id >=
|
|
rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
|
|
RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16,
|
|
(uint16_t)rx_queue_id);
|
|
return -EINVAL;
|
|
}
|
|
|
|
dev_info = &rx_adapter->eth_devices[eth_dev_id];
|
|
|
|
if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
|
|
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_add,
|
|
-ENOTSUP);
|
|
if (dev_info->rx_queue == NULL) {
|
|
dev_info->rx_queue =
|
|
rte_zmalloc_socket(rx_adapter->mem_name,
|
|
dev_info->dev->data->nb_rx_queues *
|
|
sizeof(struct eth_rx_queue_info), 0,
|
|
rx_adapter->socket_id);
|
|
if (dev_info->rx_queue == NULL)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
ret = (*dev->dev_ops->eth_rx_adapter_queue_add)(dev,
|
|
&rte_eth_devices[eth_dev_id],
|
|
rx_queue_id, queue_conf);
|
|
if (ret == 0) {
|
|
dev_info->internal_event_port = 1;
|
|
rxa_update_queue(rx_adapter,
|
|
&rx_adapter->eth_devices[eth_dev_id],
|
|
rx_queue_id,
|
|
1);
|
|
}
|
|
} else {
|
|
rte_spinlock_lock(&rx_adapter->rx_lock);
|
|
dev_info->internal_event_port = 0;
|
|
ret = rxa_init_service(rx_adapter, id);
|
|
if (ret == 0) {
|
|
uint32_t service_id = rx_adapter->service_id;
|
|
ret = rxa_sw_add(rx_adapter, eth_dev_id, rx_queue_id,
|
|
queue_conf);
|
|
rte_service_component_runstate_set(service_id,
|
|
rxa_sw_adapter_queue_count(rx_adapter));
|
|
}
|
|
rte_spinlock_unlock(&rx_adapter->rx_lock);
|
|
}
|
|
|
|
if (ret)
|
|
return ret;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_queue_del(uint8_t id, uint16_t eth_dev_id,
|
|
int32_t rx_queue_id)
|
|
{
|
|
int ret = 0;
|
|
struct rte_eventdev *dev;
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
struct eth_device_info *dev_info;
|
|
uint32_t cap;
|
|
uint32_t nb_rx_poll = 0;
|
|
uint32_t nb_wrr = 0;
|
|
uint32_t nb_rx_intr;
|
|
struct eth_rx_poll_entry *rx_poll = NULL;
|
|
uint32_t *rx_wrr = NULL;
|
|
int num_intr_vec;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL)
|
|
return -EINVAL;
|
|
|
|
dev = &rte_eventdevs[rx_adapter->eventdev_id];
|
|
ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
|
|
eth_dev_id,
|
|
&cap);
|
|
if (ret)
|
|
return ret;
|
|
|
|
if (rx_queue_id != -1 && (uint16_t)rx_queue_id >=
|
|
rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
|
|
RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16,
|
|
(uint16_t)rx_queue_id);
|
|
return -EINVAL;
|
|
}
|
|
|
|
dev_info = &rx_adapter->eth_devices[eth_dev_id];
|
|
|
|
if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
|
|
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_del,
|
|
-ENOTSUP);
|
|
ret = (*dev->dev_ops->eth_rx_adapter_queue_del)(dev,
|
|
&rte_eth_devices[eth_dev_id],
|
|
rx_queue_id);
|
|
if (ret == 0) {
|
|
rxa_update_queue(rx_adapter,
|
|
&rx_adapter->eth_devices[eth_dev_id],
|
|
rx_queue_id,
|
|
0);
|
|
if (dev_info->nb_dev_queues == 0) {
|
|
rte_free(dev_info->rx_queue);
|
|
dev_info->rx_queue = NULL;
|
|
}
|
|
}
|
|
} else {
|
|
rxa_calc_nb_post_del(rx_adapter, dev_info, rx_queue_id,
|
|
&nb_rx_poll, &nb_rx_intr, &nb_wrr);
|
|
|
|
ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
|
|
&rx_poll, &rx_wrr);
|
|
if (ret)
|
|
return ret;
|
|
|
|
rte_spinlock_lock(&rx_adapter->rx_lock);
|
|
|
|
num_intr_vec = 0;
|
|
if (rx_adapter->num_rx_intr > nb_rx_intr) {
|
|
|
|
num_intr_vec = rxa_nb_intr_vect(dev_info,
|
|
rx_queue_id, 0);
|
|
ret = rxa_del_intr_queue(rx_adapter, dev_info,
|
|
rx_queue_id);
|
|
if (ret)
|
|
goto unlock_ret;
|
|
}
|
|
|
|
if (nb_rx_intr == 0) {
|
|
ret = rxa_free_intr_resources(rx_adapter);
|
|
if (ret)
|
|
goto unlock_ret;
|
|
}
|
|
|
|
rxa_sw_del(rx_adapter, dev_info, rx_queue_id);
|
|
rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
|
|
|
|
rte_free(rx_adapter->eth_rx_poll);
|
|
rte_free(rx_adapter->wrr_sched);
|
|
|
|
if (nb_rx_intr == 0) {
|
|
rte_free(dev_info->intr_queue);
|
|
dev_info->intr_queue = NULL;
|
|
}
|
|
|
|
rx_adapter->eth_rx_poll = rx_poll;
|
|
rx_adapter->wrr_sched = rx_wrr;
|
|
rx_adapter->wrr_len = nb_wrr;
|
|
rx_adapter->num_intr_vec += num_intr_vec;
|
|
|
|
if (dev_info->nb_dev_queues == 0) {
|
|
rte_free(dev_info->rx_queue);
|
|
dev_info->rx_queue = NULL;
|
|
}
|
|
unlock_ret:
|
|
rte_spinlock_unlock(&rx_adapter->rx_lock);
|
|
if (ret) {
|
|
rte_free(rx_poll);
|
|
rte_free(rx_wrr);
|
|
return ret;
|
|
}
|
|
|
|
rte_service_component_runstate_set(rx_adapter->service_id,
|
|
rxa_sw_adapter_queue_count(rx_adapter));
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_start(uint8_t id)
|
|
{
|
|
return rxa_ctrl(id, 1);
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_stop(uint8_t id)
|
|
{
|
|
return rxa_ctrl(id, 0);
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_stats_get(uint8_t id,
|
|
struct rte_event_eth_rx_adapter_stats *stats)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
struct rte_event_eth_rx_adapter_stats dev_stats_sum = { 0 };
|
|
struct rte_event_eth_rx_adapter_stats dev_stats;
|
|
struct rte_eventdev *dev;
|
|
struct eth_device_info *dev_info;
|
|
uint32_t i;
|
|
int ret;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL || stats == NULL)
|
|
return -EINVAL;
|
|
|
|
dev = &rte_eventdevs[rx_adapter->eventdev_id];
|
|
memset(stats, 0, sizeof(*stats));
|
|
RTE_ETH_FOREACH_DEV(i) {
|
|
dev_info = &rx_adapter->eth_devices[i];
|
|
if (dev_info->internal_event_port == 0 ||
|
|
dev->dev_ops->eth_rx_adapter_stats_get == NULL)
|
|
continue;
|
|
ret = (*dev->dev_ops->eth_rx_adapter_stats_get)(dev,
|
|
&rte_eth_devices[i],
|
|
&dev_stats);
|
|
if (ret)
|
|
continue;
|
|
dev_stats_sum.rx_packets += dev_stats.rx_packets;
|
|
dev_stats_sum.rx_enq_count += dev_stats.rx_enq_count;
|
|
}
|
|
|
|
if (rx_adapter->service_inited)
|
|
*stats = rx_adapter->stats;
|
|
|
|
stats->rx_packets += dev_stats_sum.rx_packets;
|
|
stats->rx_enq_count += dev_stats_sum.rx_enq_count;
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_stats_reset(uint8_t id)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
struct rte_eventdev *dev;
|
|
struct eth_device_info *dev_info;
|
|
uint32_t i;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL)
|
|
return -EINVAL;
|
|
|
|
dev = &rte_eventdevs[rx_adapter->eventdev_id];
|
|
RTE_ETH_FOREACH_DEV(i) {
|
|
dev_info = &rx_adapter->eth_devices[i];
|
|
if (dev_info->internal_event_port == 0 ||
|
|
dev->dev_ops->eth_rx_adapter_stats_reset == NULL)
|
|
continue;
|
|
(*dev->dev_ops->eth_rx_adapter_stats_reset)(dev,
|
|
&rte_eth_devices[i]);
|
|
}
|
|
|
|
memset(&rx_adapter->stats, 0, sizeof(rx_adapter->stats));
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL || service_id == NULL)
|
|
return -EINVAL;
|
|
|
|
if (rx_adapter->service_inited)
|
|
*service_id = rx_adapter->service_id;
|
|
|
|
return rx_adapter->service_inited ? 0 : -ESRCH;
|
|
}
|
|
|
|
int rte_event_eth_rx_adapter_cb_register(uint8_t id,
|
|
uint16_t eth_dev_id,
|
|
rte_event_eth_rx_adapter_cb_fn cb_fn,
|
|
void *cb_arg)
|
|
{
|
|
struct rte_event_eth_rx_adapter *rx_adapter;
|
|
struct eth_device_info *dev_info;
|
|
uint32_t cap;
|
|
int ret;
|
|
|
|
RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
|
|
RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
|
|
|
|
rx_adapter = rxa_id_to_adapter(id);
|
|
if (rx_adapter == NULL)
|
|
return -EINVAL;
|
|
|
|
dev_info = &rx_adapter->eth_devices[eth_dev_id];
|
|
if (dev_info->rx_queue == NULL)
|
|
return -EINVAL;
|
|
|
|
ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
|
|
eth_dev_id,
|
|
&cap);
|
|
if (ret) {
|
|
RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
|
|
"eth port %" PRIu16, id, eth_dev_id);
|
|
return ret;
|
|
}
|
|
|
|
if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
|
|
RTE_EDEV_LOG_ERR("Rx callback not supported for eth port %"
|
|
PRIu16, eth_dev_id);
|
|
return -EINVAL;
|
|
}
|
|
|
|
rte_spinlock_lock(&rx_adapter->rx_lock);
|
|
dev_info->cb_fn = cb_fn;
|
|
dev_info->cb_arg = cb_arg;
|
|
rte_spinlock_unlock(&rx_adapter->rx_lock);
|
|
|
|
return 0;
|
|
}
|