app/eventdev: add pipeline queue worker functions
Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com> Acked-by: Harry van Haaren <harry.van.haaren@intel.com>
This commit is contained in:
Pavan Nikhilesh
Jerin Jacob
parent
d60b418503
commit
314bcf58ca
@ -65,6 +65,78 @@ struct test_pipeline {
|
||||
|
||||
#define BURST_SIZE 16
|
||||
|
||||
#define PIPELINE_WROKER_SINGLE_STAGE_INIT \
|
||||
struct worker_data *w = arg; \
|
||||
struct test_pipeline *t = w->t; \
|
||||
const uint8_t dev = w->dev_id; \
|
||||
const uint8_t port = w->port_id; \
|
||||
struct rte_event ev
|
||||
|
||||
#define PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT \
|
||||
int i; \
|
||||
struct worker_data *w = arg; \
|
||||
struct test_pipeline *t = w->t; \
|
||||
const uint8_t dev = w->dev_id; \
|
||||
const uint8_t port = w->port_id; \
|
||||
struct rte_event ev[BURST_SIZE + 1]
|
||||
|
||||
#define PIPELINE_WROKER_MULTI_STAGE_INIT \
|
||||
struct worker_data *w = arg; \
|
||||
struct test_pipeline *t = w->t; \
|
||||
uint8_t cq_id; \
|
||||
const uint8_t dev = w->dev_id; \
|
||||
const uint8_t port = w->port_id; \
|
||||
const uint8_t last_queue = t->opt->nb_stages - 1; \
|
||||
uint8_t *const sched_type_list = &t->sched_type_list[0]; \
|
||||
struct rte_event ev
|
||||
|
||||
#define PIPELINE_WROKER_MULTI_STAGE_BURST_INIT \
|
||||
int i; \
|
||||
struct worker_data *w = arg; \
|
||||
struct test_pipeline *t = w->t; \
|
||||
uint8_t cq_id; \
|
||||
const uint8_t dev = w->dev_id; \
|
||||
const uint8_t port = w->port_id; \
|
||||
const uint8_t last_queue = t->opt->nb_stages - 1; \
|
||||
uint8_t *const sched_type_list = &t->sched_type_list[0]; \
|
||||
struct rte_event ev[BURST_SIZE + 1]
|
||||
|
||||
static __rte_always_inline void
|
||||
pipeline_fwd_event(struct rte_event *ev, uint8_t sched)
|
||||
{
|
||||
ev->event_type = RTE_EVENT_TYPE_CPU;
|
||||
ev->op = RTE_EVENT_OP_FORWARD;
|
||||
ev->sched_type = sched;
|
||||
}
|
||||
|
||||
static __rte_always_inline void
|
||||
pipeline_event_enqueue(const uint8_t dev, const uint8_t port,
|
||||
struct rte_event *ev)
|
||||
{
|
||||
while (rte_event_enqueue_burst(dev, port, ev, 1) != 1)
|
||||
rte_pause();
|
||||
}
|
||||
|
||||
static __rte_always_inline void
|
||||
pipeline_event_enqueue_burst(const uint8_t dev, const uint8_t port,
|
||||
struct rte_event *ev, const uint16_t nb_rx)
|
||||
{
|
||||
uint16_t enq;
|
||||
|
||||
enq = rte_event_enqueue_burst(dev, port, ev, nb_rx);
|
||||
while (enq < nb_rx) {
|
||||
enq += rte_event_enqueue_burst(dev, port,
|
||||
ev + enq, nb_rx - enq);
|
||||
}
|
||||
}
|
||||
|
||||
static __rte_always_inline void
|
||||
pipeline_tx_pkt(struct rte_mbuf *mbuf)
|
||||
{
|
||||
while (rte_eth_tx_burst(mbuf->port, 0, &mbuf, 1) != 1)
|
||||
rte_pause();
|
||||
}
|
||||
|
||||
static inline int
|
||||
pipeline_nb_event_ports(struct evt_options *opt)
|
||||
{
|
||||
|
||||
@ -15,10 +15,296 @@ pipeline_queue_nb_event_queues(struct evt_options *opt)
|
||||
return (eth_count * opt->nb_stages) + eth_count;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_single_stage_tx(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_SINGLE_STAGE_INIT;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
|
||||
|
||||
if (!event) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
|
||||
pipeline_tx_pkt(ev.mbuf);
|
||||
w->processed_pkts++;
|
||||
} else {
|
||||
ev.queue_id++;
|
||||
pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
|
||||
pipeline_event_enqueue(dev, port, &ev);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_single_stage_fwd(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_SINGLE_STAGE_INIT;
|
||||
const uint8_t tx_queue = t->tx_service.queue_id;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
|
||||
|
||||
if (!event) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
ev.queue_id = tx_queue;
|
||||
pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
|
||||
pipeline_event_enqueue(dev, port, &ev);
|
||||
w->processed_pkts++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_single_stage_burst_tx(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
|
||||
BURST_SIZE, 0);
|
||||
|
||||
if (!nb_rx) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
for (i = 0; i < nb_rx; i++) {
|
||||
rte_prefetch0(ev[i + 1].mbuf);
|
||||
if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
|
||||
|
||||
pipeline_tx_pkt(ev[i].mbuf);
|
||||
ev[i].op = RTE_EVENT_OP_RELEASE;
|
||||
w->processed_pkts++;
|
||||
} else {
|
||||
ev[i].queue_id++;
|
||||
pipeline_fwd_event(&ev[i],
|
||||
RTE_SCHED_TYPE_ATOMIC);
|
||||
}
|
||||
}
|
||||
|
||||
pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_single_stage_burst_fwd(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT;
|
||||
const uint8_t tx_queue = t->tx_service.queue_id;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
|
||||
BURST_SIZE, 0);
|
||||
|
||||
if (!nb_rx) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
for (i = 0; i < nb_rx; i++) {
|
||||
rte_prefetch0(ev[i + 1].mbuf);
|
||||
ev[i].queue_id = tx_queue;
|
||||
pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC);
|
||||
w->processed_pkts++;
|
||||
}
|
||||
|
||||
pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_multi_stage_tx(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_MULTI_STAGE_INIT;
|
||||
const uint8_t nb_stages = t->opt->nb_stages + 1;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
|
||||
|
||||
if (!event) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
cq_id = ev.queue_id % nb_stages;
|
||||
|
||||
if (cq_id >= last_queue) {
|
||||
if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
|
||||
|
||||
pipeline_tx_pkt(ev.mbuf);
|
||||
w->processed_pkts++;
|
||||
continue;
|
||||
}
|
||||
ev.queue_id += (cq_id == last_queue) ? 1 : 0;
|
||||
pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
|
||||
} else {
|
||||
ev.queue_id++;
|
||||
pipeline_fwd_event(&ev, sched_type_list[cq_id]);
|
||||
}
|
||||
|
||||
pipeline_event_enqueue(dev, port, &ev);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_multi_stage_fwd(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_MULTI_STAGE_INIT;
|
||||
const uint8_t nb_stages = t->opt->nb_stages + 1;
|
||||
const uint8_t tx_queue = t->tx_service.queue_id;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
|
||||
|
||||
if (!event) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
cq_id = ev.queue_id % nb_stages;
|
||||
|
||||
if (cq_id == last_queue) {
|
||||
ev.queue_id = tx_queue;
|
||||
pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
|
||||
w->processed_pkts++;
|
||||
} else {
|
||||
ev.queue_id++;
|
||||
pipeline_fwd_event(&ev, sched_type_list[cq_id]);
|
||||
}
|
||||
|
||||
pipeline_event_enqueue(dev, port, &ev);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_multi_stage_burst_tx(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_MULTI_STAGE_BURST_INIT;
|
||||
const uint8_t nb_stages = t->opt->nb_stages + 1;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
|
||||
BURST_SIZE, 0);
|
||||
|
||||
if (!nb_rx) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
for (i = 0; i < nb_rx; i++) {
|
||||
rte_prefetch0(ev[i + 1].mbuf);
|
||||
cq_id = ev[i].queue_id % nb_stages;
|
||||
|
||||
if (cq_id >= last_queue) {
|
||||
if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
|
||||
|
||||
pipeline_tx_pkt(ev[i].mbuf);
|
||||
ev[i].op = RTE_EVENT_OP_RELEASE;
|
||||
w->processed_pkts++;
|
||||
continue;
|
||||
}
|
||||
|
||||
ev[i].queue_id += (cq_id == last_queue) ? 1 : 0;
|
||||
pipeline_fwd_event(&ev[i],
|
||||
RTE_SCHED_TYPE_ATOMIC);
|
||||
} else {
|
||||
ev[i].queue_id++;
|
||||
pipeline_fwd_event(&ev[i],
|
||||
sched_type_list[cq_id]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pipeline_queue_worker_multi_stage_burst_fwd(void *arg)
|
||||
{
|
||||
PIPELINE_WROKER_MULTI_STAGE_BURST_INIT;
|
||||
const uint8_t nb_stages = t->opt->nb_stages + 1;
|
||||
const uint8_t tx_queue = t->tx_service.queue_id;
|
||||
|
||||
while (t->done == false) {
|
||||
uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
|
||||
BURST_SIZE, 0);
|
||||
|
||||
if (!nb_rx) {
|
||||
rte_pause();
|
||||
continue;
|
||||
}
|
||||
|
||||
for (i = 0; i < nb_rx; i++) {
|
||||
rte_prefetch0(ev[i + 1].mbuf);
|
||||
cq_id = ev[i].queue_id % nb_stages;
|
||||
|
||||
if (cq_id == last_queue) {
|
||||
ev[i].queue_id = tx_queue;
|
||||
pipeline_fwd_event(&ev[i],
|
||||
RTE_SCHED_TYPE_ATOMIC);
|
||||
w->processed_pkts++;
|
||||
} else {
|
||||
ev[i].queue_id++;
|
||||
pipeline_fwd_event(&ev[i],
|
||||
sched_type_list[cq_id]);
|
||||
}
|
||||
}
|
||||
|
||||
pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
worker_wrapper(void *arg)
|
||||
{
|
||||
RTE_SET_USED(arg);
|
||||
struct worker_data *w = arg;
|
||||
struct evt_options *opt = w->t->opt;
|
||||
const bool burst = evt_has_burst_mode(w->dev_id);
|
||||
const bool mt_safe = !w->t->mt_unsafe;
|
||||
const uint8_t nb_stages = opt->nb_stages;
|
||||
RTE_SET_USED(opt);
|
||||
|
||||
if (nb_stages == 1) {
|
||||
if (!burst && mt_safe)
|
||||
return pipeline_queue_worker_single_stage_tx(arg);
|
||||
else if (!burst && !mt_safe)
|
||||
return pipeline_queue_worker_single_stage_fwd(arg);
|
||||
else if (burst && mt_safe)
|
||||
return pipeline_queue_worker_single_stage_burst_tx(arg);
|
||||
else if (burst && !mt_safe)
|
||||
return pipeline_queue_worker_single_stage_burst_fwd(
|
||||
arg);
|
||||
} else {
|
||||
if (!burst && mt_safe)
|
||||
return pipeline_queue_worker_multi_stage_tx(arg);
|
||||
else if (!burst && !mt_safe)
|
||||
return pipeline_queue_worker_multi_stage_fwd(arg);
|
||||
else if (burst && mt_safe)
|
||||
return pipeline_queue_worker_multi_stage_burst_tx(arg);
|
||||
else if (burst && !mt_safe)
|
||||
return pipeline_queue_worker_multi_stage_burst_fwd(arg);
|
||||
|
||||
}
|
||||
rte_panic("invalid worker\n");
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user