app/testpmd: support multi-process
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes: * the primary process: ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0 * the secondary process: ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1 Signed-off-by: Min Hu (Connor) <humin29@huawei.com> Signed-off-by: Lijun Ou <oulijun@huawei.com> Signed-off-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru> Acked-by: Xiaoyun Li <xiaoyun.li@intel.com> Acked-by: Ajit Khaparde <ajit.khaparde@broadcom.com> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com> Acked-by: Aman Deep Singh <aman.deep.singh@intel.com>
This commit is contained in:
Min Hu (Connor)
Ferruh Yigit
parent
36a97cd23f
commit
a550baf24a
@ -5491,6 +5491,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
|
||||
__rte_unused void *data)
|
||||
{
|
||||
struct cmd_set_flush_rx *res = parsed_result;
|
||||
|
||||
if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
|
||||
printf("multi-process doesn't support to flush Rx queues.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
|
||||
}
|
||||
|
||||
|
||||
@ -2998,6 +2998,8 @@ rss_fwd_config_setup(void)
|
||||
queueid_t rxq;
|
||||
queueid_t nb_q;
|
||||
streamid_t sm_id;
|
||||
int start;
|
||||
int end;
|
||||
|
||||
nb_q = nb_rxq;
|
||||
if (nb_q > nb_txq)
|
||||
@ -3015,7 +3017,21 @@ rss_fwd_config_setup(void)
|
||||
init_fwd_streams();
|
||||
|
||||
setup_fwd_config_of_each_lcore(&cur_fwd_config);
|
||||
rxp = 0; rxq = 0;
|
||||
|
||||
if (proc_id > 0 && nb_q % num_procs != 0)
|
||||
printf("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n");
|
||||
|
||||
/**
|
||||
* In multi-process, All queues are allocated to different
|
||||
* processes based on num_procs and proc_id. For example:
|
||||
* if supports 4 queues(nb_q), 2 processes(num_procs),
|
||||
* the 0~1 queue for primary process.
|
||||
* the 2~3 queue for secondary process.
|
||||
*/
|
||||
start = proc_id * nb_q / num_procs;
|
||||
end = start + nb_q / num_procs;
|
||||
rxp = 0;
|
||||
rxq = start;
|
||||
for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
|
||||
struct fwd_stream *fs;
|
||||
|
||||
@ -3032,6 +3048,8 @@ rss_fwd_config_setup(void)
|
||||
continue;
|
||||
rxp = 0;
|
||||
rxq++;
|
||||
if (rxq >= end)
|
||||
rxq = start;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -513,6 +513,9 @@ parse_link_speed(int n)
|
||||
void
|
||||
launch_args_parse(int argc, char** argv)
|
||||
{
|
||||
#define PARAM_PROC_ID "proc-id"
|
||||
#define PARAM_NUM_PROCS "num-procs"
|
||||
|
||||
int n, opt;
|
||||
char **argvopt;
|
||||
int opt_idx;
|
||||
@ -633,6 +636,8 @@ launch_args_parse(int argc, char** argv)
|
||||
{ "rx-mq-mode", 1, 0, 0 },
|
||||
{ "record-core-cycles", 0, 0, 0 },
|
||||
{ "record-burst-stats", 0, 0, 0 },
|
||||
{ PARAM_NUM_PROCS, 1, 0, 0 },
|
||||
{ PARAM_PROC_ID, 1, 0, 0 },
|
||||
{ 0, 0, 0, 0 },
|
||||
};
|
||||
|
||||
@ -1407,6 +1412,10 @@ launch_args_parse(int argc, char** argv)
|
||||
record_core_cycles = 1;
|
||||
if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
|
||||
record_burst_stats = 1;
|
||||
if (!strcmp(lgopts[opt_idx].name, PARAM_NUM_PROCS))
|
||||
num_procs = atoi(optarg);
|
||||
if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
|
||||
proc_id = atoi(optarg);
|
||||
break;
|
||||
case 'h':
|
||||
usage(argv[0]);
|
||||
|
||||
@ -521,6 +521,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
|
||||
*/
|
||||
uint32_t eth_link_speed;
|
||||
|
||||
/*
|
||||
* ID of the current process in multi-process, used to
|
||||
* configure the queues to be polled.
|
||||
*/
|
||||
int proc_id;
|
||||
|
||||
/*
|
||||
* Number of processes in multi-process, used to
|
||||
* configure the queues to be polled.
|
||||
*/
|
||||
unsigned int num_procs = 1;
|
||||
|
||||
static int
|
||||
eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
|
||||
const struct rte_eth_conf *dev_conf)
|
||||
{
|
||||
if (is_proc_primary())
|
||||
return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
|
||||
dev_conf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
eth_dev_start_mp(uint16_t port_id)
|
||||
{
|
||||
if (is_proc_primary())
|
||||
return rte_eth_dev_start(port_id);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
eth_dev_stop_mp(uint16_t port_id)
|
||||
{
|
||||
if (is_proc_primary())
|
||||
return rte_eth_dev_stop(port_id);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
mempool_free_mp(struct rte_mempool *mp)
|
||||
{
|
||||
if (is_proc_primary())
|
||||
rte_mempool_free(mp);
|
||||
}
|
||||
|
||||
static int
|
||||
eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
|
||||
{
|
||||
if (is_proc_primary())
|
||||
return rte_eth_dev_set_mtu(port_id, mtu);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Forward function declarations */
|
||||
static void setup_attached_port(portid_t pi);
|
||||
static void check_all_ports_link_status(uint32_t port_mask);
|
||||
@ -996,6 +1052,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
|
||||
mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
|
||||
#endif
|
||||
mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
|
||||
if (!is_proc_primary()) {
|
||||
rte_mp = rte_mempool_lookup(pool_name);
|
||||
if (rte_mp == NULL)
|
||||
rte_exit(EXIT_FAILURE,
|
||||
"Get mbuf pool for socket %u failed: %s\n",
|
||||
socket_id, rte_strerror(rte_errno));
|
||||
return rte_mp;
|
||||
}
|
||||
|
||||
TESTPMD_LOG(INFO,
|
||||
"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
|
||||
@ -1995,6 +2059,11 @@ flush_fwd_rx_queues(void)
|
||||
uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
|
||||
uint64_t timer_period;
|
||||
|
||||
if (num_procs > 1) {
|
||||
printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
/* convert to number of cycles */
|
||||
timer_period = rte_get_timer_hz(); /* 1 second timeout */
|
||||
|
||||
@ -2484,7 +2553,7 @@ start_port(portid_t pid)
|
||||
return -1;
|
||||
}
|
||||
/* configure port */
|
||||
diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
|
||||
diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
|
||||
nb_txq + nb_hairpinq,
|
||||
&(port->dev_conf));
|
||||
if (diag != 0) {
|
||||
@ -2500,7 +2569,7 @@ start_port(portid_t pid)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
if (port->need_reconfig_queues > 0) {
|
||||
if (port->need_reconfig_queues > 0 && is_proc_primary()) {
|
||||
port->need_reconfig_queues = 0;
|
||||
/* setup tx queues */
|
||||
for (qi = 0; qi < nb_txq; qi++) {
|
||||
@ -2603,7 +2672,7 @@ start_port(portid_t pid)
|
||||
cnt_pi++;
|
||||
|
||||
/* start port */
|
||||
diag = rte_eth_dev_start(pi);
|
||||
diag = eth_dev_start_mp(pi);
|
||||
if (diag < 0) {
|
||||
fprintf(stderr, "Fail to start port %d: %s\n",
|
||||
pi, rte_strerror(-diag));
|
||||
@ -2746,7 +2815,7 @@ stop_port(portid_t pid)
|
||||
if (port->flow_list)
|
||||
port_flow_flush(pi);
|
||||
|
||||
if (rte_eth_dev_stop(pi) != 0)
|
||||
if (eth_dev_stop_mp(pi) != 0)
|
||||
RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
|
||||
pi);
|
||||
|
||||
@ -2820,8 +2889,10 @@ close_port(portid_t pid)
|
||||
continue;
|
||||
}
|
||||
|
||||
port_flow_flush(pi);
|
||||
rte_eth_dev_close(pi);
|
||||
if (is_proc_primary()) {
|
||||
port_flow_flush(pi);
|
||||
rte_eth_dev_close(pi);
|
||||
}
|
||||
}
|
||||
|
||||
remove_invalid_ports();
|
||||
@ -3106,7 +3177,7 @@ pmd_test_exit(void)
|
||||
}
|
||||
for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
|
||||
if (mempools[i])
|
||||
rte_mempool_free(mempools[i]);
|
||||
mempool_free_mp(mempools[i]);
|
||||
}
|
||||
|
||||
printf("\nBye...\n");
|
||||
@ -3447,7 +3518,7 @@ update_jumbo_frame_offload(portid_t portid)
|
||||
* if unset do it here
|
||||
*/
|
||||
if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
|
||||
ret = rte_eth_dev_set_mtu(portid,
|
||||
ret = eth_dev_set_mtu_mp(portid,
|
||||
port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
|
||||
if (ret)
|
||||
fprintf(stderr,
|
||||
@ -3643,6 +3714,10 @@ init_port_dcb_config(portid_t pid,
|
||||
int retval;
|
||||
uint16_t i;
|
||||
|
||||
if (num_procs > 1) {
|
||||
printf("The multi-process feature doesn't support dcb.\n");
|
||||
return -ENOTSUP;
|
||||
}
|
||||
rte_port = &ports[pid];
|
||||
|
||||
memset(&port_conf, 0, sizeof(struct rte_eth_conf));
|
||||
@ -3812,10 +3887,6 @@ main(int argc, char** argv)
|
||||
rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
|
||||
rte_strerror(rte_errno));
|
||||
|
||||
if (rte_eal_process_type() == RTE_PROC_SECONDARY)
|
||||
rte_exit(EXIT_FAILURE,
|
||||
"Secondary process type not supported.\n");
|
||||
|
||||
ret = register_eth_event_callback();
|
||||
if (ret != 0)
|
||||
rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
|
||||
|
||||
@ -633,6 +633,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
|
||||
|
||||
extern struct rte_flow_action_conntrack conntrack_context;
|
||||
|
||||
extern int proc_id;
|
||||
extern unsigned int num_procs;
|
||||
|
||||
static inline bool
|
||||
is_proc_primary(void)
|
||||
{
|
||||
return rte_eal_process_type() == RTE_PROC_PRIMARY;
|
||||
}
|
||||
|
||||
static inline unsigned int
|
||||
lcore_num(void)
|
||||
{
|
||||
|
||||
@ -82,6 +82,11 @@ New Features
|
||||
|
||||
* Added PDCP short MAC-I support.
|
||||
|
||||
* **Added multi-process support for testpmd.**
|
||||
|
||||
Added command-line options to specify total number of processes and
|
||||
current process ID. Each process owns subset of Rx and Tx queues.
|
||||
|
||||
|
||||
Removed Items
|
||||
-------------
|
||||
|
||||
@ -550,3 +550,87 @@ The command line options are:
|
||||
bit 0 - two hairpin ports loop
|
||||
|
||||
The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
|
||||
|
||||
|
||||
Testpmd Multi-Process Command-line Options
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The following are the command-line options for testpmd multi-process support:
|
||||
|
||||
* primary process:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
|
||||
--num-procs=2 --proc-id=0
|
||||
|
||||
* secondary process:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
|
||||
--num-procs=2 --proc-id=1
|
||||
|
||||
The command line options are:
|
||||
|
||||
* ``--num-procs=N``
|
||||
|
||||
The number of processes which will be used.
|
||||
|
||||
* ``--proc-id=ID``
|
||||
|
||||
The ID of the current process (ID < num-procs). ID should be different in
|
||||
primary process and secondary process, which starts from '0'.
|
||||
|
||||
Calculation rule for queue:
|
||||
All queues are allocated to different processes based on ``proc_num`` and
|
||||
``proc_id``.
|
||||
Calculation rule for the testpmd to allocate queues to each process:
|
||||
* start(queue start id) = proc_id * nb_q / num_procs；
|
||||
|
||||
* end(queue end id) = start + nb_q / num_procs；
|
||||
|
||||
For example, if testpmd is configured to have 4 Tx and Rx queues,
|
||||
queues 0 and 1 will be used by the primary process and
|
||||
queues 2 and 3 will be used by the secondary process.
|
||||
|
||||
The number of queues should be a multiple of the number of processes. If not,
|
||||
redundant queues will exist after queues are allocated to processes. If RSS
|
||||
is enabled, packet loss occurs when traffic is sent to all processes at the same
|
||||
time. Some traffic goes to redundant queues and cannot be forwarded.
|
||||
|
||||
All the dev ops is supported in primary process. While secondary process is
|
||||
not permitted to allocate or release shared memory, so some ops are not supported
|
||||
as follows:
|
||||
|
||||
- ``dev_configure``
|
||||
- ``dev_start``
|
||||
- ``dev_stop``
|
||||
- ``rx_queue_setup``
|
||||
- ``tx_queue_setup``
|
||||
- ``rx_queue_release``
|
||||
- ``tx_queue_release``
|
||||
|
||||
So, any command from testpmd which calls those APIs will not be supported in
|
||||
secondary process, like:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
port config all rxq|txq|rxd|txd <value>
|
||||
port config <port_id> rx_offload xxx on/off
|
||||
port config <port_id> tx_offload xxx on/off
|
||||
|
||||
etc.
|
||||
|
||||
When secondary is running, port in primary is not permitted to be stopped.
|
||||
Reconfigure operation is only valid in primary.
|
||||
|
||||
Stats is supported, stats will not change when one quits and starts, as they
|
||||
share the same buffer to store the stats. Flow rules are maintained in process
|
||||
level: primary and secondary has its own flow list (but one flow list in HW).
|
||||
The two can see all the queues, so setting the flow rules for the other is OK.
|
||||
But in the testpmd primary process receiving or transmitting packets from the
|
||||
queue allocated for secondary process is not permitted, and same for secondary
|
||||
process.
|
||||
|
||||
Flow API and RSS are supported.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user