d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Tx/Rx multiqueue support to vmx(4)

Browse Source 
As a prerequisite for multiple queues, the guest must have MSIX enabled.
Unfortunately, to work around device passthrough bugs, FreeBSD disables
MSIX when running as a VMWare guest due to the hw.pci.honor_msi_blacklist
tunable; this tunable must be disabled for multiple queues.

Also included is various minor changes from the projects/vmxnet branch.

MFC after:	1 month
This commit is contained in:
Bryan Venteicher 2014-03-17 05:45:29 +00:00
parent 47c6266afd
commit e557c1dd90
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=263259
5 changed files with 734 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
share/man/man4/vmx.4
View File

@ -81,12 +81,35 @@ VMware Fusion 2.0 and newer
.Pp
For more information on configuring this device, see
.Xr ifconfig 8 .
.Sh MULTIPLE QUEUES
The
.Nm
driver supports multiple transmit and receive queues.
Multiple queues are only supported by certain VMware products, such as ESXi.
The number of queues allocated depends on the presence of MSI-X,
the number of configured CPUs,
and the tunables listed below.
.Fx
does not enable MSI-X support on VMware by default.
The
.Va hw.pci.honor_msi_blacklist
tunable must be disabled to enable MSI-X support.
.Sh LOADER TUNABLES
Tunables can be set at the
.Xr loader 8
prompt before booting the kernel or stored in
.Xr loader.conf 5 .
.Bl -tag -width indent
.It Va hw.vmx.txnqueue
.It Va hw.vmx. Ns Ar X Ns Va .txnqueue
Maximum number of transmit queues allocated by default by the driver.
The default value is 8.
The maximum supported by the VMXNET3 virtual NIC is 8.
.It Va hw.vmx.rxnqueue
.It Va hw.vmx. Ns Ar X Ns Va .rxnqueue
Maximum number of receive queues allocated by default by the driver.
The default value is 8.
The maximum supported by the VMXNET3 virtual NIC is 16.
.It Va hw.vmx.txndesc
.It Va hw.vmx. Ns Ar X Ns Va .txndesc
.Pp

720
sys/dev/vmware/vmxnet3/if_vmx.c
View File

File diff suppressed because it is too large Load Diff

27
sys/dev/vmware/vmxnet3/if_vmxreg.h
View File

@ -170,6 +170,12 @@ struct vmxnet3_rxcompdesc {
uint32_t gen:1;
} __packed;
#define VMXNET3_RCD_RSS_TYPE_NONE 0
#define VMXNET3_RCD_RSS_TYPE_IPV4 1
#define VMXNET3_RCD_RSS_TYPE_TCPIPV4 2
#define VMXNET3_RCD_RSS_TYPE_IPV6 3
#define VMXNET3_RCD_RSS_TYPE_TCPIPV6 4
#define VMXNET3_REV1_MAGIC 0XBABEFEE1
#define VMXNET3_GOS_UNKNOWN 0x00
@ -313,4 +319,25 @@ struct vmxnet3_rxq_shared {
uint8_t pad4[88];
} __packed;
#define UPT1_RSS_HASH_TYPE_NONE 0x00
#define UPT1_RSS_HASH_TYPE_IPV4 0x01
#define UPT1_RSS_HASH_TYPE_TCP_IPV4 0x02
#define UPT1_RSS_HASH_TYPE_IPV6 0x04
#define UPT1_RSS_HASH_TYPE_TCP_IPV6 0x08
#define UPT1_RSS_HASH_FUNC_NONE 0x00
#define UPT1_RSS_HASH_FUNC_TOEPLITZ 0x01
#define UPT1_RSS_MAX_KEY_SIZE 40
#define UPT1_RSS_MAX_IND_TABLE_SIZE 128
struct vmxnet3_rss_shared {
uint16_t hash_type;
uint16_t hash_func;
uint16_t hash_key_size;
uint16_t ind_table_size;
uint8_t hash_key[UPT1_RSS_MAX_KEY_SIZE];
uint8_t ind_table[UPT1_RSS_MAX_IND_TABLE_SIZE];
} __packed;
#endif /* _IF_VMXREG_H */

52
sys/dev/vmware/vmxnet3/if_vmxvar.h
View File

@ -31,10 +31,10 @@ struct vmxnet3_dma_alloc {
};
/*
* The number of Rx/Tx queues this driver supports.
* The number of Rx/Tx queues this driver prefers.
*/
#define VMXNET3_RX_QUEUES 1
#define VMXNET3_TX_QUEUES 1
#define VMXNET3_DEF_RX_QUEUES 8
#define VMXNET3_DEF_TX_QUEUES 8
/*
* The number of Rx rings in each Rx queue.
@ -119,13 +119,21 @@ struct vmxnet3_comp_ring {
};
struct vmxnet3_txq_stats {
uint64_t vtxrs_full;
uint64_t vtxrs_offload_failed;
uint64_t vmtxs_opackets; /* if_opackets */
uint64_t vmtxs_obytes; /* if_obytes */
uint64_t vmtxs_omcasts; /* if_omcasts */
uint64_t vmtxs_csum;
uint64_t vmtxs_tso;
uint64_t vmtxs_full;
uint64_t vmtxs_offload_failed;
};
struct vmxnet3_txqueue {
struct mtx vxtxq_mtx;
struct vmxnet3_softc *vxtxq_sc;
#ifndef VMXNET3_TX_LEGACY
struct buf_ring *vxtxq_br;
#endif
int vxtxq_id;
int vxtxq_intr_idx;
int vxtxq_watchdog;
@ -134,8 +142,11 @@ struct vmxnet3_txqueue {
struct vmxnet3_txq_stats vxtxq_stats;
struct vmxnet3_txq_shared *vxtxq_ts;
struct sysctl_oid_list *vxtxq_sysctl;
#ifndef VMXNET3_TX_LEGACY
struct task vxtxq_defrtask;
#endif
char vxtxq_name[16];
};
} __aligned(CACHE_LINE_SIZE);
#define VMXNET3_TXQ_LOCK(_txq) mtx_lock(&(_txq)->vxtxq_mtx)
#define VMXNET3_TXQ_TRYLOCK(_txq) mtx_trylock(&(_txq)->vxtxq_mtx)
@ -146,7 +157,10 @@ struct vmxnet3_txqueue {
mtx_assert(&(_txq)->vxtxq_mtx, MA_NOTOWNED)
struct vmxnet3_rxq_stats {
uint64_t vmrxs_ipackets; /* if_ipackets */
uint64_t vmrxs_ibytes; /* if_ibytes */
uint64_t vmrxs_iqdrops; /* if_iqdrops */
uint64_t vmrxs_ierrors; /* if_ierrors */
};
struct vmxnet3_rxqueue {
@ -160,7 +174,7 @@ struct vmxnet3_rxqueue {
struct vmxnet3_rxq_shared *vxrxq_rs;
struct sysctl_oid_list *vxrxq_sysctl;
char vxrxq_name[16];
};
} __aligned(CACHE_LINE_SIZE);
#define VMXNET3_RXQ_LOCK(_rxq) mtx_lock(&(_rxq)->vxrxq_mtx)
#define VMXNET3_RXQ_UNLOCK(_rxq) mtx_unlock(&(_rxq)->vxrxq_mtx)
@ -170,10 +184,10 @@ struct vmxnet3_rxqueue {
mtx_assert(&(_rxq)->vxrxq_mtx, MA_NOTOWNED)
struct vmxnet3_statistics {
uint32_t vmst_collapsed;
uint32_t vmst_defragged;
uint32_t vmst_defrag_failed;
uint32_t vmst_mgetcl_failed;
uint32_t vmst_mbuf_load_failed;
};
struct vmxnet3_interrupt {
@ -188,6 +202,7 @@ struct vmxnet3_softc {
struct vmxnet3_driver_shared *vmx_ds;
uint32_t vmx_flags;
#define VMXNET3_FLAG_NO_MSIX 0x0001
#define VMXNET3_FLAG_RSS 0x0002
struct vmxnet3_rxqueue *vmx_rxq;
struct vmxnet3_txqueue *vmx_txq;
@ -219,13 +234,20 @@ struct vmxnet3_softc {
struct vmxnet3_interrupt vmx_intrs[VMXNET3_MAX_INTRS];
struct mtx vmx_mtx;
#ifndef VMXNET3_LEGACY_TX
struct taskqueue *vmx_tq;
#endif
uint8_t *vmx_mcast;
void *vmx_qs;
struct vmxnet3_rss_shared *vmx_rss;
struct callout vmx_tick;
struct vmxnet3_dma_alloc vmx_ds_dma;
struct vmxnet3_dma_alloc vmx_qs_dma;
struct vmxnet3_dma_alloc vmx_mcast_dma;
struct vmxnet3_dma_alloc vmx_rss_dma;
struct ifmedia vmx_media;
int vmx_max_ntxqueues;
int vmx_max_nrxqueues;
eventhandler_tag vmx_vlan_attach;
eventhandler_tag vmx_vlan_detach;
uint32_t vmx_vlan_filter[4096/32];
@ -252,7 +274,9 @@ struct vmxnet3_softc {
* any TSO packets based on the number of segments.
*/
#define VMXNET3_TX_MAXSEGS 32
#define VMXNET3_TSO_MAXSIZE 65550
#define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES)
#define VMXNET3_TSO_MAXSIZE \
(VMXNET3_TX_MAXSIZE - sizeof(struct ether_vlan_header))
/*
* Maximum support Tx segments size. The length field in the
@ -279,6 +303,12 @@ struct vmxnet3_softc {
*/
#define VMXNET3_WATCHDOG_TIMEOUT 5
/*
* Number of slots in the Tx bufrings. This value matches most other
* multiqueue drivers.
*/
#define VMXNET3_DEF_BUFRING_SIZE 4096
/*
* IP protocols that we can perform Tx checksum offloading of.
*/

4
sys/modules/vmware/vmxnet3/Makefile
View File

@ -31,6 +31,10 @@ KMOD= if_vmx
SRCS= if_vmx.c
SRCS+= bus_if.h device_if.h pci_if.h opt_inet.h opt_inet6.h
# With VMXNET3_LEGACY_TX, the driver will use the non-multiqueue
# capable if_start interface.
#CFLAGS+= -DVMXNET3_LEGACY_TX
.if !defined(KERNBUILDDIR)
.if ${MK_INET_SUPPORT} != "no"
opt_inet.h: