Add a new driver to support the Intel Non-Transparent Bridge(NTB).

The NTB allows you to connect two systems with this device using a PCI-e
link. The driver is made of two modules:
 - ntb_hw which is a basic hardware abstraction layer for the device.
 - if_ntb which implements the ntb network device and the communication
   protocol.

The driver is limited at the moment to CPU memcpy instead of using DMA, and
only Back-to-Back mode is supported. Also the network device isn't full
featured yet. These changes will be coming soon. The DMA change will also
bring in the ioat driver from the project branch it is on now.

This is an initial port of the GPL/BSD Linux driver contributed by Jon Mason
from Intel. Any bugs are my contributions.

Sponsored by: Intel
Reviewed by: jimharris, joel (man page only)
Approved by: jimharris (mentor)
This commit is contained in:
Carl Delsey 2013-04-29 22:48:53 +00:00
parent 3c9cb0c6c9
commit e47937d1b7
Notes: svn2git 2020-12-20 02:59:44 +00:00
svn path=/head/; revision=250079
12 changed files with 3023 additions and 0 deletions

View File

@ -333,6 +333,7 @@ MAN= aac.4 \
ng_vlan.4 \
nmdm.4 \
nsp.4 \
${_ntb.4} \
null.4 \
${_nvd.4} \
${_nve.4} \
@ -647,6 +648,7 @@ MLINKS+=netintro.4 net.4 \
netintro.4 networking.4
MLINKS+=${_nfe.4} ${_if_nfe.4}
MLINKS+=nge.4 if_nge.4
MLINKS+=${_ntb.4} ${_if_ntb.4} ${_ntb_hw.4}
MLINKS+=${_nve.4} ${_if_nve.4}
MLINKS+=${_nxge.4} ${_if_nxge.4}
MLINKS+=patm.4 if_patm.4
@ -784,6 +786,9 @@ MLINKS+=lindev.4 full.4
.if ${MACHINE_CPUARCH} == "amd64"
_bhyve.4= bhyve.4
_if_ntb.4= if_ntb.4
_ntb.4= ntb.4
_ntb_hw.4= ntb_hw.4
_qlxgb.4= qlxgb.4
_sfxge.4= sfxge.4

114
share/man/man4/ntb.4 Normal file
View File

@ -0,0 +1,114 @@
.\"
.\" Copyright (c) 2013 Intel Corporation
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions, and the following disclaimer,
.\" without modification.
.\" 2. Redistributions in binary form must reproduce at minimum a disclaimer
.\" substantially similar to the "NO WARRANTY" disclaimer below
.\" ("Disclaimer") and any redistribution must be conditioned upon
.\" including a substantially similar Disclaimer requirement for further
.\" binary redistribution.
.\"
.\" NO WARRANTY
.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
.\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
.\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
.\" HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGES.
.\"
.\" ntb driver man page.
.\"
.\" Author: Carl Delsey <carl@FreeBSD.org>
.\"
.\" $FreeBSD$
.\"
.Dd Apr 11, 2013
.Dt NTB 4
.Os
.Sh NAME
.Nm ntb ,
.Nm ntb_hw ,
.Nm if_ntb
.Nd Intel(R) Non-Transparent Bridge driver
.Sh SYNOPSIS
To compile this driver into your kernel,
place the following lines in your kernel configuration file:
.Bd -ragged -offset indent
.Cd "device ntb_hw"
.Cd "device if_ntb"
.Ed
.Pp
Or, to load the driver as a module at boot, place the following line in
.Xr loader.conf 5 :
.Bd -literal -offset indent
if_ntb_load="YES"
.Ed
.Sh DESCRIPTION
The
.Nm
driver provides support for the Non-Transparent Bridge (NTB) in the Intel S1200,
Xeon E3 and Xeon E5 processor families.
.Pp
The NTB allows you to connect two computer systems using a PCI-e link if they
have the correct equipment and connectors.
.Sh CONFIGURATION
The NTB memory windows need to be configured by the BIOS.
If your BIOS allows you to set their size, you should set the size of both
memory windows to 1 MiB.
This needs to be done on both systems.
.Pp
Each system needs to have a different IP address assigned.
The MAC address is randomly generated.
Also for maximum performance, the MTU should be set to 16 kiB.
This can be down by adding the line below to
.Xr rc.conf 5 :
.Bd -literal -offset indent
ifconfig_ntb0="inet 192.168.1.10 netmask 255.255.255.0 mtu 16384"
.Ed
.Pp
And on the second system :
.Bd -literal -offset indent
ifconfig_ntb0="inet 192.168.1.11 netmask 255.255.255.0 mtu 16384"
.Ed
.Pp
If you are using the UDP protocol, you may want to increase the
.Va net.inet.udp.maxdgram
.Xr sysctl 8
variable.
.Sh SEE ALSO
.Xr rc.conf 5 ,
.Xr sysctl 8
.Sh AUTHORS
.An -nosplit
The
.Nm
driver was developed by Intel and originally written by
.An Carl Delsey Aq carl@FreeBSD.org.
.Sh BUGS
If the driver is unloaded, it cannot be reloaded without a system reboot.
.Pp
The network support is limited.
It isn't fully configurable yet.
It also isn't integrated into
.Xr netgraph 4
or
.Xr bpf 4 .
.Pp
NTB to Root Port mode is not yet supported.
.Pp
There is no way to protect your system from malicious behavior on the other
system once the link is brought up.
Anyone with root or kernel access on the other system can read or write to
any location on your system.
In other words, only connect two systems that completely trust each other.

View File

@ -366,6 +366,10 @@ device iwn6000fw
device iwn6050fw
device wpifw
# Intel Non-Transparent Bridge (NTB) hardware
device ntb_hw # Hardware Abstraction Layer for the NTB
device if_ntb # Simulated ethernet device using the NTB
#
#XXX this stores pointers in a 32bit field that is defined by the hardware
#device pst

View File

@ -211,6 +211,8 @@ dev/kbd/kbd.c optional atkbd | sc | ukbd
dev/lindev/full.c optional lindev
dev/lindev/lindev.c optional lindev
dev/nfe/if_nfe.c optional nfe pci
dev/ntb/if_ntb/if_ntb.c optional if_ntb
dev/ntb/ntb_hw/ntb_hw.c optional if_ntb ntb_hw
dev/nvd/nvd.c optional nvd nvme
dev/nve/if_nve.c optional nve pci
dev/nvme/nvme.c optional nvme

1366
sys/dev/ntb/if_ntb/if_ntb.c Normal file

File diff suppressed because it is too large Load Diff

1288
sys/dev/ntb/ntb_hw/ntb_hw.c Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,73 @@
/*-
* Copyright (C) 2013 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _NTB_HW_H_
#define _NTB_HW_H_
struct ntb_softc;
#define NTB_NUM_MW 2
#define NTB_LINK_DOWN 0
#define NTB_LINK_UP 1
enum ntb_hw_event {
NTB_EVENT_SW_EVENT0 = 0,
NTB_EVENT_SW_EVENT1,
NTB_EVENT_SW_EVENT2,
NTB_EVENT_HW_ERROR,
NTB_EVENT_HW_LINK_UP,
NTB_EVENT_HW_LINK_DOWN,
};
typedef void (*ntb_db_callback)(void *data, int db_num);
typedef void (*ntb_event_callback)(void *data, enum ntb_hw_event event);
int ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func);
void ntb_unregister_event_callback(struct ntb_softc *ntb);
int ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx,
void *data, ntb_db_callback func);
void ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx);
void *ntb_find_transport(struct ntb_softc *ntb);
struct ntb_softc *ntb_register_transport(struct ntb_softc *ntb,
void *transport);
void ntb_unregister_transport(struct ntb_softc *ntb);
int ntb_get_max_spads(struct ntb_softc *ntb);
int ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
int ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);
int ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx,
uint32_t val);
int ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx,
uint32_t *val);
void *ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw);
vm_paddr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw);
u_long ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw);
void ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr);
void ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db);
bool ntb_query_link_status(struct ntb_softc *ntb);
#endif /* _NTB_HW_H_ */

View File

@ -0,0 +1,146 @@
/*-
* Copyright (C) 2013 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _NTB_REGS_H_
#define _NTB_REGS_H_
#define NTB_LINK_ENABLE 0x0000
#define NTB_LINK_DISABLE 0x0002
#define NTB_LINK_STATUS_ACTIVE 0x2000
#define NTB_LINK_SPEED_MASK 0x000f
#define NTB_LINK_WIDTH_MASK 0x03f0
#define XEON_MSIX_CNT 4
#define XEON_MAX_SPADS 16
#define XEON_MAX_COMPAT_SPADS 8
/* Reserve the uppermost bit for link interrupt */
#define XEON_MAX_DB_BITS 15
#define XEON_DB_BITS_PER_VEC 5
#define XEON_DB_HW_LINK 0x8000
#define XEON_PCICMD_OFFSET 0x0504
#define XEON_DEVCTRL_OFFSET 0x0598
#define XEON_LINK_STATUS_OFFSET 0x01A2
#define XEON_PBAR2LMT_OFFSET 0x0000
#define XEON_PBAR4LMT_OFFSET 0x0008
#define XEON_PBAR2XLAT_OFFSET 0x0010
#define XEON_PBAR4XLAT_OFFSET 0x0018
#define XEON_SBAR2LMT_OFFSET 0x0020
#define XEON_SBAR4LMT_OFFSET 0x0028
#define XEON_SBAR2XLAT_OFFSET 0x0030
#define XEON_SBAR4XLAT_OFFSET 0x0038
#define XEON_SBAR0BASE_OFFSET 0x0040
#define XEON_SBAR2BASE_OFFSET 0x0048
#define XEON_SBAR4BASE_OFFSET 0x0050
#define XEON_NTBCNTL_OFFSET 0x0058
#define XEON_SBDF_OFFSET 0x005C
#define XEON_PDOORBELL_OFFSET 0x0060
#define XEON_PDBMSK_OFFSET 0x0062
#define XEON_SDOORBELL_OFFSET 0x0064
#define XEON_SDBMSK_OFFSET 0x0066
#define XEON_USMEMMISS 0x0070
#define XEON_SPAD_OFFSET 0x0080
#define XEON_SPADSEMA4_OFFSET 0x00c0
#define XEON_WCCNTRL_OFFSET 0x00e0
#define XEON_B2B_SPAD_OFFSET 0x0100
#define XEON_B2B_DOORBELL_OFFSET 0x0140
#define XEON_B2B_XLAT_OFFSET 0x0144
#define SOC_MSIX_CNT 34
#define SOC_MAX_SPADS 16
#define SOC_MAX_COMPAT_SPADS 16
#define SOC_MAX_DB_BITS 34
#define SOC_DB_BITS_PER_VEC 1
#define SOC_PCICMD_OFFSET 0xb004
#define SOC_MBAR23_OFFSET 0xb018
#define SOC_MBAR45_OFFSET 0xb020
#define SOC_DEVCTRL_OFFSET 0xb048
#define SOC_LINK_STATUS_OFFSET 0xb052
#define SOC_ERRCORSTS_OFFSET 0xb110
#define SOC_SBAR2XLAT_OFFSET 0x0008
#define SOC_SBAR4XLAT_OFFSET 0x0010
#define SOC_PDOORBELL_OFFSET 0x0020
#define SOC_PDBMSK_OFFSET 0x0028
#define SOC_NTBCNTL_OFFSET 0x0060
#define SOC_EBDF_OFFSET 0x0064
#define SOC_SPAD_OFFSET 0x0080
#define SOC_SPADSEMA_OFFSET 0x00c0
#define SOC_STKYSPAD_OFFSET 0x00c4
#define SOC_PBAR2XLAT_OFFSET 0x8008
#define SOC_PBAR4XLAT_OFFSET 0x8010
#define SOC_B2B_DOORBELL_OFFSET 0x8020
#define SOC_B2B_SPAD_OFFSET 0x8080
#define SOC_B2B_SPADSEMA_OFFSET 0x80c0
#define SOC_B2B_STKYSPAD_OFFSET 0x80c4
#define SOC_MODPHY_PCSREG4 0x1c004
#define SOC_MODPHY_PCSREG6 0x1c006
#define SOC_IP_BASE 0xC000
#define SOC_DESKEWSTS_OFFSET (SOC_IP_BASE + 0x3024)
#define SOC_LTSSMERRSTS0_OFFSET (SOC_IP_BASE + 0x3180)
#define SOC_LTSSMSTATEJMP_OFFSET (SOC_IP_BASE + 0x3040)
#define SOC_IBSTERRRCRVSTS0_OFFSET (SOC_IP_BASE + 0x3324)
#define SOC_DESKEWSTS_DBERR (1 << 15)
#define SOC_LTSSMERRSTS0_UNEXPECTEDEI (1 << 20)
#define SOC_LTSSMSTATEJMP_FORCEDETECT (1 << 2)
#define SOC_IBIST_ERR_OFLOW 0x7FFF7FFF
#define NTB_CNTL_BAR23_SNOOP (1 << 2)
#define NTB_CNTL_BAR45_SNOOP (1 << 6)
#define SOC_CNTL_LINK_DOWN (1 << 16)
#define NTB_PPD_OFFSET 0x00D4
#define XEON_PPD_CONN_TYPE 0x0003
#define XEON_PPD_DEV_TYPE 0x0010
#define SOC_PPD_INIT_LINK 0x0008
#define SOC_PPD_CONN_TYPE 0x0300
#define SOC_PPD_DEV_TYPE 0x1000
#define NTB_CONN_CLASSIC 0
#define NTB_CONN_B2B 1
#define NTB_CONN_RP 2
#define NTB_DEV_DSD 1
#define NTB_DEV_USD 0
#define SOC_PBAR2XLAT_USD_ADDR 0x0000004000000000
#define SOC_PBAR4XLAT_USD_ADDR 0x0000008000000000
#define SOC_MBAR23_USD_ADDR 0x000000410000000C
#define SOC_MBAR45_USD_ADDR 0x000000810000000C
#define SOC_PBAR2XLAT_DSD_ADDR 0x0000004100000000
#define SOC_PBAR4XLAT_DSD_ADDR 0x0000008100000000
#define SOC_MBAR23_DSD_ADDR 0x000000400000000C
#define SOC_MBAR45_DSD_ADDR 0x000000800000000C
#endif /* _NTB_REGS_H_ */

View File

@ -243,6 +243,7 @@ SUBDIR= \
nmdm \
${_nsp} \
nullfs \
${_ntb} \
${_nvd} \
${_nve} \
${_nvme} \
@ -693,6 +694,7 @@ _mthca= mthca
.endif
_ndis= ndis
_nfe= nfe
_ntb= ntb
_nvd= nvd
.if ${MK_SOURCELESS_HOST} != "no"
_nve= nve

5
sys/modules/ntb/Makefile Normal file
View File

@ -0,0 +1,5 @@
# $FreeBSD$
SUBDIR= ntb_hw if_ntb
.include <bsd.subdir.mk>

View File

@ -0,0 +1,9 @@
# $FreeBSD$
.PATH: ${.CURDIR}/../../../dev/ntb/if_ntb
KMOD = if_ntb
SRCS = if_ntb.c
SRCS += device_if.h bus_if.h pci_if.h
.include <bsd.kmod.mk>

View File

@ -0,0 +1,9 @@
# $FreeBSD$
.PATH: ${.CURDIR}/../../../dev/ntb/ntb_hw
KMOD = ntb_hw
SRCS = ntb_hw.c
SRCS += device_if.h bus_if.h pci_if.h
.include <bsd.kmod.mk>