d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ntb: Add Intel Xeon Gen3 support

Browse Source 
The NTB hardware starting with Skylake has some changes to the register
map and the doorbell interface.  Add a new NTB_XEON_GEN3 device type and
use it to conditionalize driver logic that differs from the existing
Xeon code.

Reviewed by:	vangyzen
Discussed with:	cem, Bret Ketchum <Bret.Ketchum@dell.com>
MFC after:	1 month
Sponsored by:	NetApp, Inc.
Sponsored by:	Klara, Inc.
Differential Revision:	https://reviews.freebsd.org/D26683
This commit is contained in:
Mark Johnston 2020-10-23 14:16:52 +00:00
parent 97441fab87
commit 6660ef6e91
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=366969
3 changed files with 448 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
share/man/man4/ntb_hw_intel.4
View File

@ -25,7 +25,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd August 30, 2017
.Dd October 11, 2020
.Dt NTB_HW_INTEL 4
.Os
.Sh NAME
@ -50,16 +50,18 @@ The
driver provides support for the Non-Transparent Bridge (NTB) hardware in
Intel Xeon E3/E5 and S1200 processor families, which allow one of their PCIe
ports to be switched from transparent to non-transparent bridge mode.
In this mode bridge looks not as a PCI bridge, but as PCI endpoint device.
In this mode the bridge looks not like a PCI bridge, but like a PCI endpoint
device.
The driver hides hardware details, exposing memory windows, scratchpads
and doorbells of the other side via hardware independent KPI to
and doorbells of the other side via a hardware independent KPI to the
.Xr ntb 4
subsystem.
.Pp
The hardware provides 2 or 3 memory windows to the other system's memory,
16 scratchpad registers and 14 or 34 doorbells to interrupt the other system.
On Xeon processors one of memory windows is typically consumed by the driver
itself to workaround multiple hardware erratas.
16 scratchpad registers and 14, 31 or 34 doorbells to interrupt the other
system, depending on the platform.
On Xeon processors one of the memory windows is typically consumed by the driver
itself to work around multiple hardware errata.
.Sh CONFIGURATION
The NTB configuration should be set by BIOS.
It includes enabling NTB, choosing between NTB-to-NTB (back-to-back) or
@ -67,9 +69,10 @@ NTB-to-Root Port mode,
enabling split BAR mode (one of two 64-bit BARs can be split into two 32-bit
ones) and configuring BAR sizes in bits (from 12 to 29/39) for both NTB sides.
.Pp
The recommended configuration is NTB-to-NTB mode, split bar is enabled and
all BAR sizes are set to 20 (1 MiB).
The recommended configuration is NTB-to-NTB mode, split bar enabled and
all BAR sizes set to 20 (1 MiB).
This needs to be done on both systems.
Note, on Xeon SkyLake and newer platforms, split bar mode is not available.
.Sh SEE ALSO
.Xr if_ntb 4 ,
.Xr ntb_transport 4 ,

398
sys/dev/ntb/ntb_hw/ntb_hw_intel.c
View File

@ -68,7 +68,8 @@ __FBSDID("$FreeBSD$");
#include "ntb_hw_intel.h"
#include "../ntb.h"
#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)
#define MAX_MSIX_INTERRUPTS \
MAX(MAX(XEON_DB_COUNT, ATOM_DB_COUNT), XEON_GEN3_DB_COUNT)
#define NTB_HB_TIMEOUT 1 /* second */
#define ATOM_LINK_RECOVERY_TIME 500 /* ms */
@ -87,7 +88,8 @@ __FBSDID("$FreeBSD$");
#define PCI_MSIX_ENTRY_DATA 8
enum ntb_device_type {
NTB_XEON,
NTB_XEON_GEN1,
NTB_XEON_GEN3,
NTB_ATOM
};
@ -338,6 +340,7 @@ static int map_memory_window_bar(struct ntb_softc *ntb,
static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);
static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
static int intel_ntb_init_isr(struct ntb_softc *ntb);
static int intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb);
static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);
@ -355,8 +358,10 @@ static void intel_ntb_exchange_msix(void *);
static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb);
static int intel_ntb_detect_atom(struct ntb_softc *ntb);
static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb);
static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
@ -368,6 +373,9 @@ static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
enum ntb_bar idx);
static int xeon_setup_b2b_mw(struct ntb_softc *,
const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
static int xeon_gen3_setup_b2b_mw(struct ntb_softc *);
static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr,
size_t size);
static inline bool link_is_up(struct ntb_softc *ntb);
static inline bool _xeon_link_is_up(struct ntb_softc *ntb);
static inline bool atom_link_is_err(struct ntb_softc *ntb);
@ -479,6 +487,7 @@ SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx,
#define NTB_B2BDOORBELL_BIT14 (1 << 3)
/* Software/configuration owns the top 16 bits. */
#define NTB_SPLIT_BAR (1ull << 16)
#define NTB_ONE_MSIX (1ull << 17)
#define NTB_FEATURES_STR \
"\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
@ -490,18 +499,21 @@ static struct ntb_hw_info pci_ids[] = {
NTB_ATOM, 0 },
{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B",
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B",
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_SB01BASE_LOCKUP },
{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B",
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
NTB_SB01BASE_LOCKUP },
{ 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B",
NTB_XEON_GEN3, 0 },
};
static const struct ntb_reg atom_reg = {
@ -586,6 +598,37 @@ static struct ntb_b2b_addr xeon_b2b_dsd_addr = {
.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
};
static const struct ntb_reg xeon_gen3_reg = {
.ntb_ctl = XEON_GEN3_REG_IMNTB_CTRL,
.lnk_sta = XEON_GEN3_INT_LNK_STS_OFFSET,
.db_size = sizeof(uint32_t),
.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
};
static const struct ntb_alt_reg xeon_gen3_pri_reg = {
.db_bell = XEON_GEN3_REG_EMDOORBELL,
.db_mask = XEON_GEN3_REG_IMINT_DISABLE,
.spad = XEON_GEN3_REG_IMSPAD,
};
static const struct ntb_alt_reg xeon_gen3_b2b_reg = {
.db_bell = XEON_GEN3_REG_IMDOORBELL,
.db_mask = XEON_GEN3_REG_EMINT_DISABLE,
.spad = XEON_GEN3_REG_IMB2B_SSPAD,
};
static const struct ntb_xlat_reg xeon_gen3_sec_xlat = {
.bar0_base = XEON_GEN3_EXT_REG_BAR0BASE,
.bar2_base = XEON_GEN3_EXT_REG_BAR1BASE,
.bar4_base = XEON_GEN3_EXT_REG_BAR2BASE,
.bar2_limit = XEON_GEN3_REG_IMBAR1XLIMIT,
.bar4_limit = XEON_GEN3_REG_IMBAR2XLIMIT,
.bar2_xlat = XEON_GEN3_REG_IMBAR1XBASE,
.bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE,
};
SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"B2B MW segment overrides -- MUST be the same on both sides");
@ -662,6 +705,8 @@ intel_ntb_attach(device_t device)
if (ntb->type == NTB_ATOM)
error = intel_ntb_detect_atom(ntb);
else if (ntb->type == NTB_XEON_GEN3)
error = intel_ntb_detect_xeon_gen3(ntb);
else
error = intel_ntb_detect_xeon(ntb);
if (error != 0)
@ -676,6 +721,8 @@ intel_ntb_attach(device_t device)
goto out;
if (ntb->type == NTB_ATOM)
error = intel_ntb_atom_init_dev(ntb);
else if (ntb->type == NTB_XEON_GEN3)
error = intel_ntb_xeon_gen3_init_dev(ntb);
else
error = intel_ntb_xeon_init_dev(ntb);
if (error != 0)
@ -715,7 +762,7 @@ intel_ntb_detach(device_t device)
callout_drain(&ntb->lr_timer);
callout_drain(&ntb->peer_msix_work);
pci_disable_busmaster(ntb->device);
if (ntb->type == NTB_XEON)
if (ntb->type == NTB_XEON_GEN1)
intel_ntb_teardown_xeon(ntb);
intel_ntb_teardown_interrupts(ntb);
@ -825,22 +872,39 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb)
rc = map_memory_window_bar(ntb, bar);
if (rc != 0)
goto out;
bar->psz_off = XEON_PBAR23SZ_OFFSET;
bar->ssz_off = XEON_SBAR23SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
if (ntb->type == NTB_XEON_GEN3) {
bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ;
bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ;
bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE;
} else {
bar->psz_off = XEON_PBAR23SZ_OFFSET;
bar->ssz_off = XEON_SBAR23SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
}
bar = &ntb->bar_info[NTB_B2B_BAR_2];
bar->pci_resource_id = PCIR_BAR(4);
rc = map_memory_window_bar(ntb, bar);
if (rc != 0)
goto out;
bar->psz_off = XEON_PBAR4SZ_OFFSET;
bar->ssz_off = XEON_SBAR4SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
if (ntb->type == NTB_XEON_GEN3) {
bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ;
bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ;
bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE;
} else {
bar->psz_off = XEON_PBAR4SZ_OFFSET;
bar->ssz_off = XEON_SBAR4SZ_OFFSET;
bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
}
if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
goto out;
if (ntb->type == NTB_XEON_GEN3) {
device_printf(ntb->device, "no split bar support\n");
return (ENXIO);
}
bar = &ntb->bar_info[NTB_B2B_BAR_3];
bar->pci_resource_id = PCIR_BAR(5);
rc = map_memory_window_bar(ntb, bar);
@ -1059,6 +1123,63 @@ intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
return (rc);
}
static int
intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb)
{
uint64_t i, reg;
uint32_t desired_vectors, num_vectors;
int rc;
ntb->allocated_interrupts = 0;
ntb->last_ts = ticks;
/* Mask all the interrupts, including hardware interrupt */
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, ~0ULL);
/* Clear Interrupt Status */
reg = intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS);
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, reg);
num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
XEON_GEN3_DB_MSIX_VECTOR_COUNT);
rc = pci_alloc_msix(ntb->device, &num_vectors);
if (rc != 0) {
device_printf(ntb->device,
"Interrupt allocation failed %d\n", rc);
return (rc);
}
if (desired_vectors != num_vectors) {
device_printf(ntb->device, "Couldn't get %d vectors\n",
XEON_GEN3_DB_MSIX_VECTOR_COUNT);
return (ENXIO);
}
/* 32 db + 1 hardware */
if (num_vectors == XEON_GEN3_DB_MSIX_VECTOR_COUNT) {
/* Program INTVECXX source register */
for (i = 0; i < XEON_GEN3_DB_MSIX_VECTOR_COUNT; i++) {
/* interrupt source i for vector i */
intel_ntb_reg_write(1, XEON_GEN3_REG_IMINTVEC00 + i, i);
if (i == (XEON_GEN3_DB_MSIX_VECTOR_COUNT - 1)) {
intel_ntb_reg_write(1,
XEON_GEN3_REG_IMINTVEC00 + i,
XEON_GEN3_LINK_VECTOR_INDEX);
}
}
intel_ntb_create_msix_vec(ntb, num_vectors);
rc = intel_ntb_setup_msix(ntb, num_vectors);
/* enable all interrupts */
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, 0ULL);
} else {
device_printf(ntb->device, "need to remap interrupts, giving up.\n");
return (ENXIO);
}
return (0);
}
static int
intel_ntb_init_isr(struct ntb_softc *ntb)
{
@ -1098,7 +1219,7 @@ intel_ntb_init_isr(struct ntb_softc *ntb)
} else
num_vectors = 1;
if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
if (ntb->type == NTB_XEON_GEN1 && num_vectors < ntb->db_vec_count) {
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
device_printf(ntb->device,
"Errata workaround does not support MSI or INTX\n");
@ -1177,20 +1298,17 @@ intel_ntb_teardown_interrupts(struct ntb_softc *ntb)
pci_release_msi(ntb->device);
}
/*
* Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it
* out to make code clearer.
*/
static inline uint64_t
db_ioread(struct ntb_softc *ntb, uint64_t regoff)
{
if (ntb->type == NTB_ATOM)
switch (ntb->type) {
case NTB_ATOM:
case NTB_XEON_GEN3:
return (intel_ntb_reg_read(8, regoff));
KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
return (intel_ntb_reg_read(2, regoff));
case NTB_XEON_GEN1:
return (intel_ntb_reg_read(2, regoff));
}
}
static inline void
@ -1211,13 +1329,15 @@ static inline void
db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
{
if (ntb->type == NTB_ATOM) {
switch (ntb->type) {
case NTB_ATOM:
case NTB_XEON_GEN3:
intel_ntb_reg_write(8, regoff, val);
return;
break;
case NTB_XEON_GEN1:
intel_ntb_reg_write(2, regoff, (uint16_t)val);
break;
}
KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
intel_ntb_reg_write(2, regoff, (uint16_t)val);
}
static void
@ -1267,8 +1387,10 @@ intel_ntb_db_read(device_t dev)
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
return (ntb->fake_db);
return (db_ioread(ntb, ntb->self_reg->db_bell));
if (ntb->type == NTB_XEON_GEN3)
return (intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
else
return (db_ioread(ntb, ntb->self_reg->db_bell));
}
static void
@ -1288,7 +1410,11 @@ intel_ntb_db_clear(device_t dev, uint64_t bits)
return;
}
db_iowrite(ntb, ntb->self_reg->db_bell, bits);
if (ntb->type == NTB_XEON_GEN3)
intel_ntb_reg_write(4, XEON_GEN3_REG_IMINT_STATUS,
(uint32_t)bits);
else
db_iowrite(ntb, ntb->self_reg->db_bell, bits);
}
static inline uint64_t
@ -1322,9 +1448,14 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
ntb->last_ts = ticks;
vec_mask = intel_ntb_vec_mask(ntb, vec);
if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX)
vec_mask |= ntb->db_link_mask;
if ((vec_mask & ntb->db_link_mask) != 0) {
if (intel_ntb_poll_link(ntb))
ntb_link_event(ntb->device);
if (ntb->type == NTB_XEON_GEN3)
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS,
intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
}
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
@ -1449,15 +1580,23 @@ static void
intel_ntb_detect_max_mw(struct ntb_softc *ntb)
{
if (ntb->type == NTB_ATOM) {
switch (ntb->type) {
case NTB_ATOM:
ntb->mw_count = ATOM_MW_COUNT;
return;
break;
case NTB_XEON_GEN1:
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
else
ntb->mw_count = XEON_SNB_MW_COUNT;
break;
case NTB_XEON_GEN3:
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT;
else
ntb->mw_count = XEON_GEN3_MW_COUNT;
break;
}
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
else
ntb->mw_count = XEON_SNB_MW_COUNT;
}
static int
@ -1533,6 +1672,54 @@ intel_ntb_detect_atom(struct ntb_softc *ntb)
return (0);
}
static int
intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb)
{
uint8_t ppd, conn_type;
ppd = pci_read_config(ntb->device, XEON_GEN3_INT_REG_PPD, 1);
ntb->ppd = ppd;
/* check port definition */
conn_type = XEON_GEN3_REG_PPD_PORT_DEF_F(ppd);
switch (conn_type) {
case NTB_CONN_B2B:
ntb->conn_type = conn_type;
break;
default:
device_printf(ntb->device, "Unsupported connection type: %u\n",
conn_type);
return (ENXIO);
}
/* check cross link configuration status */
if (XEON_GEN3_REG_PPD_CONF_STS_F(ppd)) {
/* NTB Port is configured as DSD/USP */
ntb->dev_type = NTB_DEV_DSD;
} else {
/* NTB Port is configured as USD/DSP */
ntb->dev_type = NTB_DEV_USD;
}
if (XEON_GEN3_REG_PPD_ONE_MSIX_F(ppd)) {
/*
* This bit when set, causes only a single MSI-X message to be
* generated if MSI-X is enabled.
*/
ntb->features |= NTB_ONE_MSIX;
}
if (XEON_GEN3_REG_PPD_BAR45_SPL_F(ppd)) {
/* BARs 4 and 5 are presented as two 32b non-prefetchable BARs */
ntb->features |= NTB_SPLIT_BAR;
}
device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x,"
"features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features);
return (0);
}
static int
intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
{
@ -1617,6 +1804,42 @@ intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
return (rc);
}
static int
intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb)
{
int rc;
ntb->spad_count = XEON_GEN3_SPAD_COUNT;
ntb->db_count = XEON_GEN3_DB_COUNT;
ntb->db_link_mask = XEON_GEN3_DB_LINK_BIT;
ntb->db_vec_count = XEON_GEN3_DB_MSIX_VECTOR_COUNT;
ntb->db_vec_shift = XEON_GEN3_DB_MSIX_VECTOR_SHIFT;
if (ntb->conn_type != NTB_CONN_B2B) {
device_printf(ntb->device, "Connection type %d not supported\n",
ntb->conn_type);
return (ENXIO);
}
ntb->reg = &xeon_gen3_reg;
ntb->self_reg = &xeon_gen3_pri_reg;
ntb->peer_reg = &xeon_gen3_b2b_reg;
ntb->xlat_reg = &xeon_gen3_sec_xlat;
ntb->db_valid_mask = (1ULL << ntb->db_count) - 1;
xeon_gen3_setup_b2b_mw(ntb);
/* Enable Bus Master and Memory Space on the External Side */
intel_ntb_reg_write(2, XEON_GEN3_EXT_REG_PCI_CMD,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
/* Setup Interrupt */
rc = intel_ntb_xeon_gen3_init_isr(ntb);
return (rc);
}
static int
intel_ntb_atom_init_dev(struct ntb_softc *ntb)
{
@ -1908,6 +2131,50 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
return (0);
}
static int
xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb)
{
uint64_t reg;
uint32_t embarsz, imbarsz;
/* IMBAR1SZ should be equal to EMBAR1SZ */
embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR1SZ, 1);
imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR1SZ, 1);
if (embarsz != imbarsz) {
device_printf(ntb->device,
"IMBAR1SZ (%u) should be equal to EMBAR1SZ (%u)\n",
imbarsz, embarsz);
return (EIO);
}
/* IMBAR2SZ should be equal to EMBAR2SZ */
embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR2SZ, 1);
imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR2SZ, 1);
if (embarsz != imbarsz) {
device_printf(ntb->device,
"IMBAR2SZ (%u) should be equal to EMBAR2SZ (%u)\n",
imbarsz, embarsz);
return (EIO);
}
/* Client will provide the incoming IMBAR1/2XBASE, zero it for now */
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XBASE, 0);
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XBASE, 0);
/*
* If the value in EMBAR1LIMIT is set equal to the value in EMBAR1,
* the memory window for EMBAR1 is disabled.
* Note: It is needed to avoid malacious access.
*/
reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR1BASE, 8);
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XLIMIT, reg);
reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR2BASE, 8);
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XLIMIT, reg);
return (0);
}
static inline bool
_xeon_link_is_up(struct ntb_softc *ntb)
{
@ -1921,7 +2188,7 @@ static inline bool
link_is_up(struct ntb_softc *ntb)
{
if (ntb->type == NTB_XEON)
if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3)
return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good ||
!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));
@ -2185,7 +2452,9 @@ intel_ntb_poll_link(struct ntb_softc *ntb)
ntb->ntb_ctl = ntb_cntl;
ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
} else {
db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);
if (ntb->type == NTB_XEON_GEN1)
db_iowrite_raw(ntb, ntb->self_reg->db_bell,
ntb->db_link_mask);
reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
if (reg_val == ntb->lnk_sta)
@ -2960,6 +3229,8 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
if (limit_reg != 0 && size != mw_size)
limit = base + size;
else
limit = base + mw_size;
/* Set and verify translation address */
intel_ntb_reg_write(8, xlat_reg, addr);
@ -2977,8 +3248,22 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
intel_ntb_reg_write(8, xlat_reg, 0);
return (EIO);
}
if (ntb->type == NTB_XEON_GEN3) {
limit = base + size;
/* set EMBAR1/2XLIMIT */
if (!idx)
intel_ntb_reg_write(8,
XEON_GEN3_REG_EMBAR1XLIMIT, limit);
else
intel_ntb_reg_write(8,
XEON_GEN3_REG_EMBAR2XLIMIT, limit);
}
} else {
/* Configure 32-bit (split) BAR MW */
if (ntb->type == NTB_XEON_GEN3)
return (EIO);
if ((addr & UINT32_MAX) != addr)
return (ERANGE);
@ -3062,9 +3347,15 @@ intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t m
}
static void
intel_ntb_peer_db_set(device_t dev, uint64_t bit)
intel_ntb_peer_db_set(device_t dev, uint64_t bits)
{
struct ntb_softc *ntb = device_get_softc(dev);
uint64_t db;
if ((bits & ~ntb->db_valid_mask) != 0) {
device_printf(ntb->device, "Invalid doorbell bits %lx\n", bits);
return;
}
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
struct ntb_pci_bar_info *lapic;
@ -3073,7 +3364,7 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bit)
lapic = ntb->peer_lapic_bar;
for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0)
if ((bits & intel_ntb_db_vector_mask(dev, i)) != 0)
bus_space_write_4(lapic->pci_bus_tag,
lapic->pci_bus_handle,
ntb->peer_msix_data[i].nmd_ofs,
@ -3083,11 +3374,22 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bit)
}
if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);
intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bits);
return;
}
db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
if (ntb->type == NTB_XEON_GEN3) {
while (bits != 0) {
db = ffsll(bits);
intel_ntb_reg_write(1,
ntb->peer_reg->db_bell + (db - 1) * 4, 0x1);
bits = bits & (bits - 1);
}
} else {
db_iowrite(ntb, ntb->peer_reg->db_bell, bits);
}
}
static int

87
sys/dev/ntb/ntb_hw/ntb_hw_intel.h
View File

@ -30,6 +30,25 @@
#ifndef _NTB_REGS_H_
#define _NTB_REGS_H_
#include <sys/types.h>
#include <sys/stdint.h>
/*---------------------------------------------------------------------------
* Macro: M*_M : Create a mask to isolate a bit field of a data word.
* M*_F : Extract value from a bit field of a data word.
* M*_I : Insert value into a bit field of a data word.
*
* Purpose: Bit field manipulation macros for mask, insert and extract for
* 8-bit, 16-bit, 32-bit and 64-bit data words.
*
* Params: [in] P = Bit position of start of the bit field (lsb is 0).
* [in] N = Size of the bit field in bits.
* [in] X = Value to insert or remove from the bit field.
*---------------------------------------------------------------------------
*/
#define M8_M(P, N) ((UINT8_MAX >> (8 - (N))) << (P))
#define M8_F(X, P, N) (((uint8_t)(X) & M8_M(P, N)) >> (P))
#define M8_I(X, P, N) (((uint8_t)(X) << (P)) & M8_M(P, N))
#define NTB_LINK_STATUS_ACTIVE 0x2000
#define NTB_LINK_SPEED_MASK 0x000f
@ -164,5 +183,73 @@
/* The peer ntb secondary config space is 32KB fixed size */
#define XEON_B2B_MIN_SIZE 0x8000
#define XEON_GEN3_MW_COUNT 2
#define XEON_GEN3_SPLIT_MW_COUNT 3
#define XEON_GEN3_SPAD_COUNT 16
#define XEON_GEN3_DB_COUNT 32
#define XEON_GEN3_DB_LINK 32
#define XEON_GEN3_DB_LINK_BIT (1ULL << XEON_GEN3_DB_LINK)
#define XEON_GEN3_DB_MSIX_VECTOR_COUNT 33
#define XEON_GEN3_DB_MSIX_VECTOR_SHIFT 1
#define XEON_GEN3_LINK_VECTOR_INDEX 31
/* Xeon Skylake NTB register definitions */
/*
* Internal EndPoint Configuration Registers
*/
#define XEON_GEN3_INT_REG_BAR0BASE 0x10
#define XEON_GEN3_INT_REG_BAR1BASE 0x18
#define XEON_GEN3_INT_REG_BAR2BASE 0x20
#define XEON_GEN3_INT_REG_IMBAR1SZ 0xd0
#define XEON_GEN3_INT_REG_IMBAR2SZ 0xd1
#define XEON_GEN3_INT_REG_EMBAR1SZ 0xd2
#define XEON_GEN3_INT_REG_EMBAR2SZ 0xd3
#define XEON_GEN3_INT_REG_PPD 0xd4
#define XEON_GEN3_INT_LNK_STS_OFFSET 0x01a2
/*
* External EndPoint Configuration Registers
* These are located within BAR0 of the internal endpoint.
*/
#define XEON_GEN3_EXT_REG_PCI_CMD 0x4504
#define XEON_GEN3_EXT_REG_BAR0BASE 0x4510
#define XEON_GEN3_EXT_REG_BAR1BASE 0x4518
#define XEON_GEN3_EXT_REG_BAR2BASE 0x4520
/*
* Internal Endpoint Memory Mapped Registers
*/
#define XEON_GEN3_REG_IMNTB_CTRL 0x0000
#define XEON_GEN3_REG_IMBAR1XBASE 0x0010
#define XEON_GEN3_REG_IMBAR1XLIMIT 0x0018
#define XEON_GEN3_REG_IMBAR2XBASE 0x0020
#define XEON_GEN3_REG_IMBAR2XLIMIT 0x0028
#define XEON_GEN3_REG_IMINT_STATUS 0x0040
#define XEON_GEN3_REG_IMINT_DISABLE 0x0048
#define XEON_GEN3_REG_IMSPAD 0x0080
#define XEON_GEN3_REG_IMINTVEC00 0x00d0
#define XEON_GEN3_REG_IMDOORBELL 0x0100
#define XEON_GEN3_REG_IMB2B_SSPAD 0x0180 /* Pseudo SP registers */
/*
* External Endpoint Memory Mapped Registers
*/
#define XEON_GEN3_REG_EMBAR0XBASE 0x4008
#define XEON_GEN3_REG_EMBAR1XBASE 0x4010
#define XEON_GEN3_REG_EMBAR1XLIMIT 0x4018
#define XEON_GEN3_REG_EMBAR2XBASE 0x4020
#define XEON_GEN3_REG_EMBAR2XLIMIT 0x4028
#define XEON_GEN3_REG_EMINT_STATUS 0x4040
#define XEON_GEN3_REG_EMINT_DISABLE 0x4048
#define XEON_GEN3_REG_EMSPAD 0x4080
#define XEON_GEN3_REG_EMDOORBELL 0x4100
/* XEON_GEN3_INT_REG_PPD: PPD register */
#define XEON_GEN3_REG_PPD_PORT_DEF_F(X) M8_F(X, 0, 2)
#define XEON_GEN3_REG_PPD_CONF_STS_F(X) M8_F(X, 4, 1)
#define XEON_GEN3_REG_PPD_ONE_MSIX_F(X) M8_F(X, 5, 1)
#define XEON_GEN3_REG_PPD_BAR45_SPL_F(X) M8_F(X, 6, 1)
#endif /* _NTB_REGS_H_ */