d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

NTB: (partial) MFV ed6c24ed: NTB-RP support

Browse Source 
This commit does not actually add NTB-RP support.  Mostly it serves to
shuffle code around to match the Linux driver.  Original Linux commit
log follows:

Add support for Non-Transparent Bridge connected to a PCI-E Root Port on
the remote system (also known as NTB-RP mode).  This allows for a NTB
enabled system to be connected to a non-NTB enabled system/slot.

Modifications to the registers and BARs/MWs on the Secondary side by the
remote system are reflected into registers on the Primary side for the
local system.  Similarly, modifications of registers and BARs/MWs on
Primary side by the local system are reflected into registers on the
Secondary side for the Remote System.  This allows communication between
the 2 sides via these registers and BARs/MWs.

Note: there is not a fix for the Xeon Errata (that was already worked
around in NTB-B2B mode) for NTB-RP mode.  Due to this limitation, NTB-RP
will not work on the Secondary side with the Xeon Errata workaround
enabled.  To get around this, disable the workaround via the
xeon_errata_workaround=0 modparm.  However, this can cause the hang
described in the errata.

Authored by:	Jon Mason
Obtained from:	Linux
Sponsored by:	EMC / Isilon Storage Division
This commit is contained in:
cem 2015-10-13 19:45:29 +00:00
parent c7e6de59c7
commit 45d8ace0d9
2 changed files with 46 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
sys/dev/ntb/ntb_hw/ntb_hw.c
View File

@ -709,17 +709,6 @@ ntb_setup_xeon(struct ntb_softc *ntb)
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
connection_type = val & XEON_PPD_CONN_TYPE;
switch (connection_type) {
case NTB_CONN_B2B:
ntb->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_CLASSIC:
case NTB_CONN_RP:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
connection_type);
return (ENXIO);
}
if ((val & XEON_PPD_DEV_TYPE) != 0)
ntb->dev_type = NTB_DEV_USD;
@ -728,12 +717,42 @@ ntb_setup_xeon(struct ntb_softc *ntb)
ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET;
ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET;
ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET;
ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET;
ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
switch (connection_type) {
case NTB_CONN_B2B:
ntb->conn_type = NTB_CONN_B2B;
/*
* reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored
* with the NTB_REGS_THRU_MW errata mode enabled. (See
* ntb_ring_doorbell() and ntb_read/write_remote_spad().)
*/
ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_SPADS;
break;
case NTB_CONN_RP:
/*
* Every Xeon today needs NTB_REGS_THRU_MW, so punt on RP for
* now.
*/
KASSERT(HAS_FEATURE(NTB_REGS_THRU_MW),
("Xeon without MW errata unimplemented"));
device_printf(ntb->device,
"NTB-RP disabled to due hardware errata.\n");
return (ENXIO);
case NTB_CONN_TRANSPARENT:
default:
device_printf(ntb->device, "Connection type %d not supported\n",
connection_type);
return (ENXIO);
}
/*
* There is a Xeon hardware errata related to writes to SDOORBELL or
@ -757,15 +776,9 @@ ntb_setup_xeon(struct ntb_softc *ntb)
ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
if (ntb->conn_type == NTB_CONN_B2B) {
ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_SPADS;
} else {
ntb->reg_ofs.rdb = XEON_SDOORBELL_OFFSET;
ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET;
ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS;
}
ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
ntb->limits.max_db_bits = XEON_MAX_DB_BITS;
ntb->limits.msix_cnt = XEON_MSIX_CNT;
@ -774,8 +787,9 @@ ntb_setup_xeon(struct ntb_softc *ntb)
configure_xeon_secondary_side_bars(ntb);
/* Enable Bus Master and Memory Space on the secondary side */
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
if (ntb->conn_type == NTB_CONN_B2B)
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
/* Enable link training */
ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
@ -996,7 +1010,8 @@ ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
ntb->link_status = NTB_LINK_UP;
event = NTB_EVENT_HW_LINK_UP;
if (ntb->type == NTB_SOC)
if (ntb->type == NTB_SOC ||
ntb->conn_type == NTB_CONN_TRANSPARENT)
status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
else
status = pci_read_config(ntb->device,

7
sys/dev/ntb/ntb_hw/ntb_regs.h
View File

@ -29,8 +29,6 @@
#ifndef _NTB_REGS_H_
#define _NTB_REGS_H_
#define NTB_LINK_ENABLE 0x0000
#define NTB_LINK_DISABLE 0x0002
#define NTB_LINK_STATUS_ACTIVE 0x2000
#define NTB_LINK_SPEED_MASK 0x000f
#define NTB_LINK_WIDTH_MASK 0x03f0
@ -47,6 +45,7 @@
#define XEON_PCICMD_OFFSET 0x0504
#define XEON_DEVCTRL_OFFSET 0x0598
#define XEON_LINK_STATUS_OFFSET 0x01a2
#define XEON_SLINK_STATUS_OFFSET 0x05a2
#define XEON_PBAR2LMT_OFFSET 0x0000
#define XEON_PBAR4LMT_OFFSET 0x0008
@ -117,6 +116,8 @@
#define SOC_LTSSMSTATEJMP_FORCEDETECT (1 << 2)
#define SOC_IBIST_ERR_OFLOW 0x7fff7fff
#define NTB_CNTL_CFG_LOCK (1 << 0)
#define NTB_CNTL_LINK_DISABLE (1 << 1)
#define NTB_CNTL_BAR23_SNOOP (1 << 2)
#define NTB_CNTL_BAR45_SNOOP (1 << 6)
#define SOC_CNTL_LINK_DOWN (1 << 16)
@ -130,7 +131,7 @@
#define SOC_PPD_CONN_TYPE 0x0300
#define SOC_PPD_DEV_TYPE 0x1000
#define NTB_CONN_CLASSIC 0
#define NTB_CONN_TRANSPARENT 0
#define NTB_CONN_B2B 1
#define NTB_CONN_RP 2