2a6e50d2ce
Approved by: jimharris Sponsored by: Intel
1415 lines
37 KiB
C
1415 lines
37 KiB
C
/*-
|
|
* Copyright (C) 2013 Intel Corporation
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/bus.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/module.h>
|
|
#include <sys/queue.h>
|
|
#include <sys/rman.h>
|
|
#include <vm/vm.h>
|
|
#include <vm/pmap.h>
|
|
#include <machine/bus.h>
|
|
#include <machine/pmap.h>
|
|
#include <machine/resource.h>
|
|
#include <dev/pci/pcireg.h>
|
|
#include <dev/pci/pcivar.h>
|
|
|
|
#include "ntb_regs.h"
|
|
#include "ntb_hw.h"
|
|
|
|
/*
|
|
* The Non-Transparent Bridge (NTB) is a device on some Intel processors that
|
|
* allows you to connect two systems using a PCI-e link.
|
|
*
|
|
* This module contains the hardware abstraction layer for the NTB. It allows
|
|
* you to send and recieve interrupts, map the memory windows and send and
|
|
* receive messages in the scratch-pad registers.
|
|
*
|
|
* NOTE: Much of the code in this module is shared with Linux. Any patches may
|
|
* be picked up and redistributed in Linux with a dual GPL/BSD license.
|
|
*/
|
|
|
|
#define NTB_CONFIG_BAR 0
|
|
#define NTB_B2B_BAR_1 1
|
|
#define NTB_B2B_BAR_2 2
|
|
#define NTB_MAX_BARS 3
|
|
#define NTB_MW_TO_BAR(mw) ((mw) + 1)
|
|
|
|
#define MAX_MSIX_INTERRUPTS MAX(XEON_MAX_DB_BITS, SOC_MAX_DB_BITS)
|
|
|
|
#define NTB_HB_TIMEOUT 1 /* second */
|
|
#define SOC_LINK_RECOVERY_TIME 500
|
|
|
|
#define DEVICE2SOFTC(dev) ((struct ntb_softc *) device_get_softc(dev))
|
|
|
|
enum ntb_device_type {
|
|
NTB_XEON,
|
|
NTB_SOC
|
|
};
|
|
|
|
/* Device features and workarounds */
|
|
#define HAS_FEATURE(feature) \
|
|
((ntb->features & (feature)) != 0)
|
|
|
|
#define NTB_BAR_SIZE_4K (1 << 0)
|
|
#define NTB_REGS_THRU_MW (1 << 1)
|
|
|
|
struct ntb_hw_info {
|
|
uint32_t device_id;
|
|
const char *desc;
|
|
enum ntb_device_type type;
|
|
uint64_t features;
|
|
};
|
|
|
|
struct ntb_pci_bar_info {
|
|
bus_space_tag_t pci_bus_tag;
|
|
bus_space_handle_t pci_bus_handle;
|
|
int pci_resource_id;
|
|
struct resource *pci_resource;
|
|
vm_paddr_t pbase;
|
|
void *vbase;
|
|
u_long size;
|
|
};
|
|
|
|
struct ntb_int_info {
|
|
struct resource *res;
|
|
int rid;
|
|
void *tag;
|
|
};
|
|
|
|
struct ntb_db_cb {
|
|
ntb_db_callback callback;
|
|
unsigned int db_num;
|
|
void *data;
|
|
struct ntb_softc *ntb;
|
|
};
|
|
|
|
struct ntb_softc {
|
|
device_t device;
|
|
enum ntb_device_type type;
|
|
uint64_t features;
|
|
|
|
struct ntb_pci_bar_info bar_info[NTB_MAX_BARS];
|
|
struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS];
|
|
uint32_t allocated_interrupts;
|
|
|
|
struct callout heartbeat_timer;
|
|
struct callout lr_timer;
|
|
|
|
void *ntb_transport;
|
|
ntb_event_callback event_cb;
|
|
struct ntb_db_cb *db_cb;
|
|
|
|
struct {
|
|
uint32_t max_spads;
|
|
uint32_t max_db_bits;
|
|
uint32_t msix_cnt;
|
|
} limits;
|
|
struct {
|
|
uint32_t pdb;
|
|
uint32_t pdb_mask;
|
|
uint32_t sdb;
|
|
uint32_t sbar2_xlat;
|
|
uint32_t sbar4_xlat;
|
|
uint32_t spad_remote;
|
|
uint32_t spad_local;
|
|
uint32_t lnk_cntl;
|
|
uint32_t lnk_stat;
|
|
uint32_t spci_cmd;
|
|
} reg_ofs;
|
|
uint8_t conn_type;
|
|
uint8_t dev_type;
|
|
uint8_t bits_per_vector;
|
|
uint8_t link_status;
|
|
uint8_t link_width;
|
|
uint8_t link_speed;
|
|
};
|
|
|
|
#define ntb_bar_read(SIZE, bar, offset) \
|
|
bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
|
|
ntb->bar_info[(bar)].pci_bus_handle, (offset))
|
|
#define ntb_bar_write(SIZE, bar, offset, val) \
|
|
bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
|
|
ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
|
|
#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
|
|
#define ntb_reg_write(SIZE, offset, val) \
|
|
ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
|
|
#define ntb_mw_read(SIZE, offset) ntb_bar_read(SIZE, NTB_B2B_BAR_2, offset)
|
|
#define ntb_mw_write(SIZE, offset, val) \
|
|
ntb_bar_write(SIZE, NTB_B2B_BAR_2, offset, val)
|
|
|
|
typedef int (*bar_map_strategy)(struct ntb_softc *ntb,
|
|
struct ntb_pci_bar_info *bar);
|
|
|
|
static int ntb_probe(device_t device);
|
|
static int ntb_attach(device_t device);
|
|
static int ntb_detach(device_t device);
|
|
static int ntb_map_pci_bars(struct ntb_softc *ntb);
|
|
static int map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
|
|
struct ntb_pci_bar_info *bar);
|
|
static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
|
|
static int map_memory_window_bar(struct ntb_softc *ntb,
|
|
struct ntb_pci_bar_info *bar);
|
|
static void ntb_unmap_pci_bar(struct ntb_softc *ntb);
|
|
static int ntb_setup_interrupts(struct ntb_softc *ntb);
|
|
static void ntb_teardown_interrupts(struct ntb_softc *ntb);
|
|
static void handle_soc_irq(void *arg);
|
|
static void handle_xeon_irq(void *arg);
|
|
static void handle_xeon_event_irq(void *arg);
|
|
static void ntb_handle_legacy_interrupt(void *arg);
|
|
static int ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors);
|
|
static void ntb_free_callbacks(struct ntb_softc *ntb);
|
|
static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id);
|
|
static int ntb_initialize_hw(struct ntb_softc *ntb);
|
|
static int ntb_setup_xeon(struct ntb_softc *ntb);
|
|
static int ntb_setup_soc(struct ntb_softc *ntb);
|
|
static void configure_soc_secondary_side_bars(struct ntb_softc *ntb);
|
|
static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb);
|
|
static void ntb_handle_heartbeat(void *arg);
|
|
static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state);
|
|
static void recover_soc_link(void *arg);
|
|
static int ntb_check_link_status(struct ntb_softc *ntb);
|
|
static void save_bar_parameters(struct ntb_pci_bar_info *bar);
|
|
|
|
static struct ntb_hw_info pci_ids[] = {
|
|
{ 0x3C0D8086, "Xeon E5/Core i7 Non-Transparent Bridge B2B", NTB_XEON,
|
|
NTB_REGS_THRU_MW },
|
|
{ 0x0C4E8086, "Atom Processor S1200 NTB Primary B2B", NTB_SOC, 0 },
|
|
{ 0x0E0D8086, "Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
|
|
NTB_REGS_THRU_MW | NTB_BAR_SIZE_4K },
|
|
{ 0x00000000, NULL, NTB_SOC, 0 }
|
|
};
|
|
|
|
/*
|
|
* OS <-> Driver interface structures
|
|
*/
|
|
MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
|
|
|
|
static device_method_t ntb_pci_methods[] = {
|
|
/* Device interface */
|
|
DEVMETHOD(device_probe, ntb_probe),
|
|
DEVMETHOD(device_attach, ntb_attach),
|
|
DEVMETHOD(device_detach, ntb_detach),
|
|
DEVMETHOD_END
|
|
};
|
|
|
|
static driver_t ntb_pci_driver = {
|
|
"ntb_hw",
|
|
ntb_pci_methods,
|
|
sizeof(struct ntb_softc),
|
|
};
|
|
|
|
static devclass_t ntb_devclass;
|
|
DRIVER_MODULE(ntb_hw, pci, ntb_pci_driver, ntb_devclass, NULL, NULL);
|
|
MODULE_VERSION(ntb_hw, 1);
|
|
|
|
/*
|
|
* OS <-> Driver linkage functions
|
|
*/
|
|
static int
|
|
ntb_probe(device_t device)
|
|
{
|
|
struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
|
|
|
|
if (p != NULL) {
|
|
device_set_desc(device, p->desc);
|
|
return (0);
|
|
} else
|
|
return (ENXIO);
|
|
}
|
|
|
|
#define DETACH_ON_ERROR(func) \
|
|
error = func; \
|
|
if (error < 0) { \
|
|
ntb_detach(device); \
|
|
return (error); \
|
|
}
|
|
|
|
static int
|
|
ntb_attach(device_t device)
|
|
{
|
|
struct ntb_softc *ntb = DEVICE2SOFTC(device);
|
|
struct ntb_hw_info *p = ntb_get_device_info(pci_get_devid(device));
|
|
int error;
|
|
|
|
ntb->device = device;
|
|
ntb->type = p->type;
|
|
ntb->features = p->features;
|
|
|
|
/* Heartbeat timer for NTB_SOC since there is no link interrupt */
|
|
callout_init(&ntb->heartbeat_timer, CALLOUT_MPSAFE);
|
|
callout_init(&ntb->lr_timer, CALLOUT_MPSAFE);
|
|
|
|
DETACH_ON_ERROR(ntb_map_pci_bars(ntb));
|
|
DETACH_ON_ERROR(ntb_initialize_hw(ntb));
|
|
DETACH_ON_ERROR(ntb_setup_interrupts(ntb));
|
|
|
|
pci_enable_busmaster(ntb->device);
|
|
|
|
return (error);
|
|
}
|
|
|
|
static int
|
|
ntb_detach(device_t device)
|
|
{
|
|
struct ntb_softc *ntb = DEVICE2SOFTC(device);
|
|
|
|
callout_drain(&ntb->heartbeat_timer);
|
|
callout_drain(&ntb->lr_timer);
|
|
ntb_teardown_interrupts(ntb);
|
|
ntb_unmap_pci_bar(ntb);
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
ntb_map_pci_bars(struct ntb_softc *ntb)
|
|
{
|
|
int rc;
|
|
|
|
ntb->bar_info[NTB_CONFIG_BAR].pci_resource_id = PCIR_BAR(0);
|
|
rc = map_pci_bar(ntb, map_mmr_bar, &ntb->bar_info[NTB_CONFIG_BAR]);
|
|
if (rc != 0)
|
|
return rc;
|
|
|
|
ntb->bar_info[NTB_B2B_BAR_1].pci_resource_id = PCIR_BAR(2);
|
|
rc = map_pci_bar(ntb, map_memory_window_bar,
|
|
&ntb->bar_info[NTB_B2B_BAR_1]);
|
|
if (rc != 0)
|
|
return rc;
|
|
|
|
ntb->bar_info[NTB_B2B_BAR_2].pci_resource_id = PCIR_BAR(4);
|
|
if (HAS_FEATURE(NTB_REGS_THRU_MW))
|
|
rc = map_pci_bar(ntb, map_mmr_bar,
|
|
&ntb->bar_info[NTB_B2B_BAR_2]);
|
|
else
|
|
rc = map_pci_bar(ntb, map_memory_window_bar,
|
|
&ntb->bar_info[NTB_B2B_BAR_2]);
|
|
if (rc != 0)
|
|
return rc;
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
map_pci_bar(struct ntb_softc *ntb, bar_map_strategy strategy,
|
|
struct ntb_pci_bar_info *bar)
|
|
{
|
|
int rc;
|
|
|
|
rc = strategy(ntb, bar);
|
|
if (rc != 0) {
|
|
device_printf(ntb->device,
|
|
"unable to allocate pci resource\n");
|
|
} else {
|
|
device_printf(ntb->device,
|
|
"Bar size = %lx, v %p, p %p\n",
|
|
bar->size, bar->vbase,
|
|
(void *)(bar->pbase));
|
|
}
|
|
return (rc);
|
|
}
|
|
|
|
static int
|
|
map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
|
|
{
|
|
|
|
bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
|
|
&bar->pci_resource_id, RF_ACTIVE);
|
|
|
|
if (bar->pci_resource == NULL)
|
|
return (ENXIO);
|
|
else {
|
|
save_bar_parameters(bar);
|
|
return (0);
|
|
}
|
|
}
|
|
|
|
static int
|
|
map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
|
|
{
|
|
int rc;
|
|
uint8_t bar_size_bits = 0;
|
|
|
|
bar->pci_resource = bus_alloc_resource_any(ntb->device,
|
|
SYS_RES_MEMORY, &bar->pci_resource_id, RF_ACTIVE);
|
|
|
|
if (bar->pci_resource == NULL)
|
|
return (ENXIO);
|
|
else {
|
|
save_bar_parameters(bar);
|
|
/*
|
|
* Ivytown NTB BAR sizes are misreported by the hardware due to
|
|
* a hardware issue. To work around this, query the size it
|
|
* should be configured to by the device and modify the resource
|
|
* to correspond to this new size. The BIOS on systems with this
|
|
* problem is required to provide enough address space to allow
|
|
* the driver to make this change safely.
|
|
*
|
|
* Ideally I could have just specified the size when I allocated
|
|
* the resource like:
|
|
* bus_alloc_resource(ntb->device,
|
|
* SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
|
|
* 1ul << bar_size_bits, RF_ACTIVE);
|
|
* but the PCI driver does not honor the size in this call, so
|
|
* we have to modify it after the fact.
|
|
*/
|
|
if (HAS_FEATURE(NTB_BAR_SIZE_4K)) {
|
|
if (bar->pci_resource_id == PCIR_BAR(2))
|
|
bar_size_bits = pci_read_config(ntb->device,
|
|
XEON_PBAR23SZ_OFFSET, 1);
|
|
else
|
|
bar_size_bits = pci_read_config(ntb->device,
|
|
XEON_PBAR45SZ_OFFSET, 1);
|
|
rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
|
|
bar->pci_resource, bar->pbase,
|
|
bar->pbase + (1ul << bar_size_bits) - 1);
|
|
if (rc != 0 ) {
|
|
device_printf(ntb->device,
|
|
"unable to resize bar\n");
|
|
return (rc);
|
|
} else
|
|
save_bar_parameters(bar);
|
|
}
|
|
|
|
/* Mark bar region as write combining to improve performance. */
|
|
rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size,
|
|
VM_MEMATTR_WRITE_COMBINING);
|
|
if (rc != 0) {
|
|
device_printf(ntb->device, "unable to mark bar as"
|
|
" WRITE_COMBINING\n");
|
|
return (rc);
|
|
}
|
|
}
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
ntb_unmap_pci_bar(struct ntb_softc *ntb)
|
|
{
|
|
struct ntb_pci_bar_info *current_bar;
|
|
int i;
|
|
|
|
for (i = 0; i< NTB_MAX_BARS; i++) {
|
|
current_bar = &ntb->bar_info[i];
|
|
if (current_bar->pci_resource != NULL)
|
|
bus_release_resource(ntb->device, SYS_RES_MEMORY,
|
|
current_bar->pci_resource_id,
|
|
current_bar->pci_resource);
|
|
}
|
|
}
|
|
|
|
static int
|
|
ntb_setup_interrupts(struct ntb_softc *ntb)
|
|
{
|
|
void (*interrupt_handler)(void *);
|
|
void *int_arg;
|
|
bool use_msix = 0;
|
|
uint32_t num_vectors;
|
|
int i;
|
|
|
|
ntb->allocated_interrupts = 0;
|
|
/*
|
|
* On SOC, disable all interrupts. On XEON, disable all but Link
|
|
* Interrupt. The rest will be unmasked as callbacks are registered.
|
|
*/
|
|
if (ntb->type == NTB_SOC)
|
|
ntb_reg_write(8, ntb->reg_ofs.pdb_mask, ~0);
|
|
else
|
|
ntb_reg_write(2, ntb->reg_ofs.pdb_mask,
|
|
~(1 << ntb->limits.max_db_bits));
|
|
|
|
num_vectors = MIN(pci_msix_count(ntb->device),
|
|
ntb->limits.max_db_bits);
|
|
if (num_vectors >= 1) {
|
|
pci_alloc_msix(ntb->device, &num_vectors);
|
|
if (num_vectors >= 4)
|
|
use_msix = TRUE;
|
|
}
|
|
|
|
ntb_create_callbacks(ntb, num_vectors);
|
|
if (use_msix == TRUE) {
|
|
for (i = 0; i < num_vectors; i++) {
|
|
ntb->int_info[i].rid = i + 1;
|
|
ntb->int_info[i].res = bus_alloc_resource_any(
|
|
ntb->device, SYS_RES_IRQ, &ntb->int_info[i].rid,
|
|
RF_ACTIVE);
|
|
if (ntb->int_info[i].res == NULL) {
|
|
device_printf(ntb->device,
|
|
"bus_alloc_resource failed\n");
|
|
return (-1);
|
|
}
|
|
ntb->int_info[i].tag = NULL;
|
|
ntb->allocated_interrupts++;
|
|
if (ntb->type == NTB_SOC) {
|
|
interrupt_handler = handle_soc_irq;
|
|
int_arg = &ntb->db_cb[i];
|
|
} else {
|
|
if (i == num_vectors - 1) {
|
|
interrupt_handler =
|
|
handle_xeon_event_irq;
|
|
int_arg = ntb;
|
|
} else {
|
|
interrupt_handler =
|
|
handle_xeon_irq;
|
|
int_arg = &ntb->db_cb[i];
|
|
}
|
|
}
|
|
if (bus_setup_intr(ntb->device, ntb->int_info[i].res,
|
|
INTR_MPSAFE | INTR_TYPE_MISC, NULL,
|
|
interrupt_handler, int_arg,
|
|
&ntb->int_info[i].tag) != 0) {
|
|
device_printf(ntb->device,
|
|
"bus_setup_intr failed\n");
|
|
return (ENXIO);
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
ntb->int_info[0].rid = 0;
|
|
ntb->int_info[0].res = bus_alloc_resource_any(ntb->device,
|
|
SYS_RES_IRQ, &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
|
|
interrupt_handler = ntb_handle_legacy_interrupt;
|
|
if (ntb->int_info[0].res == NULL) {
|
|
device_printf(ntb->device,
|
|
"bus_alloc_resource failed\n");
|
|
return (-1);
|
|
}
|
|
ntb->int_info[0].tag = NULL;
|
|
ntb->allocated_interrupts = 1;
|
|
|
|
if (bus_setup_intr(ntb->device, ntb->int_info[0].res,
|
|
INTR_MPSAFE | INTR_TYPE_MISC, NULL,
|
|
interrupt_handler, ntb, &ntb->int_info[0].tag) != 0) {
|
|
|
|
device_printf(ntb->device, "bus_setup_intr failed\n");
|
|
return (ENXIO);
|
|
}
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
ntb_teardown_interrupts(struct ntb_softc *ntb)
|
|
{
|
|
struct ntb_int_info *current_int;
|
|
int i;
|
|
|
|
for (i=0; i<ntb->allocated_interrupts; i++) {
|
|
current_int = &ntb->int_info[i];
|
|
if (current_int->tag != NULL)
|
|
bus_teardown_intr(ntb->device, current_int->res,
|
|
current_int->tag);
|
|
|
|
if (current_int->res != NULL)
|
|
bus_release_resource(ntb->device, SYS_RES_IRQ,
|
|
rman_get_rid(current_int->res), current_int->res);
|
|
}
|
|
|
|
ntb_free_callbacks(ntb);
|
|
pci_release_msi(ntb->device);
|
|
}
|
|
|
|
static void
|
|
handle_soc_irq(void *arg)
|
|
{
|
|
struct ntb_db_cb *db_cb = arg;
|
|
struct ntb_softc *ntb = db_cb->ntb;
|
|
|
|
ntb_reg_write(8, ntb->reg_ofs.pdb, (uint64_t) 1 << db_cb->db_num);
|
|
|
|
if (db_cb->callback != NULL)
|
|
db_cb->callback(db_cb->data, db_cb->db_num);
|
|
}
|
|
|
|
static void
|
|
handle_xeon_irq(void *arg)
|
|
{
|
|
struct ntb_db_cb *db_cb = arg;
|
|
struct ntb_softc *ntb = db_cb->ntb;
|
|
|
|
/*
|
|
* On Xeon, there are 16 bits in the interrupt register
|
|
* but only 4 vectors. So, 5 bits are assigned to the first 3
|
|
* vectors, with the 4th having a single bit for link
|
|
* interrupts.
|
|
*/
|
|
ntb_reg_write(2, ntb->reg_ofs.pdb,
|
|
((1 << ntb->bits_per_vector) - 1) <<
|
|
(db_cb->db_num * ntb->bits_per_vector));
|
|
|
|
if (db_cb->callback != NULL)
|
|
db_cb->callback(db_cb->data, db_cb->db_num);
|
|
}
|
|
|
|
/* Since we do not have a HW doorbell in SOC, this is only used in JF/JT */
|
|
static void
|
|
handle_xeon_event_irq(void *arg)
|
|
{
|
|
struct ntb_softc *ntb = arg;
|
|
int rc;
|
|
|
|
rc = ntb_check_link_status(ntb);
|
|
if (rc != 0)
|
|
device_printf(ntb->device, "Error determining link status\n");
|
|
|
|
/* bit 15 is always the link bit */
|
|
ntb_reg_write(2, ntb->reg_ofs.pdb, 1 << ntb->limits.max_db_bits);
|
|
}
|
|
|
|
static void
|
|
ntb_handle_legacy_interrupt(void *arg)
|
|
{
|
|
struct ntb_softc *ntb = arg;
|
|
unsigned int i = 0;
|
|
uint64_t pdb64;
|
|
uint16_t pdb16;
|
|
|
|
if (ntb->type == NTB_SOC) {
|
|
pdb64 = ntb_reg_read(8, ntb->reg_ofs.pdb);
|
|
|
|
while (pdb64) {
|
|
i = ffs(pdb64);
|
|
pdb64 &= pdb64 - 1;
|
|
handle_soc_irq(&ntb->db_cb[i]);
|
|
}
|
|
} else {
|
|
pdb16 = ntb_reg_read(2, ntb->reg_ofs.pdb);
|
|
|
|
if ((pdb16 & XEON_DB_HW_LINK) != 0) {
|
|
handle_xeon_event_irq(ntb);
|
|
pdb16 &= ~XEON_DB_HW_LINK;
|
|
}
|
|
|
|
while (pdb16 != 0) {
|
|
i = ffs(pdb16);
|
|
pdb16 &= pdb16 - 1;
|
|
handle_xeon_irq(&ntb->db_cb[i]);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static int
|
|
ntb_create_callbacks(struct ntb_softc *ntb, int num_vectors)
|
|
{
|
|
int i;
|
|
|
|
ntb->db_cb = malloc(num_vectors * sizeof(struct ntb_db_cb), M_NTB,
|
|
M_ZERO | M_WAITOK);
|
|
for (i = 0; i < num_vectors; i++) {
|
|
ntb->db_cb[i].db_num = i;
|
|
ntb->db_cb[i].ntb = ntb;
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
ntb_free_callbacks(struct ntb_softc *ntb)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < ntb->limits.max_db_bits; i++)
|
|
ntb_unregister_db_callback(ntb, i);
|
|
|
|
free(ntb->db_cb, M_NTB);
|
|
}
|
|
|
|
static struct ntb_hw_info *
|
|
ntb_get_device_info(uint32_t device_id)
|
|
{
|
|
struct ntb_hw_info *ep = pci_ids;
|
|
|
|
while (ep->device_id) {
|
|
if (ep->device_id == device_id)
|
|
return (ep);
|
|
++ep;
|
|
}
|
|
return (NULL);
|
|
}
|
|
|
|
static int
|
|
ntb_initialize_hw(struct ntb_softc *ntb)
|
|
{
|
|
|
|
if (ntb->type == NTB_SOC)
|
|
return (ntb_setup_soc(ntb));
|
|
else
|
|
return (ntb_setup_xeon(ntb));
|
|
}
|
|
|
|
static int
|
|
ntb_setup_xeon(struct ntb_softc *ntb)
|
|
{
|
|
uint8_t val, connection_type;
|
|
|
|
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
|
|
|
|
connection_type = val & XEON_PPD_CONN_TYPE;
|
|
switch (connection_type) {
|
|
case NTB_CONN_B2B:
|
|
ntb->conn_type = NTB_CONN_B2B;
|
|
break;
|
|
case NTB_CONN_CLASSIC:
|
|
case NTB_CONN_RP:
|
|
default:
|
|
device_printf(ntb->device, "Connection type %d not supported\n",
|
|
connection_type);
|
|
return (ENXIO);
|
|
}
|
|
|
|
if ((val & XEON_PPD_DEV_TYPE) != 0)
|
|
ntb->dev_type = NTB_DEV_DSD;
|
|
else
|
|
ntb->dev_type = NTB_DEV_USD;
|
|
|
|
ntb->reg_ofs.pdb = XEON_PDOORBELL_OFFSET;
|
|
ntb->reg_ofs.pdb_mask = XEON_PDBMSK_OFFSET;
|
|
ntb->reg_ofs.sbar2_xlat = XEON_SBAR2XLAT_OFFSET;
|
|
ntb->reg_ofs.sbar4_xlat = XEON_SBAR4XLAT_OFFSET;
|
|
ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET;
|
|
ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET;
|
|
ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET;
|
|
ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET;
|
|
|
|
if (ntb->conn_type == NTB_CONN_B2B) {
|
|
ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET;
|
|
ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET;
|
|
ntb->limits.max_spads = XEON_MAX_SPADS;
|
|
} else {
|
|
ntb->reg_ofs.sdb = XEON_SDOORBELL_OFFSET;
|
|
ntb->reg_ofs.spad_remote = XEON_SPAD_OFFSET;
|
|
ntb->limits.max_spads = XEON_MAX_COMPAT_SPADS;
|
|
}
|
|
|
|
ntb->limits.max_db_bits = XEON_MAX_DB_BITS;
|
|
ntb->limits.msix_cnt = XEON_MSIX_CNT;
|
|
ntb->bits_per_vector = XEON_DB_BITS_PER_VEC;
|
|
|
|
configure_xeon_secondary_side_bars(ntb);
|
|
/* Enable Bus Master and Memory Space on the secondary side */
|
|
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
|
|
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
|
|
|
|
/* Enable link training */
|
|
ntb_reg_write(4, ntb->reg_ofs.lnk_cntl,
|
|
NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP);
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
ntb_setup_soc(struct ntb_softc *ntb)
|
|
{
|
|
uint32_t val, connection_type;
|
|
|
|
val = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
|
|
|
|
connection_type = (val & SOC_PPD_CONN_TYPE) >> 8;
|
|
switch (connection_type) {
|
|
case NTB_CONN_B2B:
|
|
ntb->conn_type = NTB_CONN_B2B;
|
|
break;
|
|
case NTB_CONN_RP:
|
|
default:
|
|
device_printf(ntb->device, "Connection type %d not supported\n",
|
|
connection_type);
|
|
return (ENXIO);
|
|
}
|
|
|
|
if ((val & SOC_PPD_DEV_TYPE) != 0)
|
|
ntb->dev_type = NTB_DEV_DSD;
|
|
else
|
|
ntb->dev_type = NTB_DEV_USD;
|
|
|
|
/* Initiate PCI-E link training */
|
|
pci_write_config(ntb->device, NTB_PPD_OFFSET, val | SOC_PPD_INIT_LINK,
|
|
4);
|
|
|
|
ntb->reg_ofs.pdb = SOC_PDOORBELL_OFFSET;
|
|
ntb->reg_ofs.pdb_mask = SOC_PDBMSK_OFFSET;
|
|
ntb->reg_ofs.sbar2_xlat = SOC_SBAR2XLAT_OFFSET;
|
|
ntb->reg_ofs.sbar4_xlat = SOC_SBAR4XLAT_OFFSET;
|
|
ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET;
|
|
ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET;
|
|
ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET;
|
|
ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET;
|
|
|
|
if (ntb->conn_type == NTB_CONN_B2B) {
|
|
ntb->reg_ofs.sdb = SOC_B2B_DOORBELL_OFFSET;
|
|
ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET;
|
|
ntb->limits.max_spads = SOC_MAX_SPADS;
|
|
} else {
|
|
ntb->reg_ofs.sdb = SOC_PDOORBELL_OFFSET;
|
|
ntb->reg_ofs.spad_remote = SOC_SPAD_OFFSET;
|
|
ntb->limits.max_spads = SOC_MAX_COMPAT_SPADS;
|
|
}
|
|
|
|
ntb->limits.max_db_bits = SOC_MAX_DB_BITS;
|
|
ntb->limits.msix_cnt = SOC_MSIX_CNT;
|
|
ntb->bits_per_vector = SOC_DB_BITS_PER_VEC;
|
|
|
|
/*
|
|
* FIXME - MSI-X bug on early SOC HW, remove once internal issue is
|
|
* resolved. Mask transaction layer internal parity errors.
|
|
*/
|
|
pci_write_config(ntb->device, 0xFC, 0x4, 4);
|
|
|
|
configure_soc_secondary_side_bars(ntb);
|
|
|
|
/* Enable Bus Master and Memory Space on the secondary side */
|
|
ntb_reg_write(2, ntb->reg_ofs.spci_cmd,
|
|
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
|
|
callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb);
|
|
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
configure_soc_secondary_side_bars(struct ntb_softc *ntb)
|
|
{
|
|
|
|
if (ntb->dev_type == NTB_DEV_USD) {
|
|
ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
|
|
ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR);
|
|
ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_USD_ADDR);
|
|
ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_USD_ADDR);
|
|
} else {
|
|
ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
|
|
ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR);
|
|
ntb_reg_write(8, SOC_MBAR23_OFFSET, MBAR23_DSD_ADDR);
|
|
ntb_reg_write(8, SOC_MBAR45_OFFSET, MBAR45_DSD_ADDR);
|
|
}
|
|
}
|
|
|
|
static void
|
|
configure_xeon_secondary_side_bars(struct ntb_softc *ntb)
|
|
{
|
|
|
|
if (ntb->dev_type == NTB_DEV_USD) {
|
|
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_USD_ADDR);
|
|
if (HAS_FEATURE(NTB_REGS_THRU_MW))
|
|
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
|
|
MBAR01_DSD_ADDR);
|
|
else
|
|
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
|
|
PBAR4XLAT_USD_ADDR);
|
|
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR);
|
|
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR);
|
|
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR);
|
|
} else {
|
|
ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, PBAR2XLAT_DSD_ADDR);
|
|
if (HAS_FEATURE(NTB_REGS_THRU_MW))
|
|
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
|
|
MBAR01_USD_ADDR);
|
|
else
|
|
ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET,
|
|
PBAR4XLAT_DSD_ADDR);
|
|
ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR);
|
|
ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR);
|
|
ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR);
|
|
}
|
|
}
|
|
|
|
/* SOC does not have link status interrupt, poll on that platform */
|
|
static void
|
|
ntb_handle_heartbeat(void *arg)
|
|
{
|
|
struct ntb_softc *ntb = arg;
|
|
uint32_t status32;
|
|
int rc = ntb_check_link_status(ntb);
|
|
|
|
if (rc != 0)
|
|
device_printf(ntb->device,
|
|
"Error determining link status\n");
|
|
/* Check to see if a link error is the cause of the link down */
|
|
if (ntb->link_status == NTB_LINK_DOWN) {
|
|
status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
|
|
if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0) {
|
|
callout_reset(&ntb->lr_timer, 0, recover_soc_link,
|
|
ntb);
|
|
return;
|
|
}
|
|
}
|
|
|
|
callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
|
|
ntb_handle_heartbeat, ntb);
|
|
}
|
|
|
|
static void
|
|
soc_perform_link_restart(struct ntb_softc *ntb)
|
|
{
|
|
uint32_t status;
|
|
|
|
/* Driver resets the NTB ModPhy lanes - magic! */
|
|
ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0xe0);
|
|
ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x40);
|
|
ntb_reg_write(1, SOC_MODPHY_PCSREG4, 0x60);
|
|
ntb_reg_write(1, SOC_MODPHY_PCSREG6, 0x60);
|
|
|
|
/* Driver waits 100ms to allow the NTB ModPhy to settle */
|
|
pause("ModPhy", hz / 10);
|
|
|
|
/* Clear AER Errors, write to clear */
|
|
status = ntb_reg_read(4, SOC_ERRCORSTS_OFFSET);
|
|
status &= PCIM_AER_COR_REPLAY_ROLLOVER;
|
|
ntb_reg_write(4, SOC_ERRCORSTS_OFFSET, status);
|
|
|
|
/* Clear unexpected electrical idle event in LTSSM, write to clear */
|
|
status = ntb_reg_read(4, SOC_LTSSMERRSTS0_OFFSET);
|
|
status |= SOC_LTSSMERRSTS0_UNEXPECTEDEI;
|
|
ntb_reg_write(4, SOC_LTSSMERRSTS0_OFFSET, status);
|
|
|
|
/* Clear DeSkew Buffer error, write to clear */
|
|
status = ntb_reg_read(4, SOC_DESKEWSTS_OFFSET);
|
|
status |= SOC_DESKEWSTS_DBERR;
|
|
ntb_reg_write(4, SOC_DESKEWSTS_OFFSET, status);
|
|
|
|
status = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
|
|
status &= SOC_IBIST_ERR_OFLOW;
|
|
ntb_reg_write(4, SOC_IBSTERRRCRVSTS0_OFFSET, status);
|
|
|
|
/* Releases the NTB state machine to allow the link to retrain */
|
|
status = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
|
|
status &= ~SOC_LTSSMSTATEJMP_FORCEDETECT;
|
|
ntb_reg_write(4, SOC_LTSSMSTATEJMP_OFFSET, status);
|
|
}
|
|
|
|
static void
|
|
ntb_handle_link_event(struct ntb_softc *ntb, int link_state)
|
|
{
|
|
enum ntb_hw_event event;
|
|
uint16_t status;
|
|
|
|
if (ntb->link_status == link_state)
|
|
return;
|
|
|
|
if (link_state == NTB_LINK_UP) {
|
|
device_printf(ntb->device, "Link Up\n");
|
|
ntb->link_status = NTB_LINK_UP;
|
|
event = NTB_EVENT_HW_LINK_UP;
|
|
|
|
if (ntb->type == NTB_SOC)
|
|
status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
|
|
else
|
|
status = pci_read_config(ntb->device,
|
|
XEON_LINK_STATUS_OFFSET, 2);
|
|
ntb->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
|
|
ntb->link_speed = (status & NTB_LINK_SPEED_MASK);
|
|
device_printf(ntb->device, "Link Width %d, Link Speed %d\n",
|
|
ntb->link_width, ntb->link_speed);
|
|
callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
|
|
ntb_handle_heartbeat, ntb);
|
|
} else {
|
|
device_printf(ntb->device, "Link Down\n");
|
|
ntb->link_status = NTB_LINK_DOWN;
|
|
event = NTB_EVENT_HW_LINK_DOWN;
|
|
/* Do not modify link width/speed, we need it in link recovery */
|
|
}
|
|
|
|
/* notify the upper layer if we have an event change */
|
|
if (ntb->event_cb != NULL)
|
|
ntb->event_cb(ntb->ntb_transport, event);
|
|
}
|
|
|
|
static void
|
|
recover_soc_link(void *arg)
|
|
{
|
|
struct ntb_softc *ntb = arg;
|
|
uint8_t speed, width;
|
|
uint32_t status32;
|
|
uint16_t status16;
|
|
|
|
soc_perform_link_restart(ntb);
|
|
pause("Link", SOC_LINK_RECOVERY_TIME * hz / 1000);
|
|
|
|
status32 = ntb_reg_read(4, SOC_LTSSMSTATEJMP_OFFSET);
|
|
if ((status32 & SOC_LTSSMSTATEJMP_FORCEDETECT) != 0)
|
|
goto retry;
|
|
|
|
status32 = ntb_reg_read(4, SOC_IBSTERRRCRVSTS0_OFFSET);
|
|
if ((status32 & SOC_IBIST_ERR_OFLOW) != 0)
|
|
goto retry;
|
|
|
|
status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat);
|
|
width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
|
|
speed = (status16 & NTB_LINK_SPEED_MASK);
|
|
if (ntb->link_width != width || ntb->link_speed != speed)
|
|
goto retry;
|
|
|
|
callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz,
|
|
ntb_handle_heartbeat, ntb);
|
|
return;
|
|
|
|
retry:
|
|
callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_soc_link,
|
|
ntb);
|
|
}
|
|
|
|
static int
|
|
ntb_check_link_status(struct ntb_softc *ntb)
|
|
{
|
|
int link_state;
|
|
uint32_t ntb_cntl;
|
|
uint16_t status;
|
|
|
|
if (ntb->type == NTB_SOC) {
|
|
ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl);
|
|
if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0)
|
|
link_state = NTB_LINK_DOWN;
|
|
else
|
|
link_state = NTB_LINK_UP;
|
|
} else {
|
|
status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET,
|
|
2);
|
|
|
|
if ((status & NTB_LINK_STATUS_ACTIVE) != 0)
|
|
link_state = NTB_LINK_UP;
|
|
else
|
|
link_state = NTB_LINK_DOWN;
|
|
}
|
|
|
|
ntb_handle_link_event(ntb, link_state);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_register_event_callback() - register event callback
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @func: callback function to register
|
|
*
|
|
* This function registers a callback for any HW driver events such as link
|
|
* up/down, power management notices and etc.
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
int
|
|
ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func)
|
|
{
|
|
|
|
if (ntb->event_cb != NULL)
|
|
return (EINVAL);
|
|
|
|
ntb->event_cb = func;
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_unregister_event_callback() - unregisters the event callback
|
|
* @ntb: pointer to ntb_softc instance
|
|
*
|
|
* This function unregisters the existing callback from transport
|
|
*/
|
|
void
|
|
ntb_unregister_event_callback(struct ntb_softc *ntb)
|
|
{
|
|
|
|
ntb->event_cb = NULL;
|
|
}
|
|
|
|
/**
|
|
* ntb_register_db_callback() - register a callback for doorbell interrupt
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @idx: doorbell index to register callback, zero based
|
|
* @func: callback function to register
|
|
*
|
|
* This function registers a callback function for the doorbell interrupt
|
|
* on the primary side. The function will unmask the doorbell as well to
|
|
* allow interrupt.
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
int
|
|
ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx, void *data,
|
|
ntb_db_callback func)
|
|
{
|
|
uint16_t mask;
|
|
|
|
if (idx >= ntb->allocated_interrupts || ntb->db_cb[idx].callback) {
|
|
device_printf(ntb->device, "Invalid Index.\n");
|
|
return (EINVAL);
|
|
}
|
|
|
|
ntb->db_cb[idx].callback = func;
|
|
ntb->db_cb[idx].data = data;
|
|
|
|
/* unmask interrupt */
|
|
mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask);
|
|
mask &= ~(1 << (idx * ntb->bits_per_vector));
|
|
ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @idx: doorbell index to register callback, zero based
|
|
*
|
|
* This function unregisters a callback function for the doorbell interrupt
|
|
* on the primary side. The function will also mask the said doorbell.
|
|
*/
|
|
void
|
|
ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx)
|
|
{
|
|
unsigned long mask;
|
|
|
|
if (idx >= ntb->allocated_interrupts || !ntb->db_cb[idx].callback)
|
|
return;
|
|
|
|
mask = ntb_reg_read(2, ntb->reg_ofs.pdb_mask);
|
|
mask |= 1 << (idx * ntb->bits_per_vector);
|
|
ntb_reg_write(2, ntb->reg_ofs.pdb_mask, mask);
|
|
|
|
ntb->db_cb[idx].callback = NULL;
|
|
}
|
|
|
|
/**
|
|
* ntb_find_transport() - find the transport pointer
|
|
* @transport: pointer to pci device
|
|
*
|
|
* Given the pci device pointer, return the transport pointer passed in when
|
|
* the transport attached when it was inited.
|
|
*
|
|
* RETURNS: pointer to transport.
|
|
*/
|
|
void *
|
|
ntb_find_transport(struct ntb_softc *ntb)
|
|
{
|
|
|
|
return (ntb->ntb_transport);
|
|
}
|
|
|
|
/**
|
|
* ntb_register_transport() - Register NTB transport with NTB HW driver
|
|
* @transport: transport identifier
|
|
*
|
|
* This function allows a transport to reserve the hardware driver for
|
|
* NTB usage.
|
|
*
|
|
* RETURNS: pointer to ntb_softc, NULL on error.
|
|
*/
|
|
struct ntb_softc *
|
|
ntb_register_transport(struct ntb_softc *ntb, void *transport)
|
|
{
|
|
|
|
/*
|
|
* TODO: when we have more than one transport, we will need to rewrite
|
|
* this to prevent race conditions
|
|
*/
|
|
if (ntb->ntb_transport != NULL)
|
|
return (NULL);
|
|
|
|
ntb->ntb_transport = transport;
|
|
return (ntb);
|
|
}
|
|
|
|
/**
|
|
* ntb_unregister_transport() - Unregister the transport with the NTB HW driver
|
|
* @ntb - ntb_softc of the transport to be freed
|
|
*
|
|
* This function unregisters the transport from the HW driver and performs any
|
|
* necessary cleanups.
|
|
*/
|
|
void
|
|
ntb_unregister_transport(struct ntb_softc *ntb)
|
|
{
|
|
int i;
|
|
|
|
if (ntb->ntb_transport == NULL)
|
|
return;
|
|
|
|
for (i = 0; i < ntb->allocated_interrupts; i++)
|
|
ntb_unregister_db_callback(ntb, i);
|
|
|
|
ntb_unregister_event_callback(ntb);
|
|
ntb->ntb_transport = NULL;
|
|
}
|
|
|
|
/**
|
|
* ntb_get_max_spads() - get the total scratch regs usable
|
|
* @ntb: pointer to ntb_softc instance
|
|
*
|
|
* This function returns the max 32bit scratchpad registers usable by the
|
|
* upper layer.
|
|
*
|
|
* RETURNS: total number of scratch pad registers available
|
|
*/
|
|
int
|
|
ntb_get_max_spads(struct ntb_softc *ntb)
|
|
{
|
|
|
|
return (ntb->limits.max_spads);
|
|
}
|
|
|
|
/**
|
|
* ntb_write_local_spad() - write to the secondary scratchpad register
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @idx: index to the scratchpad register, 0 based
|
|
* @val: the data value to put into the register
|
|
*
|
|
* This function allows writing of a 32bit value to the indexed scratchpad
|
|
* register. The register resides on the secondary (external) side.
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
int
|
|
ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
|
|
{
|
|
|
|
if (idx >= ntb->limits.max_spads)
|
|
return (EINVAL);
|
|
|
|
ntb_reg_write(4, ntb->reg_ofs.spad_local + idx * 4, val);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_read_local_spad() - read from the primary scratchpad register
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @idx: index to scratchpad register, 0 based
|
|
* @val: pointer to 32bit integer for storing the register value
|
|
*
|
|
* This function allows reading of the 32bit scratchpad register on
|
|
* the primary (internal) side.
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
int
|
|
ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
|
|
{
|
|
|
|
if (idx >= ntb->limits.max_spads)
|
|
return (EINVAL);
|
|
|
|
*val = ntb_reg_read(4, ntb->reg_ofs.spad_local + idx * 4);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_write_remote_spad() - write to the secondary scratchpad register
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @idx: index to the scratchpad register, 0 based
|
|
* @val: the data value to put into the register
|
|
*
|
|
* This function allows writing of a 32bit value to the indexed scratchpad
|
|
* register. The register resides on the secondary (external) side.
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
int
|
|
ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val)
|
|
{
|
|
|
|
if (idx >= ntb->limits.max_spads)
|
|
return (EINVAL);
|
|
|
|
if (HAS_FEATURE(NTB_REGS_THRU_MW))
|
|
ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val);
|
|
else
|
|
ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_read_remote_spad() - read from the primary scratchpad register
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @idx: index to scratchpad register, 0 based
|
|
* @val: pointer to 32bit integer for storing the register value
|
|
*
|
|
* This function allows reading of the 32bit scratchpad register on
|
|
* the primary (internal) side.
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
int
|
|
ntb_read_remote_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
|
|
{
|
|
|
|
if (idx >= ntb->limits.max_spads)
|
|
return (EINVAL);
|
|
|
|
if (HAS_FEATURE(NTB_REGS_THRU_MW))
|
|
*val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4);
|
|
else
|
|
*val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* ntb_get_mw_vbase() - get virtual addr for the NTB memory window
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @mw: memory window number
|
|
*
|
|
* This function provides the base virtual address of the memory window
|
|
* specified.
|
|
*
|
|
* RETURNS: pointer to virtual address, or NULL on error.
|
|
*/
|
|
void *
|
|
ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw)
|
|
{
|
|
|
|
if (mw >= NTB_NUM_MW)
|
|
return (NULL);
|
|
|
|
return (ntb->bar_info[NTB_MW_TO_BAR(mw)].vbase);
|
|
}
|
|
|
|
vm_paddr_t
|
|
ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw)
|
|
{
|
|
|
|
if (mw >= NTB_NUM_MW)
|
|
return (0);
|
|
|
|
return (ntb->bar_info[NTB_MW_TO_BAR(mw)].pbase);
|
|
}
|
|
|
|
/**
|
|
* ntb_get_mw_size() - return size of NTB memory window
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @mw: memory window number
|
|
*
|
|
* This function provides the physical size of the memory window specified
|
|
*
|
|
* RETURNS: the size of the memory window or zero on error
|
|
*/
|
|
u_long
|
|
ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw)
|
|
{
|
|
|
|
if (mw >= NTB_NUM_MW)
|
|
return (0);
|
|
|
|
return (ntb->bar_info[NTB_MW_TO_BAR(mw)].size);
|
|
}
|
|
|
|
/**
|
|
* ntb_set_mw_addr - set the memory window address
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @mw: memory window number
|
|
* @addr: base address for data
|
|
*
|
|
* This function sets the base physical address of the memory window. This
|
|
* memory address is where data from the remote system will be transfered into
|
|
* or out of depending on how the transport is configured.
|
|
*/
|
|
void
|
|
ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr)
|
|
{
|
|
|
|
if (mw >= NTB_NUM_MW)
|
|
return;
|
|
|
|
switch (NTB_MW_TO_BAR(mw)) {
|
|
case NTB_B2B_BAR_1:
|
|
ntb_reg_write(8, ntb->reg_ofs.sbar2_xlat, addr);
|
|
break;
|
|
case NTB_B2B_BAR_2:
|
|
ntb_reg_write(8, ntb->reg_ofs.sbar4_xlat, addr);
|
|
break;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* ntb_ring_sdb() - Set the doorbell on the secondary/external side
|
|
* @ntb: pointer to ntb_softc instance
|
|
* @db: doorbell to ring
|
|
*
|
|
* This function allows triggering of a doorbell on the secondary/external
|
|
* side that will initiate an interrupt on the remote host
|
|
*
|
|
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
|
|
*/
|
|
void
|
|
ntb_ring_sdb(struct ntb_softc *ntb, unsigned int db)
|
|
{
|
|
|
|
if (ntb->type == NTB_SOC)
|
|
ntb_reg_write(8, ntb->reg_ofs.sdb, (uint64_t) 1 << db);
|
|
else
|
|
if (HAS_FEATURE(NTB_REGS_THRU_MW))
|
|
ntb_mw_write(2, XEON_SHADOW_PDOORBELL_OFFSET,
|
|
((1 << ntb->bits_per_vector) - 1) <<
|
|
(db * ntb->bits_per_vector));
|
|
else
|
|
ntb_reg_write(2, ntb->reg_ofs.sdb,
|
|
((1 << ntb->bits_per_vector) - 1) <<
|
|
(db * ntb->bits_per_vector));
|
|
}
|
|
|
|
/**
|
|
* ntb_query_link_status() - return the hardware link status
|
|
* @ndev: pointer to ntb_device instance
|
|
*
|
|
* Returns true if the hardware is connected to the remote system
|
|
*
|
|
* RETURNS: true or false based on the hardware link state
|
|
*/
|
|
bool
|
|
ntb_query_link_status(struct ntb_softc *ntb)
|
|
{
|
|
|
|
return (ntb->link_status == NTB_LINK_UP);
|
|
}
|
|
|
|
static void
|
|
save_bar_parameters(struct ntb_pci_bar_info *bar)
|
|
{
|
|
bar->pci_bus_tag =
|
|
rman_get_bustag(bar->pci_resource);
|
|
bar->pci_bus_handle =
|
|
rman_get_bushandle(bar->pci_resource);
|
|
bar->pbase =
|
|
rman_get_start(bar->pci_resource);
|
|
bar->size =
|
|
rman_get_size(bar->pci_resource);
|
|
bar->vbase =
|
|
rman_get_virtual(bar->pci_resource);
|
|
|
|
}
|
|
|
|
device_t ntb_get_device(struct ntb_softc *ntb)
|
|
{
|
|
|
|
return (ntb->device);
|
|
}
|