19d3b47b92
Mainly focus on files that use BSD 2-Clause license, however the tool I was using misidentified many licenses so this was mostly a manual - error prone - task. The Software Package Data Exchange (SPDX) group provides a specification to make it easier for automated tools to detect and summarize well known opensource licenses. We are gradually adopting the specification, noting that the tags are considered only advisory and do not, in any way, superceed or replace the license texts.
499 lines
13 KiB
C
499 lines
13 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
*
|
|
* Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights
|
|
* reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE
|
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* NETLOGIC_BSD */
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
#include <sys/types.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/param.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/limits.h>
|
|
#include <sys/bus.h>
|
|
#include <sys/sbuf.h>
|
|
|
|
#include <sys/ktr.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/kthread.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/resourcevar.h>
|
|
#include <sys/sched.h>
|
|
#include <sys/unistd.h>
|
|
#include <sys/sysctl.h>
|
|
#include <sys/malloc.h>
|
|
|
|
#include <machine/reg.h>
|
|
#include <machine/cpu.h>
|
|
#include <machine/hwfunc.h>
|
|
#include <machine/mips_opcode.h>
|
|
#include <machine/intr_machdep.h>
|
|
|
|
#include <mips/nlm/hal/mips-extns.h>
|
|
#include <mips/nlm/hal/haldefs.h>
|
|
#include <mips/nlm/hal/iomap.h>
|
|
#include <mips/nlm/hal/cop2.h>
|
|
#include <mips/nlm/hal/fmn.h>
|
|
#include <mips/nlm/hal/pic.h>
|
|
|
|
#include <mips/nlm/msgring.h>
|
|
#include <mips/nlm/interrupt.h>
|
|
#include <mips/nlm/xlp.h>
|
|
|
|
#define MSGRNG_NSTATIONS 1024
|
|
/*
|
|
* Keep track of our message ring handler threads, each core has a
|
|
* different message station. Ideally we will need to start a few
|
|
* message handling threads every core, and wake them up depending on
|
|
* load
|
|
*/
|
|
struct msgring_thread {
|
|
struct thread *thread; /* msgring handler threads */
|
|
int needed; /* thread needs to wake up */
|
|
};
|
|
static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static struct proc *msgring_proc; /* all threads are under a proc */
|
|
|
|
/*
|
|
* The device drivers can register a handler for the messages sent
|
|
* from a station (corresponding to the device).
|
|
*/
|
|
struct tx_stn_handler {
|
|
msgring_handler action;
|
|
void *arg;
|
|
};
|
|
static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
|
|
static struct mtx msgmap_lock;
|
|
uint32_t xlp_msg_thread_mask;
|
|
static int xlp_msg_threads_per_core = XLP_MAX_THREADS;
|
|
|
|
static void create_msgring_thread(int hwtid);
|
|
static int msgring_process_fast_intr(void *arg);
|
|
|
|
/* Debug counters */
|
|
static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4];
|
|
static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
|
|
/* Whether polled driver implementation */
|
|
static int polled = 0;
|
|
|
|
/* We do only i/o device credit setup here. CPU credit setup is now
|
|
* moved to xlp_msgring_cpu_init() so that the credits get setup
|
|
* only if the CPU exists. xlp_msgring_cpu_init() gets called from
|
|
* platform_init_ap; and this makes it easy for us to setup CMS
|
|
* credits for various types of XLP chips, with varying number of
|
|
* cpu's and cores.
|
|
*/
|
|
static void
|
|
xlp_cms_credit_setup(int credit)
|
|
{
|
|
uint64_t cmspcibase, cmsbase, pcibase;
|
|
uint32_t devoffset;
|
|
int dev, fn, maxqid;
|
|
int src, qid, i;
|
|
|
|
for (i = 0; i < XLP_MAX_NODES; i++) {
|
|
cmspcibase = nlm_get_cms_pcibase(i);
|
|
if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i)))
|
|
continue;
|
|
cmsbase = nlm_get_cms_regbase(i);
|
|
maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
|
|
for (dev = 0; dev < 8; dev++) {
|
|
for (fn = 0; fn < 8; fn++) {
|
|
devoffset = XLP_HDR_OFFSET(i, 0, dev, fn);
|
|
if (nlm_dev_exists(devoffset) == 0)
|
|
continue;
|
|
pcibase = nlm_pcicfg_base(devoffset);
|
|
src = nlm_qidstart(pcibase);
|
|
if (src == 0)
|
|
continue;
|
|
#if 0 /* Debug */
|
|
printf("Setup CMS credits for queues ");
|
|
printf("[%d to %d] from src %d\n", 0,
|
|
maxqid, src);
|
|
#endif
|
|
for (qid = 0; qid < maxqid; qid++)
|
|
nlm_cms_setup_credits(cmsbase, qid,
|
|
src, credit);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
xlp_msgring_cpu_init(int node, int cpu, int credit)
|
|
{
|
|
uint64_t cmspcibase = nlm_get_cms_pcibase(node);
|
|
uint64_t cmsbase = nlm_get_cms_regbase(node);
|
|
int qid, maxqid, src;
|
|
|
|
maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
|
|
|
|
/* cpu credit setup is done only from thread-0 of each core */
|
|
if((cpu % 4) == 0) {
|
|
src = cpu << 2; /* each thread has 4 vc's */
|
|
for (qid = 0; qid < maxqid; qid++)
|
|
nlm_cms_setup_credits(cmsbase, qid, src, credit);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Drain out max_messages for the buckets set in the bucket mask.
|
|
* Use max_msgs = 0 to drain out all messages.
|
|
*/
|
|
int
|
|
xlp_handle_msg_vc(u_int vcmask, int max_msgs)
|
|
{
|
|
struct nlm_fmn_msg msg;
|
|
int srcid = 0, size = 0, code = 0;
|
|
struct tx_stn_handler *he;
|
|
uint32_t mflags, status;
|
|
int n_msgs = 0, vc, m, hwtid;
|
|
u_int msgmask;
|
|
|
|
hwtid = nlm_cpuid();
|
|
for (;;) {
|
|
/* check if VC empty */
|
|
mflags = nlm_save_flags_cop2();
|
|
status = nlm_read_c2_msgstatus1();
|
|
nlm_restore_flags(mflags);
|
|
|
|
msgmask = ((status >> 24) & 0xf) ^ 0xf;
|
|
msgmask &= vcmask;
|
|
if (msgmask == 0)
|
|
break;
|
|
m = 0;
|
|
for (vc = 0; vc < 4; vc++) {
|
|
if ((msgmask & (1 << vc)) == 0)
|
|
continue;
|
|
|
|
mflags = nlm_save_flags_cop2();
|
|
status = nlm_fmn_msgrcv(vc, &srcid, &size, &code,
|
|
&msg);
|
|
nlm_restore_flags(mflags);
|
|
if (status != 0) /* no msg or error */
|
|
continue;
|
|
if (srcid < 0 && srcid >= 1024) {
|
|
printf("[%s]: bad src id %d\n", __func__,
|
|
srcid);
|
|
continue;
|
|
}
|
|
he = &msgmap[srcid];
|
|
if(he->action != NULL)
|
|
(he->action)(vc, size, code, srcid, &msg,
|
|
he->arg);
|
|
#if 0
|
|
else
|
|
printf("[%s]: No Handler for msg from stn %d,"
|
|
" vc=%d, size=%d, msg0=%jx, droppinge\n",
|
|
__func__, srcid, vc, size,
|
|
(uintmax_t)msg.msg[0]);
|
|
#endif
|
|
fmn_msgcount[hwtid][vc] += 1;
|
|
m++; /* msgs handled in this iter */
|
|
}
|
|
if (m == 0)
|
|
break; /* nothing done in this iter */
|
|
n_msgs += m;
|
|
if (max_msgs > 0 && n_msgs >= max_msgs)
|
|
break;
|
|
}
|
|
|
|
return (n_msgs);
|
|
}
|
|
|
|
static void
|
|
xlp_discard_msg_vc(u_int vcmask)
|
|
{
|
|
struct nlm_fmn_msg msg;
|
|
int srcid = 0, size = 0, code = 0, vc;
|
|
uint32_t mflags, status;
|
|
|
|
for (vc = 0; vc < 4; vc++) {
|
|
for (;;) {
|
|
mflags = nlm_save_flags_cop2();
|
|
status = nlm_fmn_msgrcv(vc, &srcid,
|
|
&size, &code, &msg);
|
|
nlm_restore_flags(mflags);
|
|
|
|
/* break if there is no msg or error */
|
|
if (status != 0)
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
xlp_cms_enable_intr(int node, int cpu, int type, int watermark)
|
|
{
|
|
uint64_t cmsbase;
|
|
int i, qid;
|
|
|
|
cmsbase = nlm_get_cms_regbase(node);
|
|
|
|
for (i = 0; i < 4; i++) {
|
|
qid = (i + (cpu * 4)) & 0x7f;
|
|
nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark);
|
|
nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0);
|
|
}
|
|
}
|
|
|
|
static int
|
|
msgring_process_fast_intr(void *arg)
|
|
{
|
|
struct msgring_thread *mthd;
|
|
struct thread *td;
|
|
int cpu;
|
|
|
|
cpu = nlm_cpuid();
|
|
mthd = &msgring_threads[cpu];
|
|
msgring_nintr[cpu]++;
|
|
td = mthd->thread;
|
|
|
|
/* clear pending interrupts */
|
|
nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
|
|
|
|
/* wake up the target thread */
|
|
mthd->needed = 1;
|
|
thread_lock(td);
|
|
if (TD_AWAITING_INTR(td)) {
|
|
msgring_wakeup_sleep[cpu]++;
|
|
TD_CLR_IWAIT(td);
|
|
sched_add(td, SRQ_INTR);
|
|
} else
|
|
msgring_wakeup_nosleep[cpu]++;
|
|
|
|
thread_unlock(td);
|
|
|
|
return (FILTER_HANDLED);
|
|
}
|
|
|
|
static void
|
|
msgring_process(void * arg)
|
|
{
|
|
volatile struct msgring_thread *mthd;
|
|
struct thread *td;
|
|
uint32_t mflags, msgstatus1;
|
|
int hwtid, nmsgs;
|
|
|
|
hwtid = (intptr_t)arg;
|
|
mthd = &msgring_threads[hwtid];
|
|
td = mthd->thread;
|
|
KASSERT(curthread == td,
|
|
("%s:msg_ithread and proc linkage out of sync", __func__));
|
|
|
|
/* First bind this thread to the right CPU */
|
|
thread_lock(td);
|
|
sched_bind(td, xlp_hwtid_to_cpuid[hwtid]);
|
|
thread_unlock(td);
|
|
|
|
if (hwtid != nlm_cpuid())
|
|
printf("Misscheduled hwtid %d != cpuid %d\n", hwtid,
|
|
nlm_cpuid());
|
|
|
|
xlp_discard_msg_vc(0xf);
|
|
xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT);
|
|
if (polled == 0) {
|
|
mflags = nlm_save_flags_cop2();
|
|
nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0);
|
|
nlm_restore_flags(mflags);
|
|
xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0);
|
|
/* clear pending interrupts.
|
|
* they will get re-raised if still valid */
|
|
nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
|
|
}
|
|
|
|
/* start processing messages */
|
|
for (;;) {
|
|
atomic_store_rel_int(&mthd->needed, 0);
|
|
nmsgs = xlp_handle_msg_vc(0xf, 0);
|
|
|
|
/* sleep */
|
|
if (polled == 0) {
|
|
/* clear VC-pend bits */
|
|
mflags = nlm_save_flags_cop2();
|
|
msgstatus1 = nlm_read_c2_msgstatus1();
|
|
msgstatus1 |= (0xf << 16);
|
|
nlm_write_c2_msgstatus1(msgstatus1);
|
|
nlm_restore_flags(mflags);
|
|
|
|
thread_lock(td);
|
|
if (mthd->needed) {
|
|
thread_unlock(td);
|
|
continue;
|
|
}
|
|
sched_class(td, PRI_ITHD);
|
|
TD_SET_IWAIT(td);
|
|
mi_switch(SW_VOL, NULL);
|
|
thread_unlock(td);
|
|
} else
|
|
pause("wmsg", 1);
|
|
|
|
fmn_loops[hwtid]++;
|
|
}
|
|
}
|
|
|
|
static void
|
|
create_msgring_thread(int hwtid)
|
|
{
|
|
struct msgring_thread *mthd;
|
|
struct thread *td;
|
|
int error;
|
|
|
|
mthd = &msgring_threads[hwtid];
|
|
error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid,
|
|
&msgring_proc, &td, RFSTOPPED, 2, "msgrngproc",
|
|
"msgthr%d", hwtid);
|
|
if (error)
|
|
panic("kproc_kthread_add() failed with %d", error);
|
|
mthd->thread = td;
|
|
|
|
thread_lock(td);
|
|
sched_class(td, PRI_ITHD);
|
|
sched_add(td, SRQ_INTR);
|
|
thread_unlock(td);
|
|
}
|
|
|
|
int
|
|
register_msgring_handler(int startb, int endb, msgring_handler action,
|
|
void *arg)
|
|
{
|
|
int i;
|
|
|
|
if (bootverbose)
|
|
printf("Register handler %d-%d %p(%p)\n",
|
|
startb, endb, action, arg);
|
|
KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
|
|
("Invalid value for bucket range %d,%d", startb, endb));
|
|
|
|
mtx_lock_spin(&msgmap_lock);
|
|
for (i = startb; i <= endb; i++) {
|
|
KASSERT(msgmap[i].action == NULL,
|
|
("Bucket %d already used [action %p]", i, msgmap[i].action));
|
|
msgmap[i].action = action;
|
|
msgmap[i].arg = arg;
|
|
}
|
|
mtx_unlock_spin(&msgmap_lock);
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Initialize the messaging subsystem.
|
|
*
|
|
* Message Stations are shared among all threads in a cpu core, this
|
|
* has to be called once from every core which is online.
|
|
*/
|
|
static void
|
|
xlp_msgring_config(void *arg)
|
|
{
|
|
void *cookie;
|
|
unsigned int thrmask, mask;
|
|
int i;
|
|
|
|
/* used polled handler for Ax silion */
|
|
if (nlm_is_xlp8xx_ax())
|
|
polled = 1;
|
|
|
|
/* Don't poll on all threads, if polled */
|
|
if (polled)
|
|
xlp_msg_threads_per_core -= 1;
|
|
|
|
mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
|
|
if (xlp_threads_per_core < xlp_msg_threads_per_core)
|
|
xlp_msg_threads_per_core = xlp_threads_per_core;
|
|
thrmask = ((1 << xlp_msg_threads_per_core) - 1);
|
|
mask = 0;
|
|
for (i = 0; i < XLP_MAX_CORES; i++) {
|
|
mask <<= XLP_MAX_THREADS;
|
|
mask |= thrmask;
|
|
}
|
|
xlp_msg_thread_mask = xlp_hw_thread_mask & mask;
|
|
#if 0
|
|
printf("CMS Message handler thread mask %#jx\n",
|
|
(uintmax_t)xlp_msg_thread_mask);
|
|
#endif
|
|
xlp_cms_credit_setup(CMS_DEFAULT_CREDIT);
|
|
create_msgring_thread(0);
|
|
cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL,
|
|
NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie);
|
|
}
|
|
|
|
/*
|
|
* Start message ring processing threads on other CPUs, after SMP start
|
|
*/
|
|
static void
|
|
start_msgring_threads(void *arg)
|
|
{
|
|
int hwt;
|
|
|
|
for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) {
|
|
if ((xlp_msg_thread_mask & (1 << hwt)) == 0)
|
|
continue;
|
|
create_msgring_thread(hwt);
|
|
}
|
|
}
|
|
|
|
SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST,
|
|
xlp_msgring_config, NULL);
|
|
SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
|
|
start_msgring_threads, NULL);
|
|
|
|
/*
|
|
* DEBUG support, XXX: static buffer, not locked
|
|
*/
|
|
static int
|
|
sys_print_debug(SYSCTL_HANDLER_ARGS)
|
|
{
|
|
struct sbuf sb;
|
|
int error, i;
|
|
|
|
sbuf_new_for_sysctl(&sb, NULL, 64, req);
|
|
sbuf_printf(&sb,
|
|
"\nID vc0 vc1 vc2 vc3 loops\n");
|
|
for (i = 0; i < 32; i++) {
|
|
if ((xlp_hw_thread_mask & (1 << i)) == 0)
|
|
continue;
|
|
sbuf_printf(&sb, "%2d: %8d %8d %8d %8d %8d\n", i,
|
|
fmn_msgcount[i][0], fmn_msgcount[i][1],
|
|
fmn_msgcount[i][2], fmn_msgcount[i][3],
|
|
fmn_loops[i]);
|
|
}
|
|
error = sbuf_finish(&sb);
|
|
sbuf_delete(&sb);
|
|
return (error);
|
|
}
|
|
|
|
SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
|
|
sys_print_debug, "A", "msgring debug info");
|