diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index f29dd95855a7..f5e5ee0274ab 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -80,7 +80,7 @@ dtrace_vtime_switch_func_t dtrace_vtime_switch_func; #include #include -#if defined(__sparc64__) || defined(__mips__) +#if defined(__sparc64__) #error "This architecture is not currently compatible with ULE" #endif diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index fb19b0e5835e..45a374e5a964 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -503,7 +503,10 @@ smp_topo_none(void) top = &group[0]; top->cg_parent = NULL; top->cg_child = NULL; - top->cg_mask = (1 << mp_ncpus) - 1; + if (mp_ncpus == sizeof(top->cg_mask) * 8) + top->cg_mask = -1; + else + top->cg_mask = (1 << mp_ncpus) - 1; top->cg_count = mp_ncpus; top->cg_children = 0; top->cg_level = CG_SHARE_NONE; diff --git a/sys/mips/conf/XLR b/sys/mips/conf/XLR index a0737b3b6183..9daa3562a808 100644 --- a/sys/mips/conf/XLR +++ b/sys/mips/conf/XLR @@ -59,10 +59,11 @@ include "../rmi/std.xlr" makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols #profile 2 -#options SCHED_ULE # ULE scheduler +options SCHED_ULE # ULE scheduler #options VERBOSE_SYSINIT -options SCHED_4BSD # 4BSD scheduler -#options PREEMPTION # Enable kernel thread preemption +#options SCHED_4BSD # 4BSD scheduler +options SMP +options PREEMPTION # Enable kernel thread preemption #options FULL_PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols diff --git a/sys/mips/include/locore.h b/sys/mips/include/locore.h index b1b9f5905896..7cc72c7cefff 100644 --- a/sys/mips/include/locore.h +++ b/sys/mips/include/locore.h @@ -60,6 +60,7 @@ typedef int mips_prid_t; /* 0x09 unannounced */ /* 0x0a unannounced */ #define MIPS_PRID_CID_LEXRA 0x0b /* Lexra */ +#define MIPS_PRID_CID_RMI 0x0c /* RMI */ #define MIPS_PRID_CID_CAVIUM 0x0d /* Cavium */ #define MIPS_PRID_COPTS(x) (((x) >> 24) & 0x00ff) /* Company Options */ diff --git a/sys/mips/include/param.h b/sys/mips/include/param.h index 06cdeac04832..7eae10aa5c84 100644 --- a/sys/mips/include/param.h +++ b/sys/mips/include/param.h @@ -70,7 +70,7 @@ #define MID_MACHINE 0 /* None but has to be defined */ #ifdef SMP -#define MAXSMPCPU 16 +#define MAXSMPCPU 32 #define MAXCPU MAXSMPCPU #else #define MAXSMPCPU 1 diff --git a/sys/mips/include/smp.h b/sys/mips/include/smp.h index 1cb0596bde90..216b9e72e46e 100644 --- a/sys/mips/include/smp.h +++ b/sys/mips/include/smp.h @@ -26,6 +26,7 @@ #define IPI_AST 0x0004 #define IPI_STOP 0x0008 #define IPI_STOP_HARD 0x0008 +#define IPI_PREEMPT 0x0010 #ifndef LOCORE diff --git a/sys/mips/mips/cpu.c b/sys/mips/mips/cpu.c index 4539d1e09d4a..7c61fb8219a9 100644 --- a/sys/mips/mips/cpu.c +++ b/sys/mips/mips/cpu.c @@ -178,6 +178,9 @@ cpu_identify(void) case MIPS_PRID_CID_LEXRA: printf("Lexra"); break; + case MIPS_PRID_CID_RMI: + printf("RMI"); + break; case MIPS_PRID_CID_CAVIUM: printf("Cavium"); break; diff --git a/sys/mips/mips/machdep.c b/sys/mips/mips/machdep.c index a9c484c806e1..59a7656935a1 100644 --- a/sys/mips/mips/machdep.c +++ b/sys/mips/mips/machdep.c @@ -346,7 +346,7 @@ mips_vector_init(void) bcopy(MipsTLBMiss, (void *)TLB_MISS_EXC_VEC, MipsTLBMissEnd - MipsTLBMiss); -#ifdef TARGET_OCTEON +#if defined(TARGET_OCTEON) || defined(TARGET_XLR_XLS) /* Fake, but sufficient, for the 32-bit with 64-bit hardware addresses */ bcopy(MipsTLBMiss, (void *)XTLB_MISS_EXC_VEC, MipsTLBMissEnd - MipsTLBMiss); diff --git a/sys/mips/mips/mp_machdep.c b/sys/mips/mips/mp_machdep.c index f0c142f0c3a5..3278d73f94f3 100644 --- a/sys/mips/mips/mp_machdep.c +++ b/sys/mips/mips/mp_machdep.c @@ -141,6 +141,10 @@ mips_ipi_handler(void *arg) atomic_clear_int(&stopped_cpus, cpumask); CTR0(KTR_SMP, "IPI_STOP (restart)"); break; + case IPI_PREEMPT: + CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); + sched_preempt(curthread); + break; default: panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu); } @@ -234,7 +238,9 @@ cpu_mp_start(void) void smp_init_secondary(u_int32_t cpuid) { +#ifndef TARGET_XLR_XLS int ipi_int_mask, clock_int_mask; +#endif /* TLB */ Mips_SetWIRED(0); @@ -288,12 +294,16 @@ smp_init_secondary(u_int32_t cpuid) while (smp_started == 0) ; /* nothing */ +#ifndef TARGET_XLR_XLS /* * Unmask the clock and ipi interrupts. */ clock_int_mask = hard_int_mask(5); ipi_int_mask = hard_int_mask(platform_ipi_intrnum()); set_intr_mask(ALL_INT_MASK & ~(ipi_int_mask | clock_int_mask)); +#else + platform_init_ap(cpuid); +#endif /* * Bootstrap the compare register. diff --git a/sys/mips/mips/mpboot.S b/sys/mips/mips/mpboot.S index 631099c5964b..89a1dd35e916 100644 --- a/sys/mips/mips/mpboot.S +++ b/sys/mips/mips/mpboot.S @@ -76,9 +76,10 @@ GLOBAL(mpentry) PTR_LA gp, _C_LABEL(_gp) +#ifndef TARGET_XLR_XLS jal platform_init_ap move a0, s0 - +#endif jal smp_init_secondary move a0, s0 diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c index bcca62e7ce4d..6d0c02b5b25d 100644 --- a/sys/mips/mips/pmap.c +++ b/sys/mips/mips/pmap.c @@ -80,6 +80,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -109,6 +110,10 @@ __FBSDID("$FreeBSD$"); #define PMAP_SHPGPERPROC 200 #endif +#if defined(TARGET_XLR_XLS) +#define HIGHMEM_SUPPORT +#endif + #if !defined(PMAP_DIAGNOSTIC) #define PMAP_INLINE __inline #else @@ -183,12 +188,18 @@ static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); static int init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot); static void pmap_TLB_invalidate_kernel(vm_offset_t); static void pmap_TLB_update_kernel(vm_offset_t, pt_entry_t); +static vm_page_t pmap_alloc_pte_page(pmap_t, unsigned int, int, vm_offset_t *); +static void pmap_release_pte_page(vm_page_t); #ifdef SMP static void pmap_invalidate_page_action(void *arg); static void pmap_invalidate_all_action(void *arg); static void pmap_update_page_action(void *arg); +#endif +#ifdef HIGHMEM_SUPPORT +static void * pmap_ptpgzone_allocf(uma_zone_t, int, u_int8_t*, int); +static uma_zone_t ptpgzone; #endif struct local_sysmaps { @@ -530,6 +541,12 @@ pmap_init(void) pv_entry_max = PMAP_SHPGPERPROC * maxproc + cnt.v_page_count; pv_entry_high_water = 9 * (pv_entry_max / 10); uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); + +#ifdef HIGHMEM_SUPPORT + ptpgzone = uma_zcreate("PT ENTRY", PAGE_SIZE, NULL, + NULL, NULL, NULL, PAGE_SIZE-1, UMA_ZONE_NOFREE); + uma_zone_set_allocf(ptpgzone, pmap_ptpgzone_allocf); +#endif } /*************************************************** @@ -887,7 +904,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) /* * If the page is finally unwired, simply free it. */ - vm_page_free_zero(m); + pmap_release_pte_page(m); atomic_subtract_int(&cnt.v_wire_count, 1); return (1); } @@ -947,6 +964,118 @@ pmap_pinit0(pmap_t pmap) bzero(&pmap->pm_stats, sizeof pmap->pm_stats); } +#ifdef HIGHMEM_SUPPORT +static void * +pmap_ptpgzone_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) +{ + vm_page_t m; + vm_paddr_t paddr; + + *flags = UMA_SLAB_PRIV; + m = vm_phys_alloc_contig(1, 0, MIPS_KSEG0_LARGEST_PHYS, + PAGE_SIZE, PAGE_SIZE); + if (m == NULL) + return (NULL); + + paddr = VM_PAGE_TO_PHYS(m); + return ((void *)MIPS_PHYS_TO_KSEG0(paddr)); +} + +static vm_page_t +pmap_alloc_pte_page(pmap_t pmap, unsigned int index, int wait, vm_offset_t *vap) +{ + vm_paddr_t paddr; + void *va; + vm_page_t m; + int locked; + + locked = mtx_owned(&pmap->pm_mtx); + if (locked) { + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PMAP_UNLOCK(pmap); + vm_page_unlock_queues(); + } + va = uma_zalloc(ptpgzone, wait); + if (locked) { + vm_page_lock_queues(); + PMAP_LOCK(pmap); + } + if (va == NULL) + return (NULL); + + paddr = MIPS_KSEG0_TO_PHYS(va); + m = PHYS_TO_VM_PAGE(paddr); + + if ((m->flags & PG_ZERO) == 0) + bzero(va, PAGE_SIZE); + m->pindex = index; + m->valid = VM_PAGE_BITS_ALL; + m->wire_count = 1; + atomic_add_int(&cnt.v_wire_count, 1); + *vap = (vm_offset_t)va; + return (m); +} + +static void +pmap_release_pte_page(vm_page_t m) +{ + void *va; + vm_paddr_t paddr; + + paddr = VM_PAGE_TO_PHYS(m); + va = (void *)MIPS_PHYS_TO_KSEG0(paddr); + uma_zfree(ptpgzone, va); +} +#else +static vm_page_t +pmap_alloc_pte_page(pmap_t pmap, unsigned int index, int wait, vm_offset_t *vap) +{ + vm_offset_t va; + vm_page_t m; + int locked, req; + + locked = mtx_owned(&pmap->pm_mtx); + req = VM_ALLOC_WIRED | VM_ALLOC_NOOBJ; + if (wait & M_WAITOK) + req |= VM_ALLOC_NORMAL; + else + req |= VM_ALLOC_INTERRUPT; + + m = vm_page_alloc(NULL, index, req); + if (m == NULL) { + if (wait & M_WAITOK) { + if (locked) { + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PMAP_UNLOCK(pmap); + vm_page_unlock_queues(); + } + VM_WAIT; + if (locked) { + vm_page_lock_queues(); + PMAP_LOCK(pmap); + } + } + return NULL; + } + + va = MIPS_PHYS_TO_KSEG0(VM_PAGE_TO_PHYS(m)); + if ((m->flags & PG_ZERO) == 0) + bzero((void *)va, PAGE_SIZE); + else + vm_page_flag_clear(m, PG_ZERO); + + m->valid = VM_PAGE_BITS_ALL; + *vap = (vm_offset_t)va; + return (m); +} + +static void +pmap_release_pte_page(vm_page_t m) +{ + vm_page_free(m); +} +#endif + /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. @@ -955,37 +1084,16 @@ int pmap_pinit(pmap_t pmap) { vm_offset_t ptdva; - vm_paddr_t ptdpa; vm_page_t ptdpg; int i; - int req; PMAP_LOCK_INIT(pmap); - req = VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED | - VM_ALLOC_ZERO; - /* * allocate the page directory page */ - while ((ptdpg = vm_page_alloc(NULL, NUSERPGTBLS, req)) == NULL) - VM_WAIT; - - ptdpg->valid = VM_PAGE_BITS_ALL; - - ptdpa = VM_PAGE_TO_PHYS(ptdpg); - if (ptdpa < MIPS_KSEG0_LARGEST_PHYS) { - ptdva = MIPS_PHYS_TO_KSEG0(ptdpa); - } else { - ptdva = kmem_alloc_nofault(kernel_map, PAGE_SIZE); - if (ptdva == 0) - panic("pmap_pinit: unable to allocate kva"); - pmap_kenter(ptdva, ptdpa); - } - + ptdpg = pmap_alloc_pte_page(pmap, NUSERPGTBLS, M_WAITOK, &ptdva); pmap->pm_segtab = (pd_entry_t *)ptdva; - if ((ptdpg->flags & PG_ZERO) == 0) - bzero(pmap->pm_segtab, PAGE_SIZE); pmap->pm_active = 0; pmap->pm_ptphint = NULL; @@ -1006,7 +1114,7 @@ pmap_pinit(pmap_t pmap) static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) { - vm_offset_t pteva, ptepa; + vm_offset_t pteva; vm_page_t m; int req; @@ -1018,25 +1126,9 @@ _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) /* * Find or fabricate a new pagetable page */ - if ((m = vm_page_alloc(NULL, ptepindex, req)) == NULL) { - if (flags & M_WAITOK) { - PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); - VM_WAIT; - vm_page_lock_queues(); - PMAP_LOCK(pmap); - } - /* - * Indicate the need to retry. While waiting, the page - * table page may have been allocated. - */ + m = pmap_alloc_pte_page(pmap, ptepindex, flags, &pteva); + if (m == NULL) return (NULL); - } - if ((m->flags & PG_ZERO) == 0) - pmap_zero_page(m); - - KASSERT(m->queue == PQ_NONE, - ("_pmap_allocpte: %p->queue != PQ_NONE", m)); /* * Map the pagetable page into the process address space, if it @@ -1044,34 +1136,12 @@ _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) */ pmap->pm_stats.resident_count++; - - ptepa = VM_PAGE_TO_PHYS(m); - if (ptepa < MIPS_KSEG0_LARGEST_PHYS) { - pteva = MIPS_PHYS_TO_KSEG0(ptepa); - } else { - pteva = kmem_alloc_nofault(kernel_map, PAGE_SIZE); - if (pteva == 0) - panic("_pmap_allocpte: unable to allocate kva"); - pmap_kenter(pteva, ptepa); - } - pmap->pm_segtab[ptepindex] = (pd_entry_t)pteva; /* * Set the page table hint */ pmap->pm_ptphint = m; - - /* - * Kernel page tables are allocated in pmap_bootstrap() or - * pmap_growkernel(). - */ - if (is_kernel_pmap(pmap)) - panic("_pmap_allocpte() called for kernel pmap\n"); - - m->valid = VM_PAGE_BITS_ALL; - vm_page_flag_clear(m, PG_ZERO); - return (m); } @@ -1158,17 +1228,12 @@ pmap_release(pmap_t pmap) ptdva = (vm_offset_t)pmap->pm_segtab; ptdpg = PHYS_TO_VM_PAGE(vtophys(ptdva)); - if (ptdva >= VM_MIN_KERNEL_ADDRESS) { - pmap_kremove(ptdva); - kmem_free(kernel_map, ptdva, PAGE_SIZE); - } else { - KASSERT(MIPS_IS_KSEG0_ADDR(ptdva), - ("pmap_release: 0x%0lx is not in kseg0", (long)ptdva)); - } + KASSERT(MIPS_IS_KSEG0_ADDR(ptdva), + ("pmap_release: 0x%0lx is not in kseg0", (long)ptdva)); ptdpg->wire_count--; atomic_subtract_int(&cnt.v_wire_count, 1); - vm_page_free_zero(ptdpg); + pmap_release_pte_page(ptdpg); PMAP_LOCK_DESTROY(pmap); } @@ -1178,10 +1243,10 @@ pmap_release(pmap_t pmap) void pmap_growkernel(vm_offset_t addr) { - vm_offset_t ptppaddr; + vm_offset_t pageva; vm_page_t nkpg; pt_entry_t *pte; - int i, req; + int i; mtx_assert(&kernel_map->system_mtx, MA_OWNED); if (kernel_vm_end == 0) { @@ -1213,26 +1278,13 @@ pmap_growkernel(vm_offset_t addr) /* * This index is bogus, but out of the way */ - req = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ; - nkpg = vm_page_alloc(NULL, nkpt, req); + nkpg = pmap_alloc_pte_page(kernel_pmap, nkpt, M_NOWAIT, &pageva); + if (!nkpg) panic("pmap_growkernel: no memory to grow kernel"); nkpt++; - - ptppaddr = VM_PAGE_TO_PHYS(nkpg); - if (ptppaddr >= MIPS_KSEG0_LARGEST_PHYS) { - /* - * We need to do something here, but I am not sure - * what. We can access anything in the 0 - 512Meg - * region, but if we get a page to go in the kernel - * segmap that is outside of of that we really need - * to have another mapping beyond the temporary ones - * I have. Not sure how to do this yet. FIXME FIXME. - */ - panic("Gak, can't handle a k-page table outside of lower 512Meg"); - } - pte = (pt_entry_t *)MIPS_PHYS_TO_KSEG0(ptppaddr); + pte = (pt_entry_t *)pageva; segtab_pde(kernel_segmap, kernel_vm_end) = (pd_entry_t)pte; /* @@ -1396,7 +1448,8 @@ pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va) } } - KASSERT(pv != NULL, ("pmap_remove_entry: pv not found")); + KASSERT(pv != NULL, ("pmap_remove_entry: pv not found, pa %lx va %lx", + (u_long)VM_PAGE_TO_PHYS(m), (u_long)va)); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); m->md.pv_list_count--; if (TAILQ_FIRST(&m->md.pv_list) == NULL) diff --git a/sys/mips/rmi/dev/xlr/rge.c b/sys/mips/rmi/dev/xlr/rge.c index c62c0de99b47..0a015acd715b 100644 --- a/sys/mips/rmi/dev/xlr/rge.c +++ b/sys/mips/rmi/dev/xlr/rge.c @@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$"); #define __RMAN_RESOURCE_VISIBLE #include #include +#include #include #include @@ -83,6 +84,7 @@ __FBSDID("$FreeBSD$"); #include #include #include /* for DELAY */ +#include #include /* */ #include #include @@ -112,7 +114,6 @@ MODULE_DEPEND(rge, ether, 1, 1, 1); MODULE_DEPEND(rge, miibus, 1, 1, 1); /* #define DEBUG */ -/*#define RX_COPY */ #define RGE_TX_THRESHOLD 1024 #define RGE_TX_Q_SIZE 1024 @@ -204,10 +205,18 @@ ldadd_wu(unsigned int value, unsigned long *addr) return value; } +static __inline__ uint32_t +xlr_enable_kx(void) +{ + uint32_t sr = mips_rd_status(); + + mips_wr_status((sr & ~MIPS_SR_INT_IE) | MIPS_SR_KX); + return sr; +} + /* #define mac_stats_add(x, val) ({(x) += (val);}) */ #define mac_stats_add(x, val) ldadd_wu(val, &x) - #define XLR_MAX_CORE 8 #define RGE_LOCK_INIT(_sc, _name) \ mtx_init(&(_sc)->rge_mtx, _name, MTX_NETWORK_LOCK, MTX_DEF) @@ -332,56 +341,6 @@ DRIVER_MODULE(miibus, rge, miibus_driver, miibus_devclass, 0, 0); #define STR(x) __STR(x) #endif -#define XKPHYS 0x8000000000000000 -/* -- No longer needed RRS -static __inline__ uint32_t -lw_40bit_phys(uint64_t phys, int cca) -{ - uint64_t addr; - uint32_t value = 0; - unsigned long flags; - - addr = XKPHYS | ((uint64_t) cca << 59) | (phys & 0xfffffffffcULL); - - enable_KX(flags); - __asm__ __volatile__( - ".set push\n" - ".set noreorder\n" - ".set mips64\n" - "lw %0, 0(%1) \n" - ".set pop\n" - : "=r"(value) - : "r"(addr)); - - disable_KX(flags); - return value; -} -*/ -/* -- No longer used RRS -static __inline__ uint64_t -ld_40bit_phys(uint64_t phys, int cca) -{ - uint64_t addr; - uint64_t value = 0; - unsigned long flags; - - - addr = XKPHYS | ((uint64_t) cca << 59) | (phys & 0xfffffffffcULL); - enable_KX(flags); - __asm__ __volatile__( - ".set push\n" - ".set noreorder\n" - ".set mips64\n" - "ld %0, 0(%1) \n" - ".set pop\n" - : "=r"(value) - : "r"(addr)); - - disable_KX(flags); - return value; -} -*/ - void *xlr_tx_ring_mem; struct tx_desc_node { @@ -449,7 +408,7 @@ init_p2d_allocation(void) for (i = 0; i < 32; i++) { if (cpumask & (1 << i)) { - cpu = cpu_ltop_map[i]; + cpu = i; if (!active_core[cpu / 4]) { active_core[cpu / 4] = 1; xlr_total_active_core++; @@ -507,7 +466,7 @@ get_p2d_desc(void) { struct tx_desc_node *node; struct p2d_tx_desc *tx_desc = NULL; - int cpu = xlr_cpu_id(); + int cpu = xlr_core_id(); mtx_lock_spin(&tx_desc_lock[cpu]); node = TAILQ_FIRST(&tx_frag_desc[cpu]); @@ -527,7 +486,7 @@ static void free_p2d_desc(struct p2d_tx_desc *tx_desc) { struct tx_desc_node *node; - int cpu = xlr_cpu_id(); + int cpu = xlr_core_id(); mtx_lock_spin(&tx_desc_lock[cpu]); node = TAILQ_FIRST(&free_tx_frag_desc[cpu]); @@ -553,7 +512,7 @@ build_frag_list(struct mbuf *m_head, struct msgrng_msg *p2p_msg, struct p2d_tx_d vm_offset_t taddr; uint64_t fr_stid; - fr_stid = (xlr_cpu_id() << 3) + xlr_thr_id() + 4; + fr_stid = (xlr_core_id() << 3) + xlr_thr_id() + 4; if (tx_desc == NULL) return 1; @@ -620,21 +579,6 @@ build_frag_list(struct mbuf *m_head, struct msgrng_msg *p2p_msg, struct p2d_tx_d static void release_tx_desc(struct msgrng_msg *msg, int rel_buf) { - /* - * OLD code: vm_paddr_t paddr = msg->msg0 & 0xffffffffffULL; - * uint64_t temp; struct p2d_tx_desc *tx_desc; struct mbuf *m; - * - * paddr += (XLR_MAX_TX_FRAGS * sizeof(uint64_t)); *** In o32 we will - * crash here ****** temp = ld_40bit_phys(paddr, 3); tx_desc = - * (struct p2d_tx_desc *)((vm_offset_t)temp); - * - * if (rel_buf) { paddr += sizeof(uint64_t); - * - * temp = ld_40bit_phys(paddr, 3); - * - * m = (struct mbuf *)((vm_offset_t)temp); m_freem(m); } printf("Call - * fre_p2d_desc\n"); free_p2d_desc(tx_desc); - */ struct p2d_tx_desc *tx_desc, *chk_addr; struct mbuf *m; @@ -652,52 +596,6 @@ release_tx_desc(struct msgrng_msg *msg, int rel_buf) free_p2d_desc(tx_desc); } -#ifdef RX_COPY -#define RGE_MAX_NUM_DESC (6 * MAX_NUM_DESC) -uint8_t *rge_rx_buffers[RGE_MAX_NUM_DESC]; -static struct mtx rge_rx_mtx; -int g_rx_buf_head; - -static void -init_rx_buf(void) -{ - int i; - uint8_t *buf, *start; - uint32_t size, *ptr; - - mtx_init(&rge_rx_mtx, "xlr rx_desc", NULL, MTX_SPIN); - - size = (RGE_MAX_NUM_DESC * (MAX_FRAME_SIZE + XLR_CACHELINE_SIZE)); - - start = (uint8_t *) contigmalloc(size, M_DEVBUF, M_NOWAIT | M_ZERO, - 0, 0xffffffff, XLR_CACHELINE_SIZE, 0); - if (start == NULL) - panic("NO RX BUFFERS"); - buf = start; - size = (MAX_FRAME_SIZE + XLR_CACHELINE_SIZE); - for (i = 0; i < RGE_MAX_NUM_DESC; i++) { - buf = start + (i * size); - ptr = (uint32_t *) buf; - *ptr = (uint32_t) buf; - rge_rx_buffers[i] = buf + XLR_CACHELINE_SIZE; - } -} - -static void * -get_rx_buf(void) -{ - void *ptr = NULL; - - mtx_lock_spin(&rge_rx_mtx); - if (g_rx_buf_head < RGE_MAX_NUM_DESC) { - ptr = (void *)rge_rx_buffers[g_rx_buf_head]; - g_rx_buf_head++; - } - mtx_unlock_spin(&rge_rx_mtx); - return ptr; -} - -#endif static struct mbuf * get_mbuf(void) @@ -716,23 +614,16 @@ static void free_buf(vm_paddr_t paddr) { struct mbuf *m; - uint32_t *temp; - uint32_t mag, um; + uint32_t mag, um, sr; + + sr = xlr_enable_kx(); + um = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE); + mag = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE + sizeof(uint32_t)); + mips_wr_status(sr); - /* - * This will crash I think. RRS temp = lw_40bit_phys((paddr - - * XLR_CACHELINE_SIZE), 3); m = (struct mbuf *)temp; - */ - /* - * This gets us a kseg0 address for the mbuf/magic on the ring but - * we need to get the va to free the mbuf. This is stored at *temp; - */ - temp = (uint32_t *) MIPS_PHYS_TO_KSEG0(paddr - XLR_CACHELINE_SIZE); - um = temp[0]; - mag = temp[1]; if (mag != 0xf00bad) { - printf("Something is wrong kseg:%p found mag:%x not 0xf00bad\n", - temp, mag); + printf("Something is wrong kseg:%lx found mag:%x not 0xf00bad\n", + (u_long)paddr, mag); return; } m = (struct mbuf *)um; @@ -743,19 +634,13 @@ free_buf(vm_paddr_t paddr) static void * get_buf(void) { -#ifdef RX_COPY - return get_rx_buf(); -#else struct mbuf *m_new = NULL; - + unsigned int *md; #ifdef INVARIANTS vm_paddr_t temp1, temp2; - #endif - unsigned int *md; m_new = get_mbuf(); - if (m_new == NULL) return NULL; @@ -765,8 +650,6 @@ get_buf(void) md[1] = 0xf00bad; m_adj(m_new, XLR_CACHELINE_SIZE); - - /* return (void *)m_new; */ #ifdef INVARIANTS temp1 = vtophys((vm_offset_t)m_new->m_data); temp2 = vtophys((vm_offset_t)m_new->m_data + 1536); @@ -774,7 +657,6 @@ get_buf(void) panic("ALLOCED BUFFER IS NOT CONTIGUOUS\n"); #endif return (void *)m_new->m_data; -#endif } /********************************************************************** @@ -826,13 +708,13 @@ xlr_mac_send_fr(struct driver_data *priv, { int stid = priv->rfrbucket; struct msgrng_msg msg; - int vcpu = (xlr_cpu_id() << 2) + xlr_thr_id(); + int vcpu = xlr_cpu_id(); mac_make_desc_rfr(&msg, addr); /* Send the packet to MAC */ - dbg_msg("mac_%d: Sending free packet %llx to stid %d\n", - priv->instance, addr, stid); + dbg_msg("mac_%d: Sending free packet %lx to stid %d\n", + priv->instance, (u_long)addr, stid); if (priv->type == XLR_XGMAC) { while (message_send(1, MSGRNG_CODE_XGMAC, stid, &msg)); } else { @@ -1084,18 +966,25 @@ rmi_xlr_config_pde(struct driver_data *priv) /* uint32_t desc_pack_ctrl = 0; */ uint32_t cpumask; - cpumask = PCPU_GET(cpumask) | PCPU_GET(other_cpus); + cpumask = PCPU_GET(cpumask); +#ifdef SMP + /* + * rge may be called before SMP start in a BOOTP/NFSROOT + * setup. we will distribute packets to other cpus only when + * the SMP is started. + */ + if (smp_started) + cpumask |= PCPU_GET(other_cpus); +#endif - for (i = 0; i < 32; i++) { + for (i = 0; i < MAXCPU; i++) { if (cpumask & (1 << i)) { - cpu = cpu_ltop_map[i]; + cpu = i; bucket = ((cpu >> 2) << 3); - //|(cpu & 0x03); bucket_map |= (1ULL << bucket); - dbg_msg("i=%d, cpu=%d, bucket = %d, bucket_map=%llx\n", - i, cpu, bucket, bucket_map); } } + printf("rmi_xlr_config_pde: bucket_map=%llx\n", bucket_map); /* bucket_map = 0x1; */ xlr_write_reg(priv->mmio, R_PDE_CLASS_0, (bucket_map & 0xffffffff)); @@ -1115,6 +1004,28 @@ rmi_xlr_config_pde(struct driver_data *priv) ((bucket_map >> 32) & 0xffffffff)); } +static void +rge_smp_update_pde(void *dummy __unused) +{ + int i; + struct driver_data *priv; + struct rge_softc *sc; + + printf("Updating packet distribution for SMP\n"); + for (i = 0; i < XLR_MAX_MACS; i++) { + sc = dev_mac[i]; + if (!sc) + continue; + priv = &(sc->priv); + rmi_xlr_mac_set_enable(priv, 0); + rmi_xlr_config_pde(priv); + rmi_xlr_mac_set_enable(priv, 1); + } +} + +SYSINIT(rge_smp_update_pde, SI_SUB_SMP, SI_ORDER_ANY, rge_smp_update_pde, NULL); + + static void rmi_xlr_config_parser(struct driver_data *priv) { @@ -1613,10 +1524,7 @@ rmi_xlr_mac_xmit(struct mbuf *m, struct rge_softc *sc, int len, struct p2d_tx_de static void mac_frin_replenish(void *args /* ignored */ ) { -#ifdef RX_COPY - return; -#else - int cpu = xlr_cpu_id(); + int cpu = xlr_core_id(); int done = 0; int i = 0; @@ -1685,7 +1593,6 @@ mac_frin_replenish(void *args /* ignored */ ) if (done == XLR_MAX_MACS) break; } -#endif } static volatile uint32_t g_tx_frm_tx_ok=0; @@ -1716,8 +1623,8 @@ rmi_xlr_mac_msgring_handler(int bucket, int size, int code, struct rge_softc *sc = NULL; struct driver_data *priv = 0; struct ifnet *ifp; - int cpu = xlr_cpu_id(); - int vcpu = (cpu << 2) + xlr_thr_id(); + int vcpu = xlr_cpu_id(); + int cpu = xlr_core_id(); dbg_msg("mac: bucket=%d, size=%d, code=%d, stid=%d, msg0=%llx msg1=%llx\n", bucket, size, code, stid, msg->msg0, msg->msg1); @@ -2098,80 +2005,18 @@ rge_release_resources(struct rge_softc *sc) uint32_t gmac_rx_fail[32]; uint32_t gmac_rx_pass[32]; -#ifdef RX_COPY static void rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len) { - /* - * struct mbuf *m = (struct mbuf *)*(unsigned int *)((char *)addr - - * XLR_CACHELINE_SIZE); - */ struct mbuf *m; - void *ptr; - uint32_t *temp; - struct ifnet *ifp = sc->rge_ifp; - unsigned long msgrng_flags; - int cpu = PCPU_GET(cpuid); - - - temp = (uint32_t *) MIPS_PHYS_TO_KSEG0(paddr - XLR_CACHELINE_SIZE); - - ptr = (void *)(temp + XLR_CACHELINE_SIZE); - m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); - if (m != NULL) { - m->m_len = m->m_pkthdr.len = MCLBYTES; - m_copyback(m, 0, len + BYTE_OFFSET, ptr); - /* align the data */ - m->m_data += BYTE_OFFSET; - m->m_pkthdr.len = m->m_len = len; - m->m_pkthdr.rcvif = ifp; - gmac_rx_pass[cpu]++; - } else { - gmac_rx_fail[cpu]++; - } - msgrng_access_enable(msgrng_flags); - xlr_mac_send_fr(&sc->priv, paddr, MAX_FRAME_SIZE); - msgrng_access_disable(msgrng_flags); - -#ifdef DUMP_PACKETS - { - int i = 0; - unsigned char *buf = (char *)m->m_data; - - printf("Rx Packet: length=%d\n", len); - for (i = 0; i < 64; i++) { - if (i && (i % 16) == 0) - printf("\n"); - printf("%02x ", buf[i]); - } - printf("\n"); - } -#endif - - - if (m) { - ifp->if_ipackets++; - (*ifp->if_input) (ifp, m); - } -} - -#else -static void -rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len) -{ - /* - * struct mbuf *m = (struct mbuf *)*(unsigned int *)((char *)addr - - * XLR_CACHELINE_SIZE); - */ - struct mbuf *m; - uint32_t *temp, tm, mag; - + uint32_t tm, mag, sr; struct ifnet *ifp = sc->rge_ifp; + sr = xlr_enable_kx(); + tm = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE); + mag = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE + sizeof(uint32_t)); + mips_wr_status(sr); - temp = (uint32_t *) MIPS_PHYS_TO_KSEG0(paddr - XLR_CACHELINE_SIZE); - tm = temp[0]; - mag = temp[1]; m = (struct mbuf *)tm; if (mag != 0xf00bad) { /* somebody else packet Error - FIXME in intialization */ @@ -2201,8 +2046,6 @@ rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len) (*ifp->if_input) (ifp, m); } -#endif - static void rge_intr(void *arg) { @@ -2268,8 +2111,8 @@ rge_start_locked(struct ifnet *ifp, int threshold) int prepend_pkt = 0; int i = 0; struct p2d_tx_desc *tx_desc = NULL; - int cpu = xlr_cpu_id(); - uint32_t vcpu = (cpu << 2) + xlr_thr_id(); + int cpu = xlr_core_id(); + uint32_t vcpu = xlr_cpu_id(); if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) return; @@ -2716,9 +2559,6 @@ mac_common_init(void) { init_p2d_allocation(); init_tx_ring(); -#ifdef RX_COPY - init_rx_buf(); -#endif if (xlr_board_info.is_xls) { if (register_msgring_handler(TX_STN_GMAC0, diff --git a/sys/mips/rmi/files.xlr b/sys/mips/rmi/files.xlr index fdf5b2764763..4ed243920eab 100644 --- a/sys/mips/rmi/files.xlr +++ b/sys/mips/rmi/files.xlr @@ -9,6 +9,7 @@ mips/rmi/msgring_xls.c standard mips/rmi/board.c standard mips/rmi/on_chip.c standard mips/rmi/intr_machdep.c standard +mips/rmi/mpwait.S optional smp mips/rmi/xlr_i2c.c optional iic mips/rmi/uart_bus_xlr_iodi.c optional uart mips/rmi/uart_cpu_mips_xlr.c optional uart diff --git a/sys/mips/rmi/interrupt.h b/sys/mips/rmi/interrupt.h index 247dc9f22178..edb475a32705 100644 --- a/sys/mips/rmi/interrupt.h +++ b/sys/mips/rmi/interrupt.h @@ -32,12 +32,9 @@ /* Defines for the IRQ numbers */ -#define IRQ_DUMMY_UART 2 -#define IRQ_IPI_SMP_FUNCTION 3 -#define IRQ_IPI_SMP_RESCHEDULE 4 -#define IRQ_REMOTE_DEBUG 5 -#define IRQ_MSGRING 6 -#define IRQ_TIMER 7 +#define IRQ_IPI 41 /* 8-39 are mapped by PIC intr 0-31 */ +#define IRQ_MSGRING 6 +#define IRQ_TIMER 7 /* * XLR needs custom pre and post handlers for PCI/PCI-e interrupts diff --git a/sys/mips/rmi/intr_machdep.c b/sys/mips/rmi/intr_machdep.c index 8785ac7b2960..93c11d27ac39 100644 --- a/sys/mips/rmi/intr_machdep.c +++ b/sys/mips/rmi/intr_machdep.c @@ -157,29 +157,16 @@ cpu_intr(struct trapframe *tf) for (i = sizeof(eirr) * 8 - 1; i >= 0; i--) { if ((eirr & (1ULL << i)) == 0) continue; -#ifdef SMP - /* These are reserved interrupts */ - if ((i == IPI_AST) || (i == IPI_RENDEZVOUS) || (i == IPI_STOP) - || (i == IPI_SMP_CALL_FUNCTION)) { - write_c0_eirr64(1ULL << i); - pic_ack(i, 0); - smp_handle_ipi(tf, i); - pic_delayed_ack(i, 0); - continue; - } -#ifdef XLR_PERFMON - if (i == IPI_PERFMON) { - write_c0_eirr64(1ULL << i); - pic_ack(i, 0); - xlr_perfmon_sampler(NULL); - pic_delayed_ack(i, 0); - continue; - } -#endif -#endif - ie = mips_intr_events[i]; - /* atomic_add_long(mih->cntp, 1); */ + ie = mips_intr_events[i]; + /* Don't account special IRQs */ + switch (i) { + case IRQ_IPI: + case IRQ_MSGRING: + break; + default: + mips_intrcnt_inc(mips_intr_counters[i]); + } write_c0_eirr64(1ULL << i); pic_ack(i, 0); if (!ie || TAILQ_EMPTY(&ie->ie_handlers)) { diff --git a/sys/mips/rmi/iodi.c b/sys/mips/rmi/iodi.c index 134de9b01ae0..1e8a4d19248d 100644 --- a/sys/mips/rmi/iodi.c +++ b/sys/mips/rmi/iodi.c @@ -223,6 +223,7 @@ iodi_attach(device_t dev) */ device_add_child(dev, "uart", 0); device_add_child(dev, "xlr_i2c", 0); + device_add_child(dev, "pcib", 0); if (xlr_board_info.usb) device_add_child(dev, "ehci", 0); diff --git a/sys/mips/rmi/on_chip.c b/sys/mips/rmi/on_chip.c index cfab1d173577..c4065920cc25 100644 --- a/sys/mips/rmi/on_chip.c +++ b/sys/mips/rmi/on_chip.c @@ -38,8 +38,19 @@ __FBSDID("$FreeBSD$"); #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include +#include #include #include @@ -62,6 +73,15 @@ struct tx_stn_handler { void *dev_id; }; +struct msgring_ithread { + struct thread *i_thread; + u_int i_pending; + u_int i_flags; + int i_cpu; +}; + +struct msgring_ithread *msgring_ithreads[MAXCPU]; + /* globals */ static struct tx_stn_handler tx_stn_handlers[MAX_TX_STNS]; @@ -91,8 +111,6 @@ static uint32_t msgring_thread_mask; uint32_t msgrng_msg_cycles = 0; -int xlr_counters[MAXCPU][XLR_MAX_COUNTERS] __aligned(XLR_CACHELINE_SIZE); - void xlr_msgring_handler(struct trapframe *); void @@ -103,10 +121,10 @@ xlr_msgring_cpu_init(void) int id; unsigned long flags; - /* if not thread 0 */ - if (xlr_thr_id() != 0) - return; - id = xlr_cpu_id(); + KASSERT(xlr_thr_id() == 0, + ("xlr_msgring_cpu_init from non-zero thread\n")); + + id = xlr_core_id(); bucket_sizes = xlr_board_info.bucket_sizes; cc_config = xlr_board_info.credit_configs[id]; @@ -156,10 +174,6 @@ xlr_msgring_config(void) msgring_watermark_count = 1; msgring_thread_mask = 0x01; -/* printf("[%s]: int_type = 0x%x, pop_num_buckets=%d, pop_bucket_mask=%x" */ -/* "watermark_count=%d, thread_mask=%x\n", __FUNCTION__, */ -/* msgring_int_type, msgring_pop_num_buckets, msgring_pop_bucket_mask, */ -/* msgring_watermark_count, msgring_thread_mask); */ } void @@ -172,7 +186,6 @@ xlr_msgring_handler(struct trapframe *tf) unsigned int bucket_empty_bm = 0; unsigned int status = 0; - xlr_inc_counter(MSGRNG_INT); /* TODO: not necessary to disable preemption */ msgrng_flags_save(mflags); @@ -185,8 +198,6 @@ xlr_msgring_handler(struct trapframe *tf) break; for (bucket = 0; bucket < msgring_pop_num_buckets; bucket++) { - uint32_t cycles = 0; - if ((bucket_empty_bm & (1 << bucket)) /* empty */ ) continue; @@ -194,10 +205,6 @@ xlr_msgring_handler(struct trapframe *tf) if (status) continue; - xlr_inc_counter(MSGRNG_MSG); - msgrng_msg_cycles = mips_rd_count(); - cycles = msgrng_msg_cycles; - tx_stid = xlr_board_info.msgmap[rx_stid]; if (!tx_stn_handlers[tx_stid].action) { @@ -211,17 +218,12 @@ xlr_msgring_handler(struct trapframe *tf) &msg, tx_stn_handlers[tx_stid].dev_id); msgrng_flags_save(mflags); } - xlr_set_counter(MSGRNG_MSG_CYCLES, (read_c0_count() - cycles)); } } xlr_set_counter(MSGRNG_EXIT_STATUS, msgrng_read_status()); msgrng_flags_restore(mflags); - - //dbg_msg("OUT irq=%d\n", irq); - - /* Call the msg callback */ } void @@ -249,8 +251,110 @@ disable_msgring_int(void *arg) msgrng_access_restore(&msgrng_lock, mflags); } -extern void platform_prep_smp_launch(void); -extern void msgring_process_fast_intr(void *arg); +static int +msgring_process_fast_intr(void *arg) +{ + int cpu = PCPU_GET(cpuid); + volatile struct msgring_ithread *it; + struct thread *td; + + /* wakeup an appropriate intr_thread for processing this interrupt */ + it = (volatile struct msgring_ithread *)msgring_ithreads[cpu]; + KASSERT(it != NULL, ("No interrupt thread on cpu %d", cpu)); + td = it->i_thread; + + /* + * Interrupt thread will enable the interrupts after processing all + * messages + */ + disable_msgring_int(NULL); + atomic_store_rel_int(&it->i_pending, 1); + thread_lock(td); + if (TD_AWAITING_INTR(td)) { + TD_CLR_IWAIT(td); + sched_add(td, SRQ_INTR); + } + thread_unlock(td); + return FILTER_HANDLED; +} + +static void +msgring_process(void *arg) +{ + volatile struct msgring_ithread *ithd; + struct thread *td; + struct proc *p; + + td = curthread; + p = td->td_proc; + ithd = (volatile struct msgring_ithread *)arg; + KASSERT(ithd->i_thread == td, + ("%s:msg_ithread and proc linkage out of sync", __func__)); + + /* First bind this thread to the right CPU */ + thread_lock(td); + sched_bind(td, ithd->i_cpu); + thread_unlock(td); + + atomic_store_rel_ptr((volatile uintptr_t *)&msgring_ithreads[ithd->i_cpu], + (uintptr_t)arg); + enable_msgring_int(NULL); + + while (1) { + while (ithd->i_pending) { + /* + * This might need a full read and write barrier to + * make sure that this write posts before any of the + * memory or device accesses in the handlers. + */ + xlr_msgring_handler(NULL); + atomic_store_rel_int(&ithd->i_pending, 0); + enable_msgring_int(NULL); + } + if (!ithd->i_pending) { + thread_lock(td); + if (ithd->i_pending) { + thread_unlock(td); + continue; + } + sched_class(td, PRI_ITHD); + TD_SET_IWAIT(td); + mi_switch(SW_VOL, NULL); + thread_unlock(td); + } + } + +} + +static void +create_msgring_thread(int cpu) +{ + struct msgring_ithread *ithd; + struct thread *td; + struct proc *p; + int error; + + /* Create kernel thread for message ring interrupt processing */ + /* Currently create one task for thread 0 of each core */ + ithd = malloc(sizeof(struct msgring_ithread), + M_DEVBUF, M_WAITOK | M_ZERO); + error = kproc_create(msgring_process, (void *)ithd, &p, + RFSTOPPED | RFHIGHPID, 2, "msg_intr%d", cpu); + + if (error) + panic("kproc_create() failed with %d", error); + td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */ + + ithd->i_thread = td; + ithd->i_pending = 0; + ithd->i_cpu = cpu; + + thread_lock(td); + sched_class(td, PRI_ITHD); + sched_add(td, SRQ_INTR); + thread_unlock(td); + CTR2(KTR_INTR, "%s: created %s", __func__, ithd_name[cpu]); +} int register_msgring_handler(int major, @@ -272,14 +376,10 @@ register_msgring_handler(int major, mtx_unlock_spin(&msgrng_lock); if (xlr_test_and_set(&msgring_int_enabled)) { - platform_prep_smp_launch(); - + create_msgring_thread(0); cpu_establish_hardintr("msgring", (driver_filter_t *) msgring_process_fast_intr, NULL, NULL, IRQ_MSGRING, INTR_TYPE_NET | INTR_FAST, &cookie); - - /* configure the msgring interrupt on cpu 0 */ - enable_msgring_int(NULL); } return 0; } @@ -303,7 +403,8 @@ pic_init(void) * Use local scheduling and high polarity for all IRTs * Invalidate all IRTs, by default */ - xlr_write_reg(mmio, PIC_IRT_1_BASE + i, (level << 30) | (1 << 6) | (PIC_IRQ_BASE + i)); + xlr_write_reg(mmio, PIC_IRT_1_BASE + i, (level << 30) | (1 << 6) | + (PIC_IRQ_BASE + i)); } dbg_msg("PIC init now done\n"); } @@ -311,8 +412,6 @@ pic_init(void) void on_chip_init(void) { - int i = 0, j = 0; - /* Set xlr_io_base to the run time value */ mtx_init(&msgrng_lock, "msgring", NULL, MTX_SPIN | MTX_RECURSE); mtx_init(&xlr_pic_lock, "pic", NULL, MTX_SPIN); @@ -325,8 +424,18 @@ on_chip_init(void) pic_init(); xlr_msgring_cpu_init(); - - for (i = 0; i < MAXCPU; i++) - for (j = 0; j < XLR_MAX_COUNTERS; j++) - atomic_set_int(&xlr_counters[i][j], 0); } + +static void +start_msgring_threads(void *arg) +{ + uint32_t cpu_mask; + int cpu; + + cpu_mask = PCPU_GET(cpumask) | PCPU_GET(other_cpus); + for (cpu = 4; cpu < MAXCPU; cpu += 4) + if (cpu_mask & (1<> 2) & 0x7; + + xlr_write_reg(mmio, PIC_IPI, (pid << 20) | (tid << 16) | ipi); +} + #endif /* _RMI_PIC_H_ */ diff --git a/sys/mips/rmi/xlr_machdep.c b/sys/mips/rmi/xlr_machdep.c index f66c1b0d9e7c..a01d1de0a509 100644 --- a/sys/mips/rmi/xlr_machdep.c +++ b/sys/mips/rmi/xlr_machdep.c @@ -36,19 +36,11 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include -#include -#include #include #include #include #include -#include #include -#include -#include -#include -#include #include /* cinit() */ #include @@ -87,16 +79,14 @@ __FBSDID("$FreeBSD$"); #include #endif - - -void platform_prep_smp_launch(void); +void mpwait(void); +void enable_msgring_int(void *arg); unsigned long xlr_io_base = (unsigned long)(DEFAULT_XLR_IO_BASE); /* 4KB static data aread to keep a copy of the bootload env until the dynamic kenv is setup */ char boot1_env[4096]; -extern unsigned long _gp; int rmi_spin_mutex_safe=0; /* * Parameters from boot loader @@ -108,16 +98,6 @@ int xlr_argc; char **xlr_argv, **xlr_envp; uint64_t cpu_mask_info; uint32_t xlr_online_cpumask; - -#ifdef SMP -static unsigned long xlr_secondary_gp[MAXCPU]; -static unsigned long xlr_secondary_sp[MAXCPU]; - -#endif -extern int mips_cpu_online_mask; -extern int mips_cpu_logical_mask; -uint32_t cpu_ltop_map[MAXCPU]; -uint32_t cpu_ptol_map[MAXCPU]; uint32_t xlr_core_cpu_mask = 0x1; /* Core 0 thread 0 is always there */ void @@ -130,27 +110,6 @@ platform_reset(void) mmio[8] = 0x1; } -void -platform_secondary_init(void) -{ -#ifdef SMP - xlr_msgring_cpu_init(); - - /* Setup interrupts for secondary CPUs here */ - mips_mask_hard_irq(IPI_SMP_CALL_FUNCTION); - mips_mask_hard_irq(IPI_STOP); - mips_mask_hard_irq(IPI_RENDEZVOUS); - mips_mask_hard_irq(IPI_AST); - mips_mask_hard_irq(IRQ_TIMER); -#ifdef XLR_PERFMON - mips_mask_hard_irq(IPI_PERFMON); -#endif - - return; -#endif -} - - int xlr_asid_pcpu = 256; /* This the default */ int xlr_shtlb_enabled = 0; @@ -168,13 +127,13 @@ setup_tlb_resource(void) int mmu_setup; int value = 0; uint32_t cpu_map = xlr_boot1_info.cpu_online_map; - uint32_t thr_mask = cpu_map >> (xlr_cpu_id() << 2); + uint32_t thr_mask = cpu_map >> (xlr_core_id() << 2); uint8_t core0 = xlr_boot1_info.cpu_online_map & 0xf; uint8_t core_thr_mask; int i = 0, count = 0; /* If CPU0 did not enable shared TLB, other cores need to follow */ - if ((xlr_cpu_id() != 0) && (xlr_shtlb_enabled == 0)) + if ((xlr_core_id() != 0) && (xlr_shtlb_enabled == 0)) return; /* First check if each core is brought up with the same mask */ for (i = 1; i < 8; i++) { @@ -212,12 +171,14 @@ setup_tlb_resource(void) mmu_setup |= (value << 1); /* turn on global mode */ +#ifndef SMP mmu_setup |= 0x01; - +#endif write_32bit_phnx_ctrl_reg(4, 0, mmu_setup); } + /* * Platform specific register setup for CPUs * XLR has control registers accessible with MFCR/MTCR instructions, this @@ -233,6 +194,10 @@ platform_cpu_init() uint32_t reg, val; int thr_id = xlr_thr_id(); +/* + * XXX: SMP now need different wired mappings for threads + * we cannot share TLBs. + */ if (thr_id == 0) { if ((hw_env = getenv("xlr.shtlb")) != NULL) { start = hw_env; @@ -287,22 +252,6 @@ platform_cpu_init() } -#ifdef SMP -extern void xlr_secondary_start(unsigned long, unsigned long, unsigned long); -static void -xlr_secondary_entry(void *data) -{ - unsigned long sp, gp; - unsigned int cpu = (xlr_cpu_id() << 2) + xlr_thr_id(); - - sp = xlr_secondary_sp[cpu]; - gp = xlr_secondary_gp[cpu]; - - xlr_secondary_start((unsigned long)mips_secondary_wait, sp, gp); -} - -#endif - static void xlr_set_boot_flags(void) { @@ -341,7 +290,6 @@ xlr_set_boot_flags(void) } extern uint32_t _end; - static void mips_init(void) { @@ -353,9 +301,6 @@ mips_init(void) mips_cpu_init(); pmap_bootstrap(); #ifdef DDB -#ifdef SMP - setup_nmi(); -#endif /* SMP */ kdb_init(); if (boothowto & RB_KDB) { kdb_enter("Boot flags requested debugger", NULL); @@ -521,29 +466,17 @@ platform_start(__register_t a0 __unused, for (i = 1, j = 1; i < 32; i++) { /* Allocate stack for all other cpus from fbsd kseg0 memory. */ if ((1U << i) & xlr_boot1_info.cpu_online_map) { - xlr_secondary_gp[i] = - pmap_steal_memory(PAGE_SIZE); - if (!xlr_secondary_gp[i]) - panic("Allocation failed for secondary cpu stacks"); - xlr_secondary_sp[i] = - xlr_secondary_gp[i] + PAGE_SIZE - CALLFRAME_SIZ; - xlr_secondary_gp[i] = (unsigned long)&_gp; - /* Build ltop and ptol cpu map. */ - cpu_ltop_map[j] = i; - cpu_ptol_map[i] = j; if ((i & 0x3) == 0) /* store thread0 of each core */ xlr_core_cpu_mask |= (1 << j); - mips_cpu_logical_mask |= (1 << j); j++; } } - mips_cpu_online_mask |= xlr_boot1_info.cpu_online_map; wakeup = ((void (*) (void *, void *, unsigned int)) (unsigned long)(xlr_boot1_info.wakeup)); printf("Waking up CPUs 0x%llx.\n", xlr_boot1_info.cpu_online_map & ~(0x1U)); if (xlr_boot1_info.cpu_online_map & ~(0x1U)) - wakeup(xlr_secondary_entry, 0, + wakeup(mpwait, 0, (unsigned int)xlr_boot1_info.cpu_online_map); #endif @@ -582,144 +515,59 @@ platform_trap_exit(void) { } +#ifdef SMP +int xlr_ap_release[MAXCPU]; -/* - void - platform_update_intrmask(int intr) - { - write_c0_eimr64(read_c0_eimr64() | (1ULL<i_thread; - /* - * Interrupt thread will enable the interrupts after processing all - * messages + * other cpus are enabled by the boot loader and they will be + * already looping in mpwait, release them */ - disable_msgring_int(NULL); - atomic_store_rel_int(&it->i_pending, 1); - thread_lock(td); - if (TD_AWAITING_INTR(td)) { - TD_CLR_IWAIT(td); - sched_add(td, SRQ_INTR); - } - thread_unlock(td); - return FILTER_HANDLED; + atomic_store_rel_int(&xlr_ap_release[cpuid], 1); + return 0; } -static void -msgring_process(void *arg) +void platform_init_ap(int processor_id) { - volatile struct msgring_ithread *ithd; - struct thread *td; - struct proc *p; + uint32_t stat; - td = curthread; - p = td->td_proc; - ithd = (volatile struct msgring_ithread *)arg; - KASSERT(ithd->i_thread == td, - ("%s:msg_ithread and proc linkage out of sync", __func__)); + /* Setup interrupts for secondary CPUs here */ + stat = mips_rd_status(); + stat |= MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT; + mips_wr_status(stat); - /* First bind this thread to the right CPU */ - thread_lock(td); - sched_bind(td, ithd->i_cpu); - thread_unlock(td); - - //printf("Started %s on CPU %d\n", __FUNCTION__, ithd->i_cpu); - - while (1) { - while (ithd->i_pending) { - /* - * This might need a full read and write barrier to - * make sure that this write posts before any of the - * memory or device accesses in the handlers. - */ - xlr_msgring_handler(NULL); - atomic_store_rel_int(&ithd->i_pending, 0); - enable_msgring_int(NULL); - } - if (!ithd->i_pending) { - thread_lock(td); - if (ithd->i_pending) { - thread_unlock(td); - continue; - } - sched_class(td, PRI_ITHD); - TD_SET_IWAIT(td); - mi_switch(SW_VOL, NULL); - thread_unlock(td); - } + xlr_unmask_hard_irq((void *)platform_ipi_intrnum()); + xlr_unmask_hard_irq((void *)IRQ_TIMER); + if (xlr_thr_id() == 0) { + xlr_msgring_cpu_init(); + xlr_unmask_hard_irq((void *)IRQ_MSGRING); } + return; } -void -platform_prep_smp_launch(void) + +int platform_ipi_intrnum(void) { - int cpu; - uint32_t cpu_mask; - struct msgring_ithread *ithd; - struct thread *td; - struct proc *p; - int error; - - cpu_mask = PCPU_GET(cpumask) | PCPU_GET(other_cpus); - - /* Create kernel threads for message ring interrupt processing */ - /* Currently create one task for thread 0 of each core */ - for (cpu = 0; cpu < MAXCPU; cpu += 1) { - - if (!((1 << cpu) & cpu_mask)) - continue; - - if ((cpu_ltop_map[cpu] % 4) != 0) - continue; - - ithd = &msgring_ithreads[cpu]; - sprintf(ithd_name[cpu], "msg_intr%d", cpu); - error = kproc_create(msgring_process, - (void *)ithd, - &p, - (RFSTOPPED | RFHIGHPID), - 2, - ithd_name[cpu]); - - if (error) - panic("kproc_create() failed with %d", error); - td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */ - - thread_lock(td); - sched_class(td, PRI_ITHD); - TD_SET_IWAIT(td); - thread_unlock(td); - ithd->i_thread = td; - ithd->i_pending = 0; - ithd->i_cpu = cpu; - CTR2(KTR_INTR, "%s: created %s", __func__, ithd_name[cpu]); - } + return IRQ_IPI; } + +void platform_ipi_send(int cpuid) +{ + pic_send_ipi(cpuid, platform_ipi_intrnum(), 0); +} + +void platform_ipi_clear(void) +{ +} + +int platform_processor_id(void) +{ + return xlr_cpu_id(); +} + +int platform_num_processors(void) +{ + return fls(xlr_boot1_info.cpu_online_map); +} +#endif diff --git a/sys/mips/rmi/xlr_pci.c b/sys/mips/rmi/xlr_pci.c index 5f52e9ea5436..314276f15767 100644 --- a/sys/mips/rmi/xlr_pci.c +++ b/sys/mips/rmi/xlr_pci.c @@ -636,4 +636,4 @@ static driver_t xlr_pcib_driver = { sizeof(struct xlr_pcib_softc), }; -DRIVER_MODULE(pcib, nexus, xlr_pcib_driver, pcib_devclass, 0, 0); +DRIVER_MODULE(pcib, iodi, xlr_pcib_driver, pcib_devclass, 0, 0); diff --git a/sys/mips/rmi/xlrconfig.h b/sys/mips/rmi/xlrconfig.h index 3ba96eb1a8c5..c2aee99916a7 100644 --- a/sys/mips/rmi/xlrconfig.h +++ b/sys/mips/rmi/xlrconfig.h @@ -25,6 +25,7 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. + __FBSDID("$FreeBSD$"); * * RMI_BSD */ #ifndef XLRCONFIG_H @@ -127,39 +128,35 @@ :"$8", "$9"); \ } while(0) -#if 0 -#define xlr_processor_id() \ +#define xlr_cpu_id() \ ({int __id; \ __asm__ __volatile__ ( \ ".set push\n" \ ".set noreorder\n" \ - ".word 0x40088007\n" \ - "srl $8, $8, 10\n" \ - "andi %0, $8, 0x3f\n" \ - ".set pop\n" \ - : "=r" (__id) : : "$8"); \ - __id;}) -#endif - -#define xlr_cpu_id() \ -({int __id; \ - __asm__ __volatile__ ( \ - ".set push\n" \ - ".set noreorder\n" \ - ".word 0x40088007\n" \ - "srl $8, $8, 4\n" \ - "andi %0, $8, 0x7\n" \ + "mfc0 $8, $15, 1\n" \ + "andi %0, $8, 0x1f\n" \ ".set pop\n" \ : "=r" (__id) : : "$8"); \ __id;}) -#define xlr_thr_id() \ +#define xlr_core_id() \ ({int __id; \ __asm__ __volatile__ ( \ ".set push\n" \ ".set noreorder\n" \ - ".word 0x40088007\n" \ - "andi %0, $8, 0x03\n" \ + "mfc0 $8, $15, 1\n" \ + "andi %0, $8, 0x1f\n" \ + ".set pop\n" \ + : "=r" (__id) : : "$8"); \ + __id/4;}) + +#define xlr_thr_id() \ +({int __id; \ + __asm__ __volatile__ ( \ + ".set push\n" \ + ".set noreorder\n" \ + "mfc0 $8, $15, 1\n" \ + "andi %0, $8, 0x3\n" \ ".set pop\n" \ : "=r" (__id) : : "$8"); \ __id;}) @@ -333,4 +330,26 @@ xlr_mtcr(uint32_t reg, uint32_t val) : "$8", "$9"); } +static __inline__ uint32_t +xlr_paddr_lw(uint64_t paddr) +{ + uint32_t high, low, tmp; + + high = 0x98000000 | (paddr >> 32); + low = paddr & 0xffffffff; + + __asm__ __volatile__( + ".set push \n\t" + ".set mips64 \n\t" + "dsll32 %1, %1, 0 \n\t" + "dsll32 %2, %2, 0 \n\t" /* get rid of the */ + "dsrl32 %2, %2, 0 \n\t" /* sign extend */ + "or %1, %1, %2 \n\t" + "lw %0, 0(%1) \n\t" + ".set pop \n" + : "=r"(tmp) + : "r"(high), "r"(low)); + + return tmp; +} #endif