d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework BookE pmap towards multi-core support.

Browse Source 
o Eliminate tlb0[] (a s/w copy of TLB0)
  - The table contents cannot be maintained reliably in multiple MMU
    environments, where asynchronous events (invalidations from other cores)
    can change our local TLB0 contents underneath.
  - Simplify and optimize TLB flushing: system wide invalidations are
    performed using tlbivax instruction (propagates to other cores), for
    local MMU invalidations a new optimized routine (assembly) is introduced.

o Improve and simplify TID allocation and management.
  - Let each core keep track of its TID allocations.
  - Simplify TID recycling, eliminate dead code.
  - Drop the now unused powerpc/booke/support.S file.

o Improve page tables management logic.

o Simplify TLB1 manipulation routines.

o Other improvements and polishing.

Obtained from:	Freescale, Semihalf
This commit is contained in:
Rafal Jaworowski 2009-01-13 15:41:58 +00:00
parent 8b9e2a36cc
commit b2b734e771
11 changed files with 472 additions and 759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sys/conf/files.powerpc
View File

@ -84,7 +84,6 @@ powerpc/booke/interrupt.c optional e500
powerpc/booke/locore.S optional e500 no-obj
powerpc/booke/machdep.c optional e500
powerpc/booke/pmap.c optional e500
powerpc/booke/support.S optional e500
powerpc/booke/swtch.S optional e500
powerpc/booke/trap.c optional e500
powerpc/booke/uio_machdep.c optional e500

81
sys/powerpc/booke/locore.S
View File

@ -399,6 +399,87 @@ ivor_setup:
mtspr SPR_IVOR15, %r3
blr
/*
* void tid_flush(tlbtid_t tid);
*
* Invalidate all TLB0 entries which match the given TID. Note this is
* dedicated for cases when invalidation(s) should NOT be propagated to other
* CPUs.
*
* Global vars tlb0_ways, tlb0_entries_per_way are assumed to have been set up
* correctly (by tlb0_get_tlbconf()).
*
*/
ENTRY(tid_flush)
cmpwi %r3, TID_KERNEL
beq tid_flush_end /* don't evict kernel translations */
/* Number of TLB0 ways */
lis %r4, tlb0_ways@h
ori %r4, %r4, tlb0_ways@l
lwz %r4, 0(%r4)
/* Number of entries / way */
lis %r5, tlb0_entries_per_way@h
ori %r5, %r5, tlb0_entries_per_way@l
lwz %r5, 0(%r5)
/* Disable interrupts */
mfmsr %r10
wrteei 0
li %r6, 0 /* ways counter */
loop_ways:
li %r7, 0 /* entries [per way] counter */
loop_entries:
/* Select TLB0 and ESEL (way) */
lis %r8, MAS0_TLBSEL0@h
rlwimi %r8, %r6, 16, 14, 15
mtspr SPR_MAS0, %r8
isync
/* Select EPN (entry within the way) */
rlwinm %r8, %r7, 12, 13, 19
mtspr SPR_MAS2, %r8
isync
tlbre
/* Check if valid entry */
mfspr %r8, SPR_MAS1
andis. %r9, %r8, MAS1_VALID@h
beq next_entry /* invalid entry */
/* Check if this is our TID */
rlwinm %r9, %r8, 16, 24, 31
cmplw %r9, %r3
bne next_entry /* not our TID */
/* Clear VALID bit */
rlwinm %r8, %r8, 0, 1, 31
mtspr SPR_MAS1, %r8
isync
tlbwe
isync
msync
next_entry:
addi %r7, %r7, 1
cmpw %r7, %r5
bne loop_entries
/* Next way */
addi %r6, %r6, 1
cmpw %r6, %r4
bne loop_ways
/* Restore MSR (possibly re-enable interrupts) */
mtmsr %r10
isync
tid_flush_end:
blr
/*
* Cache disable/enable/inval sequences according
* to section 2.16 of E500CORE RM.

1
sys/powerpc/booke/machdep.c
View File

@ -490,6 +490,7 @@ void
cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz)
{
pcpu->pc_tid_next = TID_MIN;
}
/* Set set up registers on exec. */

880
sys/powerpc/booke/pmap.c
View File

File diff suppressed because it is too large Load Diff

106
sys/powerpc/booke/support.S
View File

@ -1,106 +0,0 @@
/*-
* Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
* NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*/
#include "assym.s"
#include <machine/param.h>
#include <machine/asm.h>
#include <machine/spr.h>
#include <machine/psl.h>
#include <machine/pte.h>
#include <machine/trap.h>
#include <machine/vmparam.h>
#include <machine/tlb.h>
.text
/*
* void remap_ccsrbar(vm_offset_t old_ccsrbar_va, vm_offset_t new_ccsrbar_va,
* vm_offset_t new_ccsrbar_pa)
*
* r3 - old_ccsrbar_va
* r4 - new_ccsrbar_va
* r5 - new_ccsrbar_pa
*/
ENTRY(remap_ccsrbar)
/*
* CCSRBAR updating sequence according
* to section 4.3.1.1.1 of MPC8555E RM.
*/
/* Read current value of CCSRBAR */
lwz %r6, 0(%r3)
isync
/* Write new value */
rlwinm %r6, %r5, 20, 12, 23
stw %r6, 0(%r3)
/*
* Read from address that is outside of CCSRBAR space.
* We have RAM locations available at KERNBASE.
*/
lis %r7, KERNBASE@ha
addi %r7, %r7, KERNBASE@l
lwz %r6, 0(%r7)
isync
/* Read value of CCSRBAR from new location */
lwz %r6, 0(%r4)
isync
blr
/*
* void switch_to_as0(void)
*/
ENTRY(switch_to_as0)
mflr %r5 /* Save LR */
mfmsr %r3
lis %r6, (PSL_IS | PSL_DS)@ha
ori %r6, %r6, (PSL_IS | PSL_DS)@l
not %r6, %r6
and %r3, %r3, %r6 /* Clear IS/DS bits */
bl 1f
1: mflr %r4 /* Use current address */
addi %r4, %r4, 20 /* Increment to instruction after rfi */
mtspr SPR_SRR0, %r4
mtspr SPR_SRR1, %r3
rfi
mtlr %r5 /* Restore LR */
blr
/*
* void load_pid0(tlbtid_t)
*/
ENTRY(load_pid0)
mtspr SPR_PID0, %r3
isync
blr

103
sys/powerpc/booke/trap_subr.S
View File

@ -1,6 +1,6 @@
/*-
* Copyright (C) 2006-2008 Semihalf, Rafal Jaworowski <raj@semihalf.com>
* Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
* Copyright (C) 2006 Semihalf, Rafal Jaworowski <raj@semihalf.com>
* Copyright (C) 2006 Juniper Networks, Inc.
* All rights reserved.
*
@ -462,16 +462,13 @@ INTERRUPT(int_data_tlb_error)
mfdear %r31
/*
* Save MAS0-MAS2 registers. There might be another tlb miss during pte
* lookup overwriting current contents (which was hw filled).
* Save MAS0-MAS2 registers. There might be another tlb miss during
* pte lookup overwriting current contents (which was hw filled).
*/
mfspr %r29, SPR_MAS0
mfspr %r28, SPR_MAS1
mfspr %r27, SPR_MAS2
/* return tlb0 entry address in r30 */
bl get_tlb0table_entry
/* Check faulting address. */
lis %r21, VM_MAXUSER_ADDRESS@h
ori %r21, %r21, VM_MAXUSER_ADDRESS@l
@ -521,11 +518,6 @@ search_failed:
*/
lis %r23, 0xffff0000@h /* revoke all permissions */
/* Save MAS registers to tlb0[] table. */
stw %r28, TLB0TABLE_MAS1(%r30) /* write tlb0[idx].mas1 */
stw %r27, TLB0TABLE_MAS2(%r30) /* write tlb0[idx].mas2 */
stw %r23, TLB0TABLE_MAS3(%r30) /* write tlb0[idx].mas3 */
/* Load MAS registers. */
mtspr SPR_MAS0, %r29
isync
@ -541,61 +533,18 @@ search_failed:
isync
b tlb_miss_return
/******************************************************/
/*
* Calculate address of tlb0[tlb0table_idx], save it in r30
/*****************************************************************************
*
* tlb0table_idx = (way * entries_per_way) + entry_number
* entries_per_way = 128
* entry_number is defined by EPN[45:51]
*
* input: r31 - faulting address
* input: r29 - MAS0
* output: r30 - address of corresponding tlb0[] entry
*
* scratch regs used: r21-r23
*/
/******************************************************/
get_tlb0table_entry:
lis %r21, 0 /* keeps tlb0table_idx */
/* Add entry number, use DEAR from r31 (faulting va) */
rlwinm %r22, %r31, 20, 25, 31 /* get EPN[45:51] */
add %r21, %r21, %r22
/* Select way */
rlwinm %r22, %r29, 16, 30, 31 /* get way# = ESEL[0:1] */
/* Get number of entries per tlb0 way. */
lis %r23, tlb0_nentries_per_way@h
ori %r23, %r23, tlb0_nentries_per_way@l
lwz %r23, 0(%r23)
mullw %r22, %r22, %r23 /* multiply by #entries per way */
add %r21, %r21, %r22
mulli %r21, %r21, TLB0_ENTRY_SIZE /* multipy by tlb0 entry size */
/* Get tlb0[tlb0tble_idx] address, save it in r30 */
lis %r30, tlb0@h
ori %r30, %r30, tlb0@l
lwz %r30, 0(%r30)
add %r30, %r30, %r21
blr
/******************************************************/
/*
* Return pte address that corresponds to given pmap/va.
* If there is no valid entry return 0.
* Return pte address that corresponds to given pmap/va. If there is no valid
* entry return 0.
*
* input: r26 - pmap
* input: r31 - dear
* output: r25 - pte address
*
* scratch regs used: r21
*/
/******************************************************/
*
****************************************************************************/
pte_lookup:
cmpwi %r26, 0
beq 1f /* fail quickly if pmap is invalid */
@ -626,32 +575,38 @@ pte_lookup:
2:
blr
/******************************************************/
/*
* Save MAS1-MAS3 registers to tlb0[] table, write TLB entry
/*****************************************************************************
*
* Load MAS1-MAS3 registers with data, write TLB entry
*
* input:
* r29 - mas0
* r28 - mas1
* r27 - mas2
* r25 - pte
* r30 - tlb0 entry address
*
* output: none
*
* scratch regs: r21-r23
*/
/******************************************************/
*
****************************************************************************/
tlb_fill_entry:
/* Handle pte flags. */
lwz %r21, PTE_FLAGS(%r25) /* get pte->flags */
/*
* Update PTE flags: we have to do it atomically, as pmap_protect()
* running on other CPUs could attempt to update the flags at the same
* time.
*/
li %r23, PTE_FLAGS
1:
lwarx %r21, %r23, %r25 /* get pte->flags */
oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */
andi. %r22, %r21, (PTE_UW | PTE_UW)@l /* check if writable */
beq 1f
beq 2f
oris %r21, %r21, PTE_MODIFIED@h /* set modified bit */
1:
stw %r21, PTE_FLAGS(%r25) /* write it back */
2:
stwcx. %r21, %r23, %r25 /* write it back */
bne- 1b
/* Update MAS2. */
rlwimi %r27, %r21, 0, 27, 30 /* insert WIMG bits from pte */
@ -661,11 +616,6 @@ tlb_fill_entry:
rlwimi %r23, %r21, 24, 26, 31 /* insert protection bits from pte */
/* Save MAS registers to tlb0[] table. */
stw %r28, TLB0TABLE_MAS1(%r30) /* write tlb0[idx].mas1 */
stw %r27, TLB0TABLE_MAS2(%r30) /* write tlb0[idx].mas2 */
stw %r23, TLB0TABLE_MAS3(%r30) /* write tlb0[idx].mas3 */
/* Load MAS registers. */
mtspr SPR_MAS0, %r29
isync
@ -700,9 +650,6 @@ INTERRUPT(int_inst_tlb_error)
mfspr %r28, SPR_MAS1
mfspr %r27, SPR_MAS2
/* return tlb0 entry address in r30 */
bl get_tlb0table_entry
mfsrr1 %r21
mtcr %r21

3
sys/powerpc/include/pcpu.h
View File

@ -59,7 +59,8 @@ struct pmap;
register_t pc_booke_critsave[BOOKE_CRITSAVE_LEN]; \
register_t pc_booke_mchksave[CPUSAVE_LEN]; \
register_t pc_booke_tlbsave[BOOKE_TLBSAVE_LEN]; \
register_t pc_booke_tlb_level;
register_t pc_booke_tlb_level; \
int pc_tid_next;
/* Definitions for register offsets within the exception tmp save areas */
#define CPUSAVE_R28 0 /* where r28 gets saved */

14
sys/powerpc/include/pmap.h
View File

@ -111,17 +111,17 @@ struct md_page {
#else
struct pmap {
struct mtx pm_mtx; /* pmap mutex */
tlbtid_t pm_tid; /* TID to identify this pmap entries in TLB */
u_int pm_active; /* active on cpus */
int pm_refs; /* ref count */
struct pmap_statistics pm_stats;/* pmap statistics */
struct mtx pm_mtx; /* pmap mutex */
tlbtid_t pm_tid[MAXCPU]; /* TID to identify this pmap entries in TLB */
u_int pm_active; /* active on cpus */
int pm_refs; /* ref count */
struct pmap_statistics pm_stats; /* pmap statistics */
/* Page table directory, array of pointers to page tables. */
pte_t *pm_pdir[PDIR_NENTRIES];
pte_t *pm_pdir[PDIR_NENTRIES];
/* List of allocated ptbl bufs (ptbl kva regions). */
TAILQ_HEAD(, ptbl_buf) ptbl_list;
TAILQ_HEAD(, ptbl_buf) pm_ptbl_list;
};
typedef struct pmap *pmap_t;

6
sys/powerpc/include/pte.h
View File

@ -211,11 +211,11 @@ extern u_int dsisr(void);
* Page Table Entry definitions and macros.
*/
#ifndef LOCORE
struct pte_entry {
struct pte {
vm_offset_t rpn;
u_int32_t flags;
uint32_t flags;
};
typedef struct pte_entry pte_t;
typedef struct pte pte_t;
#endif
/* RPN mask, TLB0 4K pages */

29
sys/powerpc/include/tlb.h
View File

@ -46,7 +46,8 @@
#define MAS1_IPROT 0x40000000
#define MAS1_TID_MASK 0x00FF0000
#define MAS1_TID_SHIFT 16
#define MAS1_TS 0x00001000
#define MAS1_TS_MASK 0x00001000
#define MAS1_TS_SHIFT 12
#define MAS1_TSIZE_MASK 0x00000F00
#define MAS1_TSIZE_SHIFT 8
@ -62,7 +63,7 @@
#define TLB_SIZE_1G 10
#define TLB_SIZE_4G 11
#define MAS2_EPN 0xFFFFF000
#define MAS2_EPN_MASK 0xFFFFF000
#define MAS2_EPN_SHIFT 12
#define MAS2_X0 0x00000040
#define MAS2_X1 0x00000020
@ -109,31 +110,37 @@
#define MAS2_TLB0_ENTRY_IDX_SHIFT 12
/*
* Maximum number of TLB1 entries used for a permanat
* mapping of kernel region (kernel image plus statically
* allocated data.
* Maximum number of TLB1 entries used for a permanent mapping of kernel
* region (kernel image plus statically allocated data).
*/
#define KERNEL_REGION_MAX_TLB_ENTRIES 4
#define _TLB_ENTRY_IO (MAS2_I | MAS2_G)
#ifdef SMP
#define _TLB_ENTRY_MEM (MAS2_M)
#else
#define _TLB_ENTRY_MEM (0)
#endif
#define KERNEL_TID 0 /* TLB TID to use for kernel translations */
#define TID_KERNEL 0 /* TLB TID to use for kernel (shared) translations */
#define TID_KRESERVED 1 /* Number of TIDs reserved for kernel */
#define TID_URESERVED 0 /* Number of TIDs reserve for user */
#define TID_URESERVED 0 /* Number of TIDs reserved for user */
#define TID_MIN (TID_KRESERVED + TID_URESERVED)
#define TID_MAX 255
#define TID_NONE -1
#if !defined(LOCORE)
typedef struct tlb_entry {
u_int32_t mas1;
u_int32_t mas2;
u_int32_t mas3;
uint32_t mas1;
uint32_t mas2;
uint32_t mas3;
} tlb_entry_t;
typedef u_int8_t tlbtid_t;
typedef int tlbtid_t;
struct pmap;
void tlb0_print_tlbentries(void);
void tlb1_inval_entry(unsigned int);
void tlb1_init(vm_offset_t);
void tlb1_print_entries(void);

7
sys/powerpc/powerpc/genassym.c
View File

@ -108,11 +108,8 @@ ASSYM(PM_PDIR, offsetof(struct pmap, pm_pdir));
#endif
#if defined(E500)
ASSYM(PTE_RPN, offsetof(struct pte_entry, rpn));
ASSYM(PTE_FLAGS, offsetof(struct pte_entry, flags));
ASSYM(TLB0TABLE_MAS1, offsetof(struct tlb_entry, mas1));
ASSYM(TLB0TABLE_MAS2, offsetof(struct tlb_entry, mas2));
ASSYM(TLB0TABLE_MAS3, offsetof(struct tlb_entry, mas3));
ASSYM(PTE_RPN, offsetof(struct pte, rpn));
ASSYM(PTE_FLAGS, offsetof(struct pte, flags));
ASSYM(TLB0_ENTRY_SIZE, sizeof(struct tlb_entry));
#endif