powerpc: Use IFUNCs for copyin/copyout/etc
Summary: Radix on AIM, and all of Book-E (currently), can do direct addressing of user space, instead of needing to map user addresses into kernel space. Take advantage of this to optimize the copy(9) functions for this behavior, and avoid effectively NOP translations. Test Plan: Tested on powerpcspe, powerpc64/booke, powerpc64/AIM Reviewed by: bdragon Differential Revision: https://reviews.freebsd.org/D25129
This commit is contained in:
Justin Hibbits
parent
0ad202f312
commit
d31111442e
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=361861
@ -239,7 +239,7 @@ powerpc/powerpc/autoconf.c standard
|
||||
powerpc/powerpc/bus_machdep.c standard
|
||||
powerpc/powerpc/busdma_machdep.c standard
|
||||
powerpc/powerpc/clock.c standard
|
||||
powerpc/powerpc/copyinout.c standard
|
||||
powerpc/powerpc/copyinout.c optional aim
|
||||
powerpc/powerpc/cpu.c standard
|
||||
powerpc/powerpc/cpu_subr64.S optional powerpc64
|
||||
powerpc/powerpc/db_disasm.c optional ddb
|
||||
@ -274,6 +274,7 @@ powerpc/powerpc/sigcode64.S optional powerpc64
|
||||
powerpc/powerpc/swtch32.S optional powerpc | powerpcspe
|
||||
powerpc/powerpc/swtch64.S optional powerpc64
|
||||
powerpc/powerpc/stack_machdep.c optional ddb | stack
|
||||
powerpc/powerpc/support.S optional powerpc64 | booke
|
||||
powerpc/powerpc/syncicache.c standard
|
||||
powerpc/powerpc/sys_machdep.c standard
|
||||
powerpc/powerpc/trap.c standard
|
||||
|
||||
@ -412,8 +412,6 @@ void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *,
|
||||
vm_size_t);
|
||||
void mmu_radix_clear_modify(vm_page_t);
|
||||
void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
|
||||
int mmu_radix_map_user_ptr(pmap_t pm,
|
||||
volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen);
|
||||
int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *);
|
||||
int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
|
||||
void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
|
||||
@ -538,7 +536,6 @@ static struct pmap_funcs mmu_radix_methods = {
|
||||
.kextract = mmu_radix_kextract,
|
||||
.kremove = mmu_radix_kremove,
|
||||
.change_attr = mmu_radix_change_attr,
|
||||
.map_user_ptr = mmu_radix_map_user_ptr,
|
||||
.decode_kernel_ptr = mmu_radix_decode_kernel_ptr,
|
||||
|
||||
.tlbie_all = mmu_radix_tlbie_all,
|
||||
@ -5998,20 +5995,6 @@ mmu_radix_kremove(vm_offset_t va)
|
||||
pte_clear(pte);
|
||||
}
|
||||
|
||||
int mmu_radix_map_user_ptr(pmap_t pm,
|
||||
volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen)
|
||||
{
|
||||
if ((uintptr_t)uaddr + ulen >= VM_MAXUSER_ADDRESS ||
|
||||
(uintptr_t)uaddr + ulen < (uintptr_t)uaddr)
|
||||
return (EFAULT);
|
||||
|
||||
*kaddr = (void *)(uintptr_t)uaddr;
|
||||
if (klen)
|
||||
*klen = ulen;
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
mmu_radix_decode_kernel_ptr(vm_offset_t addr,
|
||||
int *is_user, vm_offset_t *decoded)
|
||||
|
||||
@ -349,8 +349,6 @@ static vm_offset_t mmu_booke_quick_enter_page(vm_page_t m);
|
||||
static void mmu_booke_quick_remove_page(vm_offset_t addr);
|
||||
static int mmu_booke_change_attr(vm_offset_t addr,
|
||||
vm_size_t sz, vm_memattr_t mode);
|
||||
static int mmu_booke_map_user_ptr(pmap_t pm,
|
||||
volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen);
|
||||
static int mmu_booke_decode_kernel_ptr(vm_offset_t addr,
|
||||
int *is_user, vm_offset_t *decoded_addr);
|
||||
static void mmu_booke_page_array_startup(long);
|
||||
@ -410,7 +408,6 @@ static struct pmap_funcs mmu_booke_methods = {
|
||||
.kremove = mmu_booke_kremove,
|
||||
.unmapdev = mmu_booke_unmapdev,
|
||||
.change_attr = mmu_booke_change_attr,
|
||||
.map_user_ptr = mmu_booke_map_user_ptr,
|
||||
.decode_kernel_ptr = mmu_booke_decode_kernel_ptr,
|
||||
|
||||
/* dumpsys() support */
|
||||
@ -1208,26 +1205,6 @@ mmu_booke_kremove(vm_offset_t va)
|
||||
mtx_unlock_spin(&tlbivax_mutex);
|
||||
}
|
||||
|
||||
/*
|
||||
* Provide a kernel pointer corresponding to a given userland pointer.
|
||||
* The returned pointer is valid until the next time this function is
|
||||
* called in this thread. This is used internally in copyin/copyout.
|
||||
*/
|
||||
int
|
||||
mmu_booke_map_user_ptr(pmap_t pm, volatile const void *uaddr,
|
||||
void **kaddr, size_t ulen, size_t *klen)
|
||||
{
|
||||
|
||||
if (trunc_page((uintptr_t)uaddr + ulen) > VM_MAXUSER_ADDRESS)
|
||||
return (EFAULT);
|
||||
|
||||
*kaddr = (void *)(uintptr_t)uaddr;
|
||||
if (klen)
|
||||
*klen = ulen;
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Figure out where a given kernel pointer (usually in a fault) points
|
||||
* to from the VM's perspective, potentially remapping into userland's
|
||||
|
||||
@ -114,6 +114,7 @@
|
||||
.p2align 4; \
|
||||
TYPE_ENTRY(name) \
|
||||
DOT_LABEL(name):
|
||||
#define _NAKED_ENTRY(name) _ENTRY(name)
|
||||
#else
|
||||
#define _ENTRY(name) \
|
||||
.text; \
|
||||
@ -124,6 +125,15 @@ name: \
|
||||
addis %r2, %r12, (.TOC.-name)@ha; \
|
||||
addi %r2, %r2, (.TOC.-name)@l; \
|
||||
.localentry name, .-name;
|
||||
|
||||
/* "Naked" function entry. No TOC prologue for ELFv2. */
|
||||
#define _NAKED_ENTRY(name) \
|
||||
.text; \
|
||||
.p2align 4; \
|
||||
.globl name; \
|
||||
.type name,@function; \
|
||||
name: \
|
||||
.localentry name, .-name;
|
||||
#endif
|
||||
|
||||
#define _END(name) \
|
||||
@ -146,6 +156,8 @@ name: \
|
||||
name:
|
||||
#define _END(name)
|
||||
|
||||
#define _NAKED_ENTRY(name) _ENTRY(name)
|
||||
|
||||
#define LOAD_ADDR(reg, var) \
|
||||
lis reg, var@ha; \
|
||||
ori reg, reg, var@l;
|
||||
|
||||
@ -182,6 +182,25 @@ struct mmu_kobj {
|
||||
|
||||
typedef struct mmu_kobj *mmu_t;
|
||||
|
||||
/* The currently installed pmap object. */
|
||||
extern mmu_t mmu_obj;
|
||||
|
||||
/*
|
||||
* Resolve a given pmap function.
|
||||
* 'func' is the function name less the 'pmap_' * prefix.
|
||||
*/
|
||||
#define PMAP_RESOLVE_FUNC(func) \
|
||||
({ \
|
||||
pmap_##func##_t f; \
|
||||
const struct mmu_kobj *mmu = mmu_obj; \
|
||||
do { \
|
||||
f = mmu->funcs->func; \
|
||||
if (f != NULL) break; \
|
||||
mmu = mmu->base; \
|
||||
} while (mmu != NULL); \
|
||||
f;})
|
||||
|
||||
|
||||
#define MMU_DEF(name, ident, methods) \
|
||||
\
|
||||
const struct mmu_kobj name = { \
|
||||
|
||||
@ -134,6 +134,9 @@
|
||||
#define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */
|
||||
#define USPACE (kstack_pages * PAGE_SIZE) /* total size of pcb */
|
||||
|
||||
#define COPYFAULT 0x1
|
||||
#define FUSUFAULT 0x2
|
||||
|
||||
/*
|
||||
* Mach derived conversion macros
|
||||
*/
|
||||
|
||||
@ -66,13 +66,102 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <vm/vm.h>
|
||||
#include <vm/pmap.h>
|
||||
#include <vm/vm_extern.h>
|
||||
#include <vm/vm_map.h>
|
||||
|
||||
#include <machine/mmuvar.h>
|
||||
#include <machine/pcb.h>
|
||||
#include <machine/vmparam.h>
|
||||
#include <machine/ifunc.h>
|
||||
|
||||
/*
|
||||
* On powerpc64 (AIM only) the copy functions are IFUNcs, selecting the best
|
||||
* option based on the PMAP in use.
|
||||
*
|
||||
* There are two options for copy functions on powerpc64:
|
||||
* - 'remap' copies, which remap userspace segments into kernel space for
|
||||
* copying. This is used by the 'oea64' pmap.
|
||||
* - 'direct' copies, which copy directly from userspace. This does not require
|
||||
* remapping user segments into kernel. This is used by the 'radix' pmap for
|
||||
* performance.
|
||||
*
|
||||
* Book-E does not use the C functions, opting instead to use the 'direct'
|
||||
* copies, directly, avoiding the IFUNC overhead.
|
||||
*
|
||||
* On 32-bit AIM these functions are direct, not IFUNCs, for performance.
|
||||
*/
|
||||
#ifdef __powerpc64__
|
||||
int subyte_remap(volatile void *addr, int byte);
|
||||
int subyte_direct(volatile void *addr, int byte);
|
||||
int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done);
|
||||
int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done);
|
||||
int copyout_remap(const void *kaddr, void *udaddr, size_t len);
|
||||
int copyout_direct(const void *kaddr, void *udaddr, size_t len);
|
||||
int copyin_remap(const void *uaddr, void *kaddr, size_t len);
|
||||
int copyin_direct(const void *uaddr, void *kaddr, size_t len);
|
||||
int suword32_remap(volatile void *addr, int word);
|
||||
int suword32_direct(volatile void *addr, int word);
|
||||
int suword_remap(volatile void *addr, long word);
|
||||
int suword_direct(volatile void *addr, long word);
|
||||
int suword64_remap(volatile void *addr, int64_t word);
|
||||
int suword64_direct(volatile void *addr, int64_t word);
|
||||
int fubyte_remap(volatile const void *addr);
|
||||
int fubyte_direct(volatile const void *addr);
|
||||
int fuword16_remap(volatile const void *addr);
|
||||
int fuword16_direct(volatile const void *addr);
|
||||
int fueword32_remap(volatile const void *addr, int32_t *val);
|
||||
int fueword32_direct(volatile const void *addr, int32_t *val);
|
||||
int fueword64_remap(volatile const void *addr, int64_t *val);
|
||||
int fueword64_direct(volatile const void *addr, int64_t *val);
|
||||
int fueword_remap(volatile const void *addr, long *val);
|
||||
int fueword_direct(volatile const void *addr, long *val);
|
||||
int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
|
||||
uint32_t new);
|
||||
int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
|
||||
uint32_t new);
|
||||
int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp,
|
||||
u_long new);
|
||||
int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp,
|
||||
u_long new);
|
||||
|
||||
/*
|
||||
* The IFUNC resolver determines the copy based on if the PMAP implementation
|
||||
* includes a pmap_map_user_ptr function.
|
||||
*/
|
||||
#define DEFINE_COPY_FUNC(ret, func, args) \
|
||||
DEFINE_IFUNC(, ret, func, args) \
|
||||
{ \
|
||||
return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \
|
||||
func##_remap : func##_direct); \
|
||||
}
|
||||
DEFINE_COPY_FUNC(int, subyte, (volatile void *, int))
|
||||
DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *))
|
||||
DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t))
|
||||
DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t))
|
||||
DEFINE_COPY_FUNC(int, suword, (volatile void *, long))
|
||||
DEFINE_COPY_FUNC(int, suword32, (volatile void *, int))
|
||||
#ifdef __powerpc64__
|
||||
DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t))
|
||||
#endif
|
||||
DEFINE_COPY_FUNC(int, fubyte, (volatile const void *))
|
||||
DEFINE_COPY_FUNC(int, fuword16, (volatile const void *))
|
||||
DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *))
|
||||
#ifdef __powerpc64__
|
||||
DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *))
|
||||
#endif
|
||||
DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *))
|
||||
DEFINE_COPY_FUNC(int, casueword32,
|
||||
(volatile uint32_t *, uint32_t, uint32_t *, uint32_t))
|
||||
DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long))
|
||||
|
||||
#define REMAP(x) x##_remap
|
||||
#else
|
||||
#define REMAP(x) x
|
||||
#endif
|
||||
|
||||
|
||||
int
|
||||
copyout(const void *kaddr, void *udaddr, size_t len)
|
||||
REMAP(copyout)(const void *kaddr, void *udaddr, size_t len)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -111,7 +200,7 @@ copyout(const void *kaddr, void *udaddr, size_t len)
|
||||
}
|
||||
|
||||
int
|
||||
copyin(const void *udaddr, void *kaddr, size_t len)
|
||||
REMAP(copyin)(const void *udaddr, void *kaddr, size_t len)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -150,7 +239,7 @@ copyin(const void *udaddr, void *kaddr, size_t len)
|
||||
}
|
||||
|
||||
int
|
||||
copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
|
||||
REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done)
|
||||
{
|
||||
const char *up;
|
||||
char *kp;
|
||||
@ -183,7 +272,7 @@ copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
|
||||
}
|
||||
|
||||
int
|
||||
subyte(volatile void *addr, int byte)
|
||||
REMAP(subyte)(volatile void *addr, int byte)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -212,7 +301,7 @@ subyte(volatile void *addr, int byte)
|
||||
|
||||
#ifdef __powerpc64__
|
||||
int
|
||||
suword32(volatile void *addr, int word)
|
||||
REMAP(suword32)(volatile void *addr, int word)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -238,10 +327,16 @@ suword32(volatile void *addr, int word)
|
||||
td->td_pcb->pcb_onfault = NULL;
|
||||
return (0);
|
||||
}
|
||||
#else
|
||||
int
|
||||
REMAP(suword32)(volatile void *addr, int32_t word)
|
||||
{
|
||||
REMAP( return (suword)(addr, (long)word));
|
||||
}
|
||||
#endif
|
||||
|
||||
int
|
||||
suword(volatile void *addr, long word)
|
||||
REMAP(suword)(volatile void *addr, long word)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -270,20 +365,14 @@ suword(volatile void *addr, long word)
|
||||
|
||||
#ifdef __powerpc64__
|
||||
int
|
||||
suword64(volatile void *addr, int64_t word)
|
||||
REMAP(suword64)(volatile void *addr, int64_t word)
|
||||
{
|
||||
return (suword(addr, (long)word));
|
||||
}
|
||||
#else
|
||||
int
|
||||
suword32(volatile void *addr, int32_t word)
|
||||
{
|
||||
return (suword(addr, (long)word));
|
||||
return (REMAP(suword)(addr, (long)word));
|
||||
}
|
||||
#endif
|
||||
|
||||
int
|
||||
fubyte(volatile const void *addr)
|
||||
REMAP(fubyte)(volatile const void *addr)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -312,7 +401,7 @@ fubyte(volatile const void *addr)
|
||||
}
|
||||
|
||||
int
|
||||
fuword16(volatile const void *addr)
|
||||
REMAP(fuword16)(volatile const void *addr)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -340,7 +429,7 @@ fuword16(volatile const void *addr)
|
||||
}
|
||||
|
||||
int
|
||||
fueword32(volatile const void *addr, int32_t *val)
|
||||
REMAP(fueword32)(volatile const void *addr, int32_t *val)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -369,7 +458,7 @@ fueword32(volatile const void *addr, int32_t *val)
|
||||
|
||||
#ifdef __powerpc64__
|
||||
int
|
||||
fueword64(volatile const void *addr, int64_t *val)
|
||||
REMAP(fueword64)(volatile const void *addr, int64_t *val)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -398,7 +487,7 @@ fueword64(volatile const void *addr, int64_t *val)
|
||||
#endif
|
||||
|
||||
int
|
||||
fueword(volatile const void *addr, long *val)
|
||||
REMAP(fueword)(volatile const void *addr, long *val)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
@ -426,7 +515,7 @@ fueword(volatile const void *addr, long *val)
|
||||
}
|
||||
|
||||
int
|
||||
casueword32(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
|
||||
REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
|
||||
uint32_t new)
|
||||
{
|
||||
struct thread *td;
|
||||
@ -474,7 +563,7 @@ casueword32(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
|
||||
|
||||
#ifndef __powerpc64__
|
||||
int
|
||||
casueword(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
|
||||
REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
|
||||
{
|
||||
|
||||
return (casueword32((volatile uint32_t *)addr, old,
|
||||
@ -482,7 +571,7 @@ casueword(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
|
||||
}
|
||||
#else
|
||||
int
|
||||
casueword(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
|
||||
REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
|
||||
{
|
||||
struct thread *td;
|
||||
pmap_t pm;
|
||||
|
||||
@ -62,7 +62,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <machine/mmuvar.h>
|
||||
#include <machine/smp.h>
|
||||
|
||||
static mmu_t mmu_obj;
|
||||
mmu_t mmu_obj;
|
||||
|
||||
/*
|
||||
* pmap globals
|
||||
@ -99,24 +99,14 @@ pmap_nomethod(void)
|
||||
|
||||
#define DEFINE_PMAP_IFUNC(ret, func, args) \
|
||||
DEFINE_IFUNC(, ret, pmap_##func, args) { \
|
||||
const struct mmu_kobj *mmu = mmu_obj; \
|
||||
pmap_##func##_t f; \
|
||||
do { \
|
||||
f = mmu->funcs->func; \
|
||||
if (f != NULL) break; \
|
||||
mmu = mmu->base; \
|
||||
} while (mmu != NULL); \
|
||||
f = PMAP_RESOLVE_FUNC(func); \
|
||||
return (f != NULL ? f : (pmap_##func##_t)pmap_nomethod);\
|
||||
}
|
||||
#define DEFINE_DUMPSYS_IFUNC(ret, func, args) \
|
||||
DEFINE_IFUNC(, ret, dumpsys_##func, args) { \
|
||||
const struct mmu_kobj *mmu = mmu_obj; \
|
||||
pmap_dumpsys_##func##_t f; \
|
||||
do { \
|
||||
f = mmu->funcs->dumpsys_##func; \
|
||||
if (f != NULL) break; \
|
||||
mmu = mmu->base; \
|
||||
} while (mmu != NULL); \
|
||||
f = PMAP_RESOLVE_FUNC(dumpsys_##func); \
|
||||
return (f != NULL ? f : (pmap_dumpsys_##func##_t)pmap_nomethod);\
|
||||
}
|
||||
|
||||
|
||||
539
sys/powerpc/powerpc/support.S
Normal file
539
sys/powerpc/powerpc/support.S
Normal file
@ -0,0 +1,539 @@
|
||||
/*-
|
||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
||||
*
|
||||
* Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*
|
||||
* $FreeBSD$
|
||||
*/
|
||||
|
||||
/*
|
||||
* Assembly variants of various functions, for those that don't need generic C
|
||||
* implementations. Currently this includes:
|
||||
*
|
||||
* - Direct-access versions of copyin/copyout methods.
|
||||
* - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid
|
||||
* unnecessary pmap_map_usr_ptr() calls.
|
||||
*/
|
||||
|
||||
#include "assym.inc"
|
||||
#include "opt_sched.h"
|
||||
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/errno.h>
|
||||
|
||||
#include <machine/param.h>
|
||||
#include <machine/asm.h>
|
||||
#include <machine/spr.h>
|
||||
#include <machine/trap.h>
|
||||
#include <machine/vmparam.h>
|
||||
|
||||
#ifdef _CALL_ELF
|
||||
.abiversion _CALL_ELF
|
||||
#endif
|
||||
|
||||
#ifdef __powerpc64__
|
||||
#define LOAD ld
|
||||
#define STORE std
|
||||
#define WORD 8
|
||||
#define CMPI cmpdi
|
||||
#define CMPLI cmpldi
|
||||
/* log_2(8 * WORD) */
|
||||
#define LOOP_LOG 6
|
||||
#define LOG_WORD 3
|
||||
#else
|
||||
#define LOAD lwz
|
||||
#define STORE stw
|
||||
#define WORD 4
|
||||
#define CMPI cmpwi
|
||||
#define CMPLI cmplwi
|
||||
/* log_2(8 * WORD) */
|
||||
#define LOOP_LOG 5
|
||||
#define LOG_WORD 2
|
||||
#endif
|
||||
|
||||
#ifdef AIM
|
||||
#define ENTRY_DIRECT(x) ENTRY(x ## _direct)
|
||||
#else
|
||||
#define ENTRY_DIRECT(x) ENTRY(x)
|
||||
#endif
|
||||
|
||||
#ifdef __powerpc64__
|
||||
#define PROLOGUE ;\
|
||||
mflr %r0 ;\
|
||||
std %r0, 16(%r1) ;\
|
||||
|
||||
#define EPILOGUE ;\
|
||||
ld %r0, 16(%r1) ;\
|
||||
mtlr %r0 ;\
|
||||
blr ;\
|
||||
nop
|
||||
|
||||
#define VALIDATE_TRUNCATE_ADDR_COPY VALIDATE_ADDR_COPY
|
||||
#define VALIDATE_ADDR_COPY(raddr, len) \
|
||||
srdi %r0, raddr, 52 ;\
|
||||
cmpwi %r0, 1 ;\
|
||||
bge- copy_fault ;\
|
||||
nop
|
||||
|
||||
#define VALIDATE_ADDR_FUSU(raddr) ;\
|
||||
srdi %r0, raddr, 52 ;\
|
||||
cmpwi %r0, 1 ;\
|
||||
bge- fusufault ;\
|
||||
nop
|
||||
|
||||
#else
|
||||
#define PROLOGUE ;\
|
||||
mflr %r0 ;\
|
||||
stw %r0, 4(%r1) ;\
|
||||
|
||||
#define EPILOGUE ;\
|
||||
lwz %r0, 4(%r1) ;\
|
||||
mtlr %r0 ;\
|
||||
blr ;\
|
||||
nop
|
||||
|
||||
/* %r0 is temporary */
|
||||
/*
|
||||
* Validate address and length are valid.
|
||||
* For VALIDATE_ADDR_COPY() have to account for wraparound.
|
||||
*/
|
||||
#define VALIDATE_ADDR_COPY(raddr, len) \
|
||||
lis %r0, VM_MAXUSER_ADDRESS@h ;\
|
||||
ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\
|
||||
cmplw %r0, raddr ;\
|
||||
blt- copy_fault ;\
|
||||
add %r0, raddr, len ;\
|
||||
cmplw 7, %r0, raddr ;\
|
||||
blt- 7, copy_fault ;\
|
||||
mtcrf 0x80, %r0 ;\
|
||||
bt- 0, copy_fault ;\
|
||||
nop
|
||||
|
||||
#define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) \
|
||||
lis %r0, VM_MAXUSER_ADDRESS@h ;\
|
||||
ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\
|
||||
cmplw %r0, raddr ;\
|
||||
blt- copy_fault ;\
|
||||
sub %r0, %r0, raddr ;\
|
||||
cmplw len, %r0 ;\
|
||||
isel len, len, %r0, 0 ;\
|
||||
|
||||
#define VALIDATE_ADDR_FUSU(raddr) \
|
||||
lis %r0, VM_MAXUSER_ADDRESS@h ;\
|
||||
ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\
|
||||
cmplw %r0, raddr ;\
|
||||
ble- fusufault
|
||||
|
||||
#endif
|
||||
|
||||
#define PCPU(reg) mfsprg reg, 0
|
||||
|
||||
#define SET_COPYFAULT(raddr, rpcb, len) \
|
||||
VALIDATE_ADDR_COPY(raddr, len) ;\
|
||||
PCPU(%r9) ;\
|
||||
li %r0, COPYFAULT ;\
|
||||
LOAD rpcb, PC_CURPCB(%r9) ;\
|
||||
STORE %r0, PCB_ONFAULT(rpcb) ;\
|
||||
|
||||
#define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\
|
||||
VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\
|
||||
PCPU(%r9) ;\
|
||||
li %r0, COPYFAULT ;\
|
||||
LOAD rpcb, PC_CURPCB(%r9) ;\
|
||||
STORE %r0, PCB_ONFAULT(rpcb)
|
||||
|
||||
#define SET_FUSUFAULT(raddr, rpcb) \
|
||||
VALIDATE_ADDR_FUSU(raddr) ;\
|
||||
PCPU(%r9) ;\
|
||||
li %r0, FUSUFAULT ;\
|
||||
LOAD rpcb, PC_CURPCB(%r9) ;\
|
||||
STORE %r0, PCB_ONFAULT(rpcb)
|
||||
|
||||
#define CLEAR_FAULT_NO_CLOBBER(rpcb) \
|
||||
PCPU(%r9) ;\
|
||||
LOAD rpcb, PC_CURPCB(%r9) ;\
|
||||
li %r0, 0 ;\
|
||||
STORE %r0, PCB_ONFAULT(rpcb)
|
||||
|
||||
#define CLEAR_FAULT(rpcb) \
|
||||
CLEAR_FAULT_NO_CLOBBER(rpcb) ;\
|
||||
li %r3, 0
|
||||
|
||||
/*
|
||||
* bcopy(src, dst, len)
|
||||
* %r3 %r4 %r5
|
||||
*
|
||||
* %r7 is the pcb pointer
|
||||
*
|
||||
* %r0 and %r8-%r10 are volatile
|
||||
* %r11 and %r12 are generally volatile, used in linking and exception
|
||||
* handling. Can be clobbered here.
|
||||
*
|
||||
* Does not allocate or use stack space, but clobbers all volatile registers.
|
||||
*/
|
||||
|
||||
#define rs %r3
|
||||
#define rd %r4
|
||||
#define rl %r5
|
||||
|
||||
#define t1 %r6
|
||||
#define t2 %r7
|
||||
#define t3 %r8
|
||||
#define t4 %r9
|
||||
#define t5 %r10
|
||||
#define t6 %r11
|
||||
#define t7 %r12
|
||||
#define t8 %r0
|
||||
|
||||
#define Thresh WORD * 8
|
||||
#define W4 3
|
||||
#define W2 2
|
||||
#define W1 1
|
||||
#define WORDS(n) (32 - LOG_WORD - W##n)
|
||||
.text
|
||||
ENTRY(bcopy_generic)
|
||||
CMPLI 0, %r5, 0
|
||||
beq .Lend
|
||||
dcbtst 0, rd
|
||||
dcbt 0, rs
|
||||
CMPLI rl, Thresh
|
||||
blt .Lsmall
|
||||
b .Llarge
|
||||
/* memcpy */
|
||||
/* ... */
|
||||
.Lsmall: /* < 8 words remaining */
|
||||
mtcrf 0x3, rl
|
||||
.Lsmall_start:
|
||||
bf WORDS(4), 0f
|
||||
LOAD t1, 0(rs)
|
||||
LOAD t2, WORD*1(rs)
|
||||
LOAD t3, WORD*2(rs)
|
||||
LOAD t4, WORD*3(rs)
|
||||
addi rs, rs, WORD*4
|
||||
STORE t1, 0(rd)
|
||||
STORE t2, WORD*1(rd)
|
||||
STORE t3, WORD*2(rd)
|
||||
STORE t4, WORD*3(rd)
|
||||
addi rd, rd, WORD*4
|
||||
0: /* < 4 words remaining */
|
||||
bf WORDS(2), 1f
|
||||
LOAD t1, 0(rs)
|
||||
LOAD t2, WORD*1(rs)
|
||||
addi rs, rs, WORD*2
|
||||
STORE t1, 0(rd)
|
||||
STORE t2, WORD*1(rd)
|
||||
addi rd, rd, WORD*2
|
||||
1: /* < 2 words remaining */
|
||||
bf WORDS(1), 2f
|
||||
LOAD t1, 0(rs)
|
||||
addi rs, rs, WORD
|
||||
STORE t1, 0(rd)
|
||||
addi rd, rd, WORD
|
||||
2: /* < 1 word remaining */
|
||||
#ifdef __powerpc64__
|
||||
bf 29, 3f
|
||||
lwz t1, 0(rs)
|
||||
addi rs, rs, 4
|
||||
stw t1, 0(rd)
|
||||
addi rd, rd, 4
|
||||
3: /* < 4 bytes remaining */
|
||||
#endif
|
||||
bf 30, 4f
|
||||
lhz t1, 0(rs)
|
||||
addi rs, rs, 2
|
||||
sth t1, 0(rd)
|
||||
addi rd, rd, 2
|
||||
4: /* < 2 bytes remaining */
|
||||
bf 31, .Lout
|
||||
lbz t1, 0(rs)
|
||||
addi rs, rs, 1
|
||||
stb t1, 0(rd)
|
||||
addi rd, rd, 1
|
||||
b .Lout
|
||||
|
||||
.align 4
|
||||
.Llarge:
|
||||
neg t3, rd
|
||||
andi. t6, t3, WORD-1 /* Align rd to word size */
|
||||
mtctr t6
|
||||
sub rl, rl, t6
|
||||
beq+ .Llargealigned
|
||||
1:
|
||||
lbz t1, 0(rs)
|
||||
addi rs, rs, 1
|
||||
stb t1, 0(rd)
|
||||
addi rd, rd, 1
|
||||
bdnz 1b
|
||||
|
||||
.Llargealigned:
|
||||
srwi. t2, rl, LOOP_LOG /* length >> log_2(loop_size) => 8W iterations */
|
||||
mtcrf 0x3, rl
|
||||
beq .Lsmall_start
|
||||
mtctr t2
|
||||
b 1f
|
||||
|
||||
.align 5
|
||||
1:
|
||||
LOAD t1, 0(rs)
|
||||
LOAD t2, WORD(rs)
|
||||
LOAD t3, WORD*2(rs)
|
||||
LOAD t4, WORD*3(rs)
|
||||
LOAD t5, WORD*4(rs)
|
||||
LOAD t6, WORD*5(rs)
|
||||
LOAD t7, WORD*6(rs)
|
||||
LOAD t8, WORD*7(rs)
|
||||
addi rs, rs, WORD*8
|
||||
STORE t1, 0(rd)
|
||||
STORE t2, WORD*1(rd)
|
||||
STORE t3, WORD*2(rd)
|
||||
STORE t4, WORD*3(rd)
|
||||
STORE t5, WORD*4(rd)
|
||||
STORE t6, WORD*5(rd)
|
||||
STORE t7, WORD*6(rd)
|
||||
STORE t8, WORD*7(rd)
|
||||
addi rd, rd, WORD*8
|
||||
bdnz 1b
|
||||
|
||||
b .Lsmall_start
|
||||
.Lout:
|
||||
/* done */
|
||||
.Lend:
|
||||
blr
|
||||
|
||||
/*
|
||||
* copyout(from_kernel, to_user, len)
|
||||
* %r3, %r4, %r5
|
||||
*/
|
||||
ENTRY_DIRECT(copyout)
|
||||
PROLOGUE
|
||||
SET_COPYFAULT(%r4, %r7, %r5)
|
||||
bl bcopy_generic
|
||||
nop
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
|
||||
/*
|
||||
* copyin(from_user, to_kernel, len)
|
||||
* %r3, %r4, %r5
|
||||
*/
|
||||
ENTRY_DIRECT(copyin)
|
||||
PROLOGUE
|
||||
SET_COPYFAULT(%r3, %r7, %r5)
|
||||
bl bcopy_generic
|
||||
nop
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
/*
|
||||
* copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
|
||||
* %r3 %r4 %r5 %r6
|
||||
*/
|
||||
|
||||
ENTRY_DIRECT(copyinstr)
|
||||
PROLOGUE
|
||||
SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5)
|
||||
addi %r9, %r5, 1
|
||||
mtctr %r9
|
||||
mr %r8, %r3
|
||||
addi %r8, %r8, -1
|
||||
addi %r4, %r4, -1
|
||||
li %r3, ENAMETOOLONG
|
||||
0:
|
||||
bdz- 2f
|
||||
lbzu %r0, 1(%r8)
|
||||
stbu %r0, 1(%r4)
|
||||
|
||||
// NULL byte reached ?
|
||||
CMPI %r0, 0
|
||||
beq- 1f
|
||||
b 0b
|
||||
1:
|
||||
li %r3, 0
|
||||
2:
|
||||
/* skip storing length if done is NULL */
|
||||
CMPI %r6, 0
|
||||
beq- 3f
|
||||
mfctr %r0
|
||||
sub %r0, %r9, %r0
|
||||
STORE %r0, 0(%r6)
|
||||
3:
|
||||
CLEAR_FAULT_NO_CLOBBER(%r7)
|
||||
EPILOGUE
|
||||
|
||||
ENTRY_DIRECT(subyte)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
stb %r4, 0(%r3)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
|
||||
#ifndef __powerpc64__
|
||||
ENTRY_DIRECT(suword)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
stw %r4, 0(%r3)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
#endif
|
||||
|
||||
ENTRY_DIRECT(suword32)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
stw %r4, 0(%r3)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
|
||||
#ifdef __powerpc64__
|
||||
ENTRY_DIRECT(suword64)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
std %r4, 0(%r3)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
ENTRY_DIRECT(suword)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
std %r4, 0(%r3)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
#endif
|
||||
|
||||
ENTRY_DIRECT(fubyte)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
lbz %r3, 0(%r3)
|
||||
CLEAR_FAULT_NO_CLOBBER(%r7)
|
||||
EPILOGUE
|
||||
|
||||
ENTRY_DIRECT(fuword16)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
lhz %r3, 0(%r3)
|
||||
CLEAR_FAULT_NO_CLOBBER(%r7)
|
||||
EPILOGUE
|
||||
|
||||
#ifndef __powerpc64__
|
||||
ENTRY_DIRECT(fueword)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
lwz %r0, 0(%r3)
|
||||
stw %r0, 0(%r4)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
#endif
|
||||
ENTRY_DIRECT(fueword32)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
lwz %r0, 0(%r3)
|
||||
stw %r0, 0(%r4)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
|
||||
#ifdef __powerpc64__
|
||||
ENTRY_DIRECT(fueword)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
ld %r0, 0(%r3)
|
||||
std %r0, 0(%r4)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
|
||||
ENTRY_DIRECT(fueword64)
|
||||
PROLOGUE
|
||||
SET_FUSUFAULT(%r3, %r7)
|
||||
ld %r0, 0(%r3)
|
||||
std %r0, 0(%r4)
|
||||
CLEAR_FAULT(%r7)
|
||||
EPILOGUE
|
||||
#endif
|
||||
|
||||
/*
|
||||
* casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new)
|
||||
* %r3 %r4 %r5 %r6
|
||||
*/
|
||||
|
||||
#define CASUEWORD32(raddr, rpcb) ;\
|
||||
PROLOGUE ;\
|
||||
SET_FUSUFAULT(raddr, rpcb) ;\
|
||||
li %r8, 0 ;\
|
||||
1: ;\
|
||||
lwarx %r0, 0, %r3 ;\
|
||||
cmplw %r4, %r0 ;\
|
||||
bne 2f ;\
|
||||
stwcx. %r6, 0, %r3 ;\
|
||||
bne- 3f ;\
|
||||
b 4f ;\
|
||||
2: ;\
|
||||
stwcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\
|
||||
3: ;\
|
||||
li %r8, 1 ;\
|
||||
4: ;\
|
||||
stw %r0, 0(%r5) ;\
|
||||
CLEAR_FAULT_NO_CLOBBER(rpcb) ;\
|
||||
mr %r3, %r8 ;\
|
||||
EPILOGUE
|
||||
|
||||
ENTRY_DIRECT(casueword32)
|
||||
CASUEWORD32(%r3, %r7)
|
||||
|
||||
#ifdef __powerpc64__
|
||||
#define CASUEWORD64(raddr, rpcb) ;\
|
||||
PROLOGUE ;\
|
||||
SET_FUSUFAULT(raddr, rpcb) ;\
|
||||
li %r8, 0 ;\
|
||||
1: ;\
|
||||
ldarx %r0, 0, %r3 ;\
|
||||
cmpld %r4, %r0 ;\
|
||||
bne 2f ;\
|
||||
stdcx. %r6, 0, %r3 ;\
|
||||
bne- 3f ;\
|
||||
b 4f ;\
|
||||
2: ;\
|
||||
stdcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\
|
||||
3: ;\
|
||||
li %r8, 1 ;\
|
||||
4: ;\
|
||||
std %r0, 0(%r5) ;\
|
||||
CLEAR_FAULT_NO_CLOBBER(rpcb) ;\
|
||||
mr %r3, %r8 ;\
|
||||
EPILOGUE
|
||||
|
||||
ENTRY_DIRECT(casueword)
|
||||
CASUEWORD64(%r3, %r7)
|
||||
|
||||
ENTRY_DIRECT(casueword64)
|
||||
CASUEWORD64(%r3, %r7)
|
||||
#else
|
||||
ENTRY_DIRECT(casueword)
|
||||
CASUEWORD32(%r3, %r7)
|
||||
#endif
|
||||
|
||||
_NAKED_ENTRY(fusufault)
|
||||
CLEAR_FAULT_NO_CLOBBER(%r7)
|
||||
li %r3, -1
|
||||
EPILOGUE
|
||||
|
||||
_NAKED_ENTRY(copy_fault)
|
||||
CLEAR_FAULT_NO_CLOBBER(%r7)
|
||||
li %r3, EFAULT
|
||||
EPILOGUE
|
||||
@ -98,6 +98,9 @@ static void normalize_inputs(void);
|
||||
|
||||
extern vm_offset_t __startkernel;
|
||||
|
||||
extern int copy_fault(void);
|
||||
extern int fusufault(void);
|
||||
|
||||
#ifdef KDB
|
||||
int db_trap_glue(struct trapframe *); /* Called from trap_subr.S */
|
||||
#endif
|
||||
@ -590,6 +593,23 @@ handle_onfault(struct trapframe *frame)
|
||||
jmp_buf *fb;
|
||||
|
||||
td = curthread;
|
||||
#if defined(__powerpc64__) || defined(BOOKE)
|
||||
uintptr_t dispatch = (uintptr_t)td->td_pcb->pcb_onfault;
|
||||
|
||||
if (dispatch == 0)
|
||||
return (0);
|
||||
/* Short-circuit radix and Book-E paths. */
|
||||
switch (dispatch) {
|
||||
case COPYFAULT:
|
||||
frame->srr0 = (uintptr_t)copy_fault;
|
||||
return (1);
|
||||
case FUSUFAULT:
|
||||
frame->srr0 = (uintptr_t)fusufault;
|
||||
return (1);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
fb = td->td_pcb->pcb_onfault;
|
||||
if (fb != NULL) {
|
||||
frame->srr0 = (*fb)->_jb[FAULTBUF_LR];
|
||||
|
||||
Loading…
Reference in New Issue
Block a user