Flags should be M_WAITOK | M_ZERO instead of just M_ZERO Reviewed by: markj MFC after: 2 days Sponsored by: Instituto de Pesquisas Eldorado (eldorado.org.br) Differential Revision: https://reviews.freebsd.org/D36865
627 lines
15 KiB
C
627 lines
15 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
*
|
|
* Copyright (c) 2010 Nathan Whitehorn
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* $FreeBSD$
|
|
*/
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/systm.h>
|
|
|
|
#include <vm/vm.h>
|
|
#include <vm/pmap.h>
|
|
#include <vm/uma.h>
|
|
#include <vm/vm.h>
|
|
#include <vm/vm_map.h>
|
|
#include <vm/vm_page.h>
|
|
#include <vm/vm_pageout.h>
|
|
|
|
#include <machine/md_var.h>
|
|
#include <machine/platform.h>
|
|
#include <machine/vmparam.h>
|
|
#include <machine/trap.h>
|
|
|
|
#include "mmu_oea64.h"
|
|
|
|
uintptr_t moea64_get_unique_vsid(void);
|
|
void moea64_release_vsid(uint64_t vsid);
|
|
static void slb_zone_init(void *);
|
|
|
|
static uma_zone_t slbt_zone;
|
|
static uma_zone_t slb_cache_zone;
|
|
int n_slbs = 64;
|
|
|
|
SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL);
|
|
|
|
struct slbtnode {
|
|
uint16_t ua_alloc;
|
|
uint8_t ua_level;
|
|
/* Only 36 bits needed for full 64-bit address space. */
|
|
uint64_t ua_base;
|
|
union {
|
|
struct slbtnode *ua_child[16];
|
|
struct slb slb_entries[16];
|
|
} u;
|
|
};
|
|
|
|
/*
|
|
* For a full 64-bit address space, there are 36 bits in play in an
|
|
* esid, so 8 levels, with the leaf being at level 0.
|
|
*
|
|
* |3333|3322|2222|2222|1111|1111|11 | | | esid
|
|
* |5432|1098|7654|3210|9876|5432|1098|7654|3210| bits
|
|
* +----+----+----+----+----+----+----+----+----+--------
|
|
* | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | level
|
|
*/
|
|
#define UAD_ROOT_LEVEL 8
|
|
#define UAD_LEAF_LEVEL 0
|
|
|
|
static inline int
|
|
esid2idx(uint64_t esid, int level)
|
|
{
|
|
int shift;
|
|
|
|
shift = level * 4;
|
|
return ((esid >> shift) & 0xF);
|
|
}
|
|
|
|
/*
|
|
* The ua_base field should have 0 bits after the first 4*(level+1)
|
|
* bits; i.e. only
|
|
*/
|
|
#define uad_baseok(ua) \
|
|
(esid2base(ua->ua_base, ua->ua_level) == ua->ua_base)
|
|
|
|
static inline uint64_t
|
|
esid2base(uint64_t esid, int level)
|
|
{
|
|
uint64_t mask;
|
|
int shift;
|
|
|
|
shift = (level + 1) * 4;
|
|
mask = ~((1ULL << shift) - 1);
|
|
return (esid & mask);
|
|
}
|
|
|
|
/*
|
|
* Allocate a new leaf node for the specified esid/vmhandle from the
|
|
* parent node.
|
|
*/
|
|
static struct slb *
|
|
make_new_leaf(uint64_t esid, uint64_t slbv, struct slbtnode *parent)
|
|
{
|
|
struct slbtnode *child;
|
|
struct slb *retval;
|
|
int idx;
|
|
|
|
idx = esid2idx(esid, parent->ua_level);
|
|
KASSERT(parent->u.ua_child[idx] == NULL, ("Child already exists!"));
|
|
|
|
/* unlock and M_WAITOK and loop? */
|
|
child = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
|
|
KASSERT(child != NULL, ("unhandled NULL case"));
|
|
|
|
child->ua_level = UAD_LEAF_LEVEL;
|
|
child->ua_base = esid2base(esid, child->ua_level);
|
|
idx = esid2idx(esid, child->ua_level);
|
|
child->u.slb_entries[idx].slbv = slbv;
|
|
child->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
|
|
setbit(&child->ua_alloc, idx);
|
|
|
|
retval = &child->u.slb_entries[idx];
|
|
|
|
/*
|
|
* The above stores must be visible before the next one, so
|
|
* that a lockless searcher always sees a valid path through
|
|
* the tree.
|
|
*/
|
|
powerpc_lwsync();
|
|
|
|
idx = esid2idx(esid, parent->ua_level);
|
|
parent->u.ua_child[idx] = child;
|
|
setbit(&parent->ua_alloc, idx);
|
|
|
|
return (retval);
|
|
}
|
|
|
|
/*
|
|
* Allocate a new intermediate node to fit between the parent and
|
|
* esid.
|
|
*/
|
|
static struct slbtnode*
|
|
make_intermediate(uint64_t esid, struct slbtnode *parent)
|
|
{
|
|
struct slbtnode *child, *inter;
|
|
int idx, level;
|
|
|
|
idx = esid2idx(esid, parent->ua_level);
|
|
child = parent->u.ua_child[idx];
|
|
KASSERT(esid2base(esid, child->ua_level) != child->ua_base,
|
|
("No need for an intermediate node?"));
|
|
|
|
/*
|
|
* Find the level where the existing child and our new esid
|
|
* meet. It must be lower than parent->ua_level or we would
|
|
* have chosen a different index in parent.
|
|
*/
|
|
level = child->ua_level + 1;
|
|
while (esid2base(esid, level) !=
|
|
esid2base(child->ua_base, level))
|
|
level++;
|
|
KASSERT(level < parent->ua_level,
|
|
("Found splitting level %d for %09jx and %09jx, "
|
|
"but it's the same as %p's",
|
|
level, esid, child->ua_base, parent));
|
|
|
|
/* unlock and M_WAITOK and loop? */
|
|
inter = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
|
|
KASSERT(inter != NULL, ("unhandled NULL case"));
|
|
|
|
/* Set up intermediate node to point to child ... */
|
|
inter->ua_level = level;
|
|
inter->ua_base = esid2base(esid, inter->ua_level);
|
|
idx = esid2idx(child->ua_base, inter->ua_level);
|
|
inter->u.ua_child[idx] = child;
|
|
setbit(&inter->ua_alloc, idx);
|
|
powerpc_lwsync();
|
|
|
|
/* Set up parent to point to intermediate node ... */
|
|
idx = esid2idx(inter->ua_base, parent->ua_level);
|
|
parent->u.ua_child[idx] = inter;
|
|
setbit(&parent->ua_alloc, idx);
|
|
|
|
return (inter);
|
|
}
|
|
|
|
uint64_t
|
|
kernel_va_to_slbv(vm_offset_t va)
|
|
{
|
|
uint64_t slbv;
|
|
|
|
/* Set kernel VSID to deterministic value */
|
|
slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT;
|
|
|
|
/*
|
|
* Figure out if this is a large-page mapping.
|
|
*/
|
|
if (hw_direct_map && va > DMAP_BASE_ADDRESS && va < DMAP_MAX_ADDRESS) {
|
|
/*
|
|
* XXX: If we have set up a direct map, assumes
|
|
* all physical memory is mapped with large pages.
|
|
*/
|
|
|
|
if (mem_valid(DMAP_TO_PHYS(va), 0) == 0)
|
|
slbv |= SLBV_L;
|
|
} else if (moea64_large_page_size != 0 &&
|
|
va >= (vm_offset_t)vm_page_array &&
|
|
va <= (uintptr_t)(&vm_page_array[vm_page_array_size]))
|
|
slbv |= SLBV_L;
|
|
|
|
return (slbv);
|
|
}
|
|
|
|
struct slb *
|
|
user_va_to_slb_entry(pmap_t pm, vm_offset_t va)
|
|
{
|
|
uint64_t esid = va >> ADDR_SR_SHFT;
|
|
struct slbtnode *ua;
|
|
int idx;
|
|
|
|
ua = pm->pm_slb_tree_root;
|
|
|
|
for (;;) {
|
|
KASSERT(uad_baseok(ua), ("uad base %016jx level %d bad!",
|
|
ua->ua_base, ua->ua_level));
|
|
idx = esid2idx(esid, ua->ua_level);
|
|
|
|
/*
|
|
* This code is specific to ppc64 where a load is
|
|
* atomic, so no need for atomic_load macro.
|
|
*/
|
|
if (ua->ua_level == UAD_LEAF_LEVEL)
|
|
return ((ua->u.slb_entries[idx].slbe & SLBE_VALID) ?
|
|
&ua->u.slb_entries[idx] : NULL);
|
|
|
|
/*
|
|
* The following accesses are implicitly ordered under the POWER
|
|
* ISA by load dependencies (the store ordering is provided by
|
|
* the powerpc_lwsync() calls elsewhere) and so are run without
|
|
* barriers.
|
|
*/
|
|
ua = ua->u.ua_child[idx];
|
|
if (ua == NULL ||
|
|
esid2base(esid, ua->ua_level) != ua->ua_base)
|
|
return (NULL);
|
|
}
|
|
|
|
return (NULL);
|
|
}
|
|
|
|
uint64_t
|
|
va_to_vsid(pmap_t pm, vm_offset_t va)
|
|
{
|
|
struct slb *entry;
|
|
|
|
/* Shortcut kernel case */
|
|
if (pm == kernel_pmap)
|
|
return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT));
|
|
|
|
/*
|
|
* If there is no vsid for this VA, we need to add a new entry
|
|
* to the PMAP's segment table.
|
|
*/
|
|
|
|
entry = user_va_to_slb_entry(pm, va);
|
|
|
|
if (entry == NULL)
|
|
return (allocate_user_vsid(pm,
|
|
(uintptr_t)va >> ADDR_SR_SHFT, 0));
|
|
|
|
return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
|
|
}
|
|
|
|
uint64_t
|
|
allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
|
|
{
|
|
uint64_t vsid, slbv;
|
|
struct slbtnode *ua, *next, *inter;
|
|
struct slb *slb;
|
|
int idx;
|
|
|
|
KASSERT(pm != kernel_pmap, ("Attempting to allocate a kernel VSID"));
|
|
|
|
PMAP_LOCK_ASSERT(pm, MA_OWNED);
|
|
vsid = moea64_get_unique_vsid();
|
|
|
|
slbv = vsid << SLBV_VSID_SHIFT;
|
|
if (large)
|
|
slbv |= SLBV_L;
|
|
|
|
ua = pm->pm_slb_tree_root;
|
|
|
|
/* Descend to the correct leaf or NULL pointer. */
|
|
for (;;) {
|
|
KASSERT(uad_baseok(ua),
|
|
("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));
|
|
idx = esid2idx(esid, ua->ua_level);
|
|
|
|
if (ua->ua_level == UAD_LEAF_LEVEL) {
|
|
ua->u.slb_entries[idx].slbv = slbv;
|
|
eieio();
|
|
ua->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT)
|
|
| SLBE_VALID;
|
|
setbit(&ua->ua_alloc, idx);
|
|
slb = &ua->u.slb_entries[idx];
|
|
break;
|
|
}
|
|
|
|
next = ua->u.ua_child[idx];
|
|
if (next == NULL) {
|
|
slb = make_new_leaf(esid, slbv, ua);
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* Check if the next item down has an okay ua_base.
|
|
* If not, we need to allocate an intermediate node.
|
|
*/
|
|
if (esid2base(esid, next->ua_level) != next->ua_base) {
|
|
inter = make_intermediate(esid, ua);
|
|
slb = make_new_leaf(esid, slbv, inter);
|
|
break;
|
|
}
|
|
|
|
ua = next;
|
|
}
|
|
|
|
/*
|
|
* Someone probably wants this soon, and it may be a wired
|
|
* SLB mapping, so pre-spill this entry.
|
|
*/
|
|
eieio();
|
|
slb_insert_user(pm, slb);
|
|
|
|
return (vsid);
|
|
}
|
|
|
|
void
|
|
free_vsid(pmap_t pm, uint64_t esid, int large)
|
|
{
|
|
struct slbtnode *ua;
|
|
int idx;
|
|
|
|
PMAP_LOCK_ASSERT(pm, MA_OWNED);
|
|
|
|
ua = pm->pm_slb_tree_root;
|
|
/* Descend to the correct leaf. */
|
|
for (;;) {
|
|
KASSERT(uad_baseok(ua),
|
|
("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));
|
|
|
|
idx = esid2idx(esid, ua->ua_level);
|
|
if (ua->ua_level == UAD_LEAF_LEVEL) {
|
|
ua->u.slb_entries[idx].slbv = 0;
|
|
eieio();
|
|
ua->u.slb_entries[idx].slbe = 0;
|
|
clrbit(&ua->ua_alloc, idx);
|
|
return;
|
|
}
|
|
|
|
ua = ua->u.ua_child[idx];
|
|
if (ua == NULL ||
|
|
esid2base(esid, ua->ua_level) != ua->ua_base) {
|
|
/* Perhaps just return instead of assert? */
|
|
KASSERT(0,
|
|
("Asked to remove an entry that was never inserted!"));
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
free_slb_tree_node(struct slbtnode *ua)
|
|
{
|
|
int idx;
|
|
|
|
for (idx = 0; idx < 16; idx++) {
|
|
if (ua->ua_level != UAD_LEAF_LEVEL) {
|
|
if (ua->u.ua_child[idx] != NULL)
|
|
free_slb_tree_node(ua->u.ua_child[idx]);
|
|
} else {
|
|
if (ua->u.slb_entries[idx].slbv != 0)
|
|
moea64_release_vsid(ua->u.slb_entries[idx].slbv
|
|
>> SLBV_VSID_SHIFT);
|
|
}
|
|
}
|
|
|
|
uma_zfree(slbt_zone, ua);
|
|
}
|
|
|
|
void
|
|
slb_free_tree(pmap_t pm)
|
|
{
|
|
|
|
free_slb_tree_node(pm->pm_slb_tree_root);
|
|
}
|
|
|
|
struct slbtnode *
|
|
slb_alloc_tree(void)
|
|
{
|
|
struct slbtnode *root;
|
|
|
|
root = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);
|
|
KASSERT(root != NULL, ("unhandled NULL case"));
|
|
root->ua_level = UAD_ROOT_LEVEL;
|
|
|
|
return (root);
|
|
}
|
|
|
|
/* Lock entries mapping kernel text and stacks */
|
|
|
|
void
|
|
slb_insert_kernel(uint64_t slbe, uint64_t slbv)
|
|
{
|
|
struct slb *slbcache;
|
|
int i;
|
|
|
|
/* We don't want to be preempted while modifying the kernel map */
|
|
critical_enter();
|
|
|
|
slbcache = PCPU_GET(aim.slb);
|
|
|
|
/* Check for an unused slot, abusing the user slot as a full flag */
|
|
if (slbcache[USER_SLB_SLOT].slbe == 0) {
|
|
for (i = 0; i < n_slbs; i++) {
|
|
if (i == USER_SLB_SLOT)
|
|
continue;
|
|
if (!(slbcache[i].slbe & SLBE_VALID))
|
|
goto fillkernslb;
|
|
}
|
|
|
|
if (i == n_slbs)
|
|
slbcache[USER_SLB_SLOT].slbe = 1;
|
|
}
|
|
|
|
i = mftb() % n_slbs;
|
|
if (i == USER_SLB_SLOT)
|
|
i = (i+1) % n_slbs;
|
|
|
|
fillkernslb:
|
|
KASSERT(i != USER_SLB_SLOT,
|
|
("Filling user SLB slot with a kernel mapping"));
|
|
slbcache[i].slbv = slbv;
|
|
slbcache[i].slbe = slbe | (uint64_t)i;
|
|
|
|
/* If it is for this CPU, put it in the SLB right away */
|
|
if (pmap_bootstrapped) {
|
|
/* slbie not required */
|
|
__asm __volatile ("slbmte %0, %1" ::
|
|
"r"(slbcache[i].slbv), "r"(slbcache[i].slbe));
|
|
}
|
|
|
|
critical_exit();
|
|
}
|
|
|
|
void
|
|
slb_insert_user(pmap_t pm, struct slb *slb)
|
|
{
|
|
int i;
|
|
|
|
PMAP_LOCK_ASSERT(pm, MA_OWNED);
|
|
|
|
if (pm->pm_slb_len < n_slbs) {
|
|
i = pm->pm_slb_len;
|
|
pm->pm_slb_len++;
|
|
} else {
|
|
i = mftb() % n_slbs;
|
|
}
|
|
|
|
/* Note that this replacement is atomic with respect to trap_subr */
|
|
pm->pm_slb[i] = slb;
|
|
}
|
|
|
|
static void *
|
|
slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, int domain,
|
|
u_int8_t *flags, int wait)
|
|
{
|
|
static vm_offset_t realmax = 0;
|
|
void *va;
|
|
vm_page_t m;
|
|
|
|
if (realmax == 0)
|
|
realmax = platform_real_maxaddr();
|
|
|
|
*flags = UMA_SLAB_PRIV;
|
|
m = vm_page_alloc_noobj_contig_domain(domain, malloc2vm_flags(wait) |
|
|
VM_ALLOC_WIRED, 1, 0, realmax, PAGE_SIZE, PAGE_SIZE,
|
|
VM_MEMATTR_DEFAULT);
|
|
if (m == NULL)
|
|
return (NULL);
|
|
|
|
if (hw_direct_map)
|
|
va = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
|
|
else {
|
|
va = (void *)(VM_PAGE_TO_PHYS(m) | DMAP_BASE_ADDRESS);
|
|
pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m));
|
|
}
|
|
|
|
return (va);
|
|
}
|
|
|
|
static void
|
|
slb_zone_init(void *dummy)
|
|
{
|
|
slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
|
|
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
|
|
UMA_ZONE_CONTIG | UMA_ZONE_VM);
|
|
slb_cache_zone = uma_zcreate("SLB cache",
|
|
(n_slbs + 1)*sizeof(struct slb *), NULL, NULL, NULL, NULL,
|
|
UMA_ALIGN_PTR, UMA_ZONE_CONTIG | UMA_ZONE_VM);
|
|
|
|
if (platform_real_maxaddr() != VM_MAX_ADDRESS) {
|
|
uma_zone_set_allocf(slb_cache_zone, slb_uma_real_alloc);
|
|
uma_zone_set_allocf(slbt_zone, slb_uma_real_alloc);
|
|
}
|
|
}
|
|
|
|
struct slb **
|
|
slb_alloc_user_cache(void)
|
|
{
|
|
return (uma_zalloc(slb_cache_zone, M_WAITOK | M_ZERO));
|
|
}
|
|
|
|
void
|
|
slb_free_user_cache(struct slb **slb)
|
|
{
|
|
uma_zfree(slb_cache_zone, slb);
|
|
}
|
|
|
|
/* Handle kernel SLB faults -- runs in real mode, all seat belts off */
|
|
void
|
|
handle_kernel_slb_spill(int type, register_t dar, register_t srr0)
|
|
{
|
|
struct slb *slbcache;
|
|
uint64_t slbe, slbv;
|
|
uint64_t esid, addr;
|
|
int i;
|
|
|
|
addr = (type == EXC_ISE) ? srr0 : dar;
|
|
slbcache = PCPU_GET(aim.slb);
|
|
esid = (uintptr_t)addr >> ADDR_SR_SHFT;
|
|
slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
|
|
|
|
/* See if the hardware flushed this somehow (can happen in LPARs) */
|
|
for (i = 0; i < n_slbs; i++)
|
|
if (slbcache[i].slbe == (slbe | (uint64_t)i))
|
|
return;
|
|
|
|
/* Not in the map, needs to actually be added */
|
|
slbv = kernel_va_to_slbv(addr);
|
|
if (slbcache[USER_SLB_SLOT].slbe == 0) {
|
|
for (i = 0; i < n_slbs; i++) {
|
|
if (i == USER_SLB_SLOT)
|
|
continue;
|
|
if (!(slbcache[i].slbe & SLBE_VALID))
|
|
goto fillkernslb;
|
|
}
|
|
|
|
if (i == n_slbs)
|
|
slbcache[USER_SLB_SLOT].slbe = 1;
|
|
}
|
|
|
|
/* Sacrifice a random SLB entry that is not the user entry */
|
|
i = mftb() % n_slbs;
|
|
if (i == USER_SLB_SLOT)
|
|
i = (i+1) % n_slbs;
|
|
|
|
fillkernslb:
|
|
/* Write new entry */
|
|
slbcache[i].slbv = slbv;
|
|
slbcache[i].slbe = slbe | (uint64_t)i;
|
|
|
|
/* Trap handler will restore from cache on exit */
|
|
}
|
|
|
|
int
|
|
handle_user_slb_spill(pmap_t pm, vm_offset_t addr)
|
|
{
|
|
struct slb *user_entry;
|
|
uint64_t esid;
|
|
int i;
|
|
|
|
if (pm->pm_slb == NULL)
|
|
return (-1);
|
|
|
|
esid = (uintptr_t)addr >> ADDR_SR_SHFT;
|
|
|
|
PMAP_LOCK(pm);
|
|
user_entry = user_va_to_slb_entry(pm, addr);
|
|
|
|
if (user_entry == NULL) {
|
|
/* allocate_vsid auto-spills it */
|
|
(void)allocate_user_vsid(pm, esid, 0);
|
|
} else {
|
|
/*
|
|
* Check that another CPU has not already mapped this.
|
|
* XXX: Per-thread SLB caches would be better.
|
|
*/
|
|
for (i = 0; i < pm->pm_slb_len; i++)
|
|
if (pm->pm_slb[i] == user_entry)
|
|
break;
|
|
|
|
if (i == pm->pm_slb_len)
|
|
slb_insert_user(pm, user_entry);
|
|
}
|
|
PMAP_UNLOCK(pm);
|
|
|
|
return (0);
|
|
}
|