This function needs to unmap all memory in a given SMMU context. Have it iterate over all page table entries to find what has been mapped rather than looking at virtual addresses. While here use SMMU specific macros. Sponsored by: Arm Ltd Differential Revision: https://reviews.freebsd.org/D39185
864 lines
21 KiB
C
864 lines
21 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*
|
|
* Copyright (c) 2020-2021 Ruslan Bukin <br@bsdpad.com>
|
|
* Copyright (c) 2014-2021 Andrew Turner
|
|
* Copyright (c) 2014-2016 The FreeBSD Foundation
|
|
* All rights reserved.
|
|
*
|
|
* This work was supported by Innovate UK project 105694, "Digital Security
|
|
* by Design (DSbD) Technology Platform Prototype".
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
/*
|
|
* Manages physical address maps for ARM SMMUv3 and ARM Mali GPU.
|
|
*/
|
|
|
|
#include "opt_vm.h"
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/ktr.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/rwlock.h>
|
|
|
|
#include <vm/vm.h>
|
|
#include <vm/vm_param.h>
|
|
#include <vm/vm_page.h>
|
|
#include <vm/vm_map.h>
|
|
#include <vm/vm_object.h>
|
|
#include <vm/vm_pageout.h>
|
|
#include <vm/vm_radix.h>
|
|
|
|
#include <machine/machdep.h>
|
|
|
|
#include <arm64/iommu/iommu_pmap.h>
|
|
#include <arm64/iommu/iommu_pte.h>
|
|
|
|
#define IOMMU_PAGE_SIZE 4096
|
|
|
|
#define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx)
|
|
#define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx)
|
|
#define SMMU_PMAP_LOCK_ASSERT(pmap, type) \
|
|
mtx_assert(&(pmap)->sp_mtx, (type))
|
|
|
|
#define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t)))
|
|
#define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t)))
|
|
#define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t)))
|
|
#define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t)))
|
|
|
|
#define NUL0E IOMMU_L0_ENTRIES
|
|
#define NUL1E (NUL0E * NL1PG)
|
|
#define NUL2E (NUL1E * NL2PG)
|
|
|
|
#define smmu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT))
|
|
#define smmu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT))
|
|
#define smmu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT)
|
|
|
|
#define smmu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK)
|
|
#define smmu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK)
|
|
#define smmu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK)
|
|
#define smmu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK)
|
|
|
|
static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex);
|
|
static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va,
|
|
vm_page_t m, struct spglist *free);
|
|
|
|
/*
|
|
* These load the old table data and store the new value.
|
|
* They need to be atomic as the System MMU may write to the table at
|
|
* the same time as the CPU.
|
|
*/
|
|
#define smmu_pmap_load(table) (*table)
|
|
#define smmu_pmap_clear(table) atomic_store_64(table, 0)
|
|
#define smmu_pmap_store(table, entry) atomic_store_64(table, entry)
|
|
|
|
/********************/
|
|
/* Inline functions */
|
|
/********************/
|
|
|
|
static __inline pd_entry_t *
|
|
smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va)
|
|
{
|
|
|
|
return (&pmap->sp_l0[smmu_l0_index(va)]);
|
|
}
|
|
|
|
static __inline pd_entry_t *
|
|
smmu_pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)
|
|
{
|
|
pd_entry_t *l1;
|
|
|
|
l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) & ~ATTR_MASK);
|
|
return (&l1[smmu_l1_index(va)]);
|
|
}
|
|
|
|
static __inline pd_entry_t *
|
|
smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va)
|
|
{
|
|
pd_entry_t *l0;
|
|
|
|
l0 = smmu_pmap_l0(pmap, va);
|
|
if ((smmu_pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE)
|
|
return (NULL);
|
|
|
|
return (smmu_pmap_l0_to_l1(l0, va));
|
|
}
|
|
|
|
static __inline pd_entry_t *
|
|
smmu_pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va)
|
|
{
|
|
pd_entry_t l1, *l2p;
|
|
|
|
l1 = smmu_pmap_load(l1p);
|
|
|
|
/*
|
|
* The valid bit may be clear if pmap_update_entry() is concurrently
|
|
* modifying the entry, so for KVA only the entry type may be checked.
|
|
*/
|
|
KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0,
|
|
("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va));
|
|
KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE,
|
|
("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va));
|
|
l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK);
|
|
return (&l2p[smmu_l2_index(va)]);
|
|
}
|
|
|
|
static __inline pd_entry_t *
|
|
smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va)
|
|
{
|
|
pd_entry_t *l1;
|
|
|
|
l1 = smmu_pmap_l1(pmap, va);
|
|
if ((smmu_pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE)
|
|
return (NULL);
|
|
|
|
return (smmu_pmap_l1_to_l2(l1, va));
|
|
}
|
|
|
|
static __inline pt_entry_t *
|
|
smmu_pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va)
|
|
{
|
|
pd_entry_t l2;
|
|
pt_entry_t *l3p;
|
|
|
|
l2 = smmu_pmap_load(l2p);
|
|
|
|
/*
|
|
* The valid bit may be clear if pmap_update_entry() is concurrently
|
|
* modifying the entry, so for KVA only the entry type may be checked.
|
|
*/
|
|
KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0,
|
|
("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va));
|
|
KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE,
|
|
("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va));
|
|
l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK);
|
|
return (&l3p[smmu_l3_index(va)]);
|
|
}
|
|
|
|
/*
|
|
* Returns the lowest valid pde for a given virtual address.
|
|
* The next level may or may not point to a valid page or block.
|
|
*/
|
|
static __inline pd_entry_t *
|
|
smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level)
|
|
{
|
|
pd_entry_t *l0, *l1, *l2, desc;
|
|
|
|
l0 = smmu_pmap_l0(pmap, va);
|
|
desc = smmu_pmap_load(l0) & ATTR_DESCR_MASK;
|
|
if (desc != IOMMU_L0_TABLE) {
|
|
*level = -1;
|
|
return (NULL);
|
|
}
|
|
|
|
l1 = smmu_pmap_l0_to_l1(l0, va);
|
|
desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK;
|
|
if (desc != IOMMU_L1_TABLE) {
|
|
*level = 0;
|
|
return (l0);
|
|
}
|
|
|
|
l2 = smmu_pmap_l1_to_l2(l1, va);
|
|
desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK;
|
|
if (desc != IOMMU_L2_TABLE) {
|
|
*level = 1;
|
|
return (l1);
|
|
}
|
|
|
|
*level = 2;
|
|
return (l2);
|
|
}
|
|
|
|
/*
|
|
* Returns the lowest valid pte block or table entry for a given virtual
|
|
* address. If there are no valid entries return NULL and set the level to
|
|
* the first invalid level.
|
|
*/
|
|
static __inline pt_entry_t *
|
|
smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level)
|
|
{
|
|
pd_entry_t *l1, *l2, desc;
|
|
pt_entry_t *l3;
|
|
|
|
l1 = smmu_pmap_l1(pmap, va);
|
|
if (l1 == NULL) {
|
|
*level = 0;
|
|
return (NULL);
|
|
}
|
|
desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK;
|
|
if (desc == IOMMU_L1_BLOCK) {
|
|
*level = 1;
|
|
return (l1);
|
|
}
|
|
|
|
if (desc != IOMMU_L1_TABLE) {
|
|
*level = 1;
|
|
return (NULL);
|
|
}
|
|
|
|
l2 = smmu_pmap_l1_to_l2(l1, va);
|
|
desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK;
|
|
if (desc == IOMMU_L2_BLOCK) {
|
|
*level = 2;
|
|
return (l2);
|
|
}
|
|
|
|
if (desc != IOMMU_L2_TABLE) {
|
|
*level = 2;
|
|
return (NULL);
|
|
}
|
|
|
|
*level = 3;
|
|
l3 = smmu_pmap_l2_to_l3(l2, va);
|
|
if ((smmu_pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE)
|
|
return (NULL);
|
|
|
|
return (l3);
|
|
}
|
|
|
|
static __inline int
|
|
smmu_pmap_l3_valid(pt_entry_t l3)
|
|
{
|
|
|
|
return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE);
|
|
}
|
|
|
|
CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK);
|
|
|
|
#ifdef INVARIANTS
|
|
static __inline void
|
|
smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)
|
|
{
|
|
|
|
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
pmap->sp_resident_count += count;
|
|
}
|
|
|
|
static __inline void
|
|
smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)
|
|
{
|
|
|
|
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
KASSERT(pmap->sp_resident_count >= count,
|
|
("pmap %p resident count underflow %ld %d", pmap,
|
|
pmap->sp_resident_count, count));
|
|
pmap->sp_resident_count -= count;
|
|
}
|
|
#else
|
|
static __inline void
|
|
smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)
|
|
{
|
|
}
|
|
|
|
static __inline void
|
|
smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
/***************************************************
|
|
* Page table page management routines.....
|
|
***************************************************/
|
|
/*
|
|
* Schedule the specified unused page table page to be freed. Specifically,
|
|
* add the page to the specified list of pages that will be released to the
|
|
* physical memory manager after the TLB has been updated.
|
|
*/
|
|
static __inline void
|
|
smmu_pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
|
|
boolean_t set_PG_ZERO)
|
|
{
|
|
|
|
if (set_PG_ZERO)
|
|
m->flags |= PG_ZERO;
|
|
else
|
|
m->flags &= ~PG_ZERO;
|
|
SLIST_INSERT_HEAD(free, m, plinks.s.ss);
|
|
}
|
|
|
|
/***************************************************
|
|
* Low level mapping routines.....
|
|
***************************************************/
|
|
|
|
/*
|
|
* Decrements a page table page's reference count, which is used to record the
|
|
* number of valid page table entries within the page. If the reference count
|
|
* drops to zero, then the page table page is unmapped. Returns TRUE if the
|
|
* page table page was unmapped and FALSE otherwise.
|
|
*/
|
|
static inline boolean_t
|
|
smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,
|
|
struct spglist *free)
|
|
{
|
|
|
|
--m->ref_count;
|
|
if (m->ref_count == 0) {
|
|
_smmu_pmap_unwire_l3(pmap, va, m, free);
|
|
return (TRUE);
|
|
} else
|
|
return (FALSE);
|
|
}
|
|
|
|
static void
|
|
_smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,
|
|
struct spglist *free)
|
|
{
|
|
|
|
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
/*
|
|
* unmap the page table page
|
|
*/
|
|
if (m->pindex >= (NUL2E + NUL1E)) {
|
|
/* l1 page */
|
|
pd_entry_t *l0;
|
|
|
|
l0 = smmu_pmap_l0(pmap, va);
|
|
smmu_pmap_clear(l0);
|
|
} else if (m->pindex >= NUL2E) {
|
|
/* l2 page */
|
|
pd_entry_t *l1;
|
|
|
|
l1 = smmu_pmap_l1(pmap, va);
|
|
smmu_pmap_clear(l1);
|
|
} else {
|
|
/* l3 page */
|
|
pd_entry_t *l2;
|
|
|
|
l2 = smmu_pmap_l2(pmap, va);
|
|
smmu_pmap_clear(l2);
|
|
}
|
|
smmu_pmap_resident_count_dec(pmap, 1);
|
|
if (m->pindex < NUL2E) {
|
|
/* We just released an l3, unhold the matching l2 */
|
|
pd_entry_t *l1, tl1;
|
|
vm_page_t l2pg;
|
|
|
|
l1 = smmu_pmap_l1(pmap, va);
|
|
tl1 = smmu_pmap_load(l1);
|
|
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
|
|
smmu_pmap_unwire_l3(pmap, va, l2pg, free);
|
|
} else if (m->pindex < (NUL2E + NUL1E)) {
|
|
/* We just released an l2, unhold the matching l1 */
|
|
pd_entry_t *l0, tl0;
|
|
vm_page_t l1pg;
|
|
|
|
l0 = smmu_pmap_l0(pmap, va);
|
|
tl0 = smmu_pmap_load(l0);
|
|
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
|
|
smmu_pmap_unwire_l3(pmap, va, l1pg, free);
|
|
}
|
|
|
|
/*
|
|
* Put page on a list so that it is released after
|
|
* *ALL* TLB shootdown is done
|
|
*/
|
|
smmu_pmap_add_delayed_free_list(m, free, TRUE);
|
|
}
|
|
|
|
int
|
|
smmu_pmap_pinit(struct smmu_pmap *pmap)
|
|
{
|
|
vm_page_t m;
|
|
|
|
/*
|
|
* allocate the l0 page
|
|
*/
|
|
m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED |
|
|
VM_ALLOC_ZERO);
|
|
pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m);
|
|
pmap->sp_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->sp_l0_paddr);
|
|
|
|
pmap->sp_resident_count = 0;
|
|
mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF);
|
|
|
|
return (1);
|
|
}
|
|
|
|
/*
|
|
* This routine is called if the desired page table page does not exist.
|
|
*
|
|
* If page table page allocation fails, this routine may sleep before
|
|
* returning NULL. It sleeps only if a lock pointer was given.
|
|
*
|
|
* Note: If a page allocation fails at page table level two or three,
|
|
* one or two pages may be held during the wait, only to be released
|
|
* afterwards. This conservative approach is easily argued to avoid
|
|
* race conditions.
|
|
*/
|
|
static vm_page_t
|
|
_pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex)
|
|
{
|
|
vm_page_t m, l1pg, l2pg;
|
|
|
|
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
|
|
|
/*
|
|
* Allocate a page table page.
|
|
*/
|
|
if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
|
|
/*
|
|
* Indicate the need to retry. While waiting, the page table
|
|
* page may have been allocated.
|
|
*/
|
|
return (NULL);
|
|
}
|
|
m->pindex = ptepindex;
|
|
|
|
/*
|
|
* Because of AArch64's weak memory consistency model, we must have a
|
|
* barrier here to ensure that the stores for zeroing "m", whether by
|
|
* pmap_zero_page() or an earlier function, are visible before adding
|
|
* "m" to the page table. Otherwise, a page table walk by another
|
|
* processor's MMU could see the mapping to "m" and a stale, non-zero
|
|
* PTE within "m".
|
|
*/
|
|
dmb(ishst);
|
|
|
|
/*
|
|
* Map the pagetable page into the process address space, if
|
|
* it isn't already there.
|
|
*/
|
|
|
|
if (ptepindex >= (NUL2E + NUL1E)) {
|
|
pd_entry_t *l0;
|
|
vm_pindex_t l0index;
|
|
|
|
l0index = ptepindex - (NUL2E + NUL1E);
|
|
l0 = &pmap->sp_l0[l0index];
|
|
smmu_pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE);
|
|
} else if (ptepindex >= NUL2E) {
|
|
vm_pindex_t l0index, l1index;
|
|
pd_entry_t *l0, *l1;
|
|
pd_entry_t tl0;
|
|
|
|
l1index = ptepindex - NUL2E;
|
|
l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT;
|
|
|
|
l0 = &pmap->sp_l0[l0index];
|
|
tl0 = smmu_pmap_load(l0);
|
|
if (tl0 == 0) {
|
|
/* recurse for allocating page dir */
|
|
if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index)
|
|
== NULL) {
|
|
vm_page_unwire_noq(m);
|
|
vm_page_free_zero(m);
|
|
return (NULL);
|
|
}
|
|
} else {
|
|
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
|
|
l1pg->ref_count++;
|
|
}
|
|
|
|
l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) &~ATTR_MASK);
|
|
l1 = &l1[ptepindex & Ln_ADDR_MASK];
|
|
smmu_pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE);
|
|
} else {
|
|
vm_pindex_t l0index, l1index;
|
|
pd_entry_t *l0, *l1, *l2;
|
|
pd_entry_t tl0, tl1;
|
|
|
|
l1index = ptepindex >> IOMMU_Ln_ENTRIES_SHIFT;
|
|
l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT;
|
|
|
|
l0 = &pmap->sp_l0[l0index];
|
|
tl0 = smmu_pmap_load(l0);
|
|
if (tl0 == 0) {
|
|
/* recurse for allocating page dir */
|
|
if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) {
|
|
vm_page_unwire_noq(m);
|
|
vm_page_free_zero(m);
|
|
return (NULL);
|
|
}
|
|
tl0 = smmu_pmap_load(l0);
|
|
l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
|
|
l1 = &l1[l1index & Ln_ADDR_MASK];
|
|
} else {
|
|
l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
|
|
l1 = &l1[l1index & Ln_ADDR_MASK];
|
|
tl1 = smmu_pmap_load(l1);
|
|
if (tl1 == 0) {
|
|
/* recurse for allocating page dir */
|
|
if (_pmap_alloc_l3(pmap, NUL2E + l1index)
|
|
== NULL) {
|
|
vm_page_unwire_noq(m);
|
|
vm_page_free_zero(m);
|
|
return (NULL);
|
|
}
|
|
} else {
|
|
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
|
|
l2pg->ref_count++;
|
|
}
|
|
}
|
|
|
|
l2 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l1) &~ATTR_MASK);
|
|
l2 = &l2[ptepindex & Ln_ADDR_MASK];
|
|
smmu_pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE);
|
|
}
|
|
|
|
smmu_pmap_resident_count_inc(pmap, 1);
|
|
|
|
return (m);
|
|
}
|
|
|
|
/***************************************************
|
|
* Pmap allocation/deallocation routines.
|
|
***************************************************/
|
|
|
|
/*
|
|
* Release any resources held by the given physical map.
|
|
* Called when a pmap initialized by pmap_pinit is being released.
|
|
* Should only be called if the map contains no valid mappings.
|
|
*/
|
|
void
|
|
smmu_pmap_release(struct smmu_pmap *pmap)
|
|
{
|
|
vm_page_t m;
|
|
|
|
KASSERT(pmap->sp_resident_count == 0,
|
|
("pmap_release: pmap resident count %ld != 0",
|
|
pmap->sp_resident_count));
|
|
|
|
m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr);
|
|
vm_page_unwire_noq(m);
|
|
vm_page_free_zero(m);
|
|
mtx_destroy(&pmap->sp_mtx);
|
|
}
|
|
|
|
/***************************************************
|
|
* page management routines.
|
|
***************************************************/
|
|
|
|
/*
|
|
* Add a single Mali GPU entry. This function does not sleep.
|
|
*/
|
|
int
|
|
pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,
|
|
vm_prot_t prot, u_int flags)
|
|
{
|
|
pd_entry_t *pde;
|
|
pt_entry_t new_l3;
|
|
pt_entry_t orig_l3 __diagused;
|
|
pt_entry_t *l3;
|
|
vm_page_t mpte;
|
|
pd_entry_t *l1p;
|
|
pd_entry_t *l2p;
|
|
int lvl;
|
|
int rv;
|
|
|
|
KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space"));
|
|
KASSERT((va & PAGE_MASK) == 0, ("va is misaligned"));
|
|
KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned"));
|
|
|
|
new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK);
|
|
|
|
if ((prot & VM_PROT_WRITE) != 0)
|
|
new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE);
|
|
if ((prot & VM_PROT_READ) != 0)
|
|
new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ);
|
|
if ((prot & VM_PROT_EXECUTE) == 0)
|
|
new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL);
|
|
|
|
CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa);
|
|
|
|
SMMU_PMAP_LOCK(pmap);
|
|
|
|
/*
|
|
* In the case that a page table page is not
|
|
* resident, we are creating it here.
|
|
*/
|
|
retry:
|
|
pde = smmu_pmap_pde(pmap, va, &lvl);
|
|
if (pde != NULL && lvl == 2) {
|
|
l3 = smmu_pmap_l2_to_l3(pde, va);
|
|
} else {
|
|
mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));
|
|
if (mpte == NULL) {
|
|
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
|
|
rv = KERN_RESOURCE_SHORTAGE;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Ensure newly created l1, l2 are visible to GPU.
|
|
* l0 is already visible by similar call in panfrost driver.
|
|
* The cache entry for l3 handled below.
|
|
*/
|
|
|
|
l1p = smmu_pmap_l1(pmap, va);
|
|
l2p = smmu_pmap_l2(pmap, va);
|
|
cpu_dcache_wb_range((vm_offset_t)l1p, sizeof(pd_entry_t));
|
|
cpu_dcache_wb_range((vm_offset_t)l2p, sizeof(pd_entry_t));
|
|
|
|
goto retry;
|
|
}
|
|
|
|
orig_l3 = smmu_pmap_load(l3);
|
|
KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid"));
|
|
|
|
/* New mapping */
|
|
smmu_pmap_store(l3, new_l3);
|
|
|
|
cpu_dcache_wb_range((vm_offset_t)l3, sizeof(pt_entry_t));
|
|
|
|
smmu_pmap_resident_count_inc(pmap, 1);
|
|
dsb(ishst);
|
|
|
|
rv = KERN_SUCCESS;
|
|
out:
|
|
SMMU_PMAP_UNLOCK(pmap);
|
|
|
|
return (rv);
|
|
}
|
|
|
|
/*
|
|
* Remove a single Mali GPU entry.
|
|
*/
|
|
int
|
|
pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va)
|
|
{
|
|
pd_entry_t *pde;
|
|
pt_entry_t *pte;
|
|
int lvl;
|
|
int rc;
|
|
|
|
KASSERT((va & PAGE_MASK) == 0, ("va is misaligned"));
|
|
|
|
SMMU_PMAP_LOCK(pmap);
|
|
|
|
pde = smmu_pmap_pde(pmap, va, &lvl);
|
|
if (pde == NULL || lvl != 2) {
|
|
rc = KERN_FAILURE;
|
|
goto out;
|
|
}
|
|
|
|
pte = smmu_pmap_l2_to_l3(pde, va);
|
|
|
|
smmu_pmap_resident_count_dec(pmap, 1);
|
|
smmu_pmap_clear(pte);
|
|
cpu_dcache_wb_range((vm_offset_t)pte, sizeof(pt_entry_t));
|
|
rc = KERN_SUCCESS;
|
|
|
|
out:
|
|
SMMU_PMAP_UNLOCK(pmap);
|
|
|
|
return (rc);
|
|
}
|
|
|
|
/*
|
|
* Add a single SMMU entry. This function does not sleep.
|
|
*/
|
|
int
|
|
smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,
|
|
vm_prot_t prot, u_int flags)
|
|
{
|
|
pd_entry_t *pde;
|
|
pt_entry_t new_l3;
|
|
pt_entry_t orig_l3 __diagused;
|
|
pt_entry_t *l3;
|
|
vm_page_t mpte;
|
|
int lvl;
|
|
int rv;
|
|
|
|
KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space"));
|
|
|
|
va = trunc_page(va);
|
|
new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT |
|
|
ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE);
|
|
if ((prot & VM_PROT_WRITE) == 0)
|
|
new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO);
|
|
new_l3 |= ATTR_S1_XN; /* Execute never. */
|
|
new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER);
|
|
new_l3 |= ATTR_S1_nG; /* Non global. */
|
|
|
|
CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa);
|
|
|
|
SMMU_PMAP_LOCK(pmap);
|
|
|
|
/*
|
|
* In the case that a page table page is not
|
|
* resident, we are creating it here.
|
|
*/
|
|
retry:
|
|
pde = smmu_pmap_pde(pmap, va, &lvl);
|
|
if (pde != NULL && lvl == 2) {
|
|
l3 = smmu_pmap_l2_to_l3(pde, va);
|
|
} else {
|
|
mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));
|
|
if (mpte == NULL) {
|
|
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
|
|
rv = KERN_RESOURCE_SHORTAGE;
|
|
goto out;
|
|
}
|
|
goto retry;
|
|
}
|
|
|
|
orig_l3 = smmu_pmap_load(l3);
|
|
KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid"));
|
|
|
|
/* New mapping */
|
|
smmu_pmap_store(l3, new_l3);
|
|
smmu_pmap_resident_count_inc(pmap, 1);
|
|
dsb(ishst);
|
|
|
|
rv = KERN_SUCCESS;
|
|
out:
|
|
SMMU_PMAP_UNLOCK(pmap);
|
|
|
|
return (rv);
|
|
}
|
|
|
|
/*
|
|
* Remove a single SMMU entry.
|
|
*/
|
|
int
|
|
smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va)
|
|
{
|
|
pt_entry_t *pte;
|
|
int lvl;
|
|
int rc;
|
|
|
|
SMMU_PMAP_LOCK(pmap);
|
|
|
|
pte = smmu_pmap_pte(pmap, va, &lvl);
|
|
KASSERT(lvl == 3,
|
|
("Invalid SMMU pagetable level: %d != 3", lvl));
|
|
|
|
if (pte != NULL) {
|
|
smmu_pmap_resident_count_dec(pmap, 1);
|
|
smmu_pmap_clear(pte);
|
|
rc = KERN_SUCCESS;
|
|
} else
|
|
rc = KERN_FAILURE;
|
|
|
|
SMMU_PMAP_UNLOCK(pmap);
|
|
|
|
return (rc);
|
|
}
|
|
|
|
/*
|
|
* Remove all the allocated L1, L2 pages from SMMU pmap.
|
|
* All the L3 entires must be cleared in advance, otherwise
|
|
* this function panics.
|
|
*/
|
|
void
|
|
smmu_pmap_remove_pages(struct smmu_pmap *pmap)
|
|
{
|
|
pd_entry_t l0e, *l1, l1e, *l2, l2e;
|
|
pt_entry_t *l3, l3e;
|
|
vm_page_t m, m0, m1;
|
|
vm_paddr_t pa;
|
|
vm_paddr_t pa0;
|
|
vm_paddr_t pa1;
|
|
int i, j, k, l;
|
|
|
|
SMMU_PMAP_LOCK(pmap);
|
|
|
|
for (i = 0; i < IOMMU_L0_ENTRIES; i++) {
|
|
l0e = pmap->sp_l0[i];
|
|
if ((l0e & ATTR_DESCR_VALID) == 0) {
|
|
continue;
|
|
}
|
|
pa0 = l0e & ~ATTR_MASK;
|
|
m0 = PHYS_TO_VM_PAGE(pa0);
|
|
l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0);
|
|
|
|
for (j = 0; j < IOMMU_Ln_ENTRIES; j++) {
|
|
l1e = l1[j];
|
|
if ((l1e & ATTR_DESCR_VALID) == 0) {
|
|
continue;
|
|
}
|
|
if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) {
|
|
continue;
|
|
}
|
|
pa1 = l1e & ~ATTR_MASK;
|
|
m1 = PHYS_TO_VM_PAGE(pa1);
|
|
l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1);
|
|
|
|
for (k = 0; k < IOMMU_Ln_ENTRIES; k++) {
|
|
l2e = l2[k];
|
|
if ((l2e & ATTR_DESCR_VALID) == 0) {
|
|
continue;
|
|
}
|
|
pa = l2e & ~ATTR_MASK;
|
|
m = PHYS_TO_VM_PAGE(pa);
|
|
l3 = (pt_entry_t *)PHYS_TO_DMAP(pa);
|
|
|
|
for (l = 0; l < IOMMU_Ln_ENTRIES; l++) {
|
|
l3e = l3[l];
|
|
if ((l3e & ATTR_DESCR_VALID) == 0)
|
|
continue;
|
|
panic(
|
|
"%s: l3e found (indexes %d %d %d %d)",
|
|
__func__, i, j, k, l);
|
|
}
|
|
|
|
vm_page_unwire_noq(m1);
|
|
vm_page_unwire_noq(m);
|
|
smmu_pmap_resident_count_dec(pmap, 1);
|
|
vm_page_free(m);
|
|
smmu_pmap_clear(&l2[k]);
|
|
}
|
|
|
|
vm_page_unwire_noq(m0);
|
|
smmu_pmap_resident_count_dec(pmap, 1);
|
|
vm_page_free(m1);
|
|
smmu_pmap_clear(&l1[j]);
|
|
}
|
|
|
|
smmu_pmap_resident_count_dec(pmap, 1);
|
|
vm_page_free(m0);
|
|
smmu_pmap_clear(&pmap->sp_l0[i]);
|
|
}
|
|
|
|
KASSERT(pmap->sp_resident_count == 0,
|
|
("Invalid resident count %jd", pmap->sp_resident_count));
|
|
|
|
SMMU_PMAP_UNLOCK(pmap);
|
|
}
|