Optimise unmapped LinuxKPI page allocations.
When allocating unmapped pages, take advantage of the direct map on AMD64 to get the virtual address corresponding to a page. Else all pages allocated must be mapped because sometimes the virtual address of a page is requested. Move all page allocation and deallocation code into an own C-file. Add support for GFP_DMA32, GFP_KERNEL, GFP_ATOMIC and __GFP_ZERO allocation flags. Make a clear separation between mapped and unmapped allocations. Obtained from: kmacy @ MFC after: 1 week Sponsored by: Mellanox Technologies
This commit is contained in:
Hans Petter Selasky
parent
8306998f5b
commit
1cdefd084d
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=314106
@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2010 Isilon Systems, Inc.
|
||||
* Copyright (c) 2010 iX Systems, Inc.
|
||||
* Copyright (c) 2010 Panasas, Inc.
|
||||
* Copyright (c) 2013 Mellanox Technologies, Ltd.
|
||||
* Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -67,94 +67,106 @@
|
||||
#define GFP_TEMPORARY M_NOWAIT
|
||||
#define GFP_NATIVE_MASK (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_ZERO)
|
||||
|
||||
static inline void *
|
||||
page_address(struct page *page)
|
||||
/*
|
||||
* Resolve a page into a virtual address:
|
||||
*
|
||||
* NOTE: This function only works for pages allocated by the kernel.
|
||||
*/
|
||||
extern void *linux_page_address(struct page *);
|
||||
|
||||
#define page_address(page) linux_page_address(page)
|
||||
|
||||
/*
|
||||
* Page management for unmapped pages:
|
||||
*/
|
||||
extern vm_page_t linux_alloc_pages(gfp_t flags, unsigned int order);
|
||||
extern void linux_free_pages(vm_page_t page, unsigned int order);
|
||||
|
||||
static inline struct page *
|
||||
alloc_page(gfp_t flags)
|
||||
{
|
||||
|
||||
if (page->object != kmem_object && page->object != kernel_object)
|
||||
return (NULL);
|
||||
return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS +
|
||||
IDX_TO_OFF(page->pindex)));
|
||||
return (linux_alloc_pages(flags, 0));
|
||||
}
|
||||
|
||||
static inline unsigned long
|
||||
linux_get_page(gfp_t mask)
|
||||
static inline struct page *
|
||||
alloc_pages(gfp_t flags, unsigned int order)
|
||||
{
|
||||
|
||||
return kmem_malloc(kmem_arena, PAGE_SIZE, mask);
|
||||
return (linux_alloc_pages(flags, order));
|
||||
}
|
||||
|
||||
#define get_zeroed_page(mask) linux_get_page((mask) | M_ZERO)
|
||||
#define alloc_page(mask) virt_to_page(linux_get_page((mask)))
|
||||
#define __get_free_page(mask) linux_get_page((mask))
|
||||
|
||||
static inline void
|
||||
free_page(unsigned long page)
|
||||
static inline struct page *
|
||||
alloc_pages_node(int node_id, gfp_t flags, unsigned int order)
|
||||
{
|
||||
|
||||
if (page == 0)
|
||||
return;
|
||||
kmem_free(kmem_arena, page, PAGE_SIZE);
|
||||
return (linux_alloc_pages(flags, order));
|
||||
}
|
||||
|
||||
static inline void
|
||||
__free_page(struct page *m)
|
||||
__free_pages(struct page *page, unsigned int order)
|
||||
{
|
||||
|
||||
if (m->object != kmem_object)
|
||||
panic("__free_page: Freed page %p not allocated via wrappers.",
|
||||
m);
|
||||
kmem_free(kmem_arena, (vm_offset_t)page_address(m), PAGE_SIZE);
|
||||
linux_free_pages(page, order);
|
||||
}
|
||||
|
||||
static inline void
|
||||
__free_pages(struct page *m, unsigned int order)
|
||||
__free_page(struct page *page)
|
||||
{
|
||||
size_t size;
|
||||
|
||||
if (m == NULL)
|
||||
return;
|
||||
size = PAGE_SIZE << order;
|
||||
kmem_free(kmem_arena, (vm_offset_t)page_address(m), size);
|
||||
}
|
||||
|
||||
static inline void free_pages(uintptr_t addr, unsigned int order)
|
||||
{
|
||||
if (addr == 0)
|
||||
return;
|
||||
__free_pages(virt_to_page((void *)addr), order);
|
||||
linux_free_pages(page, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Alloc pages allocates directly from the buddy allocator on linux so
|
||||
* order specifies a power of two bucket of pages and the results
|
||||
* are expected to be aligned on the size as well.
|
||||
* Page management for mapped pages:
|
||||
*/
|
||||
static inline struct page *
|
||||
alloc_pages(gfp_t gfp_mask, unsigned int order)
|
||||
{
|
||||
unsigned long page;
|
||||
size_t size;
|
||||
extern vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order);
|
||||
extern void linux_free_kmem(vm_offset_t, unsigned int order);
|
||||
|
||||
size = PAGE_SIZE << order;
|
||||
page = kmem_alloc_contig(kmem_arena, size, gfp_mask,
|
||||
0, ~(vm_paddr_t)0, size, 0, VM_MEMATTR_DEFAULT);
|
||||
if (page == 0)
|
||||
return (NULL);
|
||||
return (virt_to_page(page));
|
||||
static inline vm_offset_t
|
||||
get_zeroed_page(gfp_t flags)
|
||||
{
|
||||
|
||||
return (linux_alloc_kmem(flags | __GFP_ZERO, 0));
|
||||
}
|
||||
|
||||
static inline uintptr_t __get_free_pages(gfp_t gfp_mask, unsigned int order)
|
||||
static inline vm_offset_t
|
||||
__get_free_page(gfp_t flags)
|
||||
{
|
||||
struct page *page;
|
||||
|
||||
page = alloc_pages(gfp_mask, order);
|
||||
if (page == NULL)
|
||||
return (0);
|
||||
return ((uintptr_t)page_address(page));
|
||||
return (linux_alloc_kmem(flags, 0));
|
||||
}
|
||||
|
||||
#define alloc_pages_node(node, mask, order) alloc_pages(mask, order)
|
||||
static inline vm_offset_t
|
||||
__get_free_pages(gfp_t flags, unsigned int order)
|
||||
{
|
||||
|
||||
return (linux_alloc_kmem(flags, order));
|
||||
}
|
||||
|
||||
static inline void
|
||||
free_pages(uintptr_t addr, unsigned int order)
|
||||
{
|
||||
if (addr == 0)
|
||||
return;
|
||||
|
||||
linux_free_kmem(addr, order);
|
||||
}
|
||||
|
||||
static inline void
|
||||
free_page(uintptr_t addr)
|
||||
{
|
||||
if (addr == 0)
|
||||
return;
|
||||
|
||||
linux_free_kmem(addr, 0);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
gfpflags_allow_blocking(const gfp_t gfp_flags)
|
||||
{
|
||||
return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK);
|
||||
}
|
||||
|
||||
#define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask)
|
||||
|
||||
|
||||
167
sys/compat/linuxkpi/common/src/linux_page.c
Normal file
167
sys/compat/linuxkpi/common/src/linux_page.c
Normal file
@ -0,0 +1,167 @@
|
||||
/*-
|
||||
* Copyright (c) 2010 Isilon Systems, Inc.
|
||||
* Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org)
|
||||
* Copyright (c) 2017 Mellanox Technologies, Ltd.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice unmodified, this list of conditions, and the following
|
||||
* disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/malloc.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/rwlock.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/sched.h>
|
||||
|
||||
#include <machine/bus.h>
|
||||
|
||||
#include <linux/gfp.h>
|
||||
|
||||
#include <vm/vm.h>
|
||||
#include <vm/vm_page.h>
|
||||
#include <vm/vm_pageout.h>
|
||||
|
||||
void *
|
||||
linux_page_address(struct page *page)
|
||||
{
|
||||
#ifdef __amd64__
|
||||
return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)));
|
||||
#else
|
||||
if (page->object != kmem_object && page->object != kernel_object)
|
||||
return (NULL);
|
||||
return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS +
|
||||
IDX_TO_OFF(page->pindex)));
|
||||
#endif
|
||||
}
|
||||
|
||||
vm_page_t
|
||||
linux_alloc_pages(gfp_t flags, unsigned int order)
|
||||
{
|
||||
#ifdef __amd64__
|
||||
unsigned long npages = 1UL << order;
|
||||
int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ |
|
||||
VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL);
|
||||
vm_page_t page;
|
||||
|
||||
if (order == 0 && (flags & GFP_DMA32) == 0) {
|
||||
page = vm_page_alloc(NULL, 0, req);
|
||||
if (page == NULL)
|
||||
return (NULL);
|
||||
} else {
|
||||
vm_paddr_t pmax = (flags & GFP_DMA32) ?
|
||||
BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR;
|
||||
retry:
|
||||
page = vm_page_alloc_contig(NULL, 0, req,
|
||||
npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
|
||||
|
||||
if (page == NULL) {
|
||||
if (flags & M_WAITOK) {
|
||||
if (!vm_page_reclaim_contig(req,
|
||||
npages, 0, pmax, PAGE_SIZE, 0)) {
|
||||
VM_WAIT;
|
||||
}
|
||||
flags &= ~M_WAITOK;
|
||||
goto retry;
|
||||
}
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
if (flags & M_ZERO) {
|
||||
unsigned long x;
|
||||
|
||||
for (x = 0; x != npages; x++) {
|
||||
vm_page_t pgo = page + x;
|
||||
|
||||
if ((pgo->flags & PG_ZERO) == 0)
|
||||
pmap_zero_page(pgo);
|
||||
}
|
||||
}
|
||||
#else
|
||||
vm_offset_t vaddr;
|
||||
vm_page_t page;
|
||||
|
||||
vaddr = linux_alloc_kmem(flags, order);
|
||||
if (vaddr == 0)
|
||||
return (NULL);
|
||||
|
||||
page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr));
|
||||
|
||||
KASSERT(vaddr == (vm_offset_t)page_address(page),
|
||||
("Page address mismatch"));
|
||||
#endif
|
||||
return (page);
|
||||
}
|
||||
|
||||
void
|
||||
linux_free_pages(vm_page_t page, unsigned int order)
|
||||
{
|
||||
#ifdef __amd64__
|
||||
unsigned long npages = 1UL << order;
|
||||
unsigned long x;
|
||||
|
||||
for (x = 0; x != npages; x++) {
|
||||
vm_page_t pgo = page + x;
|
||||
|
||||
vm_page_lock(pgo);
|
||||
vm_page_free(pgo);
|
||||
vm_page_unlock(pgo);
|
||||
}
|
||||
#else
|
||||
vm_offset_t vaddr;
|
||||
|
||||
vaddr = (vm_offset_t)page_address(page);
|
||||
|
||||
linux_free_kmem(vaddr, order);
|
||||
#endif
|
||||
}
|
||||
|
||||
vm_offset_t
|
||||
linux_alloc_kmem(gfp_t flags, unsigned int order)
|
||||
{
|
||||
size_t size = ((size_t)PAGE_SIZE) << order;
|
||||
vm_offset_t addr;
|
||||
|
||||
if ((flags & GFP_DMA32) == 0) {
|
||||
addr = kmem_malloc(kmem_arena, size, flags & GFP_NATIVE_MASK);
|
||||
} else {
|
||||
addr = kmem_alloc_contig(kmem_arena, size,
|
||||
flags & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR_32BIT,
|
||||
PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
|
||||
}
|
||||
return (addr);
|
||||
}
|
||||
|
||||
void
|
||||
linux_free_kmem(vm_offset_t addr, unsigned int order)
|
||||
{
|
||||
size_t size = ((size_t)PAGE_SIZE) << order;
|
||||
|
||||
kmem_free(kmem_arena, addr, size);
|
||||
}
|
||||
@ -4286,6 +4286,8 @@ compat/linuxkpi/common/src/linux_current.c optional compat_linuxkpi \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_kthread.c optional compat_linuxkpi \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_page.c optional compat_linuxkpi \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_pci.c optional compat_linuxkpi pci \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_tasklet.c optional compat_linuxkpi \
|
||||
|
||||
@ -6,6 +6,7 @@ SRCS= linux_kmod.c \
|
||||
linux_compat.c \
|
||||
linux_current.c \
|
||||
linux_kthread.c \
|
||||
linux_page.c \
|
||||
linux_pci.c \
|
||||
linux_radix.c \
|
||||
linux_rcu.c \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user