Cleanup the LinuxKPI slab implementation.
Put large functions into linux_slab.c instead of declaring them static inline. Add support for more memory allocation wrappers like kmalloc_array() and __vmalloc(). Make sure either the M_WAITOK or the M_NOWAIT flag is set and mask away unused memory allocation flags before calling FreeBSD's malloc() routine. Move kmalloc_node() definition to slab.h where it belongs. Implement support for the SLAB_DESTROY_BY_RCU feature when creating a kmem_cache which basically means kmem_cache memory is freed using call_rcu(). MFC after: 1 week Sponsored by: Mellanox Technologies
This commit is contained in:
Hans Petter Selasky
parent
0e3bbe9197
commit
14c5024db8
@ -168,8 +168,6 @@ gfpflags_allow_blocking(const gfp_t gfp_flags)
|
||||
return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK);
|
||||
}
|
||||
|
||||
#define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask)
|
||||
|
||||
#define SetPageReserved(page) do { } while (0) /* NOP */
|
||||
#define ClearPageReserved(page) do { } while (0) /* NOP */
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
* Copyright (c) 2010 Isilon Systems, Inc.
|
||||
* Copyright (c) 2010 iX Systems, Inc.
|
||||
* Copyright (c) 2010 Panasas, Inc.
|
||||
* Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
|
||||
* Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -41,20 +41,19 @@
|
||||
|
||||
MALLOC_DECLARE(M_KMALLOC);
|
||||
|
||||
#define kmalloc(size, flags) malloc((size), M_KMALLOC, (flags))
|
||||
#define kvmalloc(size) kmalloc((size), 0)
|
||||
#define kzalloc(size, flags) kmalloc((size), M_ZERO | ((flags) ? (flags) : M_NOWAIT))
|
||||
#define kzalloc_node(size, flags, node) kzalloc(size, flags)
|
||||
#define kfree(ptr) free(__DECONST(void *, (ptr)), M_KMALLOC)
|
||||
#define kvmalloc(size) kmalloc(size, 0)
|
||||
#define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
|
||||
#define kzalloc_node(size, flags, node) kmalloc(size, (flags) | __GFP_ZERO)
|
||||
#define kfree_const(ptr) kfree(ptr)
|
||||
#define krealloc(ptr, size, flags) realloc((ptr), (size), M_KMALLOC, (flags))
|
||||
#define kcalloc(n, size, flags) kmalloc((n) * (size), flags | M_ZERO)
|
||||
#define vzalloc(size) kzalloc(size, GFP_KERNEL | __GFP_NOWARN)
|
||||
#define kcalloc(n, size, flags) kmalloc((n) * (size), (flags) | __GFP_ZERO)
|
||||
#define vzalloc(size) __vmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0)
|
||||
#define vfree(arg) kfree(arg)
|
||||
#define kvfree(arg) kfree(arg)
|
||||
#define vmalloc(size) kmalloc(size, GFP_KERNEL)
|
||||
#define vmalloc_node(size, node) kmalloc(size, GFP_KERNEL)
|
||||
|
||||
#define vmalloc_node(size, node) __vmalloc(size, GFP_KERNEL, 0)
|
||||
#define vmalloc_user(size) __vmalloc(size, GFP_KERNEL | __GFP_ZERO, 0)
|
||||
#define vmalloc(size) __vmalloc(size, GFP_KERNEL, 0)
|
||||
#define __kmalloc(...) kmalloc(__VA_ARGS__)
|
||||
#define kmalloc_node(chunk, flags, n) kmalloc(chunk, flags)
|
||||
|
||||
/*
|
||||
* Prefix some functions with linux_ to avoid namespace conflict
|
||||
@ -66,59 +65,94 @@ MALLOC_DECLARE(M_KMALLOC);
|
||||
#define kmem_cache_free(...) linux_kmem_cache_free(__VA_ARGS__)
|
||||
#define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__)
|
||||
|
||||
typedef void linux_kmem_ctor_t (void *);
|
||||
|
||||
struct linux_kmem_cache {
|
||||
uma_zone_t cache_zone;
|
||||
void (*cache_ctor)(void *);
|
||||
uma_zone_t cache_zone;
|
||||
linux_kmem_ctor_t *cache_ctor;
|
||||
unsigned cache_flags;
|
||||
unsigned cache_size;
|
||||
};
|
||||
|
||||
#define SLAB_HWCACHE_ALIGN 0x0001
|
||||
#define SLAB_HWCACHE_ALIGN (1 << 0)
|
||||
#define SLAB_DESTROY_BY_RCU (1 << 1)
|
||||
#define SLAB_RECLAIM_ACCOUNT (1 << 2)
|
||||
|
||||
static inline int
|
||||
linux_kmem_ctor(void *mem, int size, void *arg, int flags)
|
||||
static inline gfp_t
|
||||
linux_check_m_flags(gfp_t flags)
|
||||
{
|
||||
void (*ctor)(void *);
|
||||
const gfp_t m = M_NOWAIT | M_WAITOK;
|
||||
|
||||
ctor = arg;
|
||||
ctor(mem);
|
||||
/* make sure either M_NOWAIT or M_WAITOK is set */
|
||||
if ((flags & m) == 0)
|
||||
flags |= M_NOWAIT;
|
||||
else if ((flags & m) == m)
|
||||
flags &= ~M_WAITOK;
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static inline struct kmem_cache *
|
||||
linux_kmem_cache_create(char *name, size_t size, size_t align, u_long flags,
|
||||
void (*ctor)(void *))
|
||||
{
|
||||
struct kmem_cache *c;
|
||||
|
||||
c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK);
|
||||
if (align)
|
||||
align--;
|
||||
if (flags & SLAB_HWCACHE_ALIGN)
|
||||
align = UMA_ALIGN_CACHE;
|
||||
c->cache_zone = uma_zcreate(name, size, ctor ? linux_kmem_ctor : NULL,
|
||||
NULL, NULL, NULL, align, 0);
|
||||
c->cache_ctor = ctor;
|
||||
|
||||
return c;
|
||||
/* mask away LinuxKPI specific flags */
|
||||
return (flags & GFP_NATIVE_MASK);
|
||||
}
|
||||
|
||||
static inline void *
|
||||
linux_kmem_cache_alloc(struct kmem_cache *c, int flags)
|
||||
kmalloc(size_t size, gfp_t flags)
|
||||
{
|
||||
return uma_zalloc_arg(c->cache_zone, c->cache_ctor, flags);
|
||||
return (malloc(size, M_KMALLOC, linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
static inline void *
|
||||
__vmalloc(size_t size, gfp_t flags, int other)
|
||||
{
|
||||
return (malloc(size, M_KMALLOC, linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
static inline void *
|
||||
kmalloc_array(size_t n, size_t size, gfp_t flags)
|
||||
{
|
||||
if (size != 0 && n > (SIZE_MAX / size))
|
||||
return (NULL);
|
||||
return (malloc(n * size, M_KMALLOC, linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
static inline void *
|
||||
krealloc(void *ptr, size_t size, gfp_t flags)
|
||||
{
|
||||
return (realloc(ptr, size, M_KMALLOC, linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
static inline void
|
||||
linux_kmem_cache_free(struct kmem_cache *c, void *m)
|
||||
kfree(const void *ptr)
|
||||
{
|
||||
uma_zfree(c->cache_zone, m);
|
||||
free(__DECONST(void *, ptr), M_KMALLOC);
|
||||
}
|
||||
|
||||
extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name,
|
||||
size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor);
|
||||
|
||||
static inline void *
|
||||
linux_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags)
|
||||
{
|
||||
return (uma_zalloc_arg(c->cache_zone, c,
|
||||
linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
static inline void *
|
||||
kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags)
|
||||
{
|
||||
return (uma_zalloc_arg(c->cache_zone, c,
|
||||
linux_check_m_flags(flags | M_ZERO)));
|
||||
}
|
||||
|
||||
extern void linux_kmem_cache_free_rcu(struct linux_kmem_cache *, void *);
|
||||
|
||||
static inline void
|
||||
linux_kmem_cache_destroy(struct kmem_cache *c)
|
||||
linux_kmem_cache_free(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
uma_zdestroy(c->cache_zone);
|
||||
free(c, M_KMALLOC);
|
||||
if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU))
|
||||
linux_kmem_cache_free_rcu(c, m);
|
||||
else
|
||||
uma_zfree(c->cache_zone, m);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_SLAB_H_ */
|
||||
extern void linux_kmem_cache_destroy(struct linux_kmem_cache *);
|
||||
|
||||
#endif /* _LINUX_SLAB_H_ */
|
||||
|
||||
128
sys/compat/linuxkpi/common/src/linux_slab.c
Normal file
128
sys/compat/linuxkpi/common/src/linux_slab.c
Normal file
@ -0,0 +1,128 @@
|
||||
/*-
|
||||
* Copyright (c) 2017 Mellanox Technologies, Ltd.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice unmodified, this list of conditions, and the following
|
||||
* disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__FBSDID("$FreeBSD$");
|
||||
|
||||
#include <linux/slab.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/kernel.h>
|
||||
|
||||
struct linux_kmem_rcu {
|
||||
struct rcu_head rcu_head;
|
||||
struct linux_kmem_cache *cache;
|
||||
};
|
||||
|
||||
#define LINUX_KMEM_TO_RCU(c, m) \
|
||||
((struct linux_kmem_rcu *)((char *)(m) + \
|
||||
(c)->cache_size - sizeof(struct linux_kmem_rcu)))
|
||||
|
||||
#define LINUX_RCU_TO_KMEM(r) \
|
||||
((void *)((char *)(r) + sizeof(struct linux_kmem_rcu) - \
|
||||
(r)->cache->cache_size))
|
||||
|
||||
static int
|
||||
linux_kmem_ctor(void *mem, int size, void *arg, int flags)
|
||||
{
|
||||
struct linux_kmem_cache *c = arg;
|
||||
|
||||
if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) {
|
||||
struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, mem);
|
||||
|
||||
/* duplicate cache pointer */
|
||||
rcu->cache = c;
|
||||
}
|
||||
|
||||
/* check for constructor */
|
||||
if (likely(c->cache_ctor != NULL))
|
||||
c->cache_ctor(mem);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static void
|
||||
linux_kmem_cache_free_rcu_callback(struct rcu_head *head)
|
||||
{
|
||||
struct linux_kmem_rcu *rcu =
|
||||
container_of(head, struct linux_kmem_rcu, rcu_head);
|
||||
|
||||
uma_zfree(rcu->cache->cache_zone, LINUX_RCU_TO_KMEM(rcu));
|
||||
}
|
||||
|
||||
struct linux_kmem_cache *
|
||||
linux_kmem_cache_create(const char *name, size_t size, size_t align,
|
||||
unsigned flags, linux_kmem_ctor_t *ctor)
|
||||
{
|
||||
struct linux_kmem_cache *c;
|
||||
|
||||
c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK);
|
||||
|
||||
if (flags & SLAB_HWCACHE_ALIGN)
|
||||
align = UMA_ALIGN_CACHE;
|
||||
else if (align != 0)
|
||||
align--;
|
||||
|
||||
if (flags & SLAB_DESTROY_BY_RCU) {
|
||||
/* make room for RCU structure */
|
||||
size = ALIGN(size, sizeof(void *));
|
||||
size += sizeof(struct linux_kmem_rcu);
|
||||
|
||||
/* create cache_zone */
|
||||
c->cache_zone = uma_zcreate(name, size,
|
||||
linux_kmem_ctor, NULL, NULL, NULL,
|
||||
align, UMA_ZONE_ZINIT);
|
||||
} else {
|
||||
/* create cache_zone */
|
||||
c->cache_zone = uma_zcreate(name, size,
|
||||
ctor ? linux_kmem_ctor : NULL, NULL,
|
||||
NULL, NULL, align, 0);
|
||||
}
|
||||
|
||||
c->cache_flags = flags;
|
||||
c->cache_ctor = ctor;
|
||||
c->cache_size = size;
|
||||
return (c);
|
||||
}
|
||||
|
||||
void
|
||||
linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m);
|
||||
|
||||
call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback);
|
||||
}
|
||||
|
||||
void
|
||||
linux_kmem_cache_destroy(struct linux_kmem_cache *c)
|
||||
{
|
||||
if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) {
|
||||
/* make sure all free callbacks have been called */
|
||||
rcu_barrier();
|
||||
}
|
||||
|
||||
uma_zdestroy(c->cache_zone);
|
||||
free(c, M_KMALLOC);
|
||||
}
|
||||
@ -4296,6 +4296,8 @@ compat/linuxkpi/common/src/linux_radix.c optional compat_linuxkpi \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_rcu.c optional compat_linuxkpi \
|
||||
compile-with "${LINUXKPI_C} -I$S/contrib/ck/include"
|
||||
compat/linuxkpi/common/src/linux_slab.c optional compat_linuxkpi \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_usb.c optional compat_linuxkpi usb \
|
||||
compile-with "${LINUXKPI_C}"
|
||||
compat/linuxkpi/common/src/linux_work.c optional compat_linuxkpi \
|
||||
|
||||
@ -10,6 +10,7 @@ SRCS= linux_kmod.c \
|
||||
linux_pci.c \
|
||||
linux_radix.c \
|
||||
linux_rcu.c \
|
||||
linux_slab.c \
|
||||
linux_tasklet.c \
|
||||
linux_idr.c \
|
||||
linux_usb.c \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user