LinuxKPI: Allow kmem_cache_free() to be called from critical sections
as it is required by i915kms driver from Linux kernel v 5.5. This is done with asynchronous freeing of requested memory areas from taskqueue thread. As memory to be freed is reused to store linked list entry, backing UMA zone item size is rounded up to pointer size. While here, make struct linux_kmem_cache private to LKPI to reduce amount of BSD headers included by linux/slab.h and switch RCU code to usage of LKPI's linux_irq_work_tq taskqueue to avoid injection of current into system-wide taskqueue_fast thread context. Submitted by: nc (initial version for drm-kmod) Reviewed by: manu, nc Differential revision: https://reviews.freebsd.org/D30760
This commit is contained in:
parent
4060e77f49
commit
a2b83b59db
@ -31,12 +31,9 @@
|
||||
#ifndef _LINUX_SLAB_H_
|
||||
#define _LINUX_SLAB_H_
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/malloc.h>
|
||||
#include <sys/limits.h>
|
||||
#include <sys/proc.h>
|
||||
#include <vm/uma.h>
|
||||
|
||||
#include <linux/compat.h>
|
||||
#include <linux/types.h>
|
||||
@ -65,8 +62,9 @@ MALLOC_DECLARE(M_KMALLOC);
|
||||
*/
|
||||
#define kmem_cache linux_kmem_cache
|
||||
#define kmem_cache_create(...) linux_kmem_cache_create(__VA_ARGS__)
|
||||
#define kmem_cache_alloc(...) linux_kmem_cache_alloc(__VA_ARGS__)
|
||||
#define kmem_cache_free(...) linux_kmem_cache_free(__VA_ARGS__)
|
||||
#define kmem_cache_alloc(...) lkpi_kmem_cache_alloc(__VA_ARGS__)
|
||||
#define kmem_cache_zalloc(...) lkpi_kmem_cache_zalloc(__VA_ARGS__)
|
||||
#define kmem_cache_free(...) lkpi_kmem_cache_free(__VA_ARGS__)
|
||||
#define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__)
|
||||
|
||||
#define KMEM_CACHE(__struct, flags) \
|
||||
@ -75,12 +73,7 @@ MALLOC_DECLARE(M_KMALLOC);
|
||||
|
||||
typedef void linux_kmem_ctor_t (void *);
|
||||
|
||||
struct linux_kmem_cache {
|
||||
uma_zone_t cache_zone;
|
||||
linux_kmem_ctor_t *cache_ctor;
|
||||
unsigned cache_flags;
|
||||
unsigned cache_size;
|
||||
};
|
||||
struct linux_kmem_cache;
|
||||
|
||||
#define SLAB_HWCACHE_ALIGN (1 << 0)
|
||||
#define SLAB_TYPESAFE_BY_RCU (1 << 1)
|
||||
@ -212,32 +205,9 @@ ksize(const void *ptr)
|
||||
|
||||
extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name,
|
||||
size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor);
|
||||
|
||||
static inline void *
|
||||
linux_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags)
|
||||
{
|
||||
return (uma_zalloc_arg(c->cache_zone, c,
|
||||
linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
static inline void *
|
||||
kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags)
|
||||
{
|
||||
return (uma_zalloc_arg(c->cache_zone, c,
|
||||
linux_check_m_flags(flags | M_ZERO)));
|
||||
}
|
||||
|
||||
extern void linux_kmem_cache_free_rcu(struct linux_kmem_cache *, void *);
|
||||
|
||||
static inline void
|
||||
linux_kmem_cache_free(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU))
|
||||
linux_kmem_cache_free_rcu(c, m);
|
||||
else
|
||||
uma_zfree(c->cache_zone, m);
|
||||
}
|
||||
|
||||
extern void *lkpi_kmem_cache_alloc(struct linux_kmem_cache *, gfp_t);
|
||||
extern void *lkpi_kmem_cache_zalloc(struct linux_kmem_cache *, gfp_t);
|
||||
extern void lkpi_kmem_cache_free(struct linux_kmem_cache *, void *);
|
||||
extern void linux_kmem_cache_destroy(struct linux_kmem_cache *);
|
||||
void linux_kfree_async(void *);
|
||||
|
||||
|
@ -48,6 +48,8 @@ __FBSDID("$FreeBSD$");
|
||||
#include <linux/slab.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/compat.h>
|
||||
#include <linux/llist.h>
|
||||
#include <linux/irq_work.h>
|
||||
|
||||
/*
|
||||
* By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will
|
||||
@ -60,13 +62,15 @@ __FBSDID("$FreeBSD$");
|
||||
#endif
|
||||
|
||||
struct callback_head {
|
||||
STAILQ_ENTRY(callback_head) entry;
|
||||
union {
|
||||
STAILQ_ENTRY(callback_head) entry;
|
||||
struct llist_node node;
|
||||
};
|
||||
rcu_callback_t func;
|
||||
};
|
||||
|
||||
struct linux_epoch_head {
|
||||
STAILQ_HEAD(, callback_head) cb_head;
|
||||
struct mtx lock;
|
||||
struct llist_head cb_head;
|
||||
struct task task;
|
||||
} __aligned(CACHE_LINE_SIZE);
|
||||
|
||||
@ -120,9 +124,8 @@ linux_rcu_runtime_init(void *arg __unused)
|
||||
|
||||
head = &linux_epoch_head[j];
|
||||
|
||||
mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF);
|
||||
TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head);
|
||||
STAILQ_INIT(&head->cb_head);
|
||||
init_llist_head(&head->cb_head);
|
||||
|
||||
CPU_FOREACH(i) {
|
||||
struct linux_epoch_record *record;
|
||||
@ -139,37 +142,22 @@ linux_rcu_runtime_init(void *arg __unused)
|
||||
}
|
||||
SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL);
|
||||
|
||||
static void
|
||||
linux_rcu_runtime_uninit(void *arg __unused)
|
||||
{
|
||||
struct linux_epoch_head *head;
|
||||
int j;
|
||||
|
||||
for (j = 0; j != RCU_TYPE_MAX; j++) {
|
||||
head = &linux_epoch_head[j];
|
||||
|
||||
mtx_destroy(&head->lock);
|
||||
}
|
||||
}
|
||||
SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);
|
||||
|
||||
static void
|
||||
linux_rcu_cleaner_func(void *context, int pending __unused)
|
||||
{
|
||||
struct linux_epoch_head *head;
|
||||
struct linux_epoch_head *head = context;
|
||||
struct callback_head *rcu;
|
||||
STAILQ_HEAD(, callback_head) tmp_head;
|
||||
struct llist_node *node, *next;
|
||||
uintptr_t offset;
|
||||
|
||||
linux_set_current(curthread);
|
||||
|
||||
head = context;
|
||||
|
||||
/* move current callbacks into own queue */
|
||||
mtx_lock(&head->lock);
|
||||
STAILQ_INIT(&tmp_head);
|
||||
STAILQ_CONCAT(&tmp_head, &head->cb_head);
|
||||
mtx_unlock(&head->lock);
|
||||
llist_for_each_safe(node, next, llist_del_all(&head->cb_head)) {
|
||||
rcu = container_of(node, struct callback_head, node);
|
||||
/* re-reverse list to restore chronological order */
|
||||
STAILQ_INSERT_HEAD(&tmp_head, rcu, entry);
|
||||
}
|
||||
|
||||
/* synchronize */
|
||||
linux_synchronize_rcu(head - linux_epoch_head);
|
||||
@ -384,7 +372,7 @@ linux_rcu_barrier(unsigned type)
|
||||
head = &linux_epoch_head[type];
|
||||
|
||||
/* wait for callbacks to complete */
|
||||
taskqueue_drain(taskqueue_fast, &head->task);
|
||||
taskqueue_drain(linux_irq_work_tq, &head->task);
|
||||
}
|
||||
|
||||
void
|
||||
@ -398,11 +386,9 @@ linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func)
|
||||
rcu = (struct callback_head *)context;
|
||||
head = &linux_epoch_head[type];
|
||||
|
||||
mtx_lock(&head->lock);
|
||||
rcu->func = func;
|
||||
STAILQ_INSERT_TAIL(&head->cb_head, rcu, entry);
|
||||
taskqueue_enqueue(taskqueue_fast, &head->task);
|
||||
mtx_unlock(&head->lock);
|
||||
llist_add(&rcu->node, &head->cb_head);
|
||||
taskqueue_enqueue(linux_irq_work_tq, &head->task);
|
||||
}
|
||||
|
||||
int
|
||||
|
@ -35,12 +35,22 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/taskqueue.h>
|
||||
#include <vm/uma.h>
|
||||
|
||||
struct linux_kmem_rcu {
|
||||
struct rcu_head rcu_head;
|
||||
struct linux_kmem_cache *cache;
|
||||
};
|
||||
|
||||
struct linux_kmem_cache {
|
||||
uma_zone_t cache_zone;
|
||||
linux_kmem_ctor_t *cache_ctor;
|
||||
unsigned cache_flags;
|
||||
unsigned cache_size;
|
||||
struct llist_head cache_items;
|
||||
struct task cache_task;
|
||||
};
|
||||
|
||||
#define LINUX_KMEM_TO_RCU(c, m) \
|
||||
((struct linux_kmem_rcu *)((char *)(m) + \
|
||||
(c)->cache_size - sizeof(struct linux_kmem_rcu)))
|
||||
@ -51,6 +61,22 @@ struct linux_kmem_rcu {
|
||||
|
||||
static LLIST_HEAD(linux_kfree_async_list);
|
||||
|
||||
static void lkpi_kmem_cache_free_async_fn(void *, int);
|
||||
|
||||
void *
|
||||
lkpi_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags)
|
||||
{
|
||||
return (uma_zalloc_arg(c->cache_zone, c,
|
||||
linux_check_m_flags(flags)));
|
||||
}
|
||||
|
||||
void *
|
||||
lkpi_kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags)
|
||||
{
|
||||
return (uma_zalloc_arg(c->cache_zone, c,
|
||||
linux_check_m_flags(flags | M_ZERO)));
|
||||
}
|
||||
|
||||
static int
|
||||
linux_kmem_ctor(void *mem, int size, void *arg, int flags)
|
||||
{
|
||||
@ -102,6 +128,9 @@ linux_kmem_cache_create(const char *name, size_t size, size_t align,
|
||||
linux_kmem_ctor, NULL, NULL, NULL,
|
||||
align, UMA_ZONE_ZINIT);
|
||||
} else {
|
||||
/* make room for async task list items */
|
||||
size = MAX(size, sizeof(struct llist_node));
|
||||
|
||||
/* create cache_zone */
|
||||
c->cache_zone = uma_zcreate(name, size,
|
||||
ctor ? linux_kmem_ctor : NULL, NULL,
|
||||
@ -111,17 +140,56 @@ linux_kmem_cache_create(const char *name, size_t size, size_t align,
|
||||
c->cache_flags = flags;
|
||||
c->cache_ctor = ctor;
|
||||
c->cache_size = size;
|
||||
init_llist_head(&c->cache_items);
|
||||
TASK_INIT(&c->cache_task, 0, lkpi_kmem_cache_free_async_fn, c);
|
||||
return (c);
|
||||
}
|
||||
|
||||
void
|
||||
linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m)
|
||||
static inline void
|
||||
lkpi_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m);
|
||||
|
||||
call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback);
|
||||
}
|
||||
|
||||
static inline void
|
||||
lkpi_kmem_cache_free_sync(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
uma_zfree(c->cache_zone, m);
|
||||
}
|
||||
|
||||
static void
|
||||
lkpi_kmem_cache_free_async_fn(void *context, int pending)
|
||||
{
|
||||
struct linux_kmem_cache *c = context;
|
||||
struct llist_node *freed, *next;
|
||||
|
||||
llist_for_each_safe(freed, next, llist_del_all(&c->cache_items))
|
||||
lkpi_kmem_cache_free_sync(c, freed);
|
||||
}
|
||||
|
||||
static inline void
|
||||
lkpi_kmem_cache_free_async(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
if (m == NULL)
|
||||
return;
|
||||
|
||||
llist_add(m, &c->cache_items);
|
||||
taskqueue_enqueue(linux_irq_work_tq, &c->cache_task);
|
||||
}
|
||||
|
||||
void
|
||||
lkpi_kmem_cache_free(struct linux_kmem_cache *c, void *m)
|
||||
{
|
||||
if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU))
|
||||
lkpi_kmem_cache_free_rcu(c, m);
|
||||
else if (unlikely(curthread->td_critnest != 0))
|
||||
lkpi_kmem_cache_free_async(c, m);
|
||||
else
|
||||
lkpi_kmem_cache_free_sync(c, m);
|
||||
}
|
||||
|
||||
void
|
||||
linux_kmem_cache_destroy(struct linux_kmem_cache *c)
|
||||
{
|
||||
@ -130,6 +198,9 @@ linux_kmem_cache_destroy(struct linux_kmem_cache *c)
|
||||
rcu_barrier();
|
||||
}
|
||||
|
||||
if (!llist_empty(&c->cache_items))
|
||||
taskqueue_enqueue(linux_irq_work_tq, &c->cache_task);
|
||||
taskqueue_drain(linux_irq_work_tq, &c->cache_task);
|
||||
uma_zdestroy(c->cache_zone);
|
||||
free(c, M_KMALLOC);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user