From 4426b2e64bd6ff5b05997d985140323b3db67085 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Tue, 11 Feb 2020 18:48:07 +0000 Subject: [PATCH] Add flag to struct task to mark the task as requiring network epoch. When processing a taskqueue and a task has associated epoch, then enter for duration of the task. If consecutive tasks belong to the same epoch, batch them. Now we are talking about the network epoch only. Shrink the ta_priority size to 8-bits. No current consumers use a priority that won't fit into 8 bits. Also complexity of taskqueue_enqueue() is a square of maximum value of priority, so we unlikely ever want to go over UCHAR_MAX here. Reviewed by: hselasky Differential Revision: https://reviews.freebsd.org/D23518 --- sys/kern/subr_gtaskqueue.c | 13 +++++++++++++ sys/kern/subr_taskqueue.c | 15 ++++++++++++++- sys/sys/_task.h | 9 ++++++++- sys/sys/epoch.h | 3 +++ sys/sys/gtaskqueue.h | 6 +----- sys/sys/taskqueue.h | 24 +++++++++++++++--------- 6 files changed, 54 insertions(+), 16 deletions(-) diff --git a/sys/kern/subr_gtaskqueue.c b/sys/kern/subr_gtaskqueue.c index 16418b32253b..9bf4b0ca3ad3 100644 --- a/sys/kern/subr_gtaskqueue.c +++ b/sys/kern/subr_gtaskqueue.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -342,13 +343,16 @@ gtaskqueue_unblock(struct gtaskqueue *queue) static void gtaskqueue_run_locked(struct gtaskqueue *queue) { + struct epoch_tracker et; struct gtaskqueue_busy tb; struct gtask *gtask; + bool in_net_epoch; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); + in_net_epoch = false; while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); @@ -358,11 +362,20 @@ gtaskqueue_run_locked(struct gtaskqueue *queue) TQ_UNLOCK(queue); KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL")); + if (!in_net_epoch && TASK_IS_NET(gtask)) { + in_net_epoch = true; + NET_EPOCH_ENTER(et); + } else if (in_net_epoch && !TASK_IS_NET(gtask)) { + NET_EPOCH_EXIT(et); + in_net_epoch = false; + } gtask->ta_func(gtask->ta_context); TQ_LOCK(queue); wakeup(gtask); } + if (in_net_epoch) + NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c index 69f6c5376c9d..0f1503c7f7c7 100644 --- a/sys/kern/subr_taskqueue.c +++ b/sys/kern/subr_taskqueue.c @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -371,7 +372,7 @@ taskqueue_drain_tq_queue(struct taskqueue *queue) * anyway) so just insert it at tail while we have the * queue lock. */ - TASK_INIT(&t_barrier, USHRT_MAX, taskqueue_task_nop_fn, &t_barrier); + TASK_INIT(&t_barrier, UCHAR_MAX, taskqueue_task_nop_fn, &t_barrier); STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link); queue->tq_hint = &t_barrier; t_barrier.ta_pending = 1; @@ -442,14 +443,17 @@ taskqueue_unblock(struct taskqueue *queue) static void taskqueue_run_locked(struct taskqueue *queue) { + struct epoch_tracker et; struct taskqueue_busy tb; struct task *task; + bool in_net_epoch; int pending; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); + in_net_epoch = false; while ((task = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); @@ -462,11 +466,20 @@ taskqueue_run_locked(struct taskqueue *queue) TQ_UNLOCK(queue); KASSERT(task->ta_func != NULL, ("task->ta_func is NULL")); + if (!in_net_epoch && TASK_IS_NET(task)) { + in_net_epoch = true; + NET_EPOCH_ENTER(et); + } else if (in_net_epoch && !TASK_IS_NET(task)) { + NET_EPOCH_EXIT(et); + in_net_epoch = false; + } task->ta_func(task->ta_context, pending); TQ_LOCK(queue); wakeup(task); } + if (in_net_epoch) + NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } diff --git a/sys/sys/_task.h b/sys/sys/_task.h index 6ee48800cada..043707d3b5f0 100644 --- a/sys/sys/_task.h +++ b/sys/sys/_task.h @@ -48,11 +48,18 @@ typedef void task_fn_t(void *context, int pending); struct task { STAILQ_ENTRY(task) ta_link; /* (q) link for queue */ uint16_t ta_pending; /* (q) count times queued */ - u_short ta_priority; /* (c) Priority */ + uint8_t ta_priority; /* (c) Priority */ + uint8_t ta_flags; /* (c) Flags */ task_fn_t *ta_func; /* (c) task handler */ void *ta_context; /* (c) argument for handler */ }; +#define TASK_ENQUEUED 0x1 +#define TASK_NOENQUEUE 0x2 +#define TASK_NETWORK 0x4 + +#define TASK_IS_NET(ta) ((ta)->ta_flags & TASK_NETWORK) + #ifdef _KERNEL typedef void gtask_fn_t(void *context); diff --git a/sys/sys/epoch.h b/sys/sys/epoch.h index a6aea56b1406..25d2bb3dc6e3 100644 --- a/sys/sys/epoch.h +++ b/sys/sys/epoch.h @@ -104,6 +104,9 @@ extern epoch_t net_epoch_preempt; #define NET_EPOCH_WAIT() epoch_wait_preempt(net_epoch_preempt) #define NET_EPOCH_CALL(f, c) epoch_call(net_epoch_preempt, (f), (c)) #define NET_EPOCH_ASSERT() MPASS(in_epoch(net_epoch_preempt)) +#define NET_TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, TASK_NETWORK) +#define NET_GROUPTASK_INIT(gtask, prio, func, ctx) \ + GTASK_INIT(&(gtask)->gt_task, TASK_NETWORK, (prio), (func), (ctx)) #endif /* _KERNEL */ #endif /* _SYS_EPOCH_H_ */ diff --git a/sys/sys/gtaskqueue.h b/sys/sys/gtaskqueue.h index a03bfebc09b1..96fd57dfb76d 100644 --- a/sys/sys/gtaskqueue.h +++ b/sys/sys/gtaskqueue.h @@ -84,10 +84,6 @@ void taskqgroup_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn, const char *name); void taskqgroup_config_gtask_deinit(struct grouptask *gtask); -#define TASK_ENQUEUED 0x1 -#define TASK_SKIP_WAKEUP 0x2 -#define TASK_NOENQUEUE 0x4 - #define GTASK_INIT(gtask, flags, priority, func, context) do { \ (gtask)->ta_flags = flags; \ (gtask)->ta_priority = (priority); \ @@ -96,7 +92,7 @@ void taskqgroup_config_gtask_deinit(struct grouptask *gtask); } while (0) #define GROUPTASK_INIT(gtask, priority, func, context) \ - GTASK_INIT(&(gtask)->gt_task, TASK_SKIP_WAKEUP, priority, func, context) + GTASK_INIT(&(gtask)->gt_task, 0, priority, func, context) #define GROUPTASK_ENQUEUE(gtask) \ grouptaskqueue_enqueue((gtask)->gt_taskqueue, &(gtask)->gt_task) diff --git a/sys/sys/taskqueue.h b/sys/sys/taskqueue.h index 3f7ff1f529a7..2be3c96bdae4 100644 --- a/sys/sys/taskqueue.h +++ b/sys/sys/taskqueue.h @@ -107,8 +107,7 @@ void taskqueue_set_callback(struct taskqueue *queue, taskqueue_callback_fn callback, void *context); #define TASK_INITIALIZER(priority, func, context) \ - { .ta_pending = 0, \ - .ta_priority = (priority), \ + { .ta_priority = (priority), \ .ta_func = (func), \ .ta_context = (context) } @@ -121,18 +120,25 @@ void taskqueue_thread_enqueue(void *context); /* * Initialise a task structure. */ -#define TASK_INIT(task, priority, func, context) do { \ - (task)->ta_pending = 0; \ - (task)->ta_priority = (priority); \ - (task)->ta_func = (func); \ - (task)->ta_context = (context); \ +#define TASK_INIT_FLAGS(task, priority, func, context, flags) do { \ + MPASS((priority) >= 0 && (priority) <= 255); \ + (task)->ta_pending = 0; \ + (task)->ta_priority = (priority); \ + (task)->ta_flags = (flags); \ + (task)->ta_func = (func); \ + (task)->ta_context = (context); \ } while (0) +#define TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, 0) + void _timeout_task_init(struct taskqueue *queue, struct timeout_task *timeout_task, int priority, task_fn_t func, void *context); -#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) \ - _timeout_task_init(queue, timeout_task, priority, func, context); +#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) do { \ + _Static_assert((priority) >= 0 && (priority) <= 255, \ + "struct task priority is 8 bit in size"); \ + _timeout_task_init(queue, timeout_task, priority, func, context); \ +} while (0) /* * Declare a reference to a taskqueue.