From 511885d7061eda3eb1faf3f57dcc936ff75863f1 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Wed, 24 Jul 2019 08:23:23 -0700 Subject: lib/timerqueue: Rely on rbtree semantics for next timer Simplify the timerqueue code by using cached rbtrees and rely on the tree leftmost node semantics to get the timer with earliest expiration time. This is a drop in conversion, and therefore semantics remain untouched. The runtime overhead of cached rbtrees is be pretty much the same as the current head->next method, noting that when removing the leftmost node, a common operation for the timerqueue, the rb_next(leftmost) is O(1) as well, so the next timer will either be the right node or its parent. Therefore no extra pointer chasing. Finally, the size of the struct timerqueue_head remains the same. Passes several hours of rcutorture. Signed-off-by: Davidlohr Bueso Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20190724152323.bojciei3muvfxalm@linux-r8p5 --- include/linux/timerqueue.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/timerqueue.h b/include/linux/timerqueue.h index 78b8cc73f12f..aff122f1062a 100644 --- a/include/linux/timerqueue.h +++ b/include/linux/timerqueue.h @@ -12,8 +12,7 @@ struct timerqueue_node { }; struct timerqueue_head { - struct rb_root head; - struct timerqueue_node *next; + struct rb_root_cached rb_root; }; @@ -29,13 +28,14 @@ extern struct timerqueue_node *timerqueue_iterate_next( * * @head: head of timerqueue * - * Returns a pointer to the timer node that has the - * earliest expiration time. + * Returns a pointer to the timer node that has the earliest expiration time. */ static inline struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head) { - return head->next; + struct rb_node *leftmost = rb_first_cached(&head->rb_root); + + return rb_entry(leftmost, struct timerqueue_node, node); } static inline void timerqueue_init(struct timerqueue_node *node) @@ -45,7 +45,6 @@ static inline void timerqueue_init(struct timerqueue_node *node) static inline void timerqueue_init_head(struct timerqueue_head *head) { - head->head = RB_ROOT; - head->next = NULL; + head->rb_root = RB_ROOT_CACHED; } #endif /* _LINUX_TIMERQUEUE_H */ -- cgit v1.2.3-71-gd317 From b74494872555d1f7888dfd9225700a363f4a84fc Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 26 Jul 2019 20:30:49 +0200 Subject: hrtimer: Remove task argument from hrtimer_init_sleeper() All callers hand in 'current' and that's the only task pointer which actually makes sense. Remove the task argument and set current in the function. Signed-off-by: Thomas Gleixner Reviewed-by: Steven Rostedt (VMware) Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190726185752.791885290@linutronix.de --- block/blk-mq.c | 2 +- drivers/staging/android/vsoc.c | 2 +- include/linux/hrtimer.h | 3 +-- include/linux/wait.h | 2 +- kernel/futex.c | 2 +- kernel/time/hrtimer.c | 8 ++++---- net/core/pktgen.c | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index b038ec680e84..5f647cb8c695 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3418,7 +3418,7 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); hrtimer_set_expires(&hs.timer, kt); - hrtimer_init_sleeper(&hs, current); + hrtimer_init_sleeper(&hs); do { if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) break; diff --git a/drivers/staging/android/vsoc.c b/drivers/staging/android/vsoc.c index 00a1ec7b9154..ce480bcf20d2 100644 --- a/drivers/staging/android/vsoc.c +++ b/drivers/staging/android/vsoc.c @@ -442,7 +442,7 @@ static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg) hrtimer_set_expires_range_ns(&to->timer, wake_time, current->timer_slack_ns); - hrtimer_init_sleeper(to, current); + hrtimer_init_sleeper(to); } while (1) { diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 4971100a8cab..3c74f89367c4 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -463,8 +463,7 @@ extern long hrtimer_nanosleep(const struct timespec64 *rqtp, const enum hrtimer_mode mode, const clockid_t clockid); -extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, - struct task_struct *tsk); +extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl); extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta, const enum hrtimer_mode mode); diff --git a/include/linux/wait.h b/include/linux/wait.h index b6f77cf60dd7..d57832774ca6 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -489,7 +489,7 @@ do { \ struct hrtimer_sleeper __t; \ \ hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); \ - hrtimer_init_sleeper(&__t, current); \ + hrtimer_init_sleeper(&__t); \ if ((timeout) != KTIME_MAX) \ hrtimer_start_range_ns(&__t.timer, timeout, \ current->timer_slack_ns, \ diff --git a/kernel/futex.c b/kernel/futex.c index 6d50728ef2e7..5e9842ea4012 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -490,7 +490,7 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, hrtimer_init_on_stack(&timeout->timer, (flags & FLAGS_CLOCKRT) ? CLOCK_REALTIME : CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - hrtimer_init_sleeper(timeout, current); + hrtimer_init_sleeper(timeout); /* * If range_ns is 0, calling hrtimer_set_expires_range_ns() is diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 5ee77f1a8a92..de895d86800c 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1639,10 +1639,10 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) return HRTIMER_NORESTART; } -void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) +void hrtimer_init_sleeper(struct hrtimer_sleeper *sl) { sl->timer.function = hrtimer_wakeup; - sl->task = task; + sl->task = current; } EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); @@ -1669,7 +1669,7 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod { struct restart_block *restart; - hrtimer_init_sleeper(t, current); + hrtimer_init_sleeper(t); do { set_current_state(TASK_INTERRUPTIBLE); @@ -1930,7 +1930,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, hrtimer_init_on_stack(&t.timer, clock_id, mode); hrtimer_set_expires_range_ns(&t.timer, *expires, delta); - hrtimer_init_sleeper(&t, current); + hrtimer_init_sleeper(&t); hrtimer_start_expires(&t.timer, mode); diff --git a/net/core/pktgen.c b/net/core/pktgen.c index bb9915291644..7f3cf2381f27 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -2171,7 +2171,7 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) } while (ktime_compare(end_time, spin_until) < 0); } else { /* see do_nanosleep */ - hrtimer_init_sleeper(&t, current); + hrtimer_init_sleeper(&t); do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS); -- cgit v1.2.3-71-gd317 From dbc1625fc9deefb352f6ff26a575ae4b3ddef23a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 26 Jul 2019 20:30:50 +0200 Subject: hrtimer: Consolidate hrtimer_init() + hrtimer_init_sleeper() calls hrtimer_init_sleeper() calls require prior initialisation of the hrtimer object which is embedded into the hrtimer_sleeper. Combine the initialization and spare a function call. Fixup all call sites. This is also a preparatory change for PREEMPT_RT to do hrtimer sleeper specific initializations of the embedded hrtimer without modifying any of the call sites. No functional change. [ anna-maria: Minor cleanups ] [ tglx: Adopted to the removal of the task argument of hrtimer_init_sleeper() and trivial polishing. Folded a fix from Stephen Rothwell for the vsoc code ] Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Anna-Maria Gleixner Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190726185752.887468908@linutronix.de --- block/blk-mq.c | 3 +-- drivers/staging/android/vsoc.c | 6 ++---- include/linux/hrtimer.h | 17 ++++++++++++++--- include/linux/wait.h | 4 ++-- kernel/futex.c | 8 +++----- kernel/time/hrtimer.c | 43 +++++++++++++++++++++++++++++++----------- net/core/pktgen.c | 4 +--- 7 files changed, 55 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index 5f647cb8c695..df3fafbfe9a9 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3415,10 +3415,9 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, kt = nsecs; mode = HRTIMER_MODE_REL; - hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); + hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode); hrtimer_set_expires(&hs.timer, kt); - hrtimer_init_sleeper(&hs); do { if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE) break; diff --git a/drivers/staging/android/vsoc.c b/drivers/staging/android/vsoc.c index ce480bcf20d2..2d6b3981afb8 100644 --- a/drivers/staging/android/vsoc.c +++ b/drivers/staging/android/vsoc.c @@ -437,12 +437,10 @@ static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg) return -EINVAL; wake_time = ktime_set(arg->wake_time_sec, arg->wake_time_nsec); - hrtimer_init_on_stack(&to->timer, CLOCK_MONOTONIC, - HRTIMER_MODE_ABS); + hrtimer_init_sleeper_on_stack(to, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS); hrtimer_set_expires_range_ns(&to->timer, wake_time, current->timer_slack_ns); - - hrtimer_init_sleeper(to); } while (1) { diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 3c74f89367c4..0df373bed3d7 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -347,10 +347,15 @@ DECLARE_PER_CPU(struct tick_device, tick_cpu_device); /* Initialize timers: */ extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode); +extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id, + enum hrtimer_mode mode); #ifdef CONFIG_DEBUG_OBJECTS_TIMERS extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode); +extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, + clockid_t clock_id, + enum hrtimer_mode mode); extern void destroy_hrtimer_on_stack(struct hrtimer *timer); #else @@ -360,6 +365,14 @@ static inline void hrtimer_init_on_stack(struct hrtimer *timer, { hrtimer_init(timer, which_clock, mode); } + +static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, + clockid_t clock_id, + enum hrtimer_mode mode) +{ + hrtimer_init_sleeper(sl, clock_id, mode); +} + static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } #endif @@ -463,10 +476,8 @@ extern long hrtimer_nanosleep(const struct timespec64 *rqtp, const enum hrtimer_mode mode, const clockid_t clockid); -extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl); - extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta, - const enum hrtimer_mode mode); + const enum hrtimer_mode mode); extern int schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, const enum hrtimer_mode mode, diff --git a/include/linux/wait.h b/include/linux/wait.h index d57832774ca6..4707543ef575 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -488,8 +488,8 @@ do { \ int __ret = 0; \ struct hrtimer_sleeper __t; \ \ - hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); \ - hrtimer_init_sleeper(&__t); \ + hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \ + HRTIMER_MODE_REL); \ if ((timeout) != KTIME_MAX) \ hrtimer_start_range_ns(&__t.timer, timeout, \ current->timer_slack_ns, \ diff --git a/kernel/futex.c b/kernel/futex.c index 5e9842ea4012..c8561aa5338e 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -487,11 +487,9 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, if (!time) return NULL; - hrtimer_init_on_stack(&timeout->timer, (flags & FLAGS_CLOCKRT) ? - CLOCK_REALTIME : CLOCK_MONOTONIC, - HRTIMER_MODE_ABS); - hrtimer_init_sleeper(timeout); - + hrtimer_init_sleeper_on_stack(timeout, (flags & FLAGS_CLOCKRT) ? + CLOCK_REALTIME : CLOCK_MONOTONIC, + HRTIMER_MODE_ABS); /* * If range_ns is 0, calling hrtimer_set_expires_range_ns() is * effectively the same as calling hrtimer_set_expires(). diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index de895d86800c..bb55d62f631e 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -427,6 +427,17 @@ void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, } EXPORT_SYMBOL_GPL(hrtimer_init_on_stack); +static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl, + clockid_t clock_id, enum hrtimer_mode mode); + +void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, + clockid_t clock_id, enum hrtimer_mode mode) +{ + debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr); + __hrtimer_init_sleeper(sl, clock_id, mode); +} +EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack); + void destroy_hrtimer_on_stack(struct hrtimer *timer) { debug_object_free(timer, &hrtimer_debug_descr); @@ -1639,11 +1650,27 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) return HRTIMER_NORESTART; } -void hrtimer_init_sleeper(struct hrtimer_sleeper *sl) +static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl, + clockid_t clock_id, enum hrtimer_mode mode) { + __hrtimer_init(&sl->timer, clock_id, mode); sl->timer.function = hrtimer_wakeup; sl->task = current; } + +/** + * hrtimer_init_sleeper - initialize sleeper to the given clock + * @sl: sleeper to be initialized + * @clock_id: the clock to be used + * @mode: timer mode abs/rel + */ +void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id, + enum hrtimer_mode mode) +{ + debug_init(&sl->timer, clock_id, mode); + __hrtimer_init_sleeper(sl, clock_id, mode); + +} EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts) @@ -1669,8 +1696,6 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod { struct restart_block *restart; - hrtimer_init_sleeper(t); - do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start_expires(&t->timer, mode); @@ -1707,10 +1732,9 @@ static long __sched hrtimer_nanosleep_restart(struct restart_block *restart) struct hrtimer_sleeper t; int ret; - hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid, - HRTIMER_MODE_ABS); + hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid, + HRTIMER_MODE_ABS); hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); - ret = do_nanosleep(&t, HRTIMER_MODE_ABS); destroy_hrtimer_on_stack(&t.timer); return ret; @@ -1728,7 +1752,7 @@ long hrtimer_nanosleep(const struct timespec64 *rqtp, if (dl_task(current) || rt_task(current)) slack = 0; - hrtimer_init_on_stack(&t.timer, clockid, mode); + hrtimer_init_sleeper_on_stack(&t, clockid, mode); hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack); ret = do_nanosleep(&t, mode); if (ret != -ERESTART_RESTARTBLOCK) @@ -1927,11 +1951,8 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, return -EINTR; } - hrtimer_init_on_stack(&t.timer, clock_id, mode); + hrtimer_init_sleeper_on_stack(&t, clock_id, mode); hrtimer_set_expires_range_ns(&t.timer, *expires, delta); - - hrtimer_init_sleeper(&t); - hrtimer_start_expires(&t.timer, mode); if (likely(t.task)) diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 7f3cf2381f27..a5905975bc12 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -2156,7 +2156,7 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) s64 remaining; struct hrtimer_sleeper t; - hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); hrtimer_set_expires(&t.timer, spin_until); remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer)); @@ -2170,8 +2170,6 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) end_time = ktime_get(); } while (ktime_compare(end_time, spin_until) < 0); } else { - /* see do_nanosleep */ - hrtimer_init_sleeper(&t); do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS); -- cgit v1.2.3-71-gd317 From 01656464fce946f70b02a84ab218e562ceb1662e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 30 Jul 2019 21:03:53 +0200 Subject: hrtimer: Provide hrtimer_sleeper_start_expires() hrtimer_sleepers will gain a scheduling class dependent treatment on PREEMPT_RT. Create a wrapper around hrtimer_start_expires() to make that possible. Signed-off-by: Thomas Gleixner --- include/linux/hrtimer.h | 3 +++ kernel/time/hrtimer.c | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 0df373bed3d7..24072a0942c0 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -408,6 +408,9 @@ static inline void hrtimer_start_expires(struct hrtimer *timer, hrtimer_start_range_ns(timer, soft, delta, mode); } +void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, + enum hrtimer_mode mode); + static inline void hrtimer_restart(struct hrtimer *timer) { hrtimer_start_expires(timer, HRTIMER_MODE_ABS); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index bb55d62f631e..dab1ea1a99d0 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1650,6 +1650,21 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) return HRTIMER_NORESTART; } +/** + * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer + * @sl: sleeper to be started + * @mode: timer mode abs/rel + * + * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers + * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context) + */ +void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, + enum hrtimer_mode mode) +{ + hrtimer_start_expires(&sl->timer, mode); +} +EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires); + static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id, enum hrtimer_mode mode) { @@ -1698,7 +1713,7 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod do { set_current_state(TASK_INTERRUPTIBLE); - hrtimer_start_expires(&t->timer, mode); + hrtimer_sleeper_start_expires(t, mode); if (likely(t->task)) freezable_schedule(); @@ -1953,7 +1968,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, hrtimer_init_sleeper_on_stack(&t, clock_id, mode); hrtimer_set_expires_range_ns(&t.timer, *expires, delta); - hrtimer_start_expires(&t.timer, mode); + hrtimer_sleeper_start_expires(&t, mode); if (likely(t.task)) schedule(); -- cgit v1.2.3-71-gd317 From ae6683d815895c2be1e60e1942630fa99488055b Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 26 Jul 2019 20:30:51 +0200 Subject: hrtimer: Introduce HARD expiry mode On PREEMPT_RT not all hrtimers can be expired in hard interrupt context even if that is perfectly fine on a PREEMPT_RT=n kernel, e.g. because they take regular spinlocks. Also for latency reasons PREEMPT_RT tries to defer most hrtimers' expiry into soft interrupt context. But there are hrtimers which must be expired in hard interrupt context even when PREEMPT_RT is enabled: - hrtimers which must expiry in hard interrupt context, e.g. scheduler, perf, watchdog related hrtimers - latency critical hrtimers, e.g. nanosleep, ..., kvm lapic timer Add a new mode flag HRTIMER_MODE_HARD which allows to mark these timers so PREEMPT_RT will not move them into softirq expiry mode. [ tglx: Split out of a larger combo patch. Added changelog ] Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190726185752.981398465@linutronix.de --- include/linux/hrtimer.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 24072a0942c0..15c2ba6b6316 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -38,6 +38,7 @@ enum hrtimer_mode { HRTIMER_MODE_REL = 0x01, HRTIMER_MODE_PINNED = 0x02, HRTIMER_MODE_SOFT = 0x04, + HRTIMER_MODE_HARD = 0x08, HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED, HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED, @@ -48,6 +49,11 @@ enum hrtimer_mode { HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT, HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT, + HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD, + HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD, + + HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD, + HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD, }; /* -- cgit v1.2.3-71-gd317 From 0ab6a3ddbad40ef5b6b8c2353fd53fa4ecf9c479 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 30 Jul 2019 20:15:25 +0200 Subject: hrtimer: Make enqueue mode check work on RT hrtimer_start_range_ns() has a WARN_ONCE() which verifies that a timer which is marker for softirq expiry is not queued in the hard interrupt base and vice versa. When PREEMPT_RT is enabled, timers which are not explicitely marked to expire in hard interrupt context are deferrred to the soft interrupt. So the regular check would trigger. Change the check, so when PREEMPT_RT is enabled, it is verified that the timers marked for hard interrupt expiry are not tried to be queued for soft interrupt expiry or any of the unmarked and softirq marked is tried to be expired in hard interrupt context. Signed-off-by: Thomas Gleixner --- include/linux/hrtimer.h | 3 +++ kernel/time/hrtimer.c | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 15c2ba6b6316..7d0d0a36a8f4 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -107,6 +107,8 @@ enum hrtimer_restart { * @state: state information (See bit values above) * @is_rel: Set if the timer was armed relative * @is_soft: Set if hrtimer will be expired in soft interrupt context. + * @is_hard: Set if hrtimer will be expired in hard interrupt context + * even on RT. * * The hrtimer structure must be initialized by hrtimer_init() */ @@ -118,6 +120,7 @@ struct hrtimer { u8 state; u8 is_rel; u8 is_soft; + u8 is_hard; }; /** diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index dab1ea1a99d0..0ace301a56f4 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1107,9 +1107,13 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, /* * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft - * match. + * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard + * expiry mode because unmarked timers are moved to softirq expiry. */ - WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); + else + WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); base = lock_hrtimer_base(timer, &flags); @@ -1288,6 +1292,7 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, base += hrtimer_clockid_to_base(clock_id); timer->is_soft = softtimer; + timer->is_hard = !softtimer; timer->base = &cpu_base->clock_base[base]; timerqueue_init(&timer->node); } -- cgit v1.2.3-71-gd317 From f61eff83cec9cfab31fd30a2ca8856be379cdcd5 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Fri, 26 Jul 2019 20:30:59 +0200 Subject: hrtimer: Prepare support for PREEMPT_RT When PREEMPT_RT is enabled, the soft interrupt thread can be preempted. If the soft interrupt thread is preempted in the middle of a timer callback, then calling hrtimer_cancel() can lead to two issues: - If the caller is on a remote CPU then it has to spin wait for the timer handler to complete. This can result in unbound priority inversion. - If the caller originates from the task which preempted the timer handler on the same CPU, then spin waiting for the timer handler to complete is never going to end. To avoid these issues, add a new lock to the timer base which is held around the execution of the timer callbacks. If hrtimer_cancel() detects that the timer callback is currently running, it blocks on the expiry lock. When the callback is finished, the expiry lock is dropped by the softirq thread which wakes up the waiter and the system makes progress. This addresses both the priority inversion and the life lock issues. The same issue can happen in virtual machines when the vCPU which runs a timer callback is scheduled out. If a second vCPU of the same guest calls hrtimer_cancel() it will spin wait for the other vCPU to be scheduled back in. The expiry lock mechanism would avoid that. It'd be trivial to enable this when paravirt spinlocks are enabled in a guest, but it's not clear whether this is an actual problem in the wild, so for now it's an RT only mechanism. [ tglx: Refactored it for mainline ] Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190726185753.737767218@linutronix.de --- include/linux/hrtimer.h | 16 +++++++++ kernel/time/hrtimer.c | 95 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 105 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 7d0d0a36a8f4..5df4bcff96d5 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -192,6 +192,10 @@ enum hrtimer_base_type { * @nr_retries: Total number of hrtimer interrupt retries * @nr_hangs: Total number of hrtimer interrupt hangs * @max_hang_time: Maximum time spent in hrtimer_interrupt + * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are + * expired + * @timer_waiters: A hrtimer_cancel() invocation waits for the timer + * callback to finish. * @expires_next: absolute time of the next event, is required for remote * hrtimer enqueue; it is the total first expiry time (hard * and soft hrtimer are taken into account) @@ -218,6 +222,10 @@ struct hrtimer_cpu_base { unsigned short nr_retries; unsigned short nr_hangs; unsigned int max_hang_time; +#endif +#ifdef CONFIG_PREEMPT_RT + spinlock_t softirq_expiry_lock; + atomic_t timer_waiters; #endif ktime_t expires_next; struct hrtimer *next_timer; @@ -350,6 +358,14 @@ extern void hrtimers_resume(void); DECLARE_PER_CPU(struct tick_device, tick_cpu_device); +#ifdef CONFIG_PREEMPT_RT +void hrtimer_cancel_wait_running(const struct hrtimer *timer); +#else +static inline void hrtimer_cancel_wait_running(struct hrtimer *timer) +{ + cpu_relax(); +} +#endif /* Exported timer functions: */ diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index c101f88ae8aa..499122752649 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1162,6 +1162,82 @@ int hrtimer_try_to_cancel(struct hrtimer *timer) } EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); +#ifdef CONFIG_PREEMPT_RT +static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) +{ + spin_lock_init(&base->softirq_expiry_lock); +} + +static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) +{ + spin_lock(&base->softirq_expiry_lock); +} + +static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) +{ + spin_unlock(&base->softirq_expiry_lock); +} + +/* + * The counterpart to hrtimer_cancel_wait_running(). + * + * If there is a waiter for cpu_base->expiry_lock, then it was waiting for + * the timer callback to finish. Drop expiry_lock and reaquire it. That + * allows the waiter to acquire the lock and make progress. + */ +static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, + unsigned long flags) +{ + if (atomic_read(&cpu_base->timer_waiters)) { + raw_spin_unlock_irqrestore(&cpu_base->lock, flags); + spin_unlock(&cpu_base->softirq_expiry_lock); + spin_lock(&cpu_base->softirq_expiry_lock); + raw_spin_lock_irq(&cpu_base->lock); + } +} + +/* + * This function is called on PREEMPT_RT kernels when the fast path + * deletion of a timer failed because the timer callback function was + * running. + * + * This prevents priority inversion, if the softirq thread on a remote CPU + * got preempted, and it prevents a life lock when the task which tries to + * delete a timer preempted the softirq thread running the timer callback + * function. + */ +void hrtimer_cancel_wait_running(const struct hrtimer *timer) +{ + struct hrtimer_clock_base *base = timer->base; + + if (!timer->is_soft || !base || !base->cpu_base) { + cpu_relax(); + return; + } + + /* + * Mark the base as contended and grab the expiry lock, which is + * held by the softirq across the timer callback. Drop the lock + * immediately so the softirq can expire the next timer. In theory + * the timer could already be running again, but that's more than + * unlikely and just causes another wait loop. + */ + atomic_inc(&base->cpu_base->timer_waiters); + spin_lock_bh(&base->cpu_base->softirq_expiry_lock); + atomic_dec(&base->cpu_base->timer_waiters); + spin_unlock_bh(&base->cpu_base->softirq_expiry_lock); +} +#else +static inline void +hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { } +static inline void +hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { } +static inline void +hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { } +static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base, + unsigned long flags) { } +#endif + /** * hrtimer_cancel - cancel a timer and wait for the handler to finish. * @timer: the timer to be cancelled @@ -1172,13 +1248,15 @@ EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); */ int hrtimer_cancel(struct hrtimer *timer) { - for (;;) { - int ret = hrtimer_try_to_cancel(timer); + int ret; - if (ret >= 0) - return ret; - cpu_relax(); - } + do { + ret = hrtimer_try_to_cancel(timer); + + if (ret < 0) + hrtimer_cancel_wait_running(timer); + } while (ret < 0); + return ret; } EXPORT_SYMBOL_GPL(hrtimer_cancel); @@ -1475,6 +1553,8 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, break; __run_hrtimer(cpu_base, base, timer, &basenow, flags); + if (active_mask == HRTIMER_ACTIVE_SOFT) + hrtimer_sync_wait_running(cpu_base, flags); } } } @@ -1485,6 +1565,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h) unsigned long flags; ktime_t now; + hrtimer_cpu_base_lock_expiry(cpu_base); raw_spin_lock_irqsave(&cpu_base->lock, flags); now = hrtimer_update_base(cpu_base); @@ -1494,6 +1575,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h) hrtimer_update_softirq_timer(cpu_base, true); raw_spin_unlock_irqrestore(&cpu_base->lock, flags); + hrtimer_cpu_base_unlock_expiry(cpu_base); } #ifdef CONFIG_HIGH_RES_TIMERS @@ -1897,6 +1979,7 @@ int hrtimers_prepare_cpu(unsigned int cpu) cpu_base->softirq_next_timer = NULL; cpu_base->expires_next = KTIME_MAX; cpu_base->softirq_expires_next = KTIME_MAX; + hrtimer_cpu_base_init_expiry_lock(cpu_base); return 0; } -- cgit v1.2.3-71-gd317 From 030dcdd197d77374879bb5603d091eee7d8aba80 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Fri, 26 Jul 2019 20:31:00 +0200 Subject: timers: Prepare support for PREEMPT_RT When PREEMPT_RT is enabled, the soft interrupt thread can be preempted. If the soft interrupt thread is preempted in the middle of a timer callback, then calling del_timer_sync() can lead to two issues: - If the caller is on a remote CPU then it has to spin wait for the timer handler to complete. This can result in unbound priority inversion. - If the caller originates from the task which preempted the timer handler on the same CPU, then spin waiting for the timer handler to complete is never going to end. To avoid these issues, add a new lock to the timer base which is held around the execution of the timer callbacks. If del_timer_sync() detects that the timer callback is currently running, it blocks on the expiry lock. When the callback is finished, the expiry lock is dropped by the softirq thread which wakes up the waiter and the system makes progress. This addresses both the priority inversion and the life lock issues. This mechanism is not used for timers which are marked IRQSAFE as for those preemption is disabled accross the callback and therefore this situation cannot happen. The callbacks for such timers need to be individually audited for RT compliance. The same issue can happen in virtual machines when the vCPU which runs a timer callback is scheduled out. If a second vCPU of the same guest calls del_timer_sync() it will spin wait for the other vCPU to be scheduled back in. The expiry lock mechanism would avoid that. It'd be trivial to enable this when paravirt spinlocks are enabled in a guest, but it's not clear whether this is an actual problem in the wild, so for now it's an RT only mechanism. As the softirq thread can be preempted with PREEMPT_RT=y, the SMP variant of del_timer_sync() needs to be used on UP as well. [ tglx: Refactored it for mainline ] Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190726185753.832418500@linutronix.de --- include/linux/timer.h | 2 +- kernel/time/timer.c | 103 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 96 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/timer.h b/include/linux/timer.h index 282e4f2a532a..1e6650ed066d 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -183,7 +183,7 @@ extern void add_timer(struct timer_list *timer); extern int try_to_del_timer_sync(struct timer_list *timer); -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) extern int del_timer_sync(struct timer_list *timer); #else # define del_timer_sync(t) del_timer(t) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 343c7ba33b1c..673c6a0f0c45 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -196,6 +196,10 @@ EXPORT_SYMBOL(jiffies_64); struct timer_base { raw_spinlock_t lock; struct timer_list *running_timer; +#ifdef CONFIG_PREEMPT_RT + spinlock_t expiry_lock; + atomic_t timer_waiters; +#endif unsigned long clk; unsigned long next_expiry; unsigned int cpu; @@ -1227,7 +1231,78 @@ int try_to_del_timer_sync(struct timer_list *timer) } EXPORT_SYMBOL(try_to_del_timer_sync); -#ifdef CONFIG_SMP +#ifdef CONFIG_PREEMPT_RT +static __init void timer_base_init_expiry_lock(struct timer_base *base) +{ + spin_lock_init(&base->expiry_lock); +} + +static inline void timer_base_lock_expiry(struct timer_base *base) +{ + spin_lock(&base->expiry_lock); +} + +static inline void timer_base_unlock_expiry(struct timer_base *base) +{ + spin_unlock(&base->expiry_lock); +} + +/* + * The counterpart to del_timer_wait_running(). + * + * If there is a waiter for base->expiry_lock, then it was waiting for the + * timer callback to finish. Drop expiry_lock and reaquire it. That allows + * the waiter to acquire the lock and make progress. + */ +static void timer_sync_wait_running(struct timer_base *base) +{ + if (atomic_read(&base->timer_waiters)) { + spin_unlock(&base->expiry_lock); + spin_lock(&base->expiry_lock); + } +} + +/* + * This function is called on PREEMPT_RT kernels when the fast path + * deletion of a timer failed because the timer callback function was + * running. + * + * This prevents priority inversion, if the softirq thread on a remote CPU + * got preempted, and it prevents a life lock when the task which tries to + * delete a timer preempted the softirq thread running the timer callback + * function. + */ +static void del_timer_wait_running(struct timer_list *timer) +{ + u32 tf; + + tf = READ_ONCE(timer->flags); + if (!(tf & TIMER_MIGRATING)) { + struct timer_base *base = get_timer_base(tf); + + /* + * Mark the base as contended and grab the expiry lock, + * which is held by the softirq across the timer + * callback. Drop the lock immediately so the softirq can + * expire the next timer. In theory the timer could already + * be running again, but that's more than unlikely and just + * causes another wait loop. + */ + atomic_inc(&base->timer_waiters); + spin_lock_bh(&base->expiry_lock); + atomic_dec(&base->timer_waiters); + spin_unlock_bh(&base->expiry_lock); + } +} +#else +static inline void timer_base_init_expiry_lock(struct timer_base *base) { } +static inline void timer_base_lock_expiry(struct timer_base *base) { } +static inline void timer_base_unlock_expiry(struct timer_base *base) { } +static inline void timer_sync_wait_running(struct timer_base *base) { } +static inline void del_timer_wait_running(struct timer_list *timer) { } +#endif + +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) /** * del_timer_sync - deactivate a timer and wait for the handler to finish. * @timer: the timer to be deactivated @@ -1266,6 +1341,8 @@ EXPORT_SYMBOL(try_to_del_timer_sync); */ int del_timer_sync(struct timer_list *timer) { + int ret; + #ifdef CONFIG_LOCKDEP unsigned long flags; @@ -1283,12 +1360,17 @@ int del_timer_sync(struct timer_list *timer) * could lead to deadlock. */ WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE)); - for (;;) { - int ret = try_to_del_timer_sync(timer); - if (ret >= 0) - return ret; - cpu_relax(); - } + + do { + ret = try_to_del_timer_sync(timer); + + if (unlikely(ret < 0)) { + del_timer_wait_running(timer); + cpu_relax(); + } + } while (ret < 0); + + return ret; } EXPORT_SYMBOL(del_timer_sync); #endif @@ -1360,10 +1442,13 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head) if (timer->flags & TIMER_IRQSAFE) { raw_spin_unlock(&base->lock); call_timer_fn(timer, fn, baseclk); + base->running_timer = NULL; raw_spin_lock(&base->lock); } else { raw_spin_unlock_irq(&base->lock); call_timer_fn(timer, fn, baseclk); + base->running_timer = NULL; + timer_sync_wait_running(base); raw_spin_lock_irq(&base->lock); } } @@ -1658,6 +1743,7 @@ static inline void __run_timers(struct timer_base *base) if (!time_after_eq(jiffies, base->clk)) return; + timer_base_lock_expiry(base); raw_spin_lock_irq(&base->lock); /* @@ -1684,8 +1770,8 @@ static inline void __run_timers(struct timer_base *base) while (levels--) expire_timers(base, heads + levels); } - base->running_timer = NULL; raw_spin_unlock_irq(&base->lock); + timer_base_unlock_expiry(base); } /* @@ -1930,6 +2016,7 @@ static void __init init_timer_cpu(int cpu) base->cpu = cpu; raw_spin_lock_init(&base->lock); base->clk = jiffies; + timer_base_init_expiry_lock(base); } } -- cgit v1.2.3-71-gd317 From 5d99b32a009e900a561f6a42ea7afe5b21288b8a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 31 Jul 2019 00:33:54 +0200 Subject: posix-timers: Move rcu_head out of it union Timer deletion on PREEMPT_RT is prone to priority inversion and live locks. The hrtimer code has a synchronization mechanism for this. Posix CPU timers will grow one. But that mechanism cannot be invoked while holding the k_itimer lock because that can deadlock against the running timer callback. So the lock must be dropped which allows the timer to be freed. The timer free can be prevented by taking RCU readlock before dropping the lock, but because the rcu_head is part of the 'it' union a concurrent free will overwrite the hrtimer on which the task is trying to synchronize. Move the rcu_head out of the union to prevent this. [ tglx: Fixed up kernel-doc. Rewrote changelog ] Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190730223828.965541887@linutronix.de --- include/linux/posix-timers.h | 5 +++-- kernel/time/posix-timers.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index b20798fc5191..604cec0e41ba 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -85,7 +85,8 @@ static inline int clockid_to_fd(const clockid_t clk) * @it_process: The task to wakeup on clock_nanosleep (CPU timers) * @sigq: Pointer to preallocated sigqueue * @it: Union representing the various posix timer type - * internals. Also used for rcu freeing the timer. + * internals. + * @rcu: RCU head for freeing the timer. */ struct k_itimer { struct list_head list; @@ -114,8 +115,8 @@ struct k_itimer { struct { struct alarm alarmtimer; } alarm; - struct rcu_head rcu; } it; + struct rcu_head rcu; }; void run_posix_cpu_timers(struct task_struct *task); diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index bbe8f9686a70..3e663f982c82 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -442,7 +442,7 @@ static struct k_itimer * alloc_posix_timer(void) static void k_itimer_rcu_free(struct rcu_head *head) { - struct k_itimer *tmr = container_of(head, struct k_itimer, it.rcu); + struct k_itimer *tmr = container_of(head, struct k_itimer, rcu); kmem_cache_free(posix_timers_cache, tmr); } @@ -459,7 +459,7 @@ static void release_posix_timer(struct k_itimer *tmr, int it_id_set) } put_pid(tmr->it_pid); sigqueue_free(tmr->sigq); - call_rcu(&tmr->it.rcu, k_itimer_rcu_free); + call_rcu(&tmr->rcu, k_itimer_rcu_free); } static int common_timer_create(struct k_itimer *new_timer) -- cgit v1.2.3-71-gd317 From ce03f613461642669d6150c405dd28f4bfd54bbb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 19 Aug 2019 16:31:42 +0200 Subject: posix-timers: Cleanup forward declarations and includes - Rename struct siginfo to kernel_siginfo as that is used and required - Add a forward declaration for task_struct and remove sched.h include - Remove timex.h include as it is not needed Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190819143801.472005793@linutronix.de --- include/linux/posix-timers.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 604cec0e41ba..26c636d1485b 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -4,11 +4,10 @@ #include #include -#include -#include #include -struct siginfo; +struct kernel_siginfo; +struct task_struct; struct cpu_timer_list { struct list_head entry; -- cgit v1.2.3-71-gd317 From 3758b0f86ef502e2f342055caef6d2232c2558b7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 19 Aug 2019 16:31:43 +0200 Subject: alarmtimers: Avoid rtc.h include rtc.h is not needed in alarmtimers when a forward declaration of struct rtc_device is provided. That allows to include posix-timers.h without adding more includes to alarmtimer.h or creating circular include dependencies. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190819143801.565389536@linutronix.de --- include/linux/alarmtimer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h index 0760ca1cb009..74748e306f4b 100644 --- a/include/linux/alarmtimer.h +++ b/include/linux/alarmtimer.h @@ -5,7 +5,8 @@ #include #include #include -#include + +struct rtc_device; enum alarmtimer_type { ALARM_REALTIME, -- cgit v1.2.3-71-gd317 From dce3e8fd039cc1b62760b3ad6822cf04c262cd0e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 19 Aug 2019 16:31:47 +0200 Subject: posix-cpu-timers: Remove tsk argument from run_posix_cpu_timers() It's always current. Don't give people wrong ideas. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190819143801.945469967@linutronix.de --- include/linux/posix-timers.h | 2 +- kernel/time/posix-cpu-timers.c | 5 +++-- kernel/time/timer.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 26c636d1485b..033374b99767 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -118,7 +118,7 @@ struct k_itimer { struct rcu_head rcu; }; -void run_posix_cpu_timers(struct task_struct *task); +void run_posix_cpu_timers(void); void posix_cpu_timers_exit(struct task_struct *task); void posix_cpu_timers_exit_group(struct task_struct *task); void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx, diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 98223d2805c2..387e0e86e1b8 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -1137,11 +1137,12 @@ static inline int fastpath_timer_check(struct task_struct *tsk) * already updated our counts. We need to check if any timers fire now. * Interrupts are disabled. */ -void run_posix_cpu_timers(struct task_struct *tsk) +void run_posix_cpu_timers(void) { - LIST_HEAD(firing); + struct task_struct *tsk = current; struct k_itimer *timer, *next; unsigned long flags; + LIST_HEAD(firing); lockdep_assert_irqs_disabled(); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 673c6a0f0c45..0e315a2e77ae 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1728,7 +1728,7 @@ void update_process_times(int user_tick) #endif scheduler_tick(); if (IS_ENABLED(CONFIG_POSIX_TIMERS)) - run_posix_cpu_timers(p); + run_posix_cpu_timers(); } /** -- cgit v1.2.3-71-gd317 From 19298fbf453c90a6cf72288155f80c6f55e9139d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:08:51 +0200 Subject: posix-cpu-timers: Provide quick sample function for itimer get_itimer() needs a sample of the current thread group cputime. It invokes thread_group_cputimer() - which is a misnomer. That function also starts eventually the group cputime accouting which is bogus because the accounting is already active when a timer is armed. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192919.599658199@linutronix.de --- include/linux/sched/cputime.h | 2 +- kernel/time/posix-cpu-timers.c | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index 53f883f5a2fd..6de7b384d2be 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -62,7 +62,7 @@ extern void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, */ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); - +void thread_group_sample_cputime(struct task_struct *tsk, struct task_cputime *times); /* * The following are functions that support scheduler-internal time accounting. diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 4426a0f9c470..c22b6b604a95 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -232,6 +232,27 @@ static inline void sample_cputime_atomic(struct task_cputime *times, times->sum_exec_runtime = atomic64_read(&atomic_times->sum_exec_runtime); } +/** + * thread_group_sample_cputime - Sample cputime for a given task + * @tsk: Task for which cputime needs to be started + * @iimes: Storage for time samples + * + * Called from sys_getitimer() to calculate the expiry time of an active + * timer. That means group cputime accounting is already active. Called + * with task sighand lock held. + * + * Updates @times with an uptodate sample of the thread group cputimes. + */ +void thread_group_sample_cputime(struct task_struct *tsk, + struct task_cputime *times) +{ + struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; + + WARN_ON_ONCE(!cputimer->running); + + sample_cputime_atomic(times, &cputimer->cputime_atomic); +} + void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; -- cgit v1.2.3-71-gd317 From c506bef424ca282f2ad357e86fee940c69018974 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:08:54 +0200 Subject: posix-cpu-timers: Rename thread_group_cputimer() and make it static thread_group_cputimer() is a complete misnomer. The function does two things: - For arming process wide timers it makes sure that the atomic time storage is up to date. If no cpu timer is armed yet, then the atomic time storage is not updated by the scheduler for performance reasons. In that case a full summing up of all threads needs to be done and the update needs to be enabled. - Samples the current time into the caller supplied storage. Rename it to thread_group_start_cputime(), make it static and fixup the callsite. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192919.869350319@linutronix.de --- include/linux/sched/cputime.h | 1 - kernel/time/posix-cpu-timers.c | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index 6de7b384d2be..2638fd0ab3f2 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -61,7 +61,6 @@ extern void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, * Thread group CPU time accounting. */ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); -void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); void thread_group_sample_cputime(struct task_struct *tsk, struct task_cputime *times); /* diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index cb736787145b..def225ae069a 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -253,7 +253,20 @@ void thread_group_sample_cputime(struct task_struct *tsk, sample_cputime_atomic(times, &cputimer->cputime_atomic); } -void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) +/** + * thread_group_start_cputime - Start cputime and return a sample + * @tsk: Task for which cputime needs to be started + * @iimes: Storage for time samples + * + * The thread group cputime accouting is avoided when there are no posix + * CPU timers armed. Before starting a timer it's required to check whether + * the time accounting is active. If not, a full update of the atomic + * accounting store needs to be done and the accounting enabled. + * + * Updates @times with an uptodate sample of the thread group cputimes. + */ +static void +thread_group_start_cputime(struct task_struct *tsk, struct task_cputime *times) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; struct task_cputime sum; @@ -536,7 +549,7 @@ static int cpu_timer_sample_group(const clockid_t which_clock, { struct task_cputime cputime; - thread_group_cputimer(p, &cputime); + thread_group_start_cputime(p, &cputime); switch (CPUCLOCK_WHICH(which_clock)) { default: return -EINVAL; -- cgit v1.2.3-71-gd317 From 2b69942f9021bf75bd1b001f53bd2578361fadf3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:04 +0200 Subject: posix-cpu-timers: Create a container struct Per task/process data of posix CPU timers is all over the place which makes the code hard to follow and requires ifdeffery. Create a container to hold all this information in one place, so data is consolidated and the ifdeffery can be confined to the posix timer header file and removed from places like fork. As a first step, move the cpu_timers list head array into the new struct and clean up the initializers and simplify fork. The remaining #ifdef in fork will be removed later. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192920.819418976@linutronix.de --- include/linux/init_task.h | 11 ----------- include/linux/posix-timers.h | 34 ++++++++++++++++++++++++++++++++++ include/linux/sched.h | 4 +++- include/linux/sched/signal.h | 5 +++-- kernel/fork.c | 11 ++++------- kernel/time/posix-cpu-timers.c | 20 ++++++++++---------- 6 files changed, 54 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 6049baa5b8bc..2c620d7ac432 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -36,17 +36,6 @@ extern struct cred init_cred; #define INIT_PREV_CPUTIME(x) #endif -#ifdef CONFIG_POSIX_TIMERS -#define INIT_CPU_TIMERS(s) \ - .cpu_timers = { \ - LIST_HEAD_INIT(s.cpu_timers[0]), \ - LIST_HEAD_INIT(s.cpu_timers[1]), \ - LIST_HEAD_INIT(s.cpu_timers[2]), \ - }, -#else -#define INIT_CPU_TIMERS(s) -#endif - #define INIT_TASK_COMM "swapper" /* Attach to the init_task data structure for proper alignment */ diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 033374b99767..cdef89750b2c 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -62,6 +62,40 @@ static inline int clockid_to_fd(const clockid_t clk) return ~(clk >> 3); } +#ifdef CONFIG_POSIX_TIMERS +/** + * posix_cputimers - Container for posix CPU timer related data + * @cpu_timers: List heads to queue posix CPU timers + * + * Used in task_struct and signal_struct + */ +struct posix_cputimers { + struct list_head cpu_timers[CPUCLOCK_MAX]; +}; + +static inline void posix_cputimers_init(struct posix_cputimers *pct) +{ + INIT_LIST_HEAD(&pct->cpu_timers[0]); + INIT_LIST_HEAD(&pct->cpu_timers[1]); + INIT_LIST_HEAD(&pct->cpu_timers[2]); +} + +/* Init task static initializer */ +#define INIT_CPU_TIMERLISTS(c) { \ + LIST_HEAD_INIT(c.cpu_timers[0]), \ + LIST_HEAD_INIT(c.cpu_timers[1]), \ + LIST_HEAD_INIT(c.cpu_timers[2]), \ +} + +#define INIT_CPU_TIMERS(s) \ + .posix_cputimers = { \ + .cpu_timers = INIT_CPU_TIMERLISTS(s.posix_cputimers), \ + }, +#else +struct posix_cputimers { }; +#define INIT_CPU_TIMERS(s) +#endif + #define REQUEUE_PENDING 1 /** diff --git a/include/linux/sched.h b/include/linux/sched.h index 8dc1811487f5..fde844a3b86e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -28,6 +28,7 @@ #include #include #include +#include #include /* task_struct member predeclarations (sorted alphabetically): */ @@ -878,8 +879,9 @@ struct task_struct { #ifdef CONFIG_POSIX_TIMERS struct task_cputime cputime_expires; - struct list_head cpu_timers[3]; #endif + /* Empty if CONFIG_POSIX_CPUTIMERS=n */ + struct posix_cputimers posix_cputimers; /* Process credentials: */ diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index efd8ce7675ed..88fbb3f1c375 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -9,6 +9,7 @@ #include #include #include +#include /* * Types defining task->signal and task->sighand and APIs using them: @@ -151,9 +152,9 @@ struct signal_struct { /* Earliest-expiration cache. */ struct task_cputime cputime_expires; - struct list_head cpu_timers[3]; - #endif + /* Empty if CONFIG_POSIX_TIMERS=n */ + struct posix_cputimers posix_cputimers; /* PID/PID hash table linkage. */ struct pid *pids[PIDTYPE_MAX]; diff --git a/kernel/fork.c b/kernel/fork.c index d8ae0f1b4148..b6a135e4275b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1523,6 +1523,7 @@ void __cleanup_sighand(struct sighand_struct *sighand) */ static void posix_cpu_timers_init_group(struct signal_struct *sig) { + struct posix_cputimers *pct = &sig->posix_cputimers; unsigned long cpu_limit; cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); @@ -1531,10 +1532,7 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig) sig->cputimer.running = true; } - /* The timer lists. */ - INIT_LIST_HEAD(&sig->cpu_timers[0]); - INIT_LIST_HEAD(&sig->cpu_timers[1]); - INIT_LIST_HEAD(&sig->cpu_timers[2]); + posix_cputimers_init(pct); } #else static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { } @@ -1649,9 +1647,8 @@ static void posix_cpu_timers_init(struct task_struct *tsk) tsk->cputime_expires.prof_exp = 0; tsk->cputime_expires.virt_exp = 0; tsk->cputime_expires.sched_exp = 0; - INIT_LIST_HEAD(&tsk->cpu_timers[0]); - INIT_LIST_HEAD(&tsk->cpu_timers[1]); - INIT_LIST_HEAD(&tsk->cpu_timers[2]); + + posix_cputimers_init(&tsk->posix_cputimers); } #else static inline void posix_cpu_timers_init(struct task_struct *tsk) { } diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index b1c97664a730..849e2045fb6e 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -407,11 +407,11 @@ static void cleanup_timers_list(struct list_head *head) * * This must be called with the siglock held. */ -static void cleanup_timers(struct list_head *head) +static void cleanup_timers(struct posix_cputimers *pct) { - cleanup_timers_list(head); - cleanup_timers_list(++head); - cleanup_timers_list(++head); + cleanup_timers_list(&pct->cpu_timers[CPUCLOCK_PROF]); + cleanup_timers_list(&pct->cpu_timers[CPUCLOCK_VIRT]); + cleanup_timers_list(&pct->cpu_timers[CPUCLOCK_SCHED]); } /* @@ -421,11 +421,11 @@ static void cleanup_timers(struct list_head *head) */ void posix_cpu_timers_exit(struct task_struct *tsk) { - cleanup_timers(tsk->cpu_timers); + cleanup_timers(&tsk->posix_cputimers); } void posix_cpu_timers_exit_group(struct task_struct *tsk) { - cleanup_timers(tsk->signal->cpu_timers); + cleanup_timers(&tsk->signal->posix_cputimers); } static inline int expires_gt(u64 expires, u64 new_exp) @@ -446,10 +446,10 @@ static void arm_timer(struct k_itimer *timer) struct cpu_timer_list *next; if (CPUCLOCK_PERTHREAD(timer->it_clock)) { - head = p->cpu_timers; + head = p->posix_cputimers.cpu_timers; cputime_expires = &p->cputime_expires; } else { - head = p->signal->cpu_timers; + head = p->signal->posix_cputimers.cpu_timers; cputime_expires = &p->signal->cputime_expires; } head += CPUCLOCK_WHICH(timer->it_clock); @@ -773,8 +773,8 @@ static inline void check_dl_overrun(struct task_struct *tsk) static void check_thread_timers(struct task_struct *tsk, struct list_head *firing) { + struct list_head *timers = tsk->posix_cputimers.cpu_timers; struct task_cputime *tsk_expires = &tsk->cputime_expires; - struct list_head *timers = tsk->cpu_timers; u64 expires, stime, utime; unsigned long soft; @@ -879,9 +879,9 @@ static void check_process_timers(struct task_struct *tsk, struct list_head *firing) { struct signal_struct *const sig = tsk->signal; + struct list_head *timers = sig->posix_cputimers.cpu_timers; u64 utime, ptime, virt_expires, prof_expires; u64 sum_sched_runtime, sched_expires; - struct list_head *timers = sig->cpu_timers; struct task_cputime cputime; unsigned long soft; -- cgit v1.2.3-71-gd317 From 9eacb5c7e6607aba00a7322b21cad83fc8b101c8 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:05 +0200 Subject: sched: Move struct task_cputime to types.h For upcoming posix-timer changes to avoid include recursion hell. Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20190821192920.909530418@linutronix.de --- include/linux/sched.h | 17 +---------------- include/linux/sched/types.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 include/linux/sched/types.h (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index fde844a3b86e..37c39df9b186 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -245,22 +246,6 @@ struct prev_cputime { #endif }; -/** - * struct task_cputime - collected CPU time counts - * @utime: time spent in user mode, in nanoseconds - * @stime: time spent in kernel mode, in nanoseconds - * @sum_exec_runtime: total time spent on the CPU, in nanoseconds - * - * This structure groups together three kinds of CPU time that are tracked for - * threads and thread groups. Most things considering CPU time want to group - * these counts together and treat all three of them in parallel. - */ -struct task_cputime { - u64 utime; - u64 stime; - unsigned long long sum_exec_runtime; -}; - /* Alternate field names when used on cache expirations: */ #define virt_exp utime #define prof_exp stime diff --git a/include/linux/sched/types.h b/include/linux/sched/types.h new file mode 100644 index 000000000000..2c5c28ddd9b2 --- /dev/null +++ b/include/linux/sched/types.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_SCHED_TYPES_H +#define _LINUX_SCHED_TYPES_H + +#include + +/** + * struct task_cputime - collected CPU time counts + * @utime: time spent in user mode, in nanoseconds + * @stime: time spent in kernel mode, in nanoseconds + * @sum_exec_runtime: total time spent on the CPU, in nanoseconds + * + * This structure groups together three kinds of CPU time that are tracked for + * threads and thread groups. Most things considering CPU time want to group + * these counts together and treat all three of them in parallel. + */ +struct task_cputime { + u64 utime; + u64 stime; + unsigned long long sum_exec_runtime; +}; + +#endif -- cgit v1.2.3-71-gd317 From 3a245c0f110e2bfcf7f2cd2248a29005c78999e3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:06 +0200 Subject: posix-cpu-timers: Move expiry cache into struct posix_cputimers The expiry cache belongs into the posix_cputimers container where the other cpu timers information is. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192921.014444012@linutronix.de --- include/linux/posix-timers.h | 22 +++++++++++++++++++++ include/linux/sched.h | 8 -------- include/linux/sched/signal.h | 3 --- kernel/fork.c | 25 +++-------------------- kernel/sched/rt.c | 6 ++++-- kernel/time/posix-cpu-timers.c | 45 +++++++++++++++++++++++++----------------- 6 files changed, 56 insertions(+), 53 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index cdef89750b2c..a3731ba15bce 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -62,24 +62,43 @@ static inline int clockid_to_fd(const clockid_t clk) return ~(clk >> 3); } +/* + * Alternate field names for struct task_cputime when used on cache + * expirations. Will go away soon. + */ +#define virt_exp utime +#define prof_exp stime +#define sched_exp sum_exec_runtime + #ifdef CONFIG_POSIX_TIMERS /** * posix_cputimers - Container for posix CPU timer related data + * @cputime_expires: Earliest-expiration cache * @cpu_timers: List heads to queue posix CPU timers * * Used in task_struct and signal_struct */ struct posix_cputimers { + struct task_cputime cputime_expires; struct list_head cpu_timers[CPUCLOCK_MAX]; }; static inline void posix_cputimers_init(struct posix_cputimers *pct) { + memset(&pct->cputime_expires, 0, sizeof(pct->cputime_expires)); INIT_LIST_HEAD(&pct->cpu_timers[0]); INIT_LIST_HEAD(&pct->cpu_timers[1]); INIT_LIST_HEAD(&pct->cpu_timers[2]); } +void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); + +static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, + u64 runtime) +{ + pct->cputime_expires.sched_exp = runtime; +} + /* Init task static initializer */ #define INIT_CPU_TIMERLISTS(c) { \ LIST_HEAD_INIT(c.cpu_timers[0]), \ @@ -94,6 +113,9 @@ static inline void posix_cputimers_init(struct posix_cputimers *pct) #else struct posix_cputimers { }; #define INIT_CPU_TIMERS(s) +static inline void posix_cputimers_init(struct posix_cputimers *pct) { } +static inline void posix_cputimers_group_init(struct posix_cputimers *pct, + u64 cpu_limit) { } #endif #define REQUEUE_PENDING 1 diff --git a/include/linux/sched.h b/include/linux/sched.h index 37c39df9b186..8cc8e323093f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -246,11 +246,6 @@ struct prev_cputime { #endif }; -/* Alternate field names when used on cache expirations: */ -#define virt_exp utime -#define prof_exp stime -#define sched_exp sum_exec_runtime - enum vtime_state { /* Task is sleeping or running in a CPU with VTIME inactive: */ VTIME_INACTIVE = 0, @@ -862,9 +857,6 @@ struct task_struct { unsigned long min_flt; unsigned long maj_flt; -#ifdef CONFIG_POSIX_TIMERS - struct task_cputime cputime_expires; -#endif /* Empty if CONFIG_POSIX_CPUTIMERS=n */ struct posix_cputimers posix_cputimers; diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 88fbb3f1c375..729bd892ee45 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -149,9 +149,6 @@ struct signal_struct { */ struct thread_group_cputimer cputimer; - /* Earliest-expiration cache. */ - struct task_cputime cputime_expires; - #endif /* Empty if CONFIG_POSIX_TIMERS=n */ struct posix_cputimers posix_cputimers; diff --git a/kernel/fork.c b/kernel/fork.c index b6a135e4275b..52bfe7c20ff6 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1527,12 +1527,9 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig) unsigned long cpu_limit; cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); - if (cpu_limit != RLIM_INFINITY) { - sig->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC; + posix_cputimers_group_init(pct, cpu_limit); + if (cpu_limit != RLIM_INFINITY) sig->cputimer.running = true; - } - - posix_cputimers_init(pct); } #else static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { } @@ -1638,22 +1635,6 @@ static void rt_mutex_init_task(struct task_struct *p) #endif } -#ifdef CONFIG_POSIX_TIMERS -/* - * Initialize POSIX timer handling for a single task. - */ -static void posix_cpu_timers_init(struct task_struct *tsk) -{ - tsk->cputime_expires.prof_exp = 0; - tsk->cputime_expires.virt_exp = 0; - tsk->cputime_expires.sched_exp = 0; - - posix_cputimers_init(&tsk->posix_cputimers); -} -#else -static inline void posix_cpu_timers_init(struct task_struct *tsk) { } -#endif - static inline void init_task_pid_links(struct task_struct *task) { enum pid_type type; @@ -1932,7 +1913,7 @@ static __latent_entropy struct task_struct *copy_process( task_io_accounting_init(&p->ioac); acct_clear_integrals(p); - posix_cpu_timers_init(p); + posix_cputimers_init(&p->posix_cputimers); p->io_context = NULL; audit_set_context(p, NULL); diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index da3e85e61013..d6678f773c96 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2305,8 +2305,10 @@ static void watchdog(struct rq *rq, struct task_struct *p) } next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ); - if (p->rt.timeout > next) - p->cputime_expires.sched_exp = p->se.sum_exec_runtime; + if (p->rt.timeout > next) { + posix_cputimers_rt_watchdog(&p->posix_cputimers, + p->se.sum_exec_runtime); + } } } #else diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 849e2045fb6e..3e29d1692437 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -20,11 +20,18 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer); +void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) +{ + posix_cputimers_init(pct); + if (cpu_limit != RLIM_INFINITY) + pct->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC; +} + /* * Called after updating RLIMIT_CPU to run cpu timer and update - * tsk->signal->cputime_expires expiration cache if necessary. Needs - * siglock protection since other code may update expiration cache as - * well. + * tsk->signal->posix_cputimers.cputime_expires expiration cache if + * necessary. Needs siglock protection since other code may update + * expiration cache as well. */ void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) { @@ -447,10 +454,10 @@ static void arm_timer(struct k_itimer *timer) if (CPUCLOCK_PERTHREAD(timer->it_clock)) { head = p->posix_cputimers.cpu_timers; - cputime_expires = &p->cputime_expires; + cputime_expires = &p->posix_cputimers.cputime_expires; } else { head = p->signal->posix_cputimers.cpu_timers; - cputime_expires = &p->signal->cputime_expires; + cputime_expires = &p->signal->posix_cputimers.cputime_expires; } head += CPUCLOCK_WHICH(timer->it_clock); @@ -774,7 +781,7 @@ static void check_thread_timers(struct task_struct *tsk, struct list_head *firing) { struct list_head *timers = tsk->posix_cputimers.cpu_timers; - struct task_cputime *tsk_expires = &tsk->cputime_expires; + struct task_cputime *tsk_expires = &tsk->posix_cputimers.cputime_expires; u64 expires, stime, utime; unsigned long soft; @@ -785,7 +792,7 @@ static void check_thread_timers(struct task_struct *tsk, * If cputime_expires is zero, then there are no active * per thread CPU timers. */ - if (task_cputime_zero(&tsk->cputime_expires)) + if (task_cputime_zero(tsk_expires)) return; task_cputime(tsk, &utime, &stime); @@ -954,10 +961,10 @@ static void check_process_timers(struct task_struct *tsk, prof_expires = x; } - sig->cputime_expires.prof_exp = prof_expires; - sig->cputime_expires.virt_exp = virt_expires; - sig->cputime_expires.sched_exp = sched_expires; - if (task_cputime_zero(&sig->cputime_expires)) + sig->posix_cputimers.cputime_expires.prof_exp = prof_expires; + sig->posix_cputimers.cputime_expires.virt_exp = virt_expires; + sig->posix_cputimers.cputime_expires.sched_exp = sched_expires; + if (task_cputime_zero(&sig->posix_cputimers.cputime_expires)) stop_process_timers(sig); sig->cputimer.checking_timer = false; @@ -1058,12 +1065,13 @@ static inline int fastpath_timer_check(struct task_struct *tsk) { struct signal_struct *sig; - if (!task_cputime_zero(&tsk->cputime_expires)) { + if (!task_cputime_zero(&tsk->posix_cputimers.cputime_expires)) { struct task_cputime task_sample; task_cputime(tsk, &task_sample.utime, &task_sample.stime); task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; - if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) + if (task_cputime_expired(&task_sample, + &tsk->posix_cputimers.cputime_expires)) return 1; } @@ -1088,7 +1096,8 @@ static inline int fastpath_timer_check(struct task_struct *tsk) sample_cputime_atomic(&group_sample, &sig->cputimer.cputime_atomic); - if (task_cputime_expired(&group_sample, &sig->cputime_expires)) + if (task_cputime_expired(&group_sample, + &sig->posix_cputimers.cputime_expires)) return 1; } @@ -1204,12 +1213,12 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, */ switch (clock_idx) { case CPUCLOCK_PROF: - if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval)) - tsk->signal->cputime_expires.prof_exp = *newval; + if (expires_gt(tsk->signal->posix_cputimers.cputime_expires.prof_exp, *newval)) + tsk->signal->posix_cputimers.cputime_expires.prof_exp = *newval; break; case CPUCLOCK_VIRT: - if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval)) - tsk->signal->cputime_expires.virt_exp = *newval; + if (expires_gt(tsk->signal->posix_cputimers.cputime_expires.virt_exp, *newval)) + tsk->signal->posix_cputimers.cputime_expires.virt_exp = *newval; break; } -- cgit v1.2.3-71-gd317 From 11b8462f7e1d25f639c88949a2746a9c2667a766 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:07 +0200 Subject: posix-cpu-timers: Provide array based access to expiry cache Using struct task_cputime for the expiry cache is a pretty odd choice and comes with magic defines to rename the fields for usage in the expiry cache. struct task_cputime is basically a u64 array with 3 members, but it has distinct members. The expiry cache content is different than the content of task_cputime because expiry[PROF] = task_cputime.stime + task_cputime.utime expiry[VIRT] = task_cputime.utime expiry[SCHED] = task_cputime.sum_exec_runtime So there is no direct mapping between task_cputime and the expiry cache and the #define based remapping is just a horrible hack. Having the expiry cache array based allows further simplification of the expiry code. To avoid an all in one cleanup which is hard to review add a temporary anonymous union into struct task_cputime which allows array based access to it. That requires to reorder the members. Add a build time sanity check to validate that the members are at the same place. The union and the build time checks will be removed after conversion. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192921.105793824@linutronix.de --- include/linux/posix-timers.h | 24 ++++++++++++++++++------ include/linux/sched/types.h | 4 ++-- kernel/time/posix-cpu-timers.c | 12 +++++++++++- 3 files changed, 31 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index a3731ba15bce..ed0f6ec30aa6 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -65,27 +65,39 @@ static inline int clockid_to_fd(const clockid_t clk) /* * Alternate field names for struct task_cputime when used on cache * expirations. Will go away soon. + * + * stime corresponds to CLOCKCPU_PROF + * utime corresponds to CLOCKCPU_VIRT + * sum_exex_runtime corresponds to CLOCKCPU_SCHED + * + * The ordering is currently enforced so struct task_cputime and the + * expiries array in struct posix_cputimers are equivalent. */ -#define virt_exp utime #define prof_exp stime +#define virt_exp utime #define sched_exp sum_exec_runtime #ifdef CONFIG_POSIX_TIMERS /** * posix_cputimers - Container for posix CPU timer related data - * @cputime_expires: Earliest-expiration cache + * @cputime_expires: Earliest-expiration cache task_cputime based + * @expiries: Earliest-expiration cache array based * @cpu_timers: List heads to queue posix CPU timers * * Used in task_struct and signal_struct */ struct posix_cputimers { - struct task_cputime cputime_expires; - struct list_head cpu_timers[CPUCLOCK_MAX]; + /* Temporary union until all users are cleaned up */ + union { + struct task_cputime cputime_expires; + u64 expiries[CPUCLOCK_MAX]; + }; + struct list_head cpu_timers[CPUCLOCK_MAX]; }; static inline void posix_cputimers_init(struct posix_cputimers *pct) { - memset(&pct->cputime_expires, 0, sizeof(pct->cputime_expires)); + memset(&pct->expiries, 0, sizeof(pct->expiries)); INIT_LIST_HEAD(&pct->cpu_timers[0]); INIT_LIST_HEAD(&pct->cpu_timers[1]); INIT_LIST_HEAD(&pct->cpu_timers[2]); @@ -96,7 +108,7 @@ void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, u64 runtime) { - pct->cputime_expires.sched_exp = runtime; + pct->expiries[CPUCLOCK_SCHED] = runtime; } /* Init task static initializer */ diff --git a/include/linux/sched/types.h b/include/linux/sched/types.h index 2c5c28ddd9b2..3c3e049224ae 100644 --- a/include/linux/sched/types.h +++ b/include/linux/sched/types.h @@ -6,8 +6,8 @@ /** * struct task_cputime - collected CPU time counts - * @utime: time spent in user mode, in nanoseconds * @stime: time spent in kernel mode, in nanoseconds + * @utime: time spent in user mode, in nanoseconds * @sum_exec_runtime: total time spent on the CPU, in nanoseconds * * This structure groups together three kinds of CPU time that are tracked for @@ -15,8 +15,8 @@ * these counts together and treat all three of them in parallel. */ struct task_cputime { - u64 utime; u64 stime; + u64 utime; unsigned long long sum_exec_runtime; }; diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 3e29d1692437..a38b6d04e8b5 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -18,13 +18,23 @@ #include "posix-timers.h" +static inline void temporary_check(void) +{ + BUILD_BUG_ON(offsetof(struct task_cputime, stime) != + CPUCLOCK_PROF * sizeof(u64)); + BUILD_BUG_ON(offsetof(struct task_cputime, utime) != + CPUCLOCK_VIRT * sizeof(u64)); + BUILD_BUG_ON(offsetof(struct task_cputime, sum_exec_runtime) != + CPUCLOCK_SCHED * sizeof(u64)); +} + static void posix_cpu_timer_rearm(struct k_itimer *timer); void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) { posix_cputimers_init(pct); if (cpu_limit != RLIM_INFINITY) - pct->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC; + pct->expiries[CPUCLOCK_PROF] = cpu_limit * NSEC_PER_SEC; } /* -- cgit v1.2.3-71-gd317 From bbc9bae1e49bee9862c7f24101a728e73cd9f589 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:11 +0200 Subject: posix-cpu-timers: Remove the odd field rename defines The last users of the odd define based renaming of struct task_cputime members are gone. Good riddance. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192921.499058279@linutronix.de --- include/linux/posix-timers.h | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index ed0f6ec30aa6..e36c6fda1af9 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -62,21 +62,6 @@ static inline int clockid_to_fd(const clockid_t clk) return ~(clk >> 3); } -/* - * Alternate field names for struct task_cputime when used on cache - * expirations. Will go away soon. - * - * stime corresponds to CLOCKCPU_PROF - * utime corresponds to CLOCKCPU_VIRT - * sum_exex_runtime corresponds to CLOCKCPU_SCHED - * - * The ordering is currently enforced so struct task_cputime and the - * expiries array in struct posix_cputimers are equivalent. - */ -#define prof_exp stime -#define virt_exp utime -#define sched_exp sum_exec_runtime - #ifdef CONFIG_POSIX_TIMERS /** * posix_cputimers - Container for posix CPU timer related data -- cgit v1.2.3-71-gd317 From 46b883995c88520f2c4de6a64cccc04c69d59f83 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:14 +0200 Subject: posix-cpu-timers: Remove cputime_expires The last users of the magic struct cputime based expiry cache are gone. Remove the leftovers. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192921.790209622@linutronix.de --- include/linux/posix-timers.h | 9 ++------- kernel/time/posix-cpu-timers.c | 10 ---------- 2 files changed, 2 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index e36c6fda1af9..fd9098467d6d 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -65,19 +65,14 @@ static inline int clockid_to_fd(const clockid_t clk) #ifdef CONFIG_POSIX_TIMERS /** * posix_cputimers - Container for posix CPU timer related data - * @cputime_expires: Earliest-expiration cache task_cputime based * @expiries: Earliest-expiration cache array based * @cpu_timers: List heads to queue posix CPU timers * * Used in task_struct and signal_struct */ struct posix_cputimers { - /* Temporary union until all users are cleaned up */ - union { - struct task_cputime cputime_expires; - u64 expiries[CPUCLOCK_MAX]; - }; - struct list_head cpu_timers[CPUCLOCK_MAX]; + u64 expiries[CPUCLOCK_MAX]; + struct list_head cpu_timers[CPUCLOCK_MAX]; }; static inline void posix_cputimers_init(struct posix_cputimers *pct) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index e159b039e44a..ffd49181e23d 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -18,16 +18,6 @@ #include "posix-timers.h" -static inline void temporary_check(void) -{ - BUILD_BUG_ON(offsetof(struct task_cputime, stime) != - CPUCLOCK_PROF * sizeof(u64)); - BUILD_BUG_ON(offsetof(struct task_cputime, utime) != - CPUCLOCK_VIRT * sizeof(u64)); - BUILD_BUG_ON(offsetof(struct task_cputime, sum_exec_runtime) != - CPUCLOCK_SCHED * sizeof(u64)); -} - static void posix_cpu_timer_rearm(struct k_itimer *timer); void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) -- cgit v1.2.3-71-gd317 From 87dc64480fb19a6a0fedbdff1e2557be50673287 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 26 Aug 2019 20:22:24 +0200 Subject: posix-cpu-timers: Restructure expiry array Now that the abused struct task_cputime is gone, it's more natural to bundle the expiry cache and the list head of each clock into a struct and have an array of those structs. Follow the hrtimer naming convention of 'bases' and rename the expiry cache to 'nextevt' and adapt all usage sites. Generates also better code .text size shrinks by 80 bytes. Suggested-by: Ingo Molnar Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/alpine.DEB.2.21.1908262021140.1939@nanos.tec.linutronix.de --- include/linux/posix-timers.h | 41 ++++++++++------ kernel/time/posix-cpu-timers.c | 105 ++++++++++++++++++++++------------------- 2 files changed, 83 insertions(+), 63 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index fd9098467d6d..64bd10d251fe 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -63,24 +63,33 @@ static inline int clockid_to_fd(const clockid_t clk) } #ifdef CONFIG_POSIX_TIMERS + /** - * posix_cputimers - Container for posix CPU timer related data - * @expiries: Earliest-expiration cache array based + * posix_cputimer_base - Container per posix CPU clock + * @nextevt: Earliest-expiration cache * @cpu_timers: List heads to queue posix CPU timers + */ +struct posix_cputimer_base { + u64 nextevt; + struct list_head cpu_timers; +}; + +/** + * posix_cputimers - Container for posix CPU timer related data + * @bases: Base container for posix CPU clocks * * Used in task_struct and signal_struct */ struct posix_cputimers { - u64 expiries[CPUCLOCK_MAX]; - struct list_head cpu_timers[CPUCLOCK_MAX]; + struct posix_cputimer_base bases[CPUCLOCK_MAX]; }; static inline void posix_cputimers_init(struct posix_cputimers *pct) { - memset(&pct->expiries, 0, sizeof(pct->expiries)); - INIT_LIST_HEAD(&pct->cpu_timers[0]); - INIT_LIST_HEAD(&pct->cpu_timers[1]); - INIT_LIST_HEAD(&pct->cpu_timers[2]); + memset(pct->bases, 0, sizeof(pct->bases)); + INIT_LIST_HEAD(&pct->bases[0].cpu_timers); + INIT_LIST_HEAD(&pct->bases[1].cpu_timers); + INIT_LIST_HEAD(&pct->bases[2].cpu_timers); } void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); @@ -88,19 +97,23 @@ void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, u64 runtime) { - pct->expiries[CPUCLOCK_SCHED] = runtime; + pct->bases[CPUCLOCK_SCHED].nextevt = runtime; } /* Init task static initializer */ -#define INIT_CPU_TIMERLISTS(c) { \ - LIST_HEAD_INIT(c.cpu_timers[0]), \ - LIST_HEAD_INIT(c.cpu_timers[1]), \ - LIST_HEAD_INIT(c.cpu_timers[2]), \ +#define INIT_CPU_TIMERBASE(b) { \ + .cpu_timers = LIST_HEAD_INIT(b.cpu_timers), \ +} + +#define INIT_CPU_TIMERBASES(b) { \ + INIT_CPU_TIMERBASE(b[0]), \ + INIT_CPU_TIMERBASE(b[1]), \ + INIT_CPU_TIMERBASE(b[2]), \ } #define INIT_CPU_TIMERS(s) \ .posix_cputimers = { \ - .cpu_timers = INIT_CPU_TIMERLISTS(s.posix_cputimers), \ + .bases = INIT_CPU_TIMERBASES(s.posix_cputimers.bases), \ }, #else struct posix_cputimers { }; diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index ffd49181e23d..9ac601abc4c4 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -24,13 +24,13 @@ void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) { posix_cputimers_init(pct); if (cpu_limit != RLIM_INFINITY) - pct->expiries[CPUCLOCK_PROF] = cpu_limit * NSEC_PER_SEC; + pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC; } /* * Called after updating RLIMIT_CPU to run cpu timer and update - * tsk->signal->posix_cputimers.expiries expiration cache if - * necessary. Needs siglock protection since other code may update + * tsk->signal->posix_cputimers.bases[clock].nextevt expiration cache if + * necessary. Needs siglock protection since other code may update the * expiration cache as well. */ void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) @@ -122,9 +122,11 @@ static void bump_cpu_timer(struct k_itimer *timer, u64 now) } } -static inline bool expiry_cache_is_zero(const u64 *ec) +static inline bool expiry_cache_is_zero(const struct posix_cputimers *pct) { - return !(ec[CPUCLOCK_PROF] | ec[CPUCLOCK_VIRT] | ec[CPUCLOCK_SCHED]); + return !(pct->bases[CPUCLOCK_PROF].nextevt | + pct->bases[CPUCLOCK_VIRT].nextevt | + pct->bases[CPUCLOCK_SCHED].nextevt); } static int @@ -432,9 +434,9 @@ static void cleanup_timers_list(struct list_head *head) */ static void cleanup_timers(struct posix_cputimers *pct) { - cleanup_timers_list(&pct->cpu_timers[CPUCLOCK_PROF]); - cleanup_timers_list(&pct->cpu_timers[CPUCLOCK_VIRT]); - cleanup_timers_list(&pct->cpu_timers[CPUCLOCK_SCHED]); + cleanup_timers_list(&pct->bases[CPUCLOCK_PROF].cpu_timers); + cleanup_timers_list(&pct->bases[CPUCLOCK_VIRT].cpu_timers); + cleanup_timers_list(&pct->bases[CPUCLOCK_SCHED].cpu_timers); } /* @@ -464,21 +466,19 @@ static void arm_timer(struct k_itimer *timer) { struct cpu_timer_list *const nt = &timer->it.cpu; int clkidx = CPUCLOCK_WHICH(timer->it_clock); - u64 *cpuexp, newexp = timer->it.cpu.expires; struct task_struct *p = timer->it.cpu.task; + u64 newexp = timer->it.cpu.expires; + struct posix_cputimer_base *base; struct list_head *head, *listpos; struct cpu_timer_list *next; - if (CPUCLOCK_PERTHREAD(timer->it_clock)) { - head = p->posix_cputimers.cpu_timers + clkidx; - cpuexp = p->posix_cputimers.expiries + clkidx; - } else { - head = p->signal->posix_cputimers.cpu_timers + clkidx; - cpuexp = p->signal->posix_cputimers.expiries + clkidx; - } + if (CPUCLOCK_PERTHREAD(timer->it_clock)) + base = p->posix_cputimers.bases + clkidx; + else + base = p->signal->posix_cputimers.bases + clkidx; - listpos = head; - list_for_each_entry(next, head, entry) { + listpos = head = &base->cpu_timers; + list_for_each_entry(next,head, entry) { if (nt->expires < next->expires) break; listpos = &next->entry; @@ -494,8 +494,8 @@ static void arm_timer(struct k_itimer *timer) * for process timers we share expiration cache with itimers * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME. */ - if (expires_gt(*cpuexp, newexp)) - *cpuexp = newexp; + if (expires_gt(base->nextevt, newexp)) + base->nextevt = newexp; if (CPUCLOCK_PERTHREAD(timer->it_clock)) tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER); @@ -783,9 +783,9 @@ static inline void check_dl_overrun(struct task_struct *tsk) static void check_thread_timers(struct task_struct *tsk, struct list_head *firing) { - struct list_head *timers = tsk->posix_cputimers.cpu_timers; - u64 stime, utime, *expires = tsk->posix_cputimers.expiries; + struct posix_cputimer_base *base = tsk->posix_cputimers.bases; unsigned long soft; + u64 stime, utime; if (dl_task(tsk)) check_dl_overrun(tsk); @@ -794,14 +794,18 @@ static void check_thread_timers(struct task_struct *tsk, * If the expiry cache is zero, then there are no active per thread * CPU timers. */ - if (expiry_cache_is_zero(tsk->posix_cputimers.expiries)) + if (expiry_cache_is_zero(&tsk->posix_cputimers)) return; task_cputime(tsk, &utime, &stime); - *expires++ = check_timers_list(timers, firing, utime + stime); - *expires++ = check_timers_list(++timers, firing, utime); - *expires = check_timers_list(++timers, firing, tsk->se.sum_exec_runtime); + base->nextevt = check_timers_list(&base->cpu_timers, firing, + utime + stime); + base++; + base->nextevt = check_timers_list(&base->cpu_timers, firing, utime); + base++; + base->nextevt = check_timers_list(&base->cpu_timers, firing, + tsk->se.sum_exec_runtime); /* * Check for the special case thread timers. @@ -840,7 +844,7 @@ static void check_thread_timers(struct task_struct *tsk, } } - if (expiry_cache_is_zero(tsk->posix_cputimers.expiries)) + if (expiry_cache_is_zero(&tsk->posix_cputimers)) tick_dep_clear_task(tsk, TICK_DEP_BIT_POSIX_TIMER); } @@ -884,7 +888,7 @@ static void check_process_timers(struct task_struct *tsk, struct list_head *firing) { struct signal_struct *const sig = tsk->signal; - struct list_head *timers = sig->posix_cputimers.cpu_timers; + struct posix_cputimer_base *base = sig->posix_cputimers.bases; u64 utime, ptime, virt_expires, prof_expires; u64 sum_sched_runtime, sched_expires; struct task_cputime cputime; @@ -912,9 +916,12 @@ static void check_process_timers(struct task_struct *tsk, ptime = utime + cputime.stime; sum_sched_runtime = cputime.sum_exec_runtime; - prof_expires = check_timers_list(timers, firing, ptime); - virt_expires = check_timers_list(++timers, firing, utime); - sched_expires = check_timers_list(++timers, firing, sum_sched_runtime); + prof_expires = check_timers_list(&base[CPUCLOCK_PROF].cpu_timers, + firing, ptime); + virt_expires = check_timers_list(&base[CPUCLOCK_VIRT].cpu_timers, + firing, utime); + sched_expires = check_timers_list(&base[CPUCLOCK_SCHED].cpu_timers, + firing, sum_sched_runtime); /* * Check for the special case process timers. @@ -959,11 +966,11 @@ static void check_process_timers(struct task_struct *tsk, prof_expires = x; } - sig->posix_cputimers.expiries[CPUCLOCK_PROF] = prof_expires; - sig->posix_cputimers.expiries[CPUCLOCK_VIRT] = virt_expires; - sig->posix_cputimers.expiries[CPUCLOCK_SCHED] = sched_expires; + base[CPUCLOCK_PROF].nextevt = prof_expires; + base[CPUCLOCK_VIRT].nextevt = virt_expires; + base[CPUCLOCK_SCHED].nextevt = sched_expires; - if (expiry_cache_is_zero(sig->posix_cputimers.expiries)) + if (expiry_cache_is_zero(&sig->posix_cputimers)) stop_process_timers(sig); sig->cputimer.checking_timer = false; @@ -1028,20 +1035,21 @@ unlock: } /** - * task_cputimers_expired - Compare two task_cputime entities. + * task_cputimers_expired - Check whether posix CPU timers are expired * * @samples: Array of current samples for the CPUCLOCK clocks - * @expiries: Array of expiry values for the CPUCLOCK clocks + * @pct: Pointer to a posix_cputimers container * - * Returns true if any mmember of @samples is greater than the corresponding - * member of @expiries if that member is non zero. False otherwise + * Returns true if any member of @samples is greater than the corresponding + * member of @pct->bases[CLK].nextevt. False otherwise */ -static inline bool task_cputimers_expired(const u64 *sample, const u64 *expiries) +static inline bool +task_cputimers_expired(const u64 *sample, struct posix_cputimers *pct) { int i; for (i = 0; i < CPUCLOCK_MAX; i++) { - if (expiries[i] && sample[i] >= expiries[i]) + if (pct->bases[i].nextevt && sample[i] >= pct->bases[i].nextevt) return true; } return false; @@ -1059,14 +1067,13 @@ static inline bool task_cputimers_expired(const u64 *sample, const u64 *expiries */ static inline bool fastpath_timer_check(struct task_struct *tsk) { - u64 *expiries = tsk->posix_cputimers.expiries; struct signal_struct *sig; - if (!expiry_cache_is_zero(expiries)) { + if (!expiry_cache_is_zero(&tsk->posix_cputimers)) { u64 samples[CPUCLOCK_MAX]; task_sample_cputime(tsk, samples); - if (task_cputimers_expired(samples, expiries)) + if (task_cputimers_expired(samples, &tsk->posix_cputimers)) return true; } @@ -1092,8 +1099,7 @@ static inline bool fastpath_timer_check(struct task_struct *tsk) proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples); - if (task_cputimers_expired(samples, - sig->posix_cputimers.expiries)) + if (task_cputimers_expired(samples, &sig->posix_cputimers)) return true; } @@ -1176,11 +1182,12 @@ void run_posix_cpu_timers(void) void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid, u64 *newval, u64 *oldval) { - u64 now, *expiry = tsk->signal->posix_cputimers.expiries + clkid; + u64 now, *nextevt; if (WARN_ON_ONCE(clkid >= CPUCLOCK_SCHED)) return; + nextevt = &tsk->signal->posix_cputimers.bases[clkid].nextevt; now = cpu_clock_sample_group(clkid, tsk, true); if (oldval) { @@ -1207,8 +1214,8 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid, * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF * expiry cache is also used by RLIMIT_CPU!. */ - if (expires_gt(*expiry, *newval)) - *expiry = *newval; + if (expires_gt(*nextevt, *newval)) + *nextevt = *newval; tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER); } -- cgit v1.2.3-71-gd317 From b7be4ef1365dcb56fdffc6689e41058b23f5996d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:16 +0200 Subject: posix-cpu-timers: Switch thread group sampling to array That allows more simplifications in various places. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192921.988426956@linutronix.de --- include/linux/sched/cputime.h | 2 +- kernel/time/itimer.c | 11 ++--- kernel/time/posix-cpu-timers.c | 104 ++++++++++++++++++----------------------- 3 files changed, 49 insertions(+), 68 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index 2638fd0ab3f2..eefa5dff16b4 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -61,7 +61,7 @@ extern void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, * Thread group CPU time accounting. */ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); -void thread_group_sample_cputime(struct task_struct *tsk, struct task_cputime *times); +void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples); /* * The following are functions that support scheduler-internal time accounting. diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index ae04bc259240..77f1e5635cc1 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -55,15 +55,10 @@ static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, val = it->expires; interval = it->incr; if (val) { - struct task_cputime cputime; - u64 t; + u64 t, samples[CPUCLOCK_MAX]; - thread_group_sample_cputime(tsk, &cputime); - if (clock_id == CPUCLOCK_PROF) - t = cputime.utime + cputime.stime; - else - /* CPUCLOCK_VIRT */ - t = cputime.utime; + thread_group_sample_cputime(tsk, samples); + t = samples[clock_id]; if (val < t) /* about to fire */ diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 9ac601abc4c4..e62139a89375 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -225,22 +225,14 @@ retry: } } -static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime *sum) +static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, + struct task_cputime *sum) { __update_gt_cputime(&cputime_atomic->utime, sum->utime); __update_gt_cputime(&cputime_atomic->stime, sum->stime); __update_gt_cputime(&cputime_atomic->sum_exec_runtime, sum->sum_exec_runtime); } -/* Sample task_cputime_atomic values in "atomic_timers", store results in "times". */ -static inline void sample_cputime_atomic(struct task_cputime *times, - struct task_cputime_atomic *atomic_times) -{ - times->utime = atomic64_read(&atomic_times->utime); - times->stime = atomic64_read(&atomic_times->stime); - times->sum_exec_runtime = atomic64_read(&atomic_times->sum_exec_runtime); -} - /** * thread_group_sample_cputime - Sample cputime for a given task * @tsk: Task for which cputime needs to be started @@ -252,20 +244,19 @@ static inline void sample_cputime_atomic(struct task_cputime *times, * * Updates @times with an uptodate sample of the thread group cputimes. */ -void thread_group_sample_cputime(struct task_struct *tsk, - struct task_cputime *times) +void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; WARN_ON_ONCE(!cputimer->running); - sample_cputime_atomic(times, &cputimer->cputime_atomic); + proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); } /** * thread_group_start_cputime - Start cputime and return a sample * @tsk: Task for which cputime needs to be started - * @iimes: Storage for time samples + * @samples: Storage for time samples * * The thread group cputime accouting is avoided when there are no posix * CPU timers armed. Before starting a timer it's required to check whether @@ -274,14 +265,14 @@ void thread_group_sample_cputime(struct task_struct *tsk, * * Updates @times with an uptodate sample of the thread group cputimes. */ -static void -thread_group_start_cputime(struct task_struct *tsk, struct task_cputime *times) +static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - struct task_cputime sum; /* Check if cputimer isn't running. This is accessed without locking. */ if (!READ_ONCE(cputimer->running)) { + struct task_cputime sum; + /* * The POSIX timer interface allows for absolute time expiry * values through the TIMER_ABSTIME flag, therefore we have @@ -299,7 +290,15 @@ thread_group_start_cputime(struct task_struct *tsk, struct task_cputime *times) */ WRITE_ONCE(cputimer->running, true); } - sample_cputime_atomic(times, &cputimer->cputime_atomic); + proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); +} + +static void __thread_group_cputime(struct task_struct *tsk, u64 *samples) +{ + struct task_cputime ct; + + thread_group_cputime(tsk, &ct); + store_samples(samples, ct.stime, ct.utime, ct.sum_exec_runtime); } /* @@ -313,28 +312,18 @@ static u64 cpu_clock_sample_group(const clockid_t clkid, struct task_struct *p, bool start) { struct thread_group_cputimer *cputimer = &p->signal->cputimer; - struct task_cputime cputime; + u64 samples[CPUCLOCK_MAX]; if (!READ_ONCE(cputimer->running)) { if (start) - thread_group_start_cputime(p, &cputime); + thread_group_start_cputime(p, samples); else - thread_group_cputime(p, &cputime); + __thread_group_cputime(p, samples); } else { - sample_cputime_atomic(&cputime, &cputimer->cputime_atomic); + proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); } - switch (clkid) { - case CPUCLOCK_PROF: - return cputime.utime + cputime.stime; - case CPUCLOCK_VIRT: - return cputime.utime; - case CPUCLOCK_SCHED: - return cputime.sum_exec_runtime; - default: - WARN_ON_ONCE(1); - } - return 0; + return samples[clkid]; } static int posix_cpu_clock_get(const clockid_t clock, struct timespec64 *tp) @@ -889,9 +878,7 @@ static void check_process_timers(struct task_struct *tsk, { struct signal_struct *const sig = tsk->signal; struct posix_cputimer_base *base = sig->posix_cputimers.bases; - u64 utime, ptime, virt_expires, prof_expires; - u64 sum_sched_runtime, sched_expires; - struct task_cputime cputime; + u64 virt_exp, prof_exp, sched_exp, samples[CPUCLOCK_MAX]; unsigned long soft; /* @@ -911,30 +898,29 @@ static void check_process_timers(struct task_struct *tsk, * Collect the current process totals. Group accounting is active * so the sample can be taken directly. */ - sample_cputime_atomic(&cputime, &sig->cputimer.cputime_atomic); - utime = cputime.utime; - ptime = utime + cputime.stime; - sum_sched_runtime = cputime.sum_exec_runtime; - - prof_expires = check_timers_list(&base[CPUCLOCK_PROF].cpu_timers, - firing, ptime); - virt_expires = check_timers_list(&base[CPUCLOCK_VIRT].cpu_timers, - firing, utime); - sched_expires = check_timers_list(&base[CPUCLOCK_SCHED].cpu_timers, - firing, sum_sched_runtime); + proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples); + + prof_exp = check_timers_list(&base[CPUCLOCK_PROF].cpu_timers, + firing, samples[CPUCLOCK_PROF]); + virt_exp = check_timers_list(&base[CPUCLOCK_VIRT].cpu_timers, + firing, samples[CPUCLOCK_VIRT]); + sched_exp = check_timers_list(&base[CPUCLOCK_SCHED].cpu_timers, + firing, samples[CPUCLOCK_SCHED]); /* * Check for the special case process timers. */ - check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_expires, ptime, - SIGPROF); - check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime, - SIGVTALRM); + check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_exp, + samples[CPUCLOCK_PROF], SIGPROF); + check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_exp, + samples[CPUCLOCK_PROF], SIGVTALRM); + soft = task_rlimit(tsk, RLIMIT_CPU); if (soft != RLIM_INFINITY) { - unsigned long psecs = div_u64(ptime, NSEC_PER_SEC); + u64 softns, ptime = samples[CPUCLOCK_PROF]; unsigned long hard = task_rlimit_max(tsk, RLIMIT_CPU); - u64 x; + unsigned long psecs = div_u64(ptime, NSEC_PER_SEC); + if (psecs >= hard) { /* * At the hard limit, we just die. @@ -961,14 +947,14 @@ static void check_process_timers(struct task_struct *tsk, sig->rlim[RLIMIT_CPU].rlim_cur = soft; } } - x = soft * NSEC_PER_SEC; - if (!prof_expires || x < prof_expires) - prof_expires = x; + softns = soft * NSEC_PER_SEC; + if (!prof_exp || softns < prof_exp) + prof_exp = softns; } - base[CPUCLOCK_PROF].nextevt = prof_expires; - base[CPUCLOCK_VIRT].nextevt = virt_expires; - base[CPUCLOCK_SCHED].nextevt = sched_expires; + base[CPUCLOCK_PROF].nextevt = prof_exp; + base[CPUCLOCK_VIRT].nextevt = virt_exp; + base[CPUCLOCK_SCHED].nextevt = sched_exp; if (expiry_cache_is_zero(&sig->posix_cputimers)) stop_process_timers(sig); -- cgit v1.2.3-71-gd317 From 2bbdbdae05167c688b6d3499a7dab74208b80a22 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:19 +0200 Subject: posix-cpu-timers: Get rid of zero checks Deactivation of the expiry cache is done by setting all clock caches to 0. That requires to have a check for zero in all places which update the expiry cache: if (cache == 0 || new < cache) cache = new; Use U64_MAX as the deactivated value, which allows to remove the zero checks when updating the cache and reduces it to the obvious check: if (new < cache) cache = new; This also removes the weird workaround in do_prlimit() which was required to convert a RLIMIT_CPU value of 0 (immediate expiry) to 1 because handing in 0 to the posix CPU timer code would have effectively disarmed it. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190821192922.275086128@linutronix.de --- include/linux/posix-timers.h | 7 +++++-- kernel/sys.c | 9 --------- kernel/time/posix-cpu-timers.c | 38 +++++++++++++++----------------------- 3 files changed, 20 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 64bd10d251fe..3ea920e8fe7f 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -86,7 +86,9 @@ struct posix_cputimers { static inline void posix_cputimers_init(struct posix_cputimers *pct) { - memset(pct->bases, 0, sizeof(pct->bases)); + pct->bases[0].nextevt = U64_MAX; + pct->bases[1].nextevt = U64_MAX; + pct->bases[2].nextevt = U64_MAX; INIT_LIST_HEAD(&pct->bases[0].cpu_timers); INIT_LIST_HEAD(&pct->bases[1].cpu_timers); INIT_LIST_HEAD(&pct->bases[2].cpu_timers); @@ -102,7 +104,8 @@ static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, /* Init task static initializer */ #define INIT_CPU_TIMERBASE(b) { \ - .cpu_timers = LIST_HEAD_INIT(b.cpu_timers), \ + .nextevt = U64_MAX, \ + .cpu_timers = LIST_HEAD_INIT(b.cpu_timers), \ } #define INIT_CPU_TIMERBASES(b) { \ diff --git a/kernel/sys.c b/kernel/sys.c index c578b75d7923..2462aa84247f 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1557,15 +1557,6 @@ int do_prlimit(struct task_struct *tsk, unsigned int resource, retval = -EPERM; if (!retval) retval = security_task_setrlimit(tsk, resource, new_rlim); - if (resource == RLIMIT_CPU && new_rlim->rlim_cur == 0) { - /* - * The caller is asking for an immediate RLIMIT_CPU - * expiry. But we use the zero value to mean "it was - * never set". So let's cheat and make it one second - * instead - */ - new_rlim->rlim_cur = 1; - } } if (!retval) { if (old_rlim) diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index a738d7659915..cf85292575c5 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -122,11 +122,12 @@ static void bump_cpu_timer(struct k_itimer *timer, u64 now) } } -static inline bool expiry_cache_is_zero(const struct posix_cputimers *pct) +/* Check whether all cache entries contain U64_MAX, i.e. eternal expiry time */ +static inline bool expiry_cache_is_inactive(const struct posix_cputimers *pct) { - return !(pct->bases[CPUCLOCK_PROF].nextevt | - pct->bases[CPUCLOCK_VIRT].nextevt | - pct->bases[CPUCLOCK_SCHED].nextevt); + return !(~pct->bases[CPUCLOCK_PROF].nextevt | + ~pct->bases[CPUCLOCK_VIRT].nextevt | + ~pct->bases[CPUCLOCK_SCHED].nextevt); } static int @@ -442,11 +443,6 @@ void posix_cpu_timers_exit_group(struct task_struct *tsk) cleanup_timers(&tsk->signal->posix_cputimers); } -static inline int expires_gt(u64 expires, u64 new_exp) -{ - return expires == 0 || expires > new_exp; -} - /* * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the sighand lock held. @@ -483,7 +479,7 @@ static void arm_timer(struct k_itimer *timer) * for process timers we share expiration cache with itimers * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME. */ - if (expires_gt(base->nextevt, newexp)) + if (newexp < base->nextevt) base->nextevt = newexp; if (CPUCLOCK_PERTHREAD(timer->it_clock)) @@ -753,7 +749,7 @@ check_timers_list(struct list_head *timers, list_move_tail(&t->entry, firing); } - return 0; + return U64_MAX; } static inline void check_dl_overrun(struct task_struct *tsk) @@ -779,11 +775,7 @@ static void check_thread_timers(struct task_struct *tsk, if (dl_task(tsk)) check_dl_overrun(tsk); - /* - * If the expiry cache is zero, then there are no active per thread - * CPU timers. - */ - if (expiry_cache_is_zero(&tsk->posix_cputimers)) + if (expiry_cache_is_inactive(&tsk->posix_cputimers)) return; task_cputime(tsk, &utime, &stime); @@ -833,7 +825,7 @@ static void check_thread_timers(struct task_struct *tsk, } } - if (expiry_cache_is_zero(&tsk->posix_cputimers)) + if (expiry_cache_is_inactive(&tsk->posix_cputimers)) tick_dep_clear_task(tsk, TICK_DEP_BIT_POSIX_TIMER); } @@ -864,7 +856,7 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, __group_send_sig_info(signo, SEND_SIG_PRIV, tsk); } - if (it->expires && (!*expires || it->expires < *expires)) + if (it->expires && it->expires < *expires) *expires = it->expires; } @@ -948,7 +940,7 @@ static void check_process_timers(struct task_struct *tsk, } } softns = soft * NSEC_PER_SEC; - if (!prof_exp || softns < prof_exp) + if (softns < prof_exp) prof_exp = softns; } @@ -956,7 +948,7 @@ static void check_process_timers(struct task_struct *tsk, base[CPUCLOCK_VIRT].nextevt = virt_exp; base[CPUCLOCK_SCHED].nextevt = sched_exp; - if (expiry_cache_is_zero(&sig->posix_cputimers)) + if (expiry_cache_is_inactive(&sig->posix_cputimers)) stop_process_timers(sig); sig->cputimer.checking_timer = false; @@ -1035,7 +1027,7 @@ task_cputimers_expired(const u64 *sample, struct posix_cputimers *pct) int i; for (i = 0; i < CPUCLOCK_MAX; i++) { - if (pct->bases[i].nextevt && sample[i] >= pct->bases[i].nextevt) + if (sample[i] >= pct->bases[i].nextevt) return true; } return false; @@ -1055,7 +1047,7 @@ static inline bool fastpath_timer_check(struct task_struct *tsk) { struct signal_struct *sig; - if (!expiry_cache_is_zero(&tsk->posix_cputimers)) { + if (!expiry_cache_is_inactive(&tsk->posix_cputimers)) { u64 samples[CPUCLOCK_MAX]; task_sample_cputime(tsk, samples); @@ -1200,7 +1192,7 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid, * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF * expiry cache is also used by RLIMIT_CPU!. */ - if (expires_gt(*nextevt, *newval)) + if (*newval < *nextevt) *nextevt = *newval; tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER); -- cgit v1.2.3-71-gd317 From 244d49e30653658d4e7e9b2b8427777cbbc5affe Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Aug 2019 21:09:24 +0200 Subject: posix-cpu-timers: Move state tracking to struct posix_cputimers Put it where it belongs and clean up the ifdeffery in fork completely. Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20190821192922.743229404@linutronix.de --- include/linux/posix-timers.h | 8 +++++ include/linux/sched/cputime.h | 9 ++++-- include/linux/sched/signal.h | 6 ---- init/init_task.c | 2 -- kernel/fork.c | 6 ---- kernel/time/posix-cpu-timers.c | 73 +++++++++++++++++++++++------------------- 6 files changed, 54 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 3ea920e8fe7f..a9e3f69d2db4 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -77,15 +77,23 @@ struct posix_cputimer_base { /** * posix_cputimers - Container for posix CPU timer related data * @bases: Base container for posix CPU clocks + * @timers_active: Timers are queued. + * @expiry_active: Timer expiry is active. Used for + * process wide timers to avoid multiple + * task trying to handle expiry concurrently * * Used in task_struct and signal_struct */ struct posix_cputimers { struct posix_cputimer_base bases[CPUCLOCK_MAX]; + unsigned int timers_active; + unsigned int expiry_active; }; static inline void posix_cputimers_init(struct posix_cputimers *pct) { + pct->timers_active = 0; + pct->expiry_active = 0; pct->bases[0].nextevt = U64_MAX; pct->bases[1].nextevt = U64_MAX; pct->bases[2].nextevt = U64_MAX; diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index eefa5dff16b4..6c9f19a33865 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -70,7 +70,7 @@ void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples); */ /** - * get_running_cputimer - return &tsk->signal->cputimer if cputimer is running + * get_running_cputimer - return &tsk->signal->cputimer if cputimers are active * * @tsk: Pointer to target task. */ @@ -80,8 +80,11 @@ struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - /* Check if cputimer isn't running. This is accessed without locking. */ - if (!READ_ONCE(cputimer->running)) + /* + * Check whether posix CPU timers are active. If not the thread + * group accounting is not active either. Lockless check. + */ + if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active)) return NULL; /* diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 729bd892ee45..88050259c466 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -57,18 +57,12 @@ struct task_cputime_atomic { /** * struct thread_group_cputimer - thread group interval timer counts * @cputime_atomic: atomic thread group interval timers. - * @running: true when there are timers running and - * @cputime_atomic receives updates. - * @checking_timer: true when a thread in the group is in the - * process of checking for thread group timers. * * This structure contains the version of task_cputime, above, that is * used for thread group CPU timer calculations. */ struct thread_group_cputimer { struct task_cputime_atomic cputime_atomic; - bool running; - bool checking_timer; }; struct multiprocess_signals { diff --git a/init/init_task.c b/init/init_task.c index 7ab773b9b3cd..d49692a0ec51 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -30,8 +30,6 @@ static struct signal_struct init_signals = { .posix_timers = LIST_HEAD_INIT(init_signals.posix_timers), .cputimer = { .cputime_atomic = INIT_CPUTIME_ATOMIC, - .running = false, - .checking_timer = false, }, #endif INIT_CPU_TIMERS(init_signals) diff --git a/kernel/fork.c b/kernel/fork.c index 52bfe7c20ff6..f1228d9f0b11 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1517,7 +1517,6 @@ void __cleanup_sighand(struct sighand_struct *sighand) } } -#ifdef CONFIG_POSIX_TIMERS /* * Initialize POSIX timer handling for a thread group. */ @@ -1528,12 +1527,7 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig) cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); posix_cputimers_group_init(pct, cpu_limit); - if (cpu_limit != RLIM_INFINITY) - sig->cputimer.running = true; } -#else -static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { } -#endif static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) { diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index ef39a7a4a95c..52f4c99c1d60 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -23,8 +23,10 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer); void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) { posix_cputimers_init(pct); - if (cpu_limit != RLIM_INFINITY) + if (cpu_limit != RLIM_INFINITY) { pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC; + pct->timers_active = true; + } } /* @@ -248,8 +250,9 @@ static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; + struct posix_cputimers *pct = &tsk->signal->posix_cputimers; - WARN_ON_ONCE(!cputimer->running); + WARN_ON_ONCE(!pct->timers_active); proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); } @@ -269,9 +272,10 @@ void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples) static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; + struct posix_cputimers *pct = &tsk->signal->posix_cputimers; /* Check if cputimer isn't running. This is accessed without locking. */ - if (!READ_ONCE(cputimer->running)) { + if (!READ_ONCE(pct->timers_active)) { struct task_cputime sum; /* @@ -283,13 +287,13 @@ static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples) update_gt_cputime(&cputimer->cputime_atomic, &sum); /* - * We're setting cputimer->running without a lock. Ensure - * this only gets written to in one operation. We set - * running after update_gt_cputime() as a small optimization, - * but barriers are not required because update_gt_cputime() + * We're setting timers_active without a lock. Ensure this + * only gets written to in one operation. We set it after + * update_gt_cputime() as a small optimization, but + * barriers are not required because update_gt_cputime() * can handle concurrent updates. */ - WRITE_ONCE(cputimer->running, true); + WRITE_ONCE(pct->timers_active, true); } proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); } @@ -313,9 +317,10 @@ static u64 cpu_clock_sample_group(const clockid_t clkid, struct task_struct *p, bool start) { struct thread_group_cputimer *cputimer = &p->signal->cputimer; + struct posix_cputimers *pct = &p->signal->posix_cputimers; u64 samples[CPUCLOCK_MAX]; - if (!READ_ONCE(cputimer->running)) { + if (!READ_ONCE(pct->timers_active)) { if (start) thread_group_start_cputime(p, samples); else @@ -834,10 +839,10 @@ static void check_thread_timers(struct task_struct *tsk, static inline void stop_process_timers(struct signal_struct *sig) { - struct thread_group_cputimer *cputimer = &sig->cputimer; + struct posix_cputimers *pct = &sig->posix_cputimers; - /* Turn off cputimer->running. This is done without locking. */ - WRITE_ONCE(cputimer->running, false); + /* Turn off the active flag. This is done without locking. */ + WRITE_ONCE(pct->timers_active, false); tick_dep_clear_signal(sig, TICK_DEP_BIT_POSIX_TIMER); } @@ -877,17 +882,17 @@ static void check_process_timers(struct task_struct *tsk, unsigned long soft; /* - * If cputimer is not running, then there are no active - * process wide timers (POSIX 1.b, itimers, RLIMIT_CPU). + * If there are no active process wide timers (POSIX 1.b, itimers, + * RLIMIT_CPU) nothing to check. */ - if (!READ_ONCE(sig->cputimer.running)) + if (!READ_ONCE(pct->timers_active)) return; /* * Signify that a thread is checking for process timers. * Write access to this field is protected by the sighand lock. */ - sig->cputimer.checking_timer = true; + pct->timers_active = true; /* * Collect the current process totals. Group accounting is active @@ -933,7 +938,7 @@ static void check_process_timers(struct task_struct *tsk, if (expiry_cache_is_inactive(pct)) stop_process_timers(sig); - sig->cputimer.checking_timer = false; + pct->expiry_active = false; } /* @@ -1027,39 +1032,41 @@ task_cputimers_expired(const u64 *sample, struct posix_cputimers *pct) */ static inline bool fastpath_timer_check(struct task_struct *tsk) { + struct posix_cputimers *pct = &tsk->posix_cputimers; struct signal_struct *sig; - if (!expiry_cache_is_inactive(&tsk->posix_cputimers)) { + if (!expiry_cache_is_inactive(pct)) { u64 samples[CPUCLOCK_MAX]; task_sample_cputime(tsk, samples); - if (task_cputimers_expired(samples, &tsk->posix_cputimers)) + if (task_cputimers_expired(samples, pct)) return true; } sig = tsk->signal; + pct = &sig->posix_cputimers; /* - * Check if thread group timers expired when the cputimer is - * running and no other thread in the group is already checking - * for thread group cputimers. These fields are read without the - * sighand lock. However, this is fine because this is meant to - * be a fastpath heuristic to determine whether we should try to - * acquire the sighand lock to check/handle timers. + * Check if thread group timers expired when timers are active and + * no other thread in the group is already handling expiry for + * thread group cputimers. These fields are read without the + * sighand lock. However, this is fine because this is meant to be + * a fastpath heuristic to determine whether we should try to + * acquire the sighand lock to handle timer expiry. * - * In the worst case scenario, if 'running' or 'checking_timer' gets - * set but the current thread doesn't see the change yet, we'll wait - * until the next thread in the group gets a scheduler interrupt to - * handle the timer. This isn't an issue in practice because these - * types of delays with signals actually getting sent are expected. + * In the worst case scenario, if concurrently timers_active is set + * or expiry_active is cleared, but the current thread doesn't see + * the change yet, the timer checks are delayed until the next + * thread in the group gets a scheduler interrupt to handle the + * timer. This isn't an issue in practice because these types of + * delays with signals actually getting sent are expected. */ - if (READ_ONCE(sig->cputimer.running) && - !READ_ONCE(sig->cputimer.checking_timer)) { + if (READ_ONCE(pct->timers_active) && !READ_ONCE(pct->expiry_active)) { u64 samples[CPUCLOCK_MAX]; proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples); - if (task_cputimers_expired(samples, &sig->posix_cputimers)) + if (task_cputimers_expired(samples, pct)) return true; } -- cgit v1.2.3-71-gd317 From 60bda037f1dd8151e0c9ee5b09f0c091a0f643cd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 27 Aug 2019 21:31:02 +0200 Subject: posix-cpu-timers: Utilize timerqueue for storage Using a linear O(N) search for timer insertion affects execution time and D-cache footprint badly with a larger number of timers. Switch the storage to a timerqueue which is already used for hrtimers and alarmtimers. It does not affect the size of struct k_itimer as it.alarm is still larger. The extra list head for the expiry list will go away later once the expiry is moved into task work context. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/alpine.DEB.2.21.1908272129220.1939@nanos.tec.linutronix.de --- include/linux/posix-timers.h | 65 ++++++++++---- include/linux/timerqueue.h | 10 +++ kernel/time/posix-cpu-timers.c | 189 +++++++++++++++++++++-------------------- 3 files changed, 155 insertions(+), 109 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index a9e3f69d2db4..f9fbb4c620c1 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -5,17 +5,11 @@ #include #include #include +#include struct kernel_siginfo; struct task_struct; -struct cpu_timer_list { - struct list_head entry; - u64 expires; - struct task_struct *task; - int firing; -}; - /* * Bit fields within a clockid: * @@ -64,14 +58,58 @@ static inline int clockid_to_fd(const clockid_t clk) #ifdef CONFIG_POSIX_TIMERS +/** + * cpu_timer - Posix CPU timer representation for k_itimer + * @node: timerqueue node to queue in the task/sig + * @head: timerqueue head on which this timer is queued + * @task: Pointer to target task + * @elist: List head for the expiry list + * @firing: Timer is currently firing + */ +struct cpu_timer { + struct timerqueue_node node; + struct timerqueue_head *head; + struct task_struct *task; + struct list_head elist; + int firing; +}; + +static inline bool cpu_timer_requeue(struct cpu_timer *ctmr) +{ + return timerqueue_add(ctmr->head, &ctmr->node); +} + +static inline bool cpu_timer_enqueue(struct timerqueue_head *head, + struct cpu_timer *ctmr) +{ + ctmr->head = head; + return timerqueue_add(head, &ctmr->node); +} + +static inline void cpu_timer_dequeue(struct cpu_timer *ctmr) +{ + if (!RB_EMPTY_NODE(&ctmr->node.node)) + timerqueue_del(ctmr->head, &ctmr->node); +} + +static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr) +{ + return ctmr->node.expires; +} + +static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp) +{ + ctmr->node.expires = exp; +} + /** * posix_cputimer_base - Container per posix CPU clock * @nextevt: Earliest-expiration cache - * @cpu_timers: List heads to queue posix CPU timers + * @tqhead: timerqueue head for cpu_timers */ struct posix_cputimer_base { u64 nextevt; - struct list_head cpu_timers; + struct timerqueue_head tqhead; }; /** @@ -92,14 +130,10 @@ struct posix_cputimers { static inline void posix_cputimers_init(struct posix_cputimers *pct) { - pct->timers_active = 0; - pct->expiry_active = 0; + memset(pct, 0, sizeof(*pct)); pct->bases[0].nextevt = U64_MAX; pct->bases[1].nextevt = U64_MAX; pct->bases[2].nextevt = U64_MAX; - INIT_LIST_HEAD(&pct->bases[0].cpu_timers); - INIT_LIST_HEAD(&pct->bases[1].cpu_timers); - INIT_LIST_HEAD(&pct->bases[2].cpu_timers); } void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); @@ -113,7 +147,6 @@ static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, /* Init task static initializer */ #define INIT_CPU_TIMERBASE(b) { \ .nextevt = U64_MAX, \ - .cpu_timers = LIST_HEAD_INIT(b.cpu_timers), \ } #define INIT_CPU_TIMERBASES(b) { \ @@ -182,7 +215,7 @@ struct k_itimer { struct { struct hrtimer timer; } real; - struct cpu_timer_list cpu; + struct cpu_timer cpu; struct { struct alarm alarmtimer; } alarm; diff --git a/include/linux/timerqueue.h b/include/linux/timerqueue.h index aff122f1062a..93884086f392 100644 --- a/include/linux/timerqueue.h +++ b/include/linux/timerqueue.h @@ -43,6 +43,16 @@ static inline void timerqueue_init(struct timerqueue_node *node) RB_CLEAR_NODE(&node->node); } +static inline bool timerqueue_node_queued(struct timerqueue_node *node) +{ + return !RB_EMPTY_NODE(&node->node); +} + +static inline bool timerqueue_node_expires(struct timerqueue_node *node) +{ + return node->expires; +} + static inline void timerqueue_init_head(struct timerqueue_head *head) { head->rb_root = RB_ROOT_CACHED; diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 52f4c99c1d60..73c492ce404b 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -96,19 +96,19 @@ static inline int validate_clock_permissions(const clockid_t clock) * Update expiry time from increment, and increase overrun count, * given the current clock sample. */ -static void bump_cpu_timer(struct k_itimer *timer, u64 now) +static u64 bump_cpu_timer(struct k_itimer *timer, u64 now) { + u64 delta, incr, expires = timer->it.cpu.node.expires; int i; - u64 delta, incr; if (!timer->it_interval) - return; + return expires; - if (now < timer->it.cpu.expires) - return; + if (now < expires) + return expires; incr = timer->it_interval; - delta = now + incr - timer->it.cpu.expires; + delta = now + incr - expires; /* Don't use (incr*2 < delta), incr*2 might overflow. */ for (i = 0; incr < delta - incr; i++) @@ -118,10 +118,11 @@ static void bump_cpu_timer(struct k_itimer *timer, u64 now) if (delta < incr) continue; - timer->it.cpu.expires += incr; + timer->it.cpu.node.expires += incr; timer->it_overrun += 1LL << i; delta -= incr; } + return timer->it.cpu.node.expires; } /* Check whether all cache entries contain U64_MAX, i.e. eternal expiry time */ @@ -365,7 +366,7 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) return -EINVAL; new_timer->kclock = &clock_posix_cpu; - INIT_LIST_HEAD(&new_timer->it.cpu.entry); + timerqueue_init(&new_timer->it.cpu.node); new_timer->it.cpu.task = p; return 0; } @@ -378,10 +379,11 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) */ static int posix_cpu_timer_del(struct k_itimer *timer) { - int ret = 0; - unsigned long flags; + struct cpu_timer *ctmr = &timer->it.cpu; + struct task_struct *p = ctmr->task; struct sighand_struct *sighand; - struct task_struct *p = timer->it.cpu.task; + unsigned long flags; + int ret = 0; if (WARN_ON_ONCE(!p)) return -EINVAL; @@ -393,15 +395,15 @@ static int posix_cpu_timer_del(struct k_itimer *timer) sighand = lock_task_sighand(p, &flags); if (unlikely(sighand == NULL)) { /* - * We raced with the reaping of the task. - * The deletion should have cleared us off the list. + * This raced with the reaping of the task. The exit cleanup + * should have removed this timer from the timer queue. */ - WARN_ON_ONCE(!list_empty(&timer->it.cpu.entry)); + WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); } else { if (timer->it.cpu.firing) ret = TIMER_RETRY; else - list_del(&timer->it.cpu.entry); + cpu_timer_dequeue(ctmr); unlock_task_sighand(p, &flags); } @@ -412,12 +414,16 @@ static int posix_cpu_timer_del(struct k_itimer *timer) return ret; } -static void cleanup_timers_list(struct list_head *head) +static void cleanup_timerqueue(struct timerqueue_head *head) { - struct cpu_timer_list *timer, *next; + struct timerqueue_node *node; + struct cpu_timer *ctmr; - list_for_each_entry_safe(timer, next, head, entry) - list_del_init(&timer->entry); + while ((node = timerqueue_getnext(head))) { + timerqueue_del(head, node); + ctmr = container_of(node, struct cpu_timer, node); + ctmr->head = NULL; + } } /* @@ -429,9 +435,9 @@ static void cleanup_timers_list(struct list_head *head) */ static void cleanup_timers(struct posix_cputimers *pct) { - cleanup_timers_list(&pct->bases[CPUCLOCK_PROF].cpu_timers); - cleanup_timers_list(&pct->bases[CPUCLOCK_VIRT].cpu_timers); - cleanup_timers_list(&pct->bases[CPUCLOCK_SCHED].cpu_timers); + cleanup_timerqueue(&pct->bases[CPUCLOCK_PROF].tqhead); + cleanup_timerqueue(&pct->bases[CPUCLOCK_VIRT].tqhead); + cleanup_timerqueue(&pct->bases[CPUCLOCK_SCHED].tqhead); } /* @@ -454,28 +460,18 @@ void posix_cpu_timers_exit_group(struct task_struct *tsk) */ static void arm_timer(struct k_itimer *timer) { - struct cpu_timer_list *const nt = &timer->it.cpu; int clkidx = CPUCLOCK_WHICH(timer->it_clock); - struct task_struct *p = timer->it.cpu.task; - u64 newexp = timer->it.cpu.expires; + struct cpu_timer *ctmr = &timer->it.cpu; + u64 newexp = cpu_timer_getexpires(ctmr); + struct task_struct *p = ctmr->task; struct posix_cputimer_base *base; - struct list_head *head, *listpos; - struct cpu_timer_list *next; if (CPUCLOCK_PERTHREAD(timer->it_clock)) base = p->posix_cputimers.bases + clkidx; else base = p->signal->posix_cputimers.bases + clkidx; - listpos = head = &base->cpu_timers; - list_for_each_entry(next,head, entry) { - if (nt->expires < next->expires) - break; - listpos = &next->entry; - } - list_add(&nt->entry, listpos); - - if (listpos != head) + if (!cpu_timer_enqueue(&base->tqhead, ctmr)) return; /* @@ -498,24 +494,26 @@ static void arm_timer(struct k_itimer *timer) */ static void cpu_timer_fire(struct k_itimer *timer) { + struct cpu_timer *ctmr = &timer->it.cpu; + if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) { /* * User don't want any signal. */ - timer->it.cpu.expires = 0; + cpu_timer_setexpires(ctmr, 0); } else if (unlikely(timer->sigq == NULL)) { /* * This a special case for clock_nanosleep, * not a normal timer from sys_timer_create. */ wake_up_process(timer->it_process); - timer->it.cpu.expires = 0; + cpu_timer_setexpires(ctmr, 0); } else if (!timer->it_interval) { /* * One-shot timer. Clear it as soon as it's fired. */ posix_timer_event(timer, 0); - timer->it.cpu.expires = 0; + cpu_timer_setexpires(ctmr, 0); } else if (posix_timer_event(timer, ++timer->it_requeue_pending)) { /* * The signal did not get queued because the signal @@ -539,10 +537,11 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); u64 old_expires, new_expires, old_incr, val; - struct task_struct *p = timer->it.cpu.task; + struct cpu_timer *ctmr = &timer->it.cpu; + struct task_struct *p = ctmr->task; struct sighand_struct *sighand; unsigned long flags; - int ret; + int ret = 0; if (WARN_ON_ONCE(!p)) return -EINVAL; @@ -562,22 +561,21 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, * If p has just been reaped, we can no * longer get any information about it at all. */ - if (unlikely(sighand == NULL)) { + if (unlikely(sighand == NULL)) return -ESRCH; - } /* * Disarm any old timer after extracting its expiry time. */ - - ret = 0; old_incr = timer->it_interval; - old_expires = timer->it.cpu.expires; + old_expires = cpu_timer_getexpires(ctmr); + if (unlikely(timer->it.cpu.firing)) { timer->it.cpu.firing = -1; ret = TIMER_RETRY; - } else - list_del_init(&timer->it.cpu.entry); + } else { + cpu_timer_dequeue(ctmr); + } /* * We need to sample the current value to convert the new @@ -598,18 +596,16 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, old->it_value.tv_nsec = 0; } else { /* - * Update the timer in case it has - * overrun already. If it has, - * we'll report it as having overrun - * and with the next reloaded timer - * already ticking, though we are - * swallowing that pending - * notification here to install the - * new setting. + * Update the timer in case it has overrun already. + * If it has, we'll report it as having overrun and + * with the next reloaded timer already ticking, + * though we are swallowing that pending + * notification here to install the new setting. */ - bump_cpu_timer(timer, val); - if (val < timer->it.cpu.expires) { - old_expires = timer->it.cpu.expires - val; + u64 exp = bump_cpu_timer(timer, val); + + if (val < exp) { + old_expires = exp - val; old->it_value = ns_to_timespec64(old_expires); } else { old->it_value.tv_nsec = 1; @@ -638,7 +634,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, * For a timer with no notification action, we don't actually * arm the timer (we'll just fake it for timer_gettime). */ - timer->it.cpu.expires = new_expires; + cpu_timer_setexpires(ctmr, new_expires); if (new_expires != 0 && val < new_expires) { arm_timer(timer); } @@ -680,8 +676,9 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - struct task_struct *p = timer->it.cpu.task; - u64 now; + struct cpu_timer *ctmr = &timer->it.cpu; + u64 now, expires = cpu_timer_getexpires(ctmr); + struct task_struct *p = ctmr->task; if (WARN_ON_ONCE(!p)) return; @@ -691,7 +688,7 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp */ itp->it_interval = ktime_to_timespec64(timer->it_interval); - if (!timer->it.cpu.expires) + if (!expires) return; /* @@ -713,9 +710,9 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp /* * The process has been reaped. * We can't even collect a sample any more. - * Call the timer disarmed, nothing else to do. + * Disarm the timer, nothing else to do. */ - timer->it.cpu.expires = 0; + cpu_timer_setexpires(ctmr, 0); return; } else { now = cpu_clock_sample_group(clkid, p, false); @@ -723,8 +720,8 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp } } - if (now < timer->it.cpu.expires) { - itp->it_value = ns_to_timespec64(timer->it.cpu.expires - now); + if (now < expires) { + itp->it_value = ns_to_timespec64(expires - now); } else { /* * The timer should have expired already, but the firing @@ -735,37 +732,41 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp } } -static unsigned long long -check_timers_list(struct list_head *timers, - struct list_head *firing, - unsigned long long curr) -{ - int maxfire = 20; - - while (!list_empty(timers)) { - struct cpu_timer_list *t; - - t = list_first_entry(timers, struct cpu_timer_list, entry); +#define MAX_COLLECTED 20 - if (!--maxfire || curr < t->expires) - return t->expires; - - t->firing = 1; - list_move_tail(&t->entry, firing); +static u64 collect_timerqueue(struct timerqueue_head *head, + struct list_head *firing, u64 now) +{ + struct timerqueue_node *next; + int i = 0; + + while ((next = timerqueue_getnext(head))) { + struct cpu_timer *ctmr; + u64 expires; + + ctmr = container_of(next, struct cpu_timer, node); + expires = cpu_timer_getexpires(ctmr); + /* Limit the number of timers to expire at once */ + if (++i == MAX_COLLECTED || now < expires) + return expires; + + ctmr->firing = 1; + cpu_timer_dequeue(ctmr); + list_add_tail(&ctmr->elist, firing); } return U64_MAX; } -static void collect_posix_cputimers(struct posix_cputimers *pct, - u64 *samples, struct list_head *firing) +static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples, + struct list_head *firing) { struct posix_cputimer_base *base = pct->bases; int i; for (i = 0; i < CPUCLOCK_MAX; i++, base++) { - base->nextevt = check_timers_list(&base->cpu_timers, firing, - samples[i]); + base->nextevt = collect_timerqueue(&base->tqhead, firing, + samples[i]); } } @@ -948,7 +949,8 @@ static void check_process_timers(struct task_struct *tsk, static void posix_cpu_timer_rearm(struct k_itimer *timer) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - struct task_struct *p = timer->it.cpu.task; + struct cpu_timer *ctmr = &timer->it.cpu; + struct task_struct *p = ctmr->task; struct sighand_struct *sighand; unsigned long flags; u64 now; @@ -980,7 +982,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer) * The process has been reaped. * We can't even collect a sample any more. */ - timer->it.cpu.expires = 0; + cpu_timer_setexpires(ctmr, 0); return; } else if (unlikely(p->exit_state) && thread_group_empty(p)) { /* If the process is dying, no need to rearm */ @@ -1124,11 +1126,11 @@ void run_posix_cpu_timers(void) * each timer's lock before clearing its firing flag, so no * timer call will interfere. */ - list_for_each_entry_safe(timer, next, &firing, it.cpu.entry) { + list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) { int cpu_firing; spin_lock(&timer->it_lock); - list_del_init(&timer->it.cpu.entry); + list_del_init(&timer->it.cpu.elist); cpu_firing = timer->it.cpu.firing; timer->it.cpu.firing = 0; /* @@ -1204,6 +1206,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, timer.it_overrun = -1; error = posix_cpu_timer_create(&timer); timer.it_process = current; + if (!error) { static struct itimerspec64 zero_it; struct restart_block *restart; @@ -1219,7 +1222,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, } while (!signal_pending(current)) { - if (timer.it.cpu.expires == 0) { + if (!cpu_timer_getexpires(&timer.it.cpu)) { /* * Our timer fired and was reset, below * deletion can not fail. @@ -1241,7 +1244,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, /* * We were interrupted by a signal. */ - expires = timer.it.cpu.expires; + expires = cpu_timer_getexpires(&timer.it.cpu); error = posix_cpu_timer_set(&timer, 0, &zero_it, &it); if (!error) { /* -- cgit v1.2.3-71-gd317 From a67e408241783575716fcf3f79d0878f6cef0273 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 23 Aug 2019 13:38:44 +0200 Subject: hrtimer: Add kernel doc annotation for HRTIMER_MODE_HARD Add kernel doc annotation for HRTIMER_MODE_HARD. Fixes: ae6683d815895 ("hrtimer: Introduce HARD expiry mode") Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20190823113845.12125-2-bigeasy@linutronix.de --- include/linux/hrtimer.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 5df4bcff96d5..1b9a51a1bccb 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -32,6 +32,8 @@ struct hrtimer_cpu_base; * when starting the timer) * HRTIMER_MODE_SOFT - Timer callback function will be executed in * soft irq context + * HRTIMER_MODE_HARD - Timer callback function will be executed in + * hard irq context even on PREEMPT_RT. */ enum hrtimer_mode { HRTIMER_MODE_ABS = 0x00, -- cgit v1.2.3-71-gd317 From 8f2edb4a78f7f5fa35c025849152b1d2dfaee4eb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 29 Aug 2019 08:19:40 +0200 Subject: posix-timers: Unbreak CONFIG_POSIX_TIMERS=n build The rework of the posix-cpu-timers patch series dropped the empty declaration of struct cpu_timer for the CONFIG_POSIX_TIMERS=n case which causes the build to fail: ./include/linux/posix-timers.h:218:20: error: field 'cpu' has incomplete type Add it back. Fixes: 60bda037f1dd ("posix-cpu-timers: Utilize timerqueue for storage") Reported-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/linux/posix-timers.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index f9fbb4c620c1..e685916551bf 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -161,6 +161,7 @@ static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, }, #else struct posix_cputimers { }; +struct cpu_timer { }; #define INIT_CPU_TIMERS(s) static inline void posix_cputimers_init(struct posix_cputimers *pct) { } static inline void posix_cputimers_group_init(struct posix_cputimers *pct, -- cgit v1.2.3-71-gd317 From 00d9e47f8ec2a293db9ebed86aab0583d9a49533 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 5 Sep 2019 14:03:40 +0200 Subject: posix-cpu-timers: Always clear head pointer on dequeue The head pointer in struct cpu_timer is checked to be NULL in posix_cpu_timer_del() when the delete raced with the exit cleanup. The works correctly as long as the timer is actually dequeued via posix_cpu_timers_exit*(). But if the timer was dequeued due to expiry the head pointer is still set and triggers the warning. In fact keeping the head pointer around after any dequeue is pointless as is has no meaning at all after that. Clear the head pointer always on dequeue and remove the unused requeue function while at it. Fixes: 60bda037f1dd ("posix-cpu-timers: Utilize timerqueue for storage") Reported-by: syzbot+55acd54b57bb4b3840a4@syzkaller.appspotmail.com Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lkml.kernel.org/r/20190905120539.707986830@linutronix.de --- include/linux/posix-timers.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index e685916551bf..3d10c84a97a9 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -74,11 +74,6 @@ struct cpu_timer { int firing; }; -static inline bool cpu_timer_requeue(struct cpu_timer *ctmr) -{ - return timerqueue_add(ctmr->head, &ctmr->node); -} - static inline bool cpu_timer_enqueue(struct timerqueue_head *head, struct cpu_timer *ctmr) { @@ -88,8 +83,10 @@ static inline bool cpu_timer_enqueue(struct timerqueue_head *head, static inline void cpu_timer_dequeue(struct cpu_timer *ctmr) { - if (!RB_EMPTY_NODE(&ctmr->node.node)) + if (ctmr->head) { timerqueue_del(ctmr->head, &ctmr->node); + ctmr->head = NULL; + } } static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr) -- cgit v1.2.3-71-gd317