From 24e83d8e6fca6b689e067de7c85c17d81bbcd357 Mon Sep 17 00:00:00 2001 From: wanxiaoqing Date: Wed, 17 Jan 2024 15:53:30 +0800 Subject: [PATCH 1/6] Revert "tick/common: Align tick period with the HZ tick." This reverts commit b70fdd8acb46c638d9eb00820908daddc65a0802. Signed-off-by: wanxiaoqing --- kernel/time/tick-common.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 2b7448ae5b47..92bf99d558b4 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c @@ -216,19 +216,9 @@ static void tick_setup_device(struct tick_device *td, * this cpu: */ if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) { - ktime_t next_p; - u32 rem; - tick_do_timer_cpu = cpu; - next_p = ktime_get(); - div_u64_rem(next_p, TICK_NSEC, &rem); - if (rem) { - next_p -= rem; - next_p += TICK_NSEC; - } - - tick_next_period = next_p; + tick_next_period = ktime_get(); #ifdef CONFIG_NO_HZ_FULL /* * The boot CPU may be nohz_full, in which case set -- Gitee From b14374edb6c59194485da1558fefda252d9455b3 Mon Sep 17 00:00:00 2001 From: wanxiaoqing Date: Wed, 17 Jan 2024 15:53:45 +0800 Subject: [PATCH 2/6] Revert "tick: Get rid of tick_period" This reverts commit 9f90afdb5a379b980e77a5d1cd3555b0de7cc499. Signed-off-by: wanxiaoqing --- kernel/time/tick-broadcast.c | 2 +- kernel/time/tick-common.c | 8 +++++--- kernel/time/tick-internal.h | 1 + kernel/time/tick-sched.c | 22 +++++++++++----------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index a9530e866e5f..36d7464c8962 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -331,7 +331,7 @@ static void tick_handle_periodic_broadcast(struct clock_event_device *dev) bc_local = tick_do_periodic_broadcast(); if (clockevent_state_oneshot(dev)) { - ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC); + ktime_t next = ktime_add(dev->next_event, tick_period); clockevents_program_event(dev, next, true); } diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 92bf99d558b4..6c9c342dd0e5 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c @@ -30,6 +30,7 @@ DEFINE_PER_CPU(struct tick_device, tick_cpu_device); * Tick next event: keeps track of the tick time */ ktime_t tick_next_period; +ktime_t tick_period; /* * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR @@ -87,7 +88,7 @@ static void tick_periodic(int cpu) write_seqcount_begin(&jiffies_seq); /* Keep track of the next tick event */ - tick_next_period = ktime_add_ns(tick_next_period, TICK_NSEC); + tick_next_period = ktime_add(tick_next_period, tick_period); do_timer(1); write_seqcount_end(&jiffies_seq); @@ -126,7 +127,7 @@ void tick_handle_periodic(struct clock_event_device *dev) * Setup the next period for devices, which do not have * periodic mode: */ - next = ktime_add_ns(next, TICK_NSEC); + next = ktime_add(next, tick_period); if (!clockevents_program_event(dev, next, false)) return; @@ -172,7 +173,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) for (;;) { if (!clockevents_program_event(dev, next, false)) return; - next = ktime_add_ns(next, TICK_NSEC); + next = ktime_add(next, tick_period); } } } @@ -219,6 +220,7 @@ static void tick_setup_device(struct tick_device *td, tick_do_timer_cpu = cpu; tick_next_period = ktime_get(); + tick_period = NSEC_PER_SEC / HZ; #ifdef CONFIG_NO_HZ_FULL /* * The boot CPU may be nohz_full, in which case set diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index e61c1244e7d4..5294f5b1f955 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -15,6 +15,7 @@ DECLARE_PER_CPU(struct tick_device, tick_cpu_device); extern ktime_t tick_next_period; +extern ktime_t tick_period; extern int tick_do_timer_cpu __read_mostly; extern void tick_setup_periodic(struct clock_event_device *dev, int broadcast); diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 17dc3f53efef..5c3d4355266d 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -92,17 +92,17 @@ static void tick_do_update_jiffies64(ktime_t now) write_seqcount_begin(&jiffies_seq); delta = ktime_sub(now, tick_next_period); - if (unlikely(delta >= TICK_NSEC)) { + if (unlikely(delta >= tick_period)) { /* Slow path for long idle sleep times */ - s64 incr = TICK_NSEC; + s64 incr = ktime_to_ns(tick_period); ticks += ktime_divns(delta, incr); last_jiffies_update = ktime_add_ns(last_jiffies_update, incr * ticks); } else { - last_jiffies_update = ktime_add_ns(last_jiffies_update, - TICK_NSEC); + last_jiffies_update = ktime_add(last_jiffies_update, + tick_period); } do_timer(ticks); @@ -112,7 +112,7 @@ static void tick_do_update_jiffies64(ktime_t now) * pairs with the READ_ONCE() in the lockless quick check above. */ WRITE_ONCE(tick_next_period, - ktime_add_ns(last_jiffies_update, TICK_NSEC)); + ktime_add(last_jiffies_update, tick_period)); write_seqcount_end(&jiffies_seq); raw_spin_unlock(&jiffies_lock); @@ -688,7 +688,7 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) hrtimer_set_expires(&ts->sched_timer, ts->last_tick); /* Forward the time to expire in the future */ - hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); + hrtimer_forward(&ts->sched_timer, now, tick_period); if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { hrtimer_start_expires(&ts->sched_timer, @@ -1250,7 +1250,7 @@ static void tick_nohz_handler(struct clock_event_device *dev) if (unlikely(ts->tick_stopped)) return; - hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); + hrtimer_forward(&ts->sched_timer, now, tick_period); tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); } @@ -1287,7 +1287,7 @@ static void tick_nohz_switch_to_nohz(void) next = tick_init_jiffy_update(); hrtimer_set_expires(&ts->sched_timer, next); - hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); + hrtimer_forward_now(&ts->sched_timer, tick_period); tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); tick_nohz_activate(ts, NOHZ_MODE_LOWRES); } @@ -1353,7 +1353,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) if (unlikely(ts->tick_stopped)) return HRTIMER_NORESTART; - hrtimer_forward(timer, now, TICK_NSEC); + hrtimer_forward(timer, now, tick_period); return HRTIMER_RESTART; } @@ -1387,13 +1387,13 @@ void tick_setup_sched_timer(void) /* Offset the tick to avert jiffies_lock contention. */ if (sched_skew_tick) { - u64 offset = TICK_NSEC >> 1; + u64 offset = ktime_to_ns(tick_period) >> 1; do_div(offset, num_possible_cpus()); offset *= smp_processor_id(); hrtimer_add_expires_ns(&ts->sched_timer, offset); } - hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); + hrtimer_forward(&ts->sched_timer, now, tick_period); hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); tick_nohz_activate(ts, NOHZ_MODE_HIGHRES); } -- Gitee From 31084e66bba16c4f6d69fa9b446f38b00c3c5850 Mon Sep 17 00:00:00 2001 From: wanxiaoqing Date: Wed, 17 Jan 2024 15:54:01 +0800 Subject: [PATCH 3/6] Revert "tick/sched: Optimize tick_do_update_jiffies64() further" This reverts commit 67a5bfb2e91df724eb00b89614eeec4a82712f20. Signed-off-by: wanxiaoqing --- kernel/time/tick-sched.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 5c3d4355266d..ac9953f6f92c 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -53,7 +53,7 @@ static ktime_t last_jiffies_update; */ static void tick_do_update_jiffies64(ktime_t now) { - unsigned long ticks = 1; + unsigned long ticks = 0; ktime_t delta; /* @@ -91,21 +91,20 @@ static void tick_do_update_jiffies64(ktime_t now) write_seqcount_begin(&jiffies_seq); + last_jiffies_update = ktime_add(last_jiffies_update, tick_period); + delta = ktime_sub(now, tick_next_period); if (unlikely(delta >= tick_period)) { /* Slow path for long idle sleep times */ s64 incr = ktime_to_ns(tick_period); - ticks += ktime_divns(delta, incr); + ticks = ktime_divns(delta, incr); last_jiffies_update = ktime_add_ns(last_jiffies_update, incr * ticks); - } else { - last_jiffies_update = ktime_add(last_jiffies_update, - tick_period); } - do_timer(ticks); + do_timer(++ticks); /* * Keep the tick_next_period variable up to date. WRITE_ONCE() -- Gitee From ae22659bc00b6e6c438382c9baf2ac472706ce80 Mon Sep 17 00:00:00 2001 From: wanxiaoqing Date: Wed, 17 Jan 2024 15:54:27 +0800 Subject: [PATCH 4/6] Revert "tick/sched: Reduce seqcount held scope in tick_do_update_jiffies64()" This reverts commit f06130ad9cba912454a20103cafca5f49a2ede0e. Signed-off-by: wanxiaoqing --- kernel/time/tick-sched.c | 47 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index ac9953f6f92c..4f1b6170b3a2 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -84,35 +84,38 @@ static void tick_do_update_jiffies64(ktime_t now) /* Reevaluate with jiffies_lock held */ raw_spin_lock(&jiffies_lock); - if (ktime_before(now, tick_next_period)) { - raw_spin_unlock(&jiffies_lock); - return; - } - write_seqcount_begin(&jiffies_seq); - last_jiffies_update = ktime_add(last_jiffies_update, tick_period); + delta = ktime_sub(now, last_jiffies_update); + if (delta >= tick_period) { - delta = ktime_sub(now, tick_next_period); - if (unlikely(delta >= tick_period)) { - /* Slow path for long idle sleep times */ - s64 incr = ktime_to_ns(tick_period); + delta = ktime_sub(delta, tick_period); + last_jiffies_update = ktime_add(last_jiffies_update, + tick_period); - ticks = ktime_divns(delta, incr); + /* Slow path for long timeouts */ + if (unlikely(delta >= tick_period)) { + s64 incr = ktime_to_ns(tick_period); - last_jiffies_update = ktime_add_ns(last_jiffies_update, - incr * ticks); - } - - do_timer(++ticks); + ticks = ktime_divns(delta, incr); - /* - * Keep the tick_next_period variable up to date. WRITE_ONCE() - * pairs with the READ_ONCE() in the lockless quick check above. - */ - WRITE_ONCE(tick_next_period, - ktime_add(last_jiffies_update, tick_period)); + last_jiffies_update = ktime_add_ns(last_jiffies_update, + incr * ticks); + } + do_timer(++ticks); + /* + * Keep the tick_next_period variable up to date. + * WRITE_ONCE() pairs with the READ_ONCE() in the lockless + * quick check above. + */ + WRITE_ONCE(tick_next_period, + ktime_add(last_jiffies_update, tick_period)); + } else { + write_seqcount_end(&jiffies_seq); + raw_spin_unlock(&jiffies_lock); + return; + } write_seqcount_end(&jiffies_seq); raw_spin_unlock(&jiffies_lock); update_wall_time(); -- Gitee From 0b2b787652a3099e85fc18fff9878cf50a09ffa2 Mon Sep 17 00:00:00 2001 From: wanxiaoqing Date: Wed, 17 Jan 2024 15:54:42 +0800 Subject: [PATCH 5/6] Revert "tick/sched: Use tick_next_period for lockless quick check" This reverts commit db43cb81292559784a141a27432414026977dad7. Signed-off-by: wanxiaoqing --- kernel/time/tick-sched.c | 46 ++++++++++++---------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 4f1b6170b3a2..378096fc8c56 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -57,29 +57,11 @@ static void tick_do_update_jiffies64(ktime_t now) ktime_t delta; /* - * Do a quick check without holding jiffies_lock. The READ_ONCE() - * pairs with the update done later in this function. - * - * This is also an intentional data race which is even safe on - * 32bit in theory. If there is a concurrent update then the check - * might give a random answer. It does not matter because if it - * returns then the concurrent update is already taking care, if it - * falls through then it will pointlessly contend on jiffies_lock. - * - * Though there is one nasty case on 32bit due to store tearing of - * the 64bit value. If the first 32bit store makes the quick check - * return on all other CPUs and the writing CPU context gets - * delayed to complete the second store (scheduled out on virt) - * then jiffies can become stale for up to ~2^32 nanoseconds - * without noticing. After that point all CPUs will wait for - * jiffies lock. - * - * OTOH, this is not any different than the situation with NOHZ=off - * where one CPU is responsible for updating jiffies and - * timekeeping. If that CPU goes out for lunch then all other CPUs - * will operate on stale jiffies until it decides to come back. + * Do a quick check without holding jiffies_lock: + * The READ_ONCE() pairs with two updates done later in this function. */ - if (ktime_before(now, READ_ONCE(tick_next_period))) + delta = ktime_sub(now, READ_ONCE(last_jiffies_update)); + if (delta < tick_period) return; /* Reevaluate with jiffies_lock held */ @@ -90,8 +72,9 @@ static void tick_do_update_jiffies64(ktime_t now) if (delta >= tick_period) { delta = ktime_sub(delta, tick_period); - last_jiffies_update = ktime_add(last_jiffies_update, - tick_period); + /* Pairs with the lockless read in this function. */ + WRITE_ONCE(last_jiffies_update, + ktime_add(last_jiffies_update, tick_period)); /* Slow path for long timeouts */ if (unlikely(delta >= tick_period)) { @@ -99,18 +82,15 @@ static void tick_do_update_jiffies64(ktime_t now) ticks = ktime_divns(delta, incr); - last_jiffies_update = ktime_add_ns(last_jiffies_update, - incr * ticks); + /* Pairs with the lockless read in this function. */ + WRITE_ONCE(last_jiffies_update, + ktime_add_ns(last_jiffies_update, + incr * ticks)); } do_timer(++ticks); - /* - * Keep the tick_next_period variable up to date. - * WRITE_ONCE() pairs with the READ_ONCE() in the lockless - * quick check above. - */ - WRITE_ONCE(tick_next_period, - ktime_add(last_jiffies_update, tick_period)); + /* Keep the tick_next_period variable up to date */ + tick_next_period = ktime_add(last_jiffies_update, tick_period); } else { write_seqcount_end(&jiffies_seq); raw_spin_unlock(&jiffies_lock); -- Gitee From efdffde72119ba3001647de092d0a7b835c0707c Mon Sep 17 00:00:00 2001 From: wanxiaoqing Date: Wed, 17 Jan 2024 15:55:01 +0800 Subject: [PATCH 6/6] Revert "rcu: Fix missing TICK_DEP_MASK_RCU_EXP dependency check" This reverts commit 3ea9d8bec944cc98119d1ee206bb7d041a37bec3. Signed-off-by: wanxiaoqing --- include/trace/events/timer.h | 3 +-- kernel/time/tick-sched.c | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h index 40e9b5a12732..19abb6c3eb73 100644 --- a/include/trace/events/timer.h +++ b/include/trace/events/timer.h @@ -368,8 +368,7 @@ TRACE_EVENT(itimer_expire, tick_dep_name(PERF_EVENTS) \ tick_dep_name(SCHED) \ tick_dep_name(CLOCK_UNSTABLE) \ - tick_dep_name(RCU) \ - tick_dep_name_end(RCU_EXP) + tick_dep_name_end(RCU) #undef tick_dep_name #undef tick_dep_mask_name diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 378096fc8c56..e4e0d032126b 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -213,11 +213,6 @@ static bool check_tick_dependency(atomic_t *dep) return true; } - if (val & TICK_DEP_MASK_RCU_EXP) { - trace_tick_stop(0, TICK_DEP_MASK_RCU_EXP); - return true; - } - return false; } -- Gitee