Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland...

[linux-2.6] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f4fc867f467d71b8dae84bf2e3f0fffbab955c64..b416995b975727ca8b6dbf8b331de5351a1ef7f5 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -167,9 +167,15 @@ void tick_nohz_stop_sched_tick(void)
                goto end;
 
        cpu = smp_processor_id();
-       if (unlikely(local_softirq_pending()))
-               printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
-                      local_softirq_pending());
+       if (unlikely(local_softirq_pending())) {
+               static int ratelimit;
+
+               if (ratelimit < 10) {
+                       printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
+                              local_softirq_pending());
+                       ratelimit++;
+               }
+       }
 
        now = ktime_get();
        /*
@@ -217,6 +223,14 @@ void tick_nohz_stop_sched_tick(void)
                 * the scheduler tick in nohz_restart_sched_tick.
                 */
                if (!ts->tick_stopped) {
+                       if (select_nohz_load_balancer(1)) {
+                               /*
+                                * sched tick not stopped!
+                                */
+                               cpu_clear(cpu, nohz_cpu_mask);
+                               goto out;
+                       }
+
                        ts->idle_tick = ts->sched_timer.expires;
                        ts->tick_stopped = 1;
                        ts->idle_jiffies = last_jiffies;
@@ -233,6 +247,21 @@ void tick_nohz_stop_sched_tick(void)
                if (cpu == tick_do_timer_cpu)
                        tick_do_timer_cpu = -1;
 
+               ts->idle_sleeps++;
+
+               /*
+                * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
+                * there is no timer pending or at least extremly far
+                * into the future (12 days for HZ=1000). In this case
+                * we simply stop the tick timer:
+                */
+               if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) {
+                       ts->idle_expires.tv64 = KTIME_MAX;
+                       if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
+                               hrtimer_cancel(&ts->sched_timer);
+                       goto out;
+               }
+
                /*
                 * calculate the expiry time for the next timer wheel
                 * timer
@@ -240,7 +269,6 @@ void tick_nohz_stop_sched_tick(void)
                expires = ktime_add_ns(last_update, tick_period.tv64 *
                                       delta_jiffies);
                ts->idle_expires = expires;
-               ts->idle_sleeps++;
 
                if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
                        hrtimer_start(&ts->sched_timer, expires,
@@ -285,6 +313,7 @@ void tick_nohz_restart_sched_tick(void)
        now = ktime_get();
 
        local_irq_disable();
+       select_nohz_load_balancer(0);
        tick_do_update_jiffies64(now);
        cpu_clear(cpu, nohz_cpu_mask);
 
@@ -517,6 +546,7 @@ void tick_setup_sched_timer(void)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
        ktime_t now = ktime_get();
+       u64 offset;
 
        /*
         * Emulate tick processing via per-CPU hrtimers:
@@ -525,8 +555,12 @@ void tick_setup_sched_timer(void)
        ts->sched_timer.function = tick_sched_timer;
        ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
 
-       /* Get the next period */
+       /* Get the next period (per cpu) */
        ts->sched_timer.expires = tick_init_jiffy_update();
+       offset = ktime_to_ns(tick_period) >> 1;
+       do_div(offset, NR_CPUS);
+       offset *= smp_processor_id();
+       ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset);
 
        for (;;) {
                hrtimer_forward(&ts->sched_timer, now, tick_period);