Merge branch 'upstream-linus' of master.kernel.org:/pub/scm/linux/kernel/git/jgarzik...

[linux-2.6] / kernel / hrtimer.c

diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 62aad8e1a3835c9e390ff93e383dbceff3755d4d..067ba2c0532805f8e9e3e753eba456d6a7cbf1b9 100644 (file)
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -135,7 +135,7 @@ EXPORT_SYMBOL_GPL(ktime_get_ts);
 static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)
 {
        ktime_t xtim, tomono;
-       struct timespec xts;
+       struct timespec xts, tom;
        unsigned long seq;
 
        do {
@@ -145,10 +145,11 @@ static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)
 #else
                xts = xtime;
 #endif
+               tom = wall_to_monotonic;
        } while (read_seqretry(&xtime_lock, seq));
 
        xtim = timespec_to_ktime(xts);
-       tomono = timespec_to_ktime(wall_to_monotonic);
+       tomono = timespec_to_ktime(tom);
        base->clock_base[CLOCK_REALTIME].softirq_time = xtim;
        base->clock_base[CLOCK_MONOTONIC].softirq_time =
                ktime_add(xtim, tomono);
@@ -540,19 +541,19 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
 /*
  * Switch to high resolution mode
  */
-static void hrtimer_switch_to_hres(void)
+static int hrtimer_switch_to_hres(void)
 {
        struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases);
        unsigned long flags;
 
        if (base->hres_active)
-               return;
+               return 1;
 
        local_irq_save(flags);
 
        if (tick_init_highres()) {
                local_irq_restore(flags);
-               return;
+               return 0;
        }
        base->hres_active = 1;
        base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES;
@@ -565,13 +566,14 @@ static void hrtimer_switch_to_hres(void)
        local_irq_restore(flags);
        printk(KERN_INFO "Switched to high resolution mode on CPU %d\n",
               smp_processor_id());
+       return 1;
 }
 
 #else
 
 static inline int hrtimer_hres_active(void) { return 0; }
 static inline int hrtimer_is_hres_enabled(void) { return 0; }
-static inline void hrtimer_switch_to_hres(void) { }
+static inline int hrtimer_switch_to_hres(void) { return 0; }
 static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { }
 static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
                                            struct hrtimer_clock_base *base)
@@ -585,6 +587,18 @@ static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { }
 
 #endif /* CONFIG_HIGH_RES_TIMERS */
 
+#ifdef CONFIG_TIMER_STATS
+void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr)
+{
+       if (timer->start_site)
+               return;
+
+       timer->start_site = addr;
+       memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
+       timer->start_pid = current->pid;
+}
+#endif
+
 /*
  * Counterpart to lock_timer_base above:
  */
@@ -631,6 +645,12 @@ hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
                orun++;
        }
        timer->expires = ktime_add(timer->expires, interval);
+       /*
+        * Make sure, that the result did not wrap with a very large
+        * interval.
+        */
+       if (timer->expires.tv64 < 0)
+               timer->expires = ktime_set(KTIME_SEC_MAX, 0);
 
        return orun;
 }
@@ -743,6 +763,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
                 * reprogramming happens in the interrupt handler. This is a
                 * rare case and less expensive than a smp call.
                 */
+               timer_stats_hrtimer_clear_start_info(timer);
                reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);
                __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,
                                 reprogram);
@@ -791,7 +812,14 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
        }
        timer->expires = tim;
 
-       enqueue_hrtimer(timer, new_base, base == new_base);
+       timer_stats_hrtimer_set_start_info(timer);
+
+       /*
+        * Only allow reprogramming if the new base is on this CPU.
+        * (it might still be on another CPU if the timer was pending)
+        */
+       enqueue_hrtimer(timer, new_base,
+                       new_base->cpu_base == &__get_cpu_var(hrtimer_bases));
 
        unlock_hrtimer_base(timer, &flags);
 
@@ -925,6 +953,12 @@ void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
 
        timer->base = &cpu_base->clock_base[clock_id];
        hrtimer_init_timer_hres(timer);
+
+#ifdef CONFIG_TIMER_STATS
+       timer->start_site = NULL;
+       timer->start_pid = -1;
+       memset(timer->start_comm, 0, TASK_COMM_LEN);
+#endif
 }
 EXPORT_SYMBOL_GPL(hrtimer_init);
 
@@ -1006,6 +1040,7 @@ void hrtimer_interrupt(struct clock_event_device *dev)
 
                        __remove_hrtimer(timer, base,
                                         HRTIMER_STATE_CALLBACK, 0);
+                       timer_stats_account_hrtimer(timer);
 
                        /*
                         * Note: We clear the CALLBACK bit after
@@ -1050,6 +1085,8 @@ static void run_hrtimer_softirq(struct softirq_action *h)
                timer = list_entry(cpu_base->cb_pending.next,
                                   struct hrtimer, cb_entry);
 
+               timer_stats_account_hrtimer(timer);
+
                fn = timer->function;
                __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0);
                spin_unlock_irq(&cpu_base->lock);
@@ -1106,6 +1143,11 @@ static inline void run_hrtimer_queue(struct hrtimer_cpu_base *cpu_base,
                if (base->softirq_time.tv64 <= timer->expires.tv64)
                        break;
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+               WARN_ON_ONCE(timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ);
+#endif
+               timer_stats_account_hrtimer(timer);
+
                fn = timer->function;
                __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
                spin_unlock_irq(&cpu_base->lock);
@@ -1147,7 +1189,8 @@ void hrtimer_run_queues(void)
         * deadlock vs. xtime_lock.
         */
        if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
-               hrtimer_switch_to_hres();
+               if (hrtimer_switch_to_hres())
+                       return;
 
        hrtimer_get_softirq_time(cpu_base);
 
@@ -1329,17 +1372,16 @@ static void migrate_hrtimers(int cpu)
        tick_cancel_sched_timer(cpu);
 
        local_irq_disable();
-
-       spin_lock(&new_base->lock);
-       spin_lock(&old_base->lock);
+       double_spin_lock(&new_base->lock, &old_base->lock,
+                        smp_processor_id() < cpu);
 
        for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
                migrate_hrtimer_list(&old_base->clock_base[i],
                                     &new_base->clock_base[i]);
        }
-       spin_unlock(&old_base->lock);
-       spin_unlock(&new_base->lock);
 
+       double_spin_unlock(&new_base->lock, &old_base->lock,
+                          smp_processor_id() < cpu);
        local_irq_enable();
        put_cpu_var(hrtimer_bases);
 }