x86: idle wakeup event in the HLT loop

[linux-2.6] / arch / x86 / kernel / process_64.c

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 6309b275cb9c176f61133fa7fc2b5585c9063fd4..40fed477f3e5e488030a12735bcaa3b016dda28d 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -116,9 +116,16 @@ static void default_idle(void)
        smp_mb();
        local_irq_disable();
        if (!need_resched()) {
-               /* Enables interrupts one instruction before HLT.
-                  x86 special cases this so there is no race. */
-               safe_halt();
+               ktime_t t0, t1;
+               u64 t0n, t1n;
+
+               t0 = ktime_get();
+               t0n = ktime_to_ns(t0);
+               safe_halt();    /* enables interrupts racelessly */
+               local_irq_disable();
+               t1 = ktime_get();
+               t1n = ktime_to_ns(t1);
+               sched_clock_idle_wakeup_event(t1n - t0n);
        } else
                local_irq_enable();
        current_thread_info()->status |= TS_POLLING;
@@ -135,6 +142,10 @@ static void poll_idle (void)
        cpu_relax();
 }
 
+static void do_nothing(void *unused)
+{
+}
+
 void cpu_idle_wait(void)
 {
        unsigned int cpu, this_cpu = get_cpu();
@@ -160,6 +171,13 @@ void cpu_idle_wait(void)
                                cpu_clear(cpu, map);
                }
                cpus_and(map, map, cpu_online_map);
+               /*
+                * We waited 1 sec, if a CPU still did not call idle
+                * it may be because it is in idle and not waking up
+                * because it has nothing to do.
+                * Give all the remaining CPUS a kick.
+                */
+               smp_call_function_mask(map, do_nothing, 0, 0);
        } while (!cpus_empty(map));
 
        set_cpus_allowed(current, tmp);
@@ -196,7 +214,7 @@ static inline void play_dead(void)
  * low exit latency (ie sit in a loop waiting for
  * somebody to say that they'd like to reschedule)
  */
-void cpu_idle (void)
+void cpu_idle(void)
 {
        current_thread_info()->status |= TS_POLLING;
        /* endless idle loop with no priority at all */