x86: scale cyc_2_nsec according to CPU frequency

[linux-2.6] / arch / x86 / kernel / tsc_32.c

diff --git a/arch/x86/kernel/tsc_32.c b/arch/x86/kernel/tsc_32.c
index 9ebc0dab66b4ced5720818ce600b2dd14ccf0e18..00bb4c1c05935f60adf9a7f88b6c757428dc5d23 100644 (file)
--- a/arch/x86/kernel/tsc_32.c
+++ b/arch/x86/kernel/tsc_32.c
@@ -5,6 +5,7 @@
 #include <linux/jiffies.h>
 #include <linux/init.h>
 #include <linux/dmi.h>
+#include <linux/percpu.h>
 
 #include <asm/delay.h>
 #include <asm/tsc.h>
@@ -80,13 +81,31 @@ EXPORT_SYMBOL_GPL(check_tsc_unstable);
  *
  *                     -johnstul@us.ibm.com "math is hard, lets go shopping!"
  */
-unsigned long cyc2ns_scale __read_mostly;
 
-#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
+DEFINE_PER_CPU(unsigned long, cyc2ns);
 
-static inline void set_cyc2ns_scale(unsigned long cpu_khz)
+static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
 {
-       cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
+       unsigned long flags, prev_scale, *scale;
+       unsigned long long tsc_now, ns_now;
+
+       local_irq_save(flags);
+       sched_clock_idle_sleep_event();
+
+       scale = &per_cpu(cyc2ns, cpu);
+
+       rdtscll(tsc_now);
+       ns_now = __cycles_2_ns(tsc_now);
+
+       prev_scale = *scale;
+       if (cpu_khz)
+               *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
+
+       /*
+        * Start smoothly with the new frequency:
+        */
+       sched_clock_idle_wakeup_event(0);
+       local_irq_restore(flags);
 }
 
 /*
@@ -239,7 +258,9 @@ time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
                                                ref_freq, freq->new);
                        if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
                                tsc_khz = cpu_khz;
-                               set_cyc2ns_scale(cpu_khz);
+                               preempt_disable();
+                               set_cyc2ns_scale(cpu_khz, smp_processor_id());
+                               preempt_enable();
                                /*
                                 * TSC based sched_clock turns
                                 * to junk w/ cpufreq
@@ -367,6 +388,8 @@ static inline void check_geode_tsc_reliable(void) { }
 
 void __init tsc_init(void)
 {
+       int cpu;
+
        if (!cpu_has_tsc || tsc_disable)
                goto out_no_tsc;
 
@@ -380,7 +403,15 @@ void __init tsc_init(void)
                                (unsigned long)cpu_khz / 1000,
                                (unsigned long)cpu_khz % 1000);
 
-       set_cyc2ns_scale(cpu_khz);
+       /*
+        * Secondary CPUs do not run through tsc_init(), so set up
+        * all the scale factors for all CPUs, assuming the same
+        * speed as the bootup CPU. (cpufreq notifiers will fix this
+        * up if their speed diverges)
+        */
+       for_each_possible_cpu(cpu)
+               set_cyc2ns_scale(cpu_khz, cpu);
+
        use_tsc_delay();
 
        /* Check and install the TSC clocksource */