sys_time() speedup

Improve performance of sys_time().  sys_time() returns time in seconds, but
it does so by calling do_gettimeofday() and then returning the tv_sec
portion of the GTOD time.  But the data structure "xtime", which is updated
by every timer/scheduler tick, already offers HZ granularity time.

The patch improves the sysbench OLTP macrobenchmark significantly:

2.6.22-rc6:

#threads
   1:        transactions:                        3733   (373.21 per sec.)
   2:        transactions:                        6676   (667.46 per sec.)
   3:        transactions:                        6957   (695.50 per sec.)
   4:        transactions:                        7055   (705.48 per sec.)
   5:        transactions:                        6596   (659.33 per sec.)

2.6.22-rc6 + sys_time.patch:

   1:        transactions:                        4005   (400.47 per sec.)
   2:        transactions:                        7379   (737.77 per sec.)
   3:        transactions:                        7347   (734.49 per sec.)
   4:        transactions:                        7468   (746.65 per sec.)
   5:        transactions:                        7428   (742.47 per sec.)

Mixed API uses of gettimeofday() and time() are guaranteed to be coherent
via the use of a at-most-once-per-second slowpath that updates xtime.

[akpm@linux-foundation.org: build fixes]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Cc: John Stultz <johnstul@us.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/kernel/time.c b/kernel/time.c
index f04791f694081d16dab799ae16f5b3f405683e4d..ffe19149d77006cb6c22db392b0a828962c655d3 100644 (file)
--- a/kernel/time.c
+++ b/kernel/time.c
@@ -57,14 +57,17 @@ EXPORT_SYMBOL(sys_tz);
  */
 asmlinkage long sys_time(time_t __user * tloc)
 {
-       time_t i;
-       struct timeval tv;
+       /*
+        * We read xtime.tv_sec atomically - it's updated
+        * atomically by update_wall_time(), so no need to
+        * even read-lock the xtime seqlock:
+        */
+       time_t i = xtime.tv_sec;
 
-       do_gettimeofday(&tv);
-       i = tv.tv_sec;
+       smp_rmb(); /* sys_time() results are coherent */
 
        if (tloc) {
-               if (put_user(i,tloc))
+               if (put_user(i, tloc))
                        i = -EFAULT;
        }
        return i;
@@ -373,12 +376,25 @@ void do_gettimeofday (struct timeval *tv)
 
        tv->tv_sec = sec;
        tv->tv_usec = usec;
-}
 
+       /*
+        * Make sure xtime.tv_sec [returned by sys_time()] always
+        * follows the gettimeofday() result precisely. This
+        * condition is extremely unlikely, it can hit at most
+        * once per second:
+        */
+       if (unlikely(xtime.tv_sec != tv->tv_sec)) {
+               unsigned long flags;
+
+               write_seqlock_irqsave(&xtime_lock, flags);
+               update_wall_time();
+               write_sequnlock_irqrestore(&xtime_lock, flags);
+       }
+}
 EXPORT_SYMBOL(do_gettimeofday);
 
+#else  /* CONFIG_TIME_INTERPOLATION */
 
-#else
 #ifndef CONFIG_GENERIC_TIME
 /*
  * Simulate gettimeofday using do_gettimeofday which only allows a timeval
@@ -394,7 +410,7 @@ void getnstimeofday(struct timespec *tv)
 }
 EXPORT_SYMBOL_GPL(getnstimeofday);
 #endif
-#endif
+#endif /* CONFIG_TIME_INTERPOLATION */
 
 /* Converts Gregorian date to seconds since 1970-01-01 00:00:00.
  * Assumes input in normal date format, i.e. 1980-12-31 23:59:59