2 * linux/arch/x86_64/kernel/vsyscall.c
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright 2003 Andi Kleen, SuSE Labs.
7 * Thanks to hpa@transmeta.com for some useful hint.
8 * Special thanks to Ingo Molnar for his early experience with
9 * a different vsyscall implementation for Linux/IA32 and for the name.
11 * vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
12 * at virtual address -10Mbyte+1024bytes etc... There are at max 4
13 * vsyscalls. One vsyscall can reserve more than 1 slot to avoid
14 * jumping out of line if necessary. We cannot add more with this
15 * mechanism because older kernels won't return -ENOSYS.
16 * If we want more than four we need a vDSO.
18 * Note: the concept clashes with user mode linux. If you use UML and
19 * want per guest time just set the kernel.vsyscall64 sysctl to 0.
22 #include <linux/time.h>
23 #include <linux/init.h>
24 #include <linux/kernel.h>
25 #include <linux/timer.h>
26 #include <linux/seqlock.h>
27 #include <linux/jiffies.h>
28 #include <linux/sysctl.h>
29 #include <linux/clocksource.h>
30 #include <linux/getcpu.h>
31 #include <linux/cpu.h>
32 #include <linux/smp.h>
33 #include <linux/notifier.h>
35 #include <asm/vsyscall.h>
36 #include <asm/pgtable.h>
38 #include <asm/unistd.h>
39 #include <asm/fixmap.h>
40 #include <asm/errno.h>
42 #include <asm/segment.h>
44 #include <asm/topology.h>
46 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
47 #define __syscall_clobber "r11","rcx","memory"
48 #define __pa_vsymbol(x) \
50 extern char __vsyscall_0; \
51 asm("" : "=r" (v) : "0" (x)); \
52 ((v - VSYSCALL_FIRST_PAGE) + __pa_symbol(&__vsyscall_0)); })
54 struct vsyscall_gtod_data_t {
57 struct timeval wall_time_tv;
58 struct timezone sys_tz;
60 struct clocksource clock;
62 int __vgetcpu_mode __section_vgetcpu_mode;
64 struct vsyscall_gtod_data_t __vsyscall_gtod_data __section_vsyscall_gtod_data =
66 .lock = SEQLOCK_UNLOCKED,
70 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
74 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
75 /* copy vsyscall data */
76 vsyscall_gtod_data.clock = *clock;
77 vsyscall_gtod_data.wall_time_tv.tv_sec = wall_time->tv_sec;
78 vsyscall_gtod_data.wall_time_tv.tv_usec = wall_time->tv_nsec/1000;
79 vsyscall_gtod_data.sys_tz = sys_tz;
80 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
83 /* RED-PEN may want to readd seq locking, but then the variable should be
86 static __always_inline void do_get_tz(struct timezone * tz)
88 *tz = __vsyscall_gtod_data.sys_tz;
91 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
94 asm volatile("vsysc2: syscall"
96 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
97 : __syscall_clobber );
101 static __always_inline long time_syscall(long *t)
104 asm volatile("vsysc1: syscall"
106 : "0" (__NR_time),"D" (t) : __syscall_clobber);
110 static __always_inline void do_vgettimeofday(struct timeval * tv)
112 cycle_t now, base, mask, cycle_delta;
113 unsigned long seq, mult, shift, nsec_delta;
114 cycle_t (*vread)(void);
116 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
118 vread = __vsyscall_gtod_data.clock.vread;
119 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
120 gettimeofday(tv,NULL);
124 base = __vsyscall_gtod_data.clock.cycle_last;
125 mask = __vsyscall_gtod_data.clock.mask;
126 mult = __vsyscall_gtod_data.clock.mult;
127 shift = __vsyscall_gtod_data.clock.shift;
129 *tv = __vsyscall_gtod_data.wall_time_tv;
131 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
133 /* calculate interval: */
134 cycle_delta = (now - base) & mask;
135 /* convert to nsecs: */
136 nsec_delta = (cycle_delta * mult) >> shift;
138 /* convert to usecs and add to timespec: */
139 tv->tv_usec += nsec_delta / NSEC_PER_USEC;
140 while (tv->tv_usec > USEC_PER_SEC) {
142 tv->tv_usec -= USEC_PER_SEC;
146 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
149 do_vgettimeofday(tv);
155 /* This will break when the xtime seconds get inaccurate, but that is
157 time_t __vsyscall(1) vtime(time_t *t)
160 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
161 return time_syscall(t);
162 result = __vsyscall_gtod_data.wall_time_tv.tv_sec;
168 /* Fast way to get current CPU and node.
169 This helps to do per node and per CPU caches in user space.
170 The result is not guaranteed without CPU affinity, but usually
171 works out because the scheduler tries to keep a thread on the same
174 tcache must point to a two element sized long array.
175 All arguments can be NULL. */
177 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
179 unsigned int dummy, p;
182 /* Fast cache - only recompute value once per jiffies and avoid
183 relatively costly rdtscp/cpuid otherwise.
184 This works because the scheduler usually keeps the process
185 on the same CPU and this syscall doesn't guarantee its
187 We do this here because otherwise user space would do it on
188 its own in a likely inferior way (no access to jiffies).
189 If you don't like it pass NULL. */
190 if (tcache && tcache->blob[0] == (j = __jiffies)) {
192 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
193 /* Load per CPU data from RDTSCP */
194 rdtscp(dummy, dummy, p);
196 /* Load per CPU data from GDT */
197 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
210 long __vsyscall(3) venosys_1(void)
217 #define SYSCALL 0x050f
221 * NOP out syscall in vsyscall page when not needed.
223 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
224 void __user *buffer, size_t *lenp, loff_t *ppos)
226 extern u16 vsysc1, vsysc2;
229 int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
232 /* gcc has some trouble with __va(__pa()), so just do it this
234 map1 = ioremap(__pa_vsymbol(&vsysc1), 2);
237 map2 = ioremap(__pa_vsymbol(&vsysc2), 2);
242 if (!vsyscall_gtod_data.sysctl_enabled) {
243 writew(SYSCALL, map1);
244 writew(SYSCALL, map2);
255 static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
256 void __user *oldval, size_t __user *oldlenp,
257 void __user *newval, size_t newlen)
262 static ctl_table kernel_table2[] = {
263 { .ctl_name = 99, .procname = "vsyscall64",
264 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
266 .strategy = vsyscall_sysctl_nostrat,
267 .proc_handler = vsyscall_sysctl_change },
271 static ctl_table kernel_root_table2[] = {
272 { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
273 .child = kernel_table2 },
279 /* Assume __initcall executes before all user space. Hopefully kmod
280 doesn't violate that. We'll find out if it does. */
281 static void __cpuinit vsyscall_set_cpu(int cpu)
284 unsigned long node = 0;
286 node = cpu_to_node[cpu];
288 if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
289 write_rdtscp_aux((node << 12) | cpu);
291 /* Store cpu number in limit so that it can be loaded quickly
292 in user space in vgetcpu.
293 12 bits for the CPU and 8 bits for the node. */
294 d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
295 *d = 0x0f40000000000ULL;
297 *d |= (node & 0xf) << 12;
298 *d |= (node >> 4) << 48;
301 static void __cpuinit cpu_vsyscall_init(void *arg)
303 /* preemption should be already off */
304 vsyscall_set_cpu(raw_smp_processor_id());
308 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
310 long cpu = (long)arg;
311 if (action == CPU_ONLINE)
312 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
316 static void __init map_vsyscall(void)
318 extern char __vsyscall_0;
319 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
321 /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
322 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
325 static int __init vsyscall_init(void)
327 BUG_ON(((unsigned long) &vgettimeofday !=
328 VSYSCALL_ADDR(__NR_vgettimeofday)));
329 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
330 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
331 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
334 register_sysctl_table(kernel_root_table2);
336 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
337 hotcpu_notifier(cpu_vsyscall_notifier, 0);
341 __initcall(vsyscall_init);