2 * linux/arch/x86_64/kernel/vsyscall.c
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright 2003 Andi Kleen, SuSE Labs.
7 * Thanks to hpa@transmeta.com for some useful hint.
8 * Special thanks to Ingo Molnar for his early experience with
9 * a different vsyscall implementation for Linux/IA32 and for the name.
11 * vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
12 * at virtual address -10Mbyte+1024bytes etc... There are at max 4
13 * vsyscalls. One vsyscall can reserve more than 1 slot to avoid
14 * jumping out of line if necessary. We cannot add more with this
15 * mechanism because older kernels won't return -ENOSYS.
16 * If we want more than four we need a vDSO.
18 * Note: the concept clashes with user mode linux. If you use UML and
19 * want per guest time just set the kernel.vsyscall64 sysctl to 0.
22 #include <linux/time.h>
23 #include <linux/init.h>
24 #include <linux/kernel.h>
25 #include <linux/timer.h>
26 #include <linux/seqlock.h>
27 #include <linux/jiffies.h>
28 #include <linux/sysctl.h>
29 #include <linux/clocksource.h>
30 #include <linux/getcpu.h>
31 #include <linux/cpu.h>
32 #include <linux/smp.h>
33 #include <linux/notifier.h>
35 #include <asm/vsyscall.h>
36 #include <asm/pgtable.h>
38 #include <asm/unistd.h>
39 #include <asm/fixmap.h>
40 #include <asm/errno.h>
42 #include <asm/segment.h>
44 #include <asm/topology.h>
46 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
47 #define __syscall_clobber "r11","rcx","memory"
49 struct vsyscall_gtod_data_t {
52 struct timeval wall_time_tv;
53 struct timezone sys_tz;
55 struct clocksource clock;
57 int __vgetcpu_mode __section_vgetcpu_mode;
59 struct vsyscall_gtod_data_t __vsyscall_gtod_data __section_vsyscall_gtod_data =
61 .lock = SEQLOCK_UNLOCKED,
65 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
69 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
70 /* copy vsyscall data */
71 vsyscall_gtod_data.clock = *clock;
72 vsyscall_gtod_data.wall_time_tv.tv_sec = wall_time->tv_sec;
73 vsyscall_gtod_data.wall_time_tv.tv_usec = wall_time->tv_nsec/1000;
74 vsyscall_gtod_data.sys_tz = sys_tz;
75 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
78 /* RED-PEN may want to readd seq locking, but then the variable should be
81 static __always_inline void do_get_tz(struct timezone * tz)
83 *tz = __vsyscall_gtod_data.sys_tz;
86 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
89 asm volatile("vsysc2: syscall"
91 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
92 : __syscall_clobber );
96 static __always_inline long time_syscall(long *t)
99 asm volatile("vsysc1: syscall"
101 : "0" (__NR_time),"D" (t) : __syscall_clobber);
105 static __always_inline void do_vgettimeofday(struct timeval * tv)
107 cycle_t now, base, mask, cycle_delta;
108 unsigned long seq, mult, shift, nsec_delta;
109 cycle_t (*vread)(void);
111 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
113 vread = __vsyscall_gtod_data.clock.vread;
114 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
115 gettimeofday(tv,NULL);
119 base = __vsyscall_gtod_data.clock.cycle_last;
120 mask = __vsyscall_gtod_data.clock.mask;
121 mult = __vsyscall_gtod_data.clock.mult;
122 shift = __vsyscall_gtod_data.clock.shift;
124 *tv = __vsyscall_gtod_data.wall_time_tv;
126 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
128 /* calculate interval: */
129 cycle_delta = (now - base) & mask;
130 /* convert to nsecs: */
131 nsec_delta = (cycle_delta * mult) >> shift;
133 /* convert to usecs and add to timespec: */
134 tv->tv_usec += nsec_delta / NSEC_PER_USEC;
135 while (tv->tv_usec > USEC_PER_SEC) {
137 tv->tv_usec -= USEC_PER_SEC;
141 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
144 do_vgettimeofday(tv);
150 /* This will break when the xtime seconds get inaccurate, but that is
152 time_t __vsyscall(1) vtime(time_t *t)
154 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
155 return time_syscall(t);
157 *t = __vsyscall_gtod_data.wall_time_tv.tv_sec;
158 return __vsyscall_gtod_data.wall_time_tv.tv_sec;
161 /* Fast way to get current CPU and node.
162 This helps to do per node and per CPU caches in user space.
163 The result is not guaranteed without CPU affinity, but usually
164 works out because the scheduler tries to keep a thread on the same
167 tcache must point to a two element sized long array.
168 All arguments can be NULL. */
170 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
172 unsigned int dummy, p;
175 /* Fast cache - only recompute value once per jiffies and avoid
176 relatively costly rdtscp/cpuid otherwise.
177 This works because the scheduler usually keeps the process
178 on the same CPU and this syscall doesn't guarantee its
180 We do this here because otherwise user space would do it on
181 its own in a likely inferior way (no access to jiffies).
182 If you don't like it pass NULL. */
183 if (tcache && tcache->blob[0] == (j = __jiffies)) {
185 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
186 /* Load per CPU data from RDTSCP */
187 rdtscp(dummy, dummy, p);
189 /* Load per CPU data from GDT */
190 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
203 long __vsyscall(3) venosys_1(void)
210 #define SYSCALL 0x050f
214 * NOP out syscall in vsyscall page when not needed.
216 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
217 void __user *buffer, size_t *lenp, loff_t *ppos)
219 extern u16 vsysc1, vsysc2;
222 int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
225 /* gcc has some trouble with __va(__pa()), so just do it this
227 map1 = ioremap(__pa_symbol(&vsysc1), 2);
230 map2 = ioremap(__pa_symbol(&vsysc2), 2);
235 if (!vsyscall_gtod_data.sysctl_enabled) {
236 writew(SYSCALL, map1);
237 writew(SYSCALL, map2);
248 static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
249 void __user *oldval, size_t __user *oldlenp,
250 void __user *newval, size_t newlen)
255 static ctl_table kernel_table2[] = {
256 { .ctl_name = 99, .procname = "vsyscall64",
257 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
259 .strategy = vsyscall_sysctl_nostrat,
260 .proc_handler = vsyscall_sysctl_change },
264 static ctl_table kernel_root_table2[] = {
265 { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
266 .child = kernel_table2 },
272 /* Assume __initcall executes before all user space. Hopefully kmod
273 doesn't violate that. We'll find out if it does. */
274 static void __cpuinit vsyscall_set_cpu(int cpu)
277 unsigned long node = 0;
279 node = cpu_to_node[cpu];
281 if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
282 write_rdtscp_aux((node << 12) | cpu);
284 /* Store cpu number in limit so that it can be loaded quickly
285 in user space in vgetcpu.
286 12 bits for the CPU and 8 bits for the node. */
287 d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
288 *d = 0x0f40000000000ULL;
290 *d |= (node & 0xf) << 12;
291 *d |= (node >> 4) << 48;
294 static void __cpuinit cpu_vsyscall_init(void *arg)
296 /* preemption should be already off */
297 vsyscall_set_cpu(raw_smp_processor_id());
301 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
303 long cpu = (long)arg;
304 if (action == CPU_ONLINE)
305 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
309 static void __init map_vsyscall(void)
311 extern char __vsyscall_0;
312 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
314 /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
315 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
318 static int __init vsyscall_init(void)
320 BUG_ON(((unsigned long) &vgettimeofday !=
321 VSYSCALL_ADDR(__NR_vgettimeofday)));
322 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
323 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
324 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
327 register_sysctl_table(kernel_root_table2);
329 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
330 hotcpu_notifier(cpu_vsyscall_notifier, 0);
334 __initcall(vsyscall_init);