2 * linux/arch/x86_64/kernel/vsyscall.c
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright 2003 Andi Kleen, SuSE Labs.
7 * Thanks to hpa@transmeta.com for some useful hint.
8 * Special thanks to Ingo Molnar for his early experience with
9 * a different vsyscall implementation for Linux/IA32 and for the name.
11 * vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
12 * at virtual address -10Mbyte+1024bytes etc... There are at max 4
13 * vsyscalls. One vsyscall can reserve more than 1 slot to avoid
14 * jumping out of line if necessary. We cannot add more with this
15 * mechanism because older kernels won't return -ENOSYS.
16 * If we want more than four we need a vDSO.
18 * Note: the concept clashes with user mode linux. If you use UML and
19 * want per guest time just set the kernel.vsyscall64 sysctl to 0.
22 #include <linux/time.h>
23 #include <linux/init.h>
24 #include <linux/kernel.h>
25 #include <linux/timer.h>
26 #include <linux/seqlock.h>
27 #include <linux/jiffies.h>
28 #include <linux/sysctl.h>
29 #include <linux/clocksource.h>
30 #include <linux/getcpu.h>
31 #include <linux/cpu.h>
32 #include <linux/smp.h>
33 #include <linux/notifier.h>
35 #include <asm/vsyscall.h>
36 #include <asm/pgtable.h>
38 #include <asm/unistd.h>
39 #include <asm/fixmap.h>
40 #include <asm/errno.h>
42 #include <asm/segment.h>
44 #include <asm/topology.h>
46 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
47 #define __syscall_clobber "r11","rcx","memory"
48 #define __pa_vsymbol(x) \
50 extern char __vsyscall_0; \
51 asm("" : "=r" (v) : "0" (x)); \
52 ((v - VSYSCALL_FIRST_PAGE) + __pa_symbol(&__vsyscall_0)); })
55 * vsyscall_gtod_data contains data that is :
56 * - readonly from vsyscalls
57 * - writen by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
58 * Try to keep this structure as small as possible to avoid cache line ping pongs
60 struct vsyscall_gtod_data_t {
63 /* open coded 'struct timespec' */
68 struct timezone sys_tz;
69 struct { /* extract of a clocksource struct */
70 cycle_t (*vread)(void);
77 int __vgetcpu_mode __section_vgetcpu_mode;
79 struct vsyscall_gtod_data_t __vsyscall_gtod_data __section_vsyscall_gtod_data =
81 .lock = SEQLOCK_UNLOCKED,
85 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
89 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
90 /* copy vsyscall data */
91 vsyscall_gtod_data.clock.vread = clock->vread;
92 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
93 vsyscall_gtod_data.clock.mask = clock->mask;
94 vsyscall_gtod_data.clock.mult = clock->mult;
95 vsyscall_gtod_data.clock.shift = clock->shift;
96 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
97 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
98 vsyscall_gtod_data.sys_tz = sys_tz;
99 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
102 /* RED-PEN may want to readd seq locking, but then the variable should be
105 static __always_inline void do_get_tz(struct timezone * tz)
107 *tz = __vsyscall_gtod_data.sys_tz;
110 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
113 asm volatile("vsysc2: syscall"
115 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
116 : __syscall_clobber );
120 static __always_inline long time_syscall(long *t)
123 asm volatile("vsysc1: syscall"
125 : "0" (__NR_time),"D" (t) : __syscall_clobber);
129 static __always_inline void do_vgettimeofday(struct timeval * tv)
131 cycle_t now, base, mask, cycle_delta;
133 unsigned long mult, shift, nsec;
134 cycle_t (*vread)(void);
136 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
138 vread = __vsyscall_gtod_data.clock.vread;
139 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
140 gettimeofday(tv,NULL);
144 base = __vsyscall_gtod_data.clock.cycle_last;
145 mask = __vsyscall_gtod_data.clock.mask;
146 mult = __vsyscall_gtod_data.clock.mult;
147 shift = __vsyscall_gtod_data.clock.shift;
149 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
150 nsec = __vsyscall_gtod_data.wall_time_nsec;
151 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
153 /* calculate interval: */
154 cycle_delta = (now - base) & mask;
155 /* convert to nsecs: */
156 nsec += (cycle_delta * mult) >> shift;
158 while (nsec >= NSEC_PER_SEC) {
160 nsec -= NSEC_PER_SEC;
162 tv->tv_usec = nsec / NSEC_PER_USEC;
165 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
168 do_vgettimeofday(tv);
174 /* This will break when the xtime seconds get inaccurate, but that is
176 time_t __vsyscall(1) vtime(time_t *t)
180 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
181 return time_syscall(t);
183 vgettimeofday(&tv, 0);
190 /* Fast way to get current CPU and node.
191 This helps to do per node and per CPU caches in user space.
192 The result is not guaranteed without CPU affinity, but usually
193 works out because the scheduler tries to keep a thread on the same
196 tcache must point to a two element sized long array.
197 All arguments can be NULL. */
199 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
201 unsigned int dummy, p;
204 /* Fast cache - only recompute value once per jiffies and avoid
205 relatively costly rdtscp/cpuid otherwise.
206 This works because the scheduler usually keeps the process
207 on the same CPU and this syscall doesn't guarantee its
209 We do this here because otherwise user space would do it on
210 its own in a likely inferior way (no access to jiffies).
211 If you don't like it pass NULL. */
212 if (tcache && tcache->blob[0] == (j = __jiffies)) {
214 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
215 /* Load per CPU data from RDTSCP */
216 rdtscp(dummy, dummy, p);
218 /* Load per CPU data from GDT */
219 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
232 long __vsyscall(3) venosys_1(void)
239 #define SYSCALL 0x050f
243 * NOP out syscall in vsyscall page when not needed.
245 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
246 void __user *buffer, size_t *lenp, loff_t *ppos)
248 extern u16 vsysc1, vsysc2;
251 int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
254 /* gcc has some trouble with __va(__pa()), so just do it this
256 map1 = ioremap(__pa_vsymbol(&vsysc1), 2);
259 map2 = ioremap(__pa_vsymbol(&vsysc2), 2);
264 if (!vsyscall_gtod_data.sysctl_enabled) {
265 writew(SYSCALL, map1);
266 writew(SYSCALL, map2);
277 static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
278 void __user *oldval, size_t __user *oldlenp,
279 void __user *newval, size_t newlen)
284 static ctl_table kernel_table2[] = {
285 { .ctl_name = 99, .procname = "vsyscall64",
286 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
288 .strategy = vsyscall_sysctl_nostrat,
289 .proc_handler = vsyscall_sysctl_change },
293 static ctl_table kernel_root_table2[] = {
294 { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
295 .child = kernel_table2 },
301 /* Assume __initcall executes before all user space. Hopefully kmod
302 doesn't violate that. We'll find out if it does. */
303 static void __cpuinit vsyscall_set_cpu(int cpu)
306 unsigned long node = 0;
308 node = cpu_to_node[cpu];
310 if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
311 write_rdtscp_aux((node << 12) | cpu);
313 /* Store cpu number in limit so that it can be loaded quickly
314 in user space in vgetcpu.
315 12 bits for the CPU and 8 bits for the node. */
316 d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
317 *d = 0x0f40000000000ULL;
319 *d |= (node & 0xf) << 12;
320 *d |= (node >> 4) << 48;
323 static void __cpuinit cpu_vsyscall_init(void *arg)
325 /* preemption should be already off */
326 vsyscall_set_cpu(raw_smp_processor_id());
330 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
332 long cpu = (long)arg;
333 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
334 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
338 static void __init map_vsyscall(void)
340 extern char __vsyscall_0;
341 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
343 /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
344 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
347 static int __init vsyscall_init(void)
349 BUG_ON(((unsigned long) &vgettimeofday !=
350 VSYSCALL_ADDR(__NR_vgettimeofday)));
351 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
352 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
353 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
356 register_sysctl_table(kernel_root_table2);
358 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
359 hotcpu_notifier(cpu_vsyscall_notifier, 0);
363 __initcall(vsyscall_init);