2 * linux/arch/x86-64/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
8 #include <linux/config.h>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/smp.h>
19 #include <linux/smp_lock.h>
20 #include <linux/interrupt.h>
21 #include <linux/init.h>
22 #include <linux/tty.h>
23 #include <linux/vt_kern.h> /* For unblank_screen() */
24 #include <linux/compiler.h>
25 #include <linux/module.h>
26 #include <linux/kprobes.h>
28 #include <asm/system.h>
29 #include <asm/uaccess.h>
30 #include <asm/pgalloc.h>
32 #include <asm/tlbflush.h>
33 #include <asm/proto.h>
34 #include <asm/kdebug.h>
35 #include <asm-generic/sections.h>
36 #include <asm/kdebug.h>
38 void bust_spinlocks(int yes)
40 int loglevel_save = console_loglevel;
49 * OK, the message is on the console. Now we call printk()
50 * without oops_in_progress set so that printk will give klogd
51 * a poke. Hold onto your hats...
53 console_loglevel = 15; /* NMI oopser may have shut the console up */
55 console_loglevel = loglevel_save;
59 /* Sometimes the CPU reports invalid exceptions on prefetch.
60 Check that here and ignore.
61 Opcode checker based on code by Richard Brunner */
62 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
63 unsigned long error_code)
65 unsigned char *instr = (unsigned char *)(regs->rip);
68 unsigned char *max_instr = instr + 15;
70 /* If it was a exec fault ignore */
71 if (error_code & (1<<4))
74 /* Code segments in LDT could have a non zero base. Don't check
75 when that's possible */
76 if (regs->cs & (1<<2))
79 if ((regs->cs & 3) != 0 && regs->rip >= TASK_SIZE)
82 while (scan_more && instr < max_instr) {
84 unsigned char instr_hi;
85 unsigned char instr_lo;
87 if (__get_user(opcode, instr))
90 instr_hi = opcode & 0xf0;
91 instr_lo = opcode & 0x0f;
97 /* Values 0x26,0x2E,0x36,0x3E are valid x86
98 prefixes. In long mode, the CPU will signal
99 invalid opcode if some of these prefixes are
100 present so we will never get here anyway */
101 scan_more = ((instr_lo & 7) == 0x6);
105 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
106 Need to figure out under what instruction mode the
107 instruction was issued ... */
108 /* Could check the LDT for lm, but for now it's good
109 enough to assume that long mode only uses well known
110 segments or kernel. */
111 scan_more = ((regs->cs & 3) == 0) || (regs->cs == __USER_CS);
115 /* 0x64 thru 0x67 are valid prefixes in all modes. */
116 scan_more = (instr_lo & 0xC) == 0x4;
119 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
120 scan_more = !instr_lo || (instr_lo>>1) == 1;
123 /* Prefetch instruction is 0x0F0D or 0x0F18 */
125 if (__get_user(opcode, instr))
127 prefetch = (instr_lo == 0xF) &&
128 (opcode == 0x0D || opcode == 0x18);
138 static int bad_address(void *p)
141 return __get_user(dummy, (unsigned long *)p);
144 void dump_pagetable(unsigned long address)
151 asm("movq %%cr3,%0" : "=r" (pgd));
153 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
154 pgd += pgd_index(address);
155 printk("PGD %lx ", pgd_val(*pgd));
156 if (bad_address(pgd)) goto bad;
157 if (!pgd_present(*pgd)) goto ret;
159 pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
160 if (bad_address(pud)) goto bad;
161 printk("PUD %lx ", pud_val(*pud));
162 if (!pud_present(*pud)) goto ret;
164 pmd = pmd_offset(pud, address);
165 if (bad_address(pmd)) goto bad;
166 printk("PMD %lx ", pmd_val(*pmd));
167 if (!pmd_present(*pmd)) goto ret;
169 pte = pte_offset_kernel(pmd, address);
170 if (bad_address(pte)) goto bad;
171 printk("PTE %lx", pte_val(*pte));
179 static const char errata93_warning[] =
180 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
181 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
182 KERN_ERR "******* Please consider a BIOS update.\n"
183 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
185 /* Workaround for K8 erratum #93 & buggy BIOS.
186 BIOS SMM functions are required to use a specific workaround
187 to avoid corruption of the 64bit RIP register on C stepping K8.
188 A lot of BIOS that didn't get tested properly miss this.
189 The OS sees this as a page fault with the upper 32bits of RIP cleared.
190 Try to work around it here.
191 Note we only handle faults in kernel here. */
193 static int is_errata93(struct pt_regs *regs, unsigned long address)
196 if (address != regs->rip)
198 if ((address >> 32) != 0)
200 address |= 0xffffffffUL << 32;
201 if ((address >= (u64)_stext && address <= (u64)_etext) ||
202 (address >= MODULES_VADDR && address <= MODULES_END)) {
204 printk(errata93_warning);
213 int unhandled_signal(struct task_struct *tsk, int sig)
217 /* Warn for strace, but not for gdb */
218 if (!test_ti_thread_flag(tsk->thread_info, TIF_SYSCALL_TRACE) &&
219 (tsk->ptrace & PT_PTRACED))
221 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
222 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
225 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
226 unsigned long error_code)
229 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
230 current->comm, address);
231 dump_pagetable(address);
232 __die("Bad pagetable", regs, error_code);
238 * Handle a fault on the vmalloc or module mapping area
240 static int vmalloc_fault(unsigned long address)
242 pgd_t *pgd, *pgd_ref;
243 pud_t *pud, *pud_ref;
244 pmd_t *pmd, *pmd_ref;
245 pte_t *pte, *pte_ref;
247 /* Copy kernel mappings over when needed. This can also
248 happen within a race in page table update. In the later
251 pgd = pgd_offset(current->mm ?: &init_mm, address);
252 pgd_ref = pgd_offset_k(address);
253 if (pgd_none(*pgd_ref))
256 set_pgd(pgd, *pgd_ref);
258 /* Below here mismatches are bugs because these lower tables
261 pud = pud_offset(pgd, address);
262 pud_ref = pud_offset(pgd_ref, address);
263 if (pud_none(*pud_ref))
265 if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
267 pmd = pmd_offset(pud, address);
268 pmd_ref = pmd_offset(pud_ref, address);
269 if (pmd_none(*pmd_ref))
271 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
273 pte_ref = pte_offset_kernel(pmd_ref, address);
274 if (!pte_present(*pte_ref))
276 pte = pte_offset_kernel(pmd, address);
277 if (!pte_present(*pte) || pte_page(*pte) != pte_page(*pte_ref))
283 int page_fault_trace = 0;
284 int exception_trace = 1;
287 * This routine handles page faults. It determines the address,
288 * and the problem, and then passes it off to one of the appropriate
292 * bit 0 == 0 means no page found, 1 means protection fault
293 * bit 1 == 0 means read, 1 means write
294 * bit 2 == 0 means kernel, 1 means user-mode
295 * bit 3 == 1 means fault was an instruction fetch
297 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
299 struct task_struct *tsk;
300 struct mm_struct *mm;
301 struct vm_area_struct * vma;
302 unsigned long address;
303 const struct exception_table_entry *fixup;
307 #ifdef CONFIG_CHECKING
310 struct x8664_pda *pda = cpu_pda + stack_smp_processor_id();
311 rdmsrl(MSR_GS_BASE, gs);
312 if (gs != (unsigned long)pda) {
313 wrmsrl(MSR_GS_BASE, pda);
314 printk("page_fault: wrong gs %lx expected %p\n", gs, pda);
319 /* get the address */
320 __asm__("movq %%cr2,%0":"=r" (address));
321 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
322 SIGSEGV) == NOTIFY_STOP)
325 if (likely(regs->eflags & X86_EFLAGS_IF))
328 if (unlikely(page_fault_trace))
329 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
330 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
334 info.si_code = SEGV_MAPERR;
338 * We fault-in kernel-space virtual memory on-demand. The
339 * 'reference' page table is init_mm.pgd.
341 * NOTE! We MUST NOT take any locks for this case. We may
342 * be in an interrupt or a critical region, and should
343 * only copy the information from the master page table,
346 * This verifies that the fault happens in kernel space
347 * (error_code & 4) == 0, and that the fault was not a
348 * protection error (error_code & 1) == 0.
350 if (unlikely(address >= TASK_SIZE)) {
351 if (!(error_code & 5)) {
352 if (vmalloc_fault(address) < 0)
353 goto bad_area_nosemaphore;
357 * Don't take the mm semaphore here. If we fixup a prefetch
358 * fault we could otherwise deadlock.
360 goto bad_area_nosemaphore;
363 if (unlikely(error_code & (1 << 3)))
364 pgtable_bad(address, regs, error_code);
367 * If we're in an interrupt or have no user
368 * context, we must not take the fault..
370 if (unlikely(in_atomic() || !mm))
371 goto bad_area_nosemaphore;
374 /* When running in the kernel we expect faults to occur only to
375 * addresses in user space. All other faults represent errors in the
376 * kernel and should generate an OOPS. Unfortunatly, in the case of an
377 * erroneous fault occuring in a code path which already holds mmap_sem
378 * we will deadlock attempting to validate the fault against the
379 * address space. Luckily the kernel only validly references user
380 * space from well defined areas of code, which are listed in the
383 * As the vast majority of faults will be valid we will only perform
384 * the source reference check when there is a possibilty of a deadlock.
385 * Attempt to lock the address space, if we cannot we then validate the
386 * source. If this is invalid we can skip the address space check,
387 * thus avoiding the deadlock.
389 if (!down_read_trylock(&mm->mmap_sem)) {
390 if ((error_code & 4) == 0 &&
391 !search_exception_tables(regs->rip))
392 goto bad_area_nosemaphore;
393 down_read(&mm->mmap_sem);
396 vma = find_vma(mm, address);
399 if (likely(vma->vm_start <= address))
401 if (!(vma->vm_flags & VM_GROWSDOWN))
403 if (error_code & 4) {
404 // XXX: align red zone size with ABI
405 if (address + 128 < regs->rsp)
408 if (expand_stack(vma, address))
411 * Ok, we have a good vm_area for this memory access, so
415 info.si_code = SEGV_ACCERR;
417 switch (error_code & 3) {
418 default: /* 3: write, present */
420 case 2: /* write, not present */
421 if (!(vma->vm_flags & VM_WRITE))
425 case 1: /* read, present */
427 case 0: /* read, not present */
428 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
433 * If for any reason at all we couldn't handle the fault,
434 * make sure we exit gracefully rather than endlessly redo
437 switch (handle_mm_fault(mm, vma, address, write)) {
450 up_read(&mm->mmap_sem);
454 * Something tried to access memory that isn't in our memory map..
455 * Fix it, but check if it's kernel or user first..
458 up_read(&mm->mmap_sem);
460 bad_area_nosemaphore:
461 /* User mode accesses just cause a SIGSEGV */
462 if (error_code & 4) {
463 if (is_prefetch(regs, address, error_code))
466 /* Work around K8 erratum #100 K8 in compat mode
467 occasionally jumps to illegal addresses >4GB. We
468 catch this here in the page fault handler because
469 these addresses are not reachable. Just detect this
470 case and return. Any code segment in LDT is
471 compatibility mode. */
472 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
476 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
478 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
479 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
480 tsk->comm, tsk->pid, address, regs->rip,
481 regs->rsp, error_code);
484 tsk->thread.cr2 = address;
485 /* Kernel addresses are always protection faults */
486 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
487 tsk->thread.trap_no = 14;
488 info.si_signo = SIGSEGV;
490 /* info.si_code has been set above */
491 info.si_addr = (void __user *)address;
492 force_sig_info(SIGSEGV, &info, tsk);
498 /* Are we prepared to handle this kernel fault? */
499 fixup = search_exception_tables(regs->rip);
501 regs->rip = fixup->fixup;
506 * Hall of shame of CPU/BIOS bugs.
509 if (is_prefetch(regs, address, error_code))
512 if (is_errata93(regs, address))
516 * Oops. The kernel tried to access some bad page. We'll have to
517 * terminate things with extreme prejudice.
522 if (address < PAGE_SIZE)
523 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
525 printk(KERN_ALERT "Unable to handle kernel paging request");
526 printk(" at %016lx RIP: \n" KERN_ALERT,address);
527 printk_address(regs->rip);
529 dump_pagetable(address);
530 __die("Oops", regs, error_code);
531 /* Executive summary in case the body of the oops scrolled away */
532 printk(KERN_EMERG "CR2: %016lx\n", address);
537 * We ran out of memory, or some other thing happened to us that made
538 * us unable to handle the page fault gracefully.
541 up_read(&mm->mmap_sem);
542 if (current->pid == 1) {
546 printk("VM: killing process %s\n", tsk->comm);
552 up_read(&mm->mmap_sem);
554 /* Kernel mode? Handle exceptions or die */
555 if (!(error_code & 4))
558 tsk->thread.cr2 = address;
559 tsk->thread.error_code = error_code;
560 tsk->thread.trap_no = 14;
561 info.si_signo = SIGBUS;
563 info.si_code = BUS_ADRERR;
564 info.si_addr = (void __user *)address;
565 force_sig_info(SIGBUS, &info, tsk);