2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
8 * Copyright (C) 2000, 2001 Paolo Alberelli
9 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
10 * Copyright (C) 2003 Paul Mundt
14 #include <linux/signal.h>
15 #include <linux/rwsem.h>
16 #include <linux/sched.h>
17 #include <linux/kernel.h>
18 #include <linux/errno.h>
19 #include <linux/string.h>
20 #include <linux/types.h>
21 #include <linux/ptrace.h>
22 #include <linux/mman.h>
24 #include <linux/smp.h>
25 #include <linux/interrupt.h>
27 #include <asm/system.h>
30 #include <asm/uaccess.h>
31 #include <asm/pgalloc.h>
32 #include <asm/mmu_context.h>
33 #include <asm/registers.h> /* required by inline asm statements */
35 #if defined(CONFIG_SH64_PROC_TLB)
36 #include <linux/init.h>
37 #include <linux/proc_fs.h>
38 /* Count numbers of tlb refills in each region */
39 static unsigned long long calls_to_update_mmu_cache = 0ULL;
40 static unsigned long long calls_to_flush_tlb_page = 0ULL;
41 static unsigned long long calls_to_flush_tlb_range = 0ULL;
42 static unsigned long long calls_to_flush_tlb_mm = 0ULL;
43 static unsigned long long calls_to_flush_tlb_all = 0ULL;
44 unsigned long long calls_to_do_slow_page_fault = 0ULL;
45 unsigned long long calls_to_do_fast_page_fault = 0ULL;
47 /* Count size of ranges for flush_tlb_range */
48 static unsigned long long flush_tlb_range_1 = 0ULL;
49 static unsigned long long flush_tlb_range_2 = 0ULL;
50 static unsigned long long flush_tlb_range_3_4 = 0ULL;
51 static unsigned long long flush_tlb_range_5_7 = 0ULL;
52 static unsigned long long flush_tlb_range_8_11 = 0ULL;
53 static unsigned long long flush_tlb_range_12_15 = 0ULL;
54 static unsigned long long flush_tlb_range_16_up = 0ULL;
56 static unsigned long long page_not_present = 0ULL;
60 extern void die(const char *,struct pt_regs *,long);
62 #define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
63 #define PPROT(flag) PFLAG(pgprot_val(prot),flag)
65 static inline void print_prots(pgprot_t prot)
67 printk("prot is 0x%08lx\n",pgprot_val(prot));
69 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
70 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
73 static inline void print_vma(struct vm_area_struct *vma)
75 printk("vma start 0x%08lx\n", vma->vm_start);
76 printk("vma end 0x%08lx\n", vma->vm_end);
78 print_prots(vma->vm_page_prot);
79 printk("vm_flags 0x%08lx\n", vma->vm_flags);
82 static inline void print_task(struct task_struct *tsk)
84 printk("Task pid %d\n", task_pid_nr(tsk));
87 static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
94 dir = pgd_offset(mm, address);
99 pmd = pmd_offset(dir, address);
100 if (pmd_none(*pmd)) {
104 pte = pte_offset_kernel(pmd, address);
107 if (pte_none(entry)) {
110 if (!pte_present(entry)) {
118 * This routine handles page faults. It determines the address,
119 * and the problem, and then passes it off to one of the appropriate
122 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
123 unsigned long textaccess, unsigned long address)
125 struct task_struct *tsk;
126 struct mm_struct *mm;
127 struct vm_area_struct * vma;
128 const struct exception_table_entry *fixup;
132 #if defined(CONFIG_SH64_PROC_TLB)
133 ++calls_to_do_slow_page_fault;
137 * Note this is now called with interrupts still disabled
138 * This is to cope with being called for a missing IO port
139 * address with interrupts disabled. This should be fixed as
140 * soon as we have a better 'fast path' miss handler.
142 * Plus take care how you try and debug this stuff.
143 * For example, writing debug data to a port which you
144 * have just faulted on is not going to work.
150 /* Not an IO address, so reenable interrupts */
154 * If we're in an interrupt or have no user
155 * context, we must not take the fault..
157 if (in_atomic() || !mm)
160 /* TLB misses upon some cache flushes get done under cli() */
161 down_read(&mm->mmap_sem);
163 vma = find_vma(mm, address);
168 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
169 __FUNCTION__,__LINE__,
170 address,regs->pc,textaccess,writeaccess);
175 if (vma->vm_start <= address) {
179 if (!(vma->vm_flags & VM_GROWSDOWN)) {
182 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
183 __FUNCTION__,__LINE__,
184 address,regs->pc,textaccess,writeaccess);
191 if (expand_stack(vma, address)) {
194 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
195 __FUNCTION__,__LINE__,
196 address,regs->pc,textaccess,writeaccess);
202 * Ok, we have a good vm_area for this memory access, so
207 if (!(vma->vm_flags & VM_EXEC))
211 if (!(vma->vm_flags & VM_WRITE))
214 if (!(vma->vm_flags & VM_READ))
220 * If for any reason at all we couldn't handle the fault,
221 * make sure we exit gracefully rather than endlessly redo
225 fault = handle_mm_fault(mm, vma, address, writeaccess);
226 if (unlikely(fault & VM_FAULT_ERROR)) {
227 if (fault & VM_FAULT_OOM)
229 else if (fault & VM_FAULT_SIGBUS)
233 if (fault & VM_FAULT_MAJOR)
238 /* If we get here, the page fault has been handled. Do the TLB refill
239 now from the newly-setup PTE, to avoid having to fault again right
240 away on the same instruction. */
241 pte = lookup_pte (mm, address);
243 /* From empirical evidence, we can get here, due to
244 !pte_present(pte). (e.g. if a swap-in occurs, and the page
245 is swapped back out again before the process that wanted it
246 gets rescheduled?) */
250 __do_tlb_refill(address, textaccess, pte);
254 up_read(&mm->mmap_sem);
258 * Something tried to access memory that isn't in our memory map..
259 * Fix it, but check if it's kernel or user first..
263 printk("fault:bad area\n");
265 up_read(&mm->mmap_sem);
267 if (user_mode(regs)) {
271 /* This is really to help debug faults when starting
272 * usermode, so only need a few */
274 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
275 address, task_pid_nr(current), current->comm,
276 (unsigned long) regs->pc);
281 if (is_global_init(tsk)) {
282 panic("INIT had user mode bad_area\n");
284 tsk->thread.address = address;
285 tsk->thread.error_code = writeaccess;
286 info.si_signo = SIGSEGV;
288 info.si_addr = (void *) address;
289 force_sig_info(SIGSEGV, &info, tsk);
295 printk("fault:No context\n");
297 /* Are we prepared to handle this kernel fault? */
298 fixup = search_exception_tables(regs->pc);
300 regs->pc = fixup->fixup;
305 * Oops. The kernel tried to access some bad page. We'll have to
306 * terminate things with extreme prejudice.
309 if (address < PAGE_SIZE)
310 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
312 printk(KERN_ALERT "Unable to handle kernel paging request");
313 printk(" at virtual address %08lx\n", address);
314 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
315 die("Oops", regs, writeaccess);
319 * We ran out of memory, or some other thing happened to us that made
320 * us unable to handle the page fault gracefully.
323 if (is_global_init(current)) {
324 panic("INIT out of memory\n");
328 printk("fault:Out of memory\n");
329 up_read(&mm->mmap_sem);
330 if (is_global_init(current)) {
332 down_read(&mm->mmap_sem);
335 printk("VM: killing process %s\n", tsk->comm);
337 do_group_exit(SIGKILL);
341 printk("fault:Do sigbus\n");
342 up_read(&mm->mmap_sem);
345 * Send a sigbus, regardless of whether we were in kernel
348 tsk->thread.address = address;
349 tsk->thread.error_code = writeaccess;
350 tsk->thread.trap_no = 14;
351 force_sig(SIGBUS, tsk);
353 /* Kernel mode? Handle exceptions or die */
354 if (!user_mode(regs))
359 void flush_tlb_all(void);
361 void update_mmu_cache(struct vm_area_struct * vma,
362 unsigned long address, pte_t pte)
364 #if defined(CONFIG_SH64_PROC_TLB)
365 ++calls_to_update_mmu_cache;
369 * This appears to get called once for every pte entry that gets
370 * established => I don't think it's efficient to try refilling the
371 * TLBs with the pages - some may not get accessed even. Also, for
372 * executable pages, it is impossible to determine reliably here which
373 * TLB they should be mapped into (or both even).
375 * So, just do nothing here and handle faults on demand. In the
376 * TLBMISS handling case, the refill is now done anyway after the pte
377 * has been fixed up, so that deals with most useful cases.
381 static void __flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
383 unsigned long long match, pteh=0, lpage;
385 struct mm_struct *mm;
389 if (mm->context == NO_CONTEXT)
393 * Sign-extend based on neff.
395 lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page;
396 match = ((mm->context & MMU_CONTEXT_ASID_MASK) << PTEH_ASID_SHIFT) | PTEH_VALID;
399 /* Do ITLB : don't bother for pages in non-exectutable VMAs */
400 if (vma->vm_flags & VM_EXEC) {
401 for_each_itlb_entry(tlb) {
402 asm volatile ("getcfg %1, 0, %0"
407 __flush_tlb_slot(tlb);
414 /* Do DTLB : any page could potentially be in here. */
415 for_each_dtlb_entry(tlb) {
416 asm volatile ("getcfg %1, 0, %0"
421 __flush_tlb_slot(tlb);
428 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
432 #if defined(CONFIG_SH64_PROC_TLB)
433 ++calls_to_flush_tlb_page;
438 local_irq_save(flags);
439 __flush_tlb_page(vma, page);
440 local_irq_restore(flags);
444 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
448 unsigned long long match, pteh=0, pteh_epn, pteh_low;
450 struct mm_struct *mm;
454 #if defined(CONFIG_SH64_PROC_TLB)
455 ++calls_to_flush_tlb_range;
458 unsigned long size = (end - 1) - start;
459 size >>= 12; /* divide by PAGE_SIZE */
460 size++; /* end=start+4096 => 1 page */
462 case 1 : flush_tlb_range_1++; break;
463 case 2 : flush_tlb_range_2++; break;
464 case 3 ... 4 : flush_tlb_range_3_4++; break;
465 case 5 ... 7 : flush_tlb_range_5_7++; break;
466 case 8 ... 11 : flush_tlb_range_8_11++; break;
467 case 12 ... 15 : flush_tlb_range_12_15++; break;
468 default : flush_tlb_range_16_up++; break;
473 if (mm->context == NO_CONTEXT)
476 local_irq_save(flags);
481 match = ((mm->context & MMU_CONTEXT_ASID_MASK) << PTEH_ASID_SHIFT) | PTEH_VALID;
484 for_each_itlb_entry(tlb) {
485 asm volatile ("getcfg %1, 0, %0"
489 pteh_epn = pteh & PAGE_MASK;
490 pteh_low = pteh & ~PAGE_MASK;
492 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
493 __flush_tlb_slot(tlb);
497 for_each_dtlb_entry(tlb) {
498 asm volatile ("getcfg %1, 0, %0"
502 pteh_epn = pteh & PAGE_MASK;
503 pteh_low = pteh & ~PAGE_MASK;
505 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
506 __flush_tlb_slot(tlb);
509 local_irq_restore(flags);
512 void flush_tlb_mm(struct mm_struct *mm)
516 #if defined(CONFIG_SH64_PROC_TLB)
517 ++calls_to_flush_tlb_mm;
520 if (mm->context == NO_CONTEXT)
523 local_irq_save(flags);
525 mm->context=NO_CONTEXT;
527 activate_context(mm);
529 local_irq_restore(flags);
533 void flush_tlb_all(void)
535 /* Invalidate all, including shared pages, excluding fixed TLBs */
537 unsigned long flags, tlb;
539 #if defined(CONFIG_SH64_PROC_TLB)
540 ++calls_to_flush_tlb_all;
543 local_irq_save(flags);
545 /* Flush each ITLB entry */
546 for_each_itlb_entry(tlb) {
547 __flush_tlb_slot(tlb);
550 /* Flush each DTLB entry */
551 for_each_dtlb_entry(tlb) {
552 __flush_tlb_slot(tlb);
555 local_irq_restore(flags);
558 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
560 /* FIXME: Optimize this later.. */
564 #if defined(CONFIG_SH64_PROC_TLB)
565 /* Procfs interface to read the performance information */
568 tlb_proc_info(char *buf, char **start, off_t fpos, int length, int *eof, void *data)
571 len += sprintf(buf+len, "do_fast_page_fault called %12lld times\n", calls_to_do_fast_page_fault);
572 len += sprintf(buf+len, "do_slow_page_fault called %12lld times\n", calls_to_do_slow_page_fault);
573 len += sprintf(buf+len, "update_mmu_cache called %12lld times\n", calls_to_update_mmu_cache);
574 len += sprintf(buf+len, "flush_tlb_page called %12lld times\n", calls_to_flush_tlb_page);
575 len += sprintf(buf+len, "flush_tlb_range called %12lld times\n", calls_to_flush_tlb_range);
576 len += sprintf(buf+len, "flush_tlb_mm called %12lld times\n", calls_to_flush_tlb_mm);
577 len += sprintf(buf+len, "flush_tlb_all called %12lld times\n", calls_to_flush_tlb_all);
578 len += sprintf(buf+len, "flush_tlb_range_sizes\n"
586 flush_tlb_range_1, flush_tlb_range_2, flush_tlb_range_3_4,
587 flush_tlb_range_5_7, flush_tlb_range_8_11, flush_tlb_range_12_15,
588 flush_tlb_range_16_up);
589 len += sprintf(buf+len, "page not present %12lld times\n", page_not_present);
594 static int __init register_proc_tlb(void)
596 create_proc_read_entry("tlb", 0, NULL, tlb_proc_info, NULL);
600 __initcall(register_proc_tlb);