2 * ultra.S: Don't expand these all over the place...
4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
8 #include <asm/pgtable.h>
10 #include <asm/spitfire.h>
11 #include <asm/mmu_context.h>
15 #include <asm/thread_info.h>
16 #include <asm/cacheflush.h>
17 #include <asm/hypervisor.h>
18 #include <asm/cpudata.h>
20 /* Basically, most of the Spitfire vs. Cheetah madness
21 * has to do with the fact that Cheetah does not support
22 * IMMU flushes out of the secondary context. Someone needs
23 * to throw a south lake birthday party for the folks
24 * in Microelectronics who refused to fix this shit.
27 /* This file is meant to be read efficiently by the CPU, not humans.
28 * Staraj sie tego nikomu nie pierdolnac...
33 __flush_tlb_mm: /* 18 insns */
34 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
35 ldxa [%o1] ASI_DMMU, %g2
37 bne,pn %icc, __spitfire_flush_tlb_mm_slow
39 stxa %g0, [%g3] ASI_DMMU_DEMAP
40 stxa %g0, [%g3] ASI_IMMU_DEMAP
41 sethi %hi(KERNBASE), %g3
56 .globl __flush_tlb_pending
57 __flush_tlb_pending: /* 26 insns */
58 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
61 andn %g7, PSTATE_IE, %g2
63 mov SECONDARY_CONTEXT, %o4
64 ldxa [%o4] ASI_DMMU, %g2
65 stxa %o0, [%o4] ASI_DMMU
66 1: sub %o1, (1 << 3), %o1
72 stxa %g0, [%o3] ASI_IMMU_DEMAP
73 2: stxa %g0, [%o3] ASI_DMMU_DEMAP
77 stxa %g2, [%o4] ASI_DMMU
78 sethi %hi(KERNBASE), %o4
81 wrpr %g7, 0x0, %pstate
88 .globl __flush_tlb_kernel_range
89 __flush_tlb_kernel_range: /* 16 insns */
90 /* %o0=start, %o1=end */
93 sethi %hi(PAGE_SIZE), %o4
96 or %o0, 0x20, %o0 ! Nucleus
97 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
98 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
102 2: sethi %hi(KERNBASE), %o3
108 __spitfire_flush_tlb_mm_slow:
110 wrpr %g1, PSTATE_IE, %pstate
111 stxa %o0, [%o1] ASI_DMMU
112 stxa %g0, [%g3] ASI_DMMU_DEMAP
113 stxa %g0, [%g3] ASI_IMMU_DEMAP
115 stxa %g2, [%o1] ASI_DMMU
116 sethi %hi(KERNBASE), %o1
122 * The following code flushes one page_size worth.
124 .section .kprobes.text, "ax"
126 .globl __flush_icache_page
127 __flush_icache_page: /* %o0 = phys_page */
129 srlx %o0, PAGE_SHIFT, %o0
130 sethi %uhi(PAGE_OFFSET), %g1
131 sllx %o0, PAGE_SHIFT, %o0
132 sethi %hi(PAGE_SIZE), %g2
135 1: subcc %g2, 32, %g2
141 #ifdef DCACHE_ALIASING_POSSIBLE
143 #if (PAGE_SHIFT != 13)
144 #error only page shift of 13 is supported by dcache flush
147 #define DTAG_MASK 0x3
149 /* This routine is Spitfire specific so the hardcoded
150 * D-cache size and line-size are OK.
153 .globl __flush_dcache_page
154 __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
155 sethi %uhi(PAGE_OFFSET), %g1
157 sub %o0, %g1, %o0 ! physical address
158 srlx %o0, 11, %o0 ! make D-cache TAG
159 sethi %hi(1 << 14), %o2 ! D-cache size
160 sub %o2, (1 << 5), %o2 ! D-cache line size
161 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG
162 andcc %o3, DTAG_MASK, %g0 ! Valid?
163 be,pn %xcc, 2f ! Nope, branch
164 andn %o3, DTAG_MASK, %o3 ! Clear valid bits
165 cmp %o3, %o0 ! TAG match?
166 bne,pt %xcc, 2f ! Nope, branch
168 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG
171 sub %o2, (1 << 5), %o2 ! D-cache line size
173 /* The I-cache does not snoop local stores so we
174 * better flush that too when necessary.
176 brnz,pt %o1, __flush_icache_page
181 #endif /* DCACHE_ALIASING_POSSIBLE */
185 /* Cheetah specific versions, patched at boot time. */
186 __cheetah_flush_tlb_mm: /* 19 insns */
188 andn %g7, PSTATE_IE, %g2
189 wrpr %g2, 0x0, %pstate
191 mov PRIMARY_CONTEXT, %o2
193 ldxa [%o2] ASI_DMMU, %g2
194 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1
195 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1
196 or %o0, %o1, %o0 /* Preserve nucleus page size fields */
197 stxa %o0, [%o2] ASI_DMMU
198 stxa %g0, [%g3] ASI_DMMU_DEMAP
199 stxa %g0, [%g3] ASI_IMMU_DEMAP
200 stxa %g2, [%o2] ASI_DMMU
201 sethi %hi(KERNBASE), %o2
205 wrpr %g7, 0x0, %pstate
207 __cheetah_flush_tlb_pending: /* 27 insns */
208 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
211 andn %g7, PSTATE_IE, %g2
212 wrpr %g2, 0x0, %pstate
214 mov PRIMARY_CONTEXT, %o4
215 ldxa [%o4] ASI_DMMU, %g2
216 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
217 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
218 or %o0, %o3, %o0 /* Preserve nucleus page size fields */
219 stxa %o0, [%o4] ASI_DMMU
220 1: sub %o1, (1 << 3), %o1
225 stxa %g0, [%o3] ASI_IMMU_DEMAP
226 2: stxa %g0, [%o3] ASI_DMMU_DEMAP
230 stxa %g2, [%o4] ASI_DMMU
231 sethi %hi(KERNBASE), %o4
235 wrpr %g7, 0x0, %pstate
237 #ifdef DCACHE_ALIASING_POSSIBLE
238 __cheetah_flush_dcache_page: /* 11 insns */
239 sethi %uhi(PAGE_OFFSET), %g1
242 sethi %hi(PAGE_SIZE), %o4
243 1: subcc %o4, (1 << 5), %o4
244 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
248 retl /* I-cache flush never needed on Cheetah, see callers. */
250 #endif /* DCACHE_ALIASING_POSSIBLE */
252 /* Hypervisor specific versions, patched at boot time. */
253 __hypervisor_tlb_tl0_error:
256 call hypervisor_tlbop_error
261 __hypervisor_flush_tlb_mm: /* 10 insns */
262 mov %o0, %o2 /* ARG2: mmu context */
263 mov 0, %o0 /* ARG0: CPU lists unimplemented */
264 mov 0, %o1 /* ARG1: CPU lists unimplemented */
265 mov HV_MMU_ALL, %o3 /* ARG3: flags */
266 mov HV_FAST_MMU_DEMAP_CTX, %o5
268 brnz,pn %o0, __hypervisor_tlb_tl0_error
269 mov HV_FAST_MMU_DEMAP_CTX, %o1
273 __hypervisor_flush_tlb_pending: /* 16 insns */
274 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
278 1: sub %g1, (1 << 3), %g1
279 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */
280 mov %g3, %o1 /* ARG1: mmu context */
281 mov HV_MMU_ALL, %o2 /* ARG2: flags */
282 srlx %o0, PAGE_SHIFT, %o0
283 sllx %o0, PAGE_SHIFT, %o0
284 ta HV_MMU_UNMAP_ADDR_TRAP
285 brnz,pn %o0, __hypervisor_tlb_tl0_error
286 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
292 __hypervisor_flush_tlb_kernel_range: /* 16 insns */
293 /* %o0=start, %o1=end */
296 sethi %hi(PAGE_SIZE), %g3
300 1: add %g1, %g2, %o0 /* ARG0: virtual address */
301 mov 0, %o1 /* ARG1: mmu context */
302 mov HV_MMU_ALL, %o2 /* ARG2: flags */
303 ta HV_MMU_UNMAP_ADDR_TRAP
304 brnz,pn %o0, __hypervisor_tlb_tl0_error
305 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
311 #ifdef DCACHE_ALIASING_POSSIBLE
312 /* XXX Niagara and friends have an 8K cache, so no aliasing is
313 * XXX possible, but nothing explicit in the Hypervisor API
314 * XXX guarantees this.
316 __hypervisor_flush_dcache_page: /* 2 insns */
332 .globl cheetah_patch_cachetlbops
333 cheetah_patch_cachetlbops:
336 sethi %hi(__flush_tlb_mm), %o0
337 or %o0, %lo(__flush_tlb_mm), %o0
338 sethi %hi(__cheetah_flush_tlb_mm), %o1
339 or %o1, %lo(__cheetah_flush_tlb_mm), %o1
343 sethi %hi(__flush_tlb_pending), %o0
344 or %o0, %lo(__flush_tlb_pending), %o0
345 sethi %hi(__cheetah_flush_tlb_pending), %o1
346 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
350 #ifdef DCACHE_ALIASING_POSSIBLE
351 sethi %hi(__flush_dcache_page), %o0
352 or %o0, %lo(__flush_dcache_page), %o0
353 sethi %hi(__cheetah_flush_dcache_page), %o1
354 or %o1, %lo(__cheetah_flush_dcache_page), %o1
357 #endif /* DCACHE_ALIASING_POSSIBLE */
363 /* These are all called by the slaves of a cross call, at
364 * trap level 1, with interrupts fully disabled.
367 * %g5 mm->context (all tlb flushes)
368 * %g1 address arg 1 (tlb page and range flushes)
369 * %g7 address arg 2 (tlb range flush only)
377 .globl xcall_flush_tlb_mm
378 xcall_flush_tlb_mm: /* 21 insns */
379 mov PRIMARY_CONTEXT, %g2
380 ldxa [%g2] ASI_DMMU, %g3
381 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
382 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
383 or %g5, %g4, %g5 /* Preserve nucleus page size fields */
384 stxa %g5, [%g2] ASI_DMMU
386 stxa %g0, [%g4] ASI_DMMU_DEMAP
387 stxa %g0, [%g4] ASI_IMMU_DEMAP
388 stxa %g3, [%g2] ASI_DMMU
401 .globl xcall_flush_tlb_pending
402 xcall_flush_tlb_pending: /* 21 insns */
403 /* %g5=context, %g1=nr, %g7=vaddrs[] */
405 mov PRIMARY_CONTEXT, %g4
406 ldxa [%g4] ASI_DMMU, %g2
407 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
408 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
410 mov PRIMARY_CONTEXT, %g4
411 stxa %g5, [%g4] ASI_DMMU
412 1: sub %g1, (1 << 3), %g1
418 stxa %g0, [%g5] ASI_IMMU_DEMAP
419 2: stxa %g0, [%g5] ASI_DMMU_DEMAP
423 stxa %g2, [%g4] ASI_DMMU
427 .globl xcall_flush_tlb_kernel_range
428 xcall_flush_tlb_kernel_range: /* 25 insns */
429 sethi %hi(PAGE_SIZE - 1), %g2
430 or %g2, %lo(PAGE_SIZE - 1), %g2
436 or %g1, 0x20, %g1 ! Nucleus
437 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
438 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
455 /* This runs in a very controlled environment, so we do
456 * not need to worry about BH races etc.
458 .globl xcall_sync_tick
461 661: rdpr %pstate, %g2
462 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
463 .section .sun4v_2insn_patch, "ax"
473 109: or %g7, %lo(109b), %g7
474 #ifdef CONFIG_TRACE_IRQFLAGS
475 call trace_hardirqs_off
478 call smp_synchronize_tick_client
481 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
483 /* NOTE: This is SPECIAL!! We do etrap/rtrap however
484 * we choose to deal with the "BH's run with
485 * %pil==15" problem (described in asm/pil.h)
486 * by just invoking rtrap directly past where
487 * BH's are checked for.
489 * We do it like this because we do not want %pil==15
490 * lockups to prevent regs being reported.
492 .globl xcall_report_regs
495 661: rdpr %pstate, %g2
496 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
497 .section .sun4v_2insn_patch, "ax"
507 109: or %g7, %lo(109b), %g7
508 #ifdef CONFIG_TRACE_IRQFLAGS
509 call trace_hardirqs_off
513 add %sp, PTREGS_OFF, %o0
514 /* Has to be a non-v9 branch due to the large distance. */
516 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
518 #ifdef CONFIG_MAGIC_SYSRQ
519 .globl xcall_fetch_glob_regs
520 xcall_fetch_glob_regs:
521 sethi %hi(global_reg_snapshot), %g1
522 or %g1, %lo(global_reg_snapshot), %g1
527 stx %g7, [%g1 + GR_SNAP_TSTATE]
529 stx %g7, [%g1 + GR_SNAP_TPC]
531 stx %g7, [%g1 + GR_SNAP_TNPC]
532 stx %o7, [%g1 + GR_SNAP_O7]
533 stx %i7, [%g1 + GR_SNAP_I7]
534 /* Don't try this at home kids... */
540 stx %g7, [%g1 + GR_SNAP_RPC]
541 sethi %hi(trap_block), %g7
542 or %g7, %lo(trap_block), %g7
543 sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2
545 ldx [%g7 + TRAP_PER_CPU_THREAD], %g3
547 stx %g3, [%g1 + GR_SNAP_THREAD]
549 #endif /* CONFIG_MAGIC_SYSRQ */
551 #ifdef DCACHE_ALIASING_POSSIBLE
553 .globl xcall_flush_dcache_page_cheetah
554 xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
555 sethi %hi(PAGE_SIZE), %g3
556 1: subcc %g3, (1 << 5), %g3
557 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
563 #endif /* DCACHE_ALIASING_POSSIBLE */
565 .globl xcall_flush_dcache_page_spitfire
566 xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
567 %g7 == kernel page virtual address
568 %g5 == (page->mapping != NULL) */
569 #ifdef DCACHE_ALIASING_POSSIBLE
570 srlx %g1, (13 - 2), %g1 ! Form tag comparitor
571 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
572 sub %g3, (1 << 5), %g3 ! D$ linesize == 32
573 1: ldxa [%g3] ASI_DCACHE_TAG, %g2
581 stxa %g0, [%g3] ASI_DCACHE_TAG
585 sub %g3, (1 << 5), %g3
588 #endif /* DCACHE_ALIASING_POSSIBLE */
589 sethi %hi(PAGE_SIZE), %g3
592 subcc %g3, (1 << 5), %g3
594 add %g7, (1 << 5), %g7
603 __hypervisor_tlb_xcall_error:
609 call hypervisor_tlbop_error_xcall
613 .globl __hypervisor_xcall_flush_tlb_mm
614 __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
615 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
621 clr %o0 /* ARG0: CPU lists unimplemented */
622 clr %o1 /* ARG1: CPU lists unimplemented */
623 mov %g5, %o2 /* ARG2: mmu context */
624 mov HV_MMU_ALL, %o3 /* ARG3: flags */
625 mov HV_FAST_MMU_DEMAP_CTX, %o5
627 mov HV_FAST_MMU_DEMAP_CTX, %g6
628 brnz,pn %o0, __hypervisor_tlb_xcall_error
638 .globl __hypervisor_xcall_flush_tlb_pending
639 __hypervisor_xcall_flush_tlb_pending: /* 21 insns */
640 /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */
645 1: sub %g1, (1 << 3), %g1
646 ldx [%g7 + %g1], %o0 /* ARG0: virtual address */
647 mov %g5, %o1 /* ARG1: mmu context */
648 mov HV_MMU_ALL, %o2 /* ARG2: flags */
649 srlx %o0, PAGE_SHIFT, %o0
650 sllx %o0, PAGE_SHIFT, %o0
651 ta HV_MMU_UNMAP_ADDR_TRAP
652 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
653 brnz,a,pn %o0, __hypervisor_tlb_xcall_error
663 .globl __hypervisor_xcall_flush_tlb_kernel_range
664 __hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
665 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
666 sethi %hi(PAGE_SIZE - 1), %g2
667 or %g2, %lo(PAGE_SIZE - 1), %g2
676 1: add %g1, %g3, %o0 /* ARG0: virtual address */
677 mov 0, %o1 /* ARG1: mmu context */
678 mov HV_MMU_ALL, %o2 /* ARG2: flags */
679 ta HV_MMU_UNMAP_ADDR_TRAP
680 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
681 brnz,pn %o0, __hypervisor_tlb_xcall_error
683 sethi %hi(PAGE_SIZE), %o2
692 /* These just get rescheduled to PIL vectors. */
693 .globl xcall_call_function
695 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
698 .globl xcall_call_function_single
699 xcall_call_function_single:
700 wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
703 .globl xcall_receive_signal
704 xcall_receive_signal:
705 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
710 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint
713 .globl xcall_new_mmu_context_version
714 xcall_new_mmu_context_version:
715 wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint
719 .globl xcall_kgdb_capture
721 661: rdpr %pstate, %g2
722 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
723 .section .sun4v_2insn_patch, "ax"
732 ba,pt %xcc, etrap_irq
733 109: or %g7, %lo(109b), %g7
734 #ifdef CONFIG_TRACE_IRQFLAGS
735 call trace_hardirqs_off
738 call smp_kgdb_capture_client
739 add %sp, PTREGS_OFF, %o0
740 /* Has to be a non-v9 branch due to the large distance. */
742 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
745 #endif /* CONFIG_SMP */
748 .globl hypervisor_patch_cachetlbops
749 hypervisor_patch_cachetlbops:
752 sethi %hi(__flush_tlb_mm), %o0
753 or %o0, %lo(__flush_tlb_mm), %o0
754 sethi %hi(__hypervisor_flush_tlb_mm), %o1
755 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
759 sethi %hi(__flush_tlb_pending), %o0
760 or %o0, %lo(__flush_tlb_pending), %o0
761 sethi %hi(__hypervisor_flush_tlb_pending), %o1
762 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
766 sethi %hi(__flush_tlb_kernel_range), %o0
767 or %o0, %lo(__flush_tlb_kernel_range), %o0
768 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
769 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
773 #ifdef DCACHE_ALIASING_POSSIBLE
774 sethi %hi(__flush_dcache_page), %o0
775 or %o0, %lo(__flush_dcache_page), %o0
776 sethi %hi(__hypervisor_flush_dcache_page), %o1
777 or %o1, %lo(__hypervisor_flush_dcache_page), %o1
780 #endif /* DCACHE_ALIASING_POSSIBLE */
783 sethi %hi(xcall_flush_tlb_mm), %o0
784 or %o0, %lo(xcall_flush_tlb_mm), %o0
785 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
786 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
790 sethi %hi(xcall_flush_tlb_pending), %o0
791 or %o0, %lo(xcall_flush_tlb_pending), %o0
792 sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1
793 or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1
797 sethi %hi(xcall_flush_tlb_kernel_range), %o0
798 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
799 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
800 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
803 #endif /* CONFIG_SMP */