1 /* $Id: ultra.S,v 1.72 2002/02/09 19:49:31 davem Exp $
2 * ultra.S: Don't expand these all over the place...
4 * Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com)
8 #include <asm/pgtable.h>
10 #include <asm/spitfire.h>
11 #include <asm/mmu_context.h>
15 #include <asm/thread_info.h>
16 #include <asm/cacheflush.h>
17 #include <asm/hypervisor.h>
19 /* Basically, most of the Spitfire vs. Cheetah madness
20 * has to do with the fact that Cheetah does not support
21 * IMMU flushes out of the secondary context. Someone needs
22 * to throw a south lake birthday party for the folks
23 * in Microelectronics who refused to fix this shit.
26 /* This file is meant to be read efficiently by the CPU, not humans.
27 * Staraj sie tego nikomu nie pierdolnac...
32 __flush_tlb_mm: /* 18 insns */
33 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
34 ldxa [%o1] ASI_DMMU, %g2
36 bne,pn %icc, __spitfire_flush_tlb_mm_slow
38 stxa %g0, [%g3] ASI_DMMU_DEMAP
39 stxa %g0, [%g3] ASI_IMMU_DEMAP
40 sethi %hi(KERNBASE), %g3
55 .globl __flush_tlb_pending
56 __flush_tlb_pending: /* 26 insns */
57 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
60 andn %g7, PSTATE_IE, %g2
62 mov SECONDARY_CONTEXT, %o4
63 ldxa [%o4] ASI_DMMU, %g2
64 stxa %o0, [%o4] ASI_DMMU
65 1: sub %o1, (1 << 3), %o1
71 stxa %g0, [%o3] ASI_IMMU_DEMAP
72 2: stxa %g0, [%o3] ASI_DMMU_DEMAP
76 stxa %g2, [%o4] ASI_DMMU
77 sethi %hi(KERNBASE), %o4
80 wrpr %g7, 0x0, %pstate
87 .globl __flush_tlb_kernel_range
88 __flush_tlb_kernel_range: /* 16 insns */
89 /* %o0=start, %o1=end */
92 sethi %hi(PAGE_SIZE), %o4
95 or %o0, 0x20, %o0 ! Nucleus
96 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
97 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
101 2: sethi %hi(KERNBASE), %o3
107 __spitfire_flush_tlb_mm_slow:
109 wrpr %g1, PSTATE_IE, %pstate
110 stxa %o0, [%o1] ASI_DMMU
111 stxa %g0, [%g3] ASI_DMMU_DEMAP
112 stxa %g0, [%g3] ASI_IMMU_DEMAP
114 stxa %g2, [%o1] ASI_DMMU
115 sethi %hi(KERNBASE), %o1
121 * The following code flushes one page_size worth.
123 #if (PAGE_SHIFT == 13)
124 #define ITAG_MASK 0xfe
125 #elif (PAGE_SHIFT == 16)
126 #define ITAG_MASK 0x7fe
128 #error unsupported PAGE_SIZE
130 .section .kprobes.text, "ax"
132 .globl __flush_icache_page
133 __flush_icache_page: /* %o0 = phys_page */
135 srlx %o0, PAGE_SHIFT, %o0
136 sethi %uhi(PAGE_OFFSET), %g1
137 sllx %o0, PAGE_SHIFT, %o0
138 sethi %hi(PAGE_SIZE), %g2
141 1: subcc %g2, 32, %g2
147 #ifdef DCACHE_ALIASING_POSSIBLE
149 #if (PAGE_SHIFT != 13)
150 #error only page shift of 13 is supported by dcache flush
153 #define DTAG_MASK 0x3
155 /* This routine is Spitfire specific so the hardcoded
156 * D-cache size and line-size are OK.
159 .globl __flush_dcache_page
160 __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
161 sethi %uhi(PAGE_OFFSET), %g1
163 sub %o0, %g1, %o0 ! physical address
164 srlx %o0, 11, %o0 ! make D-cache TAG
165 sethi %hi(1 << 14), %o2 ! D-cache size
166 sub %o2, (1 << 5), %o2 ! D-cache line size
167 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG
168 andcc %o3, DTAG_MASK, %g0 ! Valid?
169 be,pn %xcc, 2f ! Nope, branch
170 andn %o3, DTAG_MASK, %o3 ! Clear valid bits
171 cmp %o3, %o0 ! TAG match?
172 bne,pt %xcc, 2f ! Nope, branch
174 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG
177 sub %o2, (1 << 5), %o2 ! D-cache line size
179 /* The I-cache does not snoop local stores so we
180 * better flush that too when necessary.
182 brnz,pt %o1, __flush_icache_page
187 #endif /* DCACHE_ALIASING_POSSIBLE */
191 /* Cheetah specific versions, patched at boot time. */
192 __cheetah_flush_tlb_mm: /* 19 insns */
194 andn %g7, PSTATE_IE, %g2
195 wrpr %g2, 0x0, %pstate
197 mov PRIMARY_CONTEXT, %o2
199 ldxa [%o2] ASI_DMMU, %g2
200 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1
201 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1
202 or %o0, %o1, %o0 /* Preserve nucleus page size fields */
203 stxa %o0, [%o2] ASI_DMMU
204 stxa %g0, [%g3] ASI_DMMU_DEMAP
205 stxa %g0, [%g3] ASI_IMMU_DEMAP
206 stxa %g2, [%o2] ASI_DMMU
207 sethi %hi(KERNBASE), %o2
211 wrpr %g7, 0x0, %pstate
213 __cheetah_flush_tlb_pending: /* 27 insns */
214 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
217 andn %g7, PSTATE_IE, %g2
218 wrpr %g2, 0x0, %pstate
220 mov PRIMARY_CONTEXT, %o4
221 ldxa [%o4] ASI_DMMU, %g2
222 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
223 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
224 or %o0, %o3, %o0 /* Preserve nucleus page size fields */
225 stxa %o0, [%o4] ASI_DMMU
226 1: sub %o1, (1 << 3), %o1
231 stxa %g0, [%o3] ASI_IMMU_DEMAP
232 2: stxa %g0, [%o3] ASI_DMMU_DEMAP
236 stxa %g2, [%o4] ASI_DMMU
237 sethi %hi(KERNBASE), %o4
241 wrpr %g7, 0x0, %pstate
243 #ifdef DCACHE_ALIASING_POSSIBLE
244 __cheetah_flush_dcache_page: /* 11 insns */
245 sethi %uhi(PAGE_OFFSET), %g1
248 sethi %hi(PAGE_SIZE), %o4
249 1: subcc %o4, (1 << 5), %o4
250 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
254 retl /* I-cache flush never needed on Cheetah, see callers. */
256 #endif /* DCACHE_ALIASING_POSSIBLE */
258 /* Hypervisor specific versions, patched at boot time. */
259 __hypervisor_tlb_tl0_error:
262 call hypervisor_tlbop_error
267 __hypervisor_flush_tlb_mm: /* 10 insns */
268 mov %o0, %o2 /* ARG2: mmu context */
269 mov 0, %o0 /* ARG0: CPU lists unimplemented */
270 mov 0, %o1 /* ARG1: CPU lists unimplemented */
271 mov HV_MMU_ALL, %o3 /* ARG3: flags */
272 mov HV_FAST_MMU_DEMAP_CTX, %o5
274 brnz,pn %o0, __hypervisor_tlb_tl0_error
275 mov HV_FAST_MMU_DEMAP_CTX, %o1
279 __hypervisor_flush_tlb_pending: /* 16 insns */
280 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
284 1: sub %g1, (1 << 3), %g1
285 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */
286 mov %g3, %o1 /* ARG1: mmu context */
287 mov HV_MMU_ALL, %o2 /* ARG2: flags */
288 srlx %o0, PAGE_SHIFT, %o0
289 sllx %o0, PAGE_SHIFT, %o0
290 ta HV_MMU_UNMAP_ADDR_TRAP
291 brnz,pn %o0, __hypervisor_tlb_tl0_error
292 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
298 __hypervisor_flush_tlb_kernel_range: /* 16 insns */
299 /* %o0=start, %o1=end */
302 sethi %hi(PAGE_SIZE), %g3
306 1: add %g1, %g2, %o0 /* ARG0: virtual address */
307 mov 0, %o1 /* ARG1: mmu context */
308 mov HV_MMU_ALL, %o2 /* ARG2: flags */
309 ta HV_MMU_UNMAP_ADDR_TRAP
310 brnz,pn %o0, __hypervisor_tlb_tl0_error
311 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
317 #ifdef DCACHE_ALIASING_POSSIBLE
318 /* XXX Niagara and friends have an 8K cache, so no aliasing is
319 * XXX possible, but nothing explicit in the Hypervisor API
320 * XXX guarantees this.
322 __hypervisor_flush_dcache_page: /* 2 insns */
338 .globl cheetah_patch_cachetlbops
339 cheetah_patch_cachetlbops:
342 sethi %hi(__flush_tlb_mm), %o0
343 or %o0, %lo(__flush_tlb_mm), %o0
344 sethi %hi(__cheetah_flush_tlb_mm), %o1
345 or %o1, %lo(__cheetah_flush_tlb_mm), %o1
349 sethi %hi(__flush_tlb_pending), %o0
350 or %o0, %lo(__flush_tlb_pending), %o0
351 sethi %hi(__cheetah_flush_tlb_pending), %o1
352 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
356 #ifdef DCACHE_ALIASING_POSSIBLE
357 sethi %hi(__flush_dcache_page), %o0
358 or %o0, %lo(__flush_dcache_page), %o0
359 sethi %hi(__cheetah_flush_dcache_page), %o1
360 or %o1, %lo(__cheetah_flush_dcache_page), %o1
363 #endif /* DCACHE_ALIASING_POSSIBLE */
369 /* These are all called by the slaves of a cross call, at
370 * trap level 1, with interrupts fully disabled.
373 * %g5 mm->context (all tlb flushes)
374 * %g1 address arg 1 (tlb page and range flushes)
375 * %g7 address arg 2 (tlb range flush only)
383 .globl xcall_flush_tlb_mm
384 xcall_flush_tlb_mm: /* 21 insns */
385 mov PRIMARY_CONTEXT, %g2
386 ldxa [%g2] ASI_DMMU, %g3
387 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
388 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
389 or %g5, %g4, %g5 /* Preserve nucleus page size fields */
390 stxa %g5, [%g2] ASI_DMMU
392 stxa %g0, [%g4] ASI_DMMU_DEMAP
393 stxa %g0, [%g4] ASI_IMMU_DEMAP
394 stxa %g3, [%g2] ASI_DMMU
407 .globl xcall_flush_tlb_pending
408 xcall_flush_tlb_pending: /* 21 insns */
409 /* %g5=context, %g1=nr, %g7=vaddrs[] */
411 mov PRIMARY_CONTEXT, %g4
412 ldxa [%g4] ASI_DMMU, %g2
413 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
414 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
416 mov PRIMARY_CONTEXT, %g4
417 stxa %g5, [%g4] ASI_DMMU
418 1: sub %g1, (1 << 3), %g1
424 stxa %g0, [%g5] ASI_IMMU_DEMAP
425 2: stxa %g0, [%g5] ASI_DMMU_DEMAP
429 stxa %g2, [%g4] ASI_DMMU
433 .globl xcall_flush_tlb_kernel_range
434 xcall_flush_tlb_kernel_range: /* 25 insns */
435 sethi %hi(PAGE_SIZE - 1), %g2
436 or %g2, %lo(PAGE_SIZE - 1), %g2
442 or %g1, 0x20, %g1 ! Nucleus
443 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
444 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
461 /* This runs in a very controlled environment, so we do
462 * not need to worry about BH races etc.
464 .globl xcall_sync_tick
467 661: rdpr %pstate, %g2
468 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
469 .section .sun4v_2insn_patch, "ax"
479 109: or %g7, %lo(109b), %g7
480 #ifdef CONFIG_TRACE_IRQFLAGS
481 call trace_hardirqs_off
484 call smp_synchronize_tick_client
488 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
490 /* NOTE: This is SPECIAL!! We do etrap/rtrap however
491 * we choose to deal with the "BH's run with
492 * %pil==15" problem (described in asm/pil.h)
493 * by just invoking rtrap directly past where
494 * BH's are checked for.
496 * We do it like this because we do not want %pil==15
497 * lockups to prevent regs being reported.
499 .globl xcall_report_regs
502 661: rdpr %pstate, %g2
503 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
504 .section .sun4v_2insn_patch, "ax"
514 109: or %g7, %lo(109b), %g7
515 #ifdef CONFIG_TRACE_IRQFLAGS
516 call trace_hardirqs_off
520 add %sp, PTREGS_OFF, %o0
522 /* Has to be a non-v9 branch due to the large distance. */
524 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
526 #ifdef DCACHE_ALIASING_POSSIBLE
528 .globl xcall_flush_dcache_page_cheetah
529 xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
530 sethi %hi(PAGE_SIZE), %g3
531 1: subcc %g3, (1 << 5), %g3
532 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
538 #endif /* DCACHE_ALIASING_POSSIBLE */
540 .globl xcall_flush_dcache_page_spitfire
541 xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
542 %g7 == kernel page virtual address
543 %g5 == (page->mapping != NULL) */
544 #ifdef DCACHE_ALIASING_POSSIBLE
545 srlx %g1, (13 - 2), %g1 ! Form tag comparitor
546 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
547 sub %g3, (1 << 5), %g3 ! D$ linesize == 32
548 1: ldxa [%g3] ASI_DCACHE_TAG, %g2
556 stxa %g0, [%g3] ASI_DCACHE_TAG
560 sub %g3, (1 << 5), %g3
563 #endif /* DCACHE_ALIASING_POSSIBLE */
564 sethi %hi(PAGE_SIZE), %g3
567 subcc %g3, (1 << 5), %g3
569 add %g7, (1 << 5), %g7
578 __hypervisor_tlb_xcall_error:
584 call hypervisor_tlbop_error_xcall
586 ba,a,pt %xcc, rtrap_clr_l6
588 .globl __hypervisor_xcall_flush_tlb_mm
589 __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
590 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
596 clr %o0 /* ARG0: CPU lists unimplemented */
597 clr %o1 /* ARG1: CPU lists unimplemented */
598 mov %g5, %o2 /* ARG2: mmu context */
599 mov HV_MMU_ALL, %o3 /* ARG3: flags */
600 mov HV_FAST_MMU_DEMAP_CTX, %o5
602 mov HV_FAST_MMU_DEMAP_CTX, %g6
603 brnz,pn %o0, __hypervisor_tlb_xcall_error
613 .globl __hypervisor_xcall_flush_tlb_pending
614 __hypervisor_xcall_flush_tlb_pending: /* 21 insns */
615 /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */
620 1: sub %g1, (1 << 3), %g1
621 ldx [%g7 + %g1], %o0 /* ARG0: virtual address */
622 mov %g5, %o1 /* ARG1: mmu context */
623 mov HV_MMU_ALL, %o2 /* ARG2: flags */
624 srlx %o0, PAGE_SHIFT, %o0
625 sllx %o0, PAGE_SHIFT, %o0
626 ta HV_MMU_UNMAP_ADDR_TRAP
627 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
628 brnz,a,pn %o0, __hypervisor_tlb_xcall_error
638 .globl __hypervisor_xcall_flush_tlb_kernel_range
639 __hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
640 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
641 sethi %hi(PAGE_SIZE - 1), %g2
642 or %g2, %lo(PAGE_SIZE - 1), %g2
651 1: add %g1, %g3, %o0 /* ARG0: virtual address */
652 mov 0, %o1 /* ARG1: mmu context */
653 mov HV_MMU_ALL, %o2 /* ARG2: flags */
654 ta HV_MMU_UNMAP_ADDR_TRAP
655 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
656 brnz,pn %o0, __hypervisor_tlb_xcall_error
658 sethi %hi(PAGE_SIZE), %o2
667 /* These just get rescheduled to PIL vectors. */
668 .globl xcall_call_function
670 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
673 .globl xcall_receive_signal
674 xcall_receive_signal:
675 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
680 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint
683 .globl xcall_new_mmu_context_version
684 xcall_new_mmu_context_version:
685 wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint
688 #endif /* CONFIG_SMP */
691 .globl hypervisor_patch_cachetlbops
692 hypervisor_patch_cachetlbops:
695 sethi %hi(__flush_tlb_mm), %o0
696 or %o0, %lo(__flush_tlb_mm), %o0
697 sethi %hi(__hypervisor_flush_tlb_mm), %o1
698 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
702 sethi %hi(__flush_tlb_pending), %o0
703 or %o0, %lo(__flush_tlb_pending), %o0
704 sethi %hi(__hypervisor_flush_tlb_pending), %o1
705 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
709 sethi %hi(__flush_tlb_kernel_range), %o0
710 or %o0, %lo(__flush_tlb_kernel_range), %o0
711 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
712 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
716 #ifdef DCACHE_ALIASING_POSSIBLE
717 sethi %hi(__flush_dcache_page), %o0
718 or %o0, %lo(__flush_dcache_page), %o0
719 sethi %hi(__hypervisor_flush_dcache_page), %o1
720 or %o1, %lo(__hypervisor_flush_dcache_page), %o1
723 #endif /* DCACHE_ALIASING_POSSIBLE */
726 sethi %hi(xcall_flush_tlb_mm), %o0
727 or %o0, %lo(xcall_flush_tlb_mm), %o0
728 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
729 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
733 sethi %hi(xcall_flush_tlb_pending), %o0
734 or %o0, %lo(xcall_flush_tlb_pending), %o0
735 sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1
736 or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1
740 sethi %hi(xcall_flush_tlb_kernel_range), %o0
741 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
742 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
743 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
746 #endif /* CONFIG_SMP */