1 /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
3 * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
9 #include <linux/config.h>
13 #include <asm/pgtable.h>
21 mov TLB_TAG_ACCESS, %g4
22 ldxa [%g4] ASI_IMMU, %g4
24 /* sun4v_itlb_miss branches here with the missing virtual
25 * address already loaded into %g4
30 /* Catch kernel NULL pointer calls. */
31 sethi %hi(PAGE_SIZE), %g5
33 bleu,pn %xcc, kvmap_dtlb_longpath
36 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
39 sethi %hi(LOW_OBP_ADDRESS), %g5
41 blu,pn %xcc, kvmap_itlb_vmalloc_addr
45 blu,pn %xcc, kvmap_itlb_obp
48 kvmap_itlb_vmalloc_addr:
49 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
51 KTSB_LOCK_TAG(%g1, %g2, %g7)
53 /* Load and check PTE. */
54 ldxa [%g5] ASI_PHYS_USE_EC, %g5
56 sllx %g7, TSB_TAG_INVALID_BIT, %g7
57 brgez,a,pn %g5, kvmap_itlb_longpath
60 KTSB_WRITE(%g1, %g5, %g6)
62 /* fallthrough to TLB load */
66 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
68 .section .sun4v_2insn_patch, "ax"
74 /* For sun4v the ASI_ITLB_DATA_IN store and the retry
75 * instruction get nop'd out and we get here to branch
76 * to the sun4v tlb load code. The registers are setup
83 * The sun4v TLB load wants the PTE in %g3 so we fix that
86 ba,pt %xcc, sun4v_itlb_load
91 661: rdpr %pstate, %g5
92 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
93 .section .sun4v_2insn_patch, "ax"
100 ba,pt %xcc, sparc64_realfault_common
101 mov FAULT_CODE_ITLB, %g4
104 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
106 KTSB_LOCK_TAG(%g1, %g2, %g7)
108 KTSB_WRITE(%g1, %g5, %g6)
110 ba,pt %xcc, kvmap_itlb_load
114 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
116 KTSB_LOCK_TAG(%g1, %g2, %g7)
118 KTSB_WRITE(%g1, %g5, %g6)
120 ba,pt %xcc, kvmap_dtlb_load
125 /* %g6: TAG TARGET */
126 mov TLB_TAG_ACCESS, %g4
127 ldxa [%g4] ASI_DMMU, %g4
129 /* sun4v_dtlb_miss branches here with the missing virtual
130 * address already loaded into %g4
133 brgez,pn %g4, kvmap_dtlb_nonlinear
136 sethi %hi(kpte_linear_bitmap), %g2
137 or %g2, %lo(kpte_linear_bitmap), %g2
139 /* Clear the PAGE_OFFSET top virtual bits, then shift
140 * down to get a 256MB physical address index.
144 srlx %g5, 21 + 28, %g5
146 /* Don't try this at home kids... this depends upon srlx
147 * only taking the low 6 bits of the shift count in %g5.
151 /* Divide by 64 to get the offset into the bitmask. */
154 /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */
157 sethi %hi(kern_linear_pte_xor), %g5
158 or %g5, %lo(kern_linear_pte_xor), %g5
164 .globl kvmap_linear_patch
166 ba,pt %xcc, kvmap_dtlb_load
169 kvmap_dtlb_vmalloc_addr:
170 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
172 KTSB_LOCK_TAG(%g1, %g2, %g7)
174 /* Load and check PTE. */
175 ldxa [%g5] ASI_PHYS_USE_EC, %g5
177 sllx %g7, TSB_TAG_INVALID_BIT, %g7
178 brgez,a,pn %g5, kvmap_dtlb_longpath
181 KTSB_WRITE(%g1, %g5, %g6)
183 /* fallthrough to TLB load */
187 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
189 .section .sun4v_2insn_patch, "ax"
195 /* For sun4v the ASI_DTLB_DATA_IN store and the retry
196 * instruction get nop'd out and we get here to branch
197 * to the sun4v tlb load code. The registers are setup
204 * The sun4v TLB load wants the PTE in %g3 so we fix that
207 ba,pt %xcc, sun4v_dtlb_load
210 kvmap_dtlb_nonlinear:
211 /* Catch kernel NULL pointer derefs. */
212 sethi %hi(PAGE_SIZE), %g5
214 bleu,pn %xcc, kvmap_dtlb_longpath
217 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
220 sethi %hi(MODULES_VADDR), %g5
222 blu,pn %xcc, kvmap_dtlb_longpath
223 mov (VMALLOC_END >> 24), %g5
226 bgeu,pn %xcc, kvmap_dtlb_longpath
230 sethi %hi(LOW_OBP_ADDRESS), %g5
232 blu,pn %xcc, kvmap_dtlb_vmalloc_addr
236 blu,pn %xcc, kvmap_dtlb_obp
238 ba,pt %xcc, kvmap_dtlb_vmalloc_addr
243 661: rdpr %pstate, %g5
244 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
245 .section .sun4v_2insn_patch, "ax"
248 ldxa [%g0] ASI_SCRATCHPAD, %g5
254 661: mov TLB_TAG_ACCESS, %g4
255 ldxa [%g4] ASI_DMMU, %g5
256 .section .sun4v_2insn_patch, "ax"
258 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
262 be,pt %xcc, sparc64_realfault_common
263 mov FAULT_CODE_DTLB, %g4
264 ba,pt %xcc, winfix_trampoline