1 /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
3 * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
9 #include <linux/config.h>
13 #include <asm/pgtable.h>
21 mov TLB_TAG_ACCESS, %g4
22 ldxa [%g4] ASI_IMMU, %g4
24 /* sun4v_itlb_miss branches here with the missing virtual
25 * address already loaded into %g4
30 /* Catch kernel NULL pointer calls. */
31 sethi %hi(PAGE_SIZE), %g5
33 bleu,pn %xcc, kvmap_dtlb_longpath
36 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
39 sethi %hi(LOW_OBP_ADDRESS), %g5
41 blu,pn %xcc, kvmap_itlb_vmalloc_addr
45 blu,pn %xcc, kvmap_itlb_obp
48 kvmap_itlb_vmalloc_addr:
49 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
51 KTSB_LOCK_TAG(%g1, %g2, %g7)
53 /* Load and check PTE. */
54 ldxa [%g5] ASI_PHYS_USE_EC, %g5
56 sllx %g7, TSB_TAG_INVALID_BIT, %g7
57 brgez,a,pn %g5, kvmap_itlb_longpath
60 KTSB_WRITE(%g1, %g5, %g6)
62 /* fallthrough to TLB load */
66 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
68 .section .sun4v_2insn_patch, "ax"
74 /* For sun4v the ASI_ITLB_DATA_IN store and the retry
75 * instruction get nop'd out and we get here to branch
76 * to the sun4v tlb load code. The registers are setup
83 * The sun4v TLB load wants the PTE in %g3 so we fix that
86 ba,pt %xcc, sun4v_itlb_load
91 661: rdpr %pstate, %g5
92 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
93 .section .sun4v_2insn_patch, "ax"
100 ba,pt %xcc, sparc64_realfault_common
101 mov FAULT_CODE_ITLB, %g4
104 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
106 KTSB_LOCK_TAG(%g1, %g2, %g7)
108 KTSB_WRITE(%g1, %g5, %g6)
110 ba,pt %xcc, kvmap_itlb_load
114 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
116 KTSB_LOCK_TAG(%g1, %g2, %g7)
118 KTSB_WRITE(%g1, %g5, %g6)
120 ba,pt %xcc, kvmap_dtlb_load
124 kvmap_dtlb_tsb4m_load:
125 KTSB_LOCK_TAG(%g1, %g2, %g7)
126 KTSB_WRITE(%g1, %g5, %g6)
127 ba,pt %xcc, kvmap_dtlb_load
131 /* %g6: TAG TARGET */
132 mov TLB_TAG_ACCESS, %g4
133 ldxa [%g4] ASI_DMMU, %g4
135 /* sun4v_dtlb_miss branches here with the missing virtual
136 * address already loaded into %g4
139 brgez,pn %g4, kvmap_dtlb_nonlinear
142 /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
143 KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
145 /* TSB entry address left in %g1, lookup linear PTE.
146 * Must preserve %g1 and %g6 (TAG).
148 kvmap_dtlb_tsb4m_miss:
149 sethi %hi(kpte_linear_bitmap), %g2
150 or %g2, %lo(kpte_linear_bitmap), %g2
152 /* Clear the PAGE_OFFSET top virtual bits, then shift
153 * down to get a 256MB physical address index.
157 srlx %g5, 21 + 28, %g5
159 /* Don't try this at home kids... this depends upon srlx
160 * only taking the low 6 bits of the shift count in %g5.
164 /* Divide by 64 to get the offset into the bitmask. */
167 /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */
170 sethi %hi(kern_linear_pte_xor), %g5
171 or %g5, %lo(kern_linear_pte_xor), %g5
177 .globl kvmap_linear_patch
179 ba,pt %xcc, kvmap_dtlb_tsb4m_load
182 kvmap_dtlb_vmalloc_addr:
183 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
185 KTSB_LOCK_TAG(%g1, %g2, %g7)
187 /* Load and check PTE. */
188 ldxa [%g5] ASI_PHYS_USE_EC, %g5
190 sllx %g7, TSB_TAG_INVALID_BIT, %g7
191 brgez,a,pn %g5, kvmap_dtlb_longpath
194 KTSB_WRITE(%g1, %g5, %g6)
196 /* fallthrough to TLB load */
200 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
202 .section .sun4v_2insn_patch, "ax"
208 /* For sun4v the ASI_DTLB_DATA_IN store and the retry
209 * instruction get nop'd out and we get here to branch
210 * to the sun4v tlb load code. The registers are setup
217 * The sun4v TLB load wants the PTE in %g3 so we fix that
220 ba,pt %xcc, sun4v_dtlb_load
223 kvmap_dtlb_nonlinear:
224 /* Catch kernel NULL pointer derefs. */
225 sethi %hi(PAGE_SIZE), %g5
227 bleu,pn %xcc, kvmap_dtlb_longpath
230 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
233 sethi %hi(MODULES_VADDR), %g5
235 blu,pn %xcc, kvmap_dtlb_longpath
236 mov (VMALLOC_END >> 24), %g5
239 bgeu,pn %xcc, kvmap_dtlb_longpath
243 sethi %hi(LOW_OBP_ADDRESS), %g5
245 blu,pn %xcc, kvmap_dtlb_vmalloc_addr
249 blu,pn %xcc, kvmap_dtlb_obp
251 ba,pt %xcc, kvmap_dtlb_vmalloc_addr
256 661: rdpr %pstate, %g5
257 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
258 .section .sun4v_2insn_patch, "ax"
261 ldxa [%g0] ASI_SCRATCHPAD, %g5
267 661: mov TLB_TAG_ACCESS, %g4
268 ldxa [%g4] ASI_DMMU, %g5
269 .section .sun4v_2insn_patch, "ax"
271 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
275 be,pt %xcc, sparc64_realfault_common
276 mov FAULT_CODE_DTLB, %g4
277 ba,pt %xcc, winfix_trampoline