2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Synthesize TLB refill handlers at runtime.
8 * Copyright (C) 2004,2005,2006 by Thiemo Seufer
9 * Copyright (C) 2005, 2007 Maciej W. Rozycki
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
12 * ... and the days got worse and worse and now you see
13 * I've gone completly out of my mind.
15 * They're coming to take me a away haha
16 * they're coming to take me a away hoho hihi haha
17 * to the funny farm where code is beautiful all the time ...
19 * (Condolences to Napoleon XIV)
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/string.h>
25 #include <linux/init.h>
28 #include <asm/mmu_context.h>
33 static inline int r45k_bvahwbug(void)
35 /* XXX: We should probe for the presence of this bug, but we don't. */
39 static inline int r4k_250MHZhwbug(void)
41 /* XXX: We should probe for the presence of this bug, but we don't. */
45 static inline int __maybe_unused bcm1250_m3_war(void)
47 return BCM1250_M3_WAR;
50 static inline int __maybe_unused r10000_llsc_war(void)
52 return R10000_LLSC_WAR;
56 * Found by experiment: At least some revisions of the 4kc throw under
57 * some circumstances a machine check exception, triggered by invalid
58 * values in the index register. Delaying the tlbp instruction until
59 * after the next branch, plus adding an additional nop in front of
60 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
61 * why; it's not an issue caused by the core RTL.
64 static int __init m4kc_tlbp_war(void)
66 return (current_cpu_data.processor_id & 0xffff00) ==
67 (PRID_COMP_MIPS | PRID_IMP_4KC);
71 * A little micro-assembler, intended for TLB refill handler
72 * synthesizing. It is intentionally kept simple, does only support
73 * a subset of instructions, and does not try to hide pipeline effects
74 * like branch delay slots.
101 #define IMM_MASK 0xffff
103 #define JIMM_MASK 0x3ffffff
105 #define FUNC_MASK 0x3f
112 insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
113 insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
114 insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
115 insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
116 insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
117 insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
118 insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
119 insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
120 insn_tlbwr, insn_xor, insn_xori
129 /* This macro sets the non-variable bits of an instruction. */
130 #define M(a, b, c, d, e, f) \
138 static struct insn insn_table[] __initdata = {
139 { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
140 { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
141 { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD },
142 { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
143 { insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
144 { insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
145 { insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM },
146 { insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM },
147 { insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM },
148 { insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM },
149 { insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
150 { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
151 { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD },
152 { insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
153 { insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
154 { insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE },
155 { insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE },
156 { insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE },
157 { insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE },
158 { insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE },
159 { insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD },
160 { insn_eret, M(cop0_op, cop_op, 0, 0, 0, eret_op), 0 },
161 { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM },
162 { insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM },
163 { insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS },
164 { insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
165 { insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
166 { insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
167 { insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM },
168 { insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
169 { insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET},
170 { insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET},
171 { insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
172 { insn_rfe, M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0 },
173 { insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
174 { insn_scd, M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
175 { insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
176 { insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE },
177 { insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE },
178 { insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE },
179 { insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD },
180 { insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
181 { insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 },
182 { insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 },
183 { insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 },
184 { insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD },
185 { insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
186 { insn_invalid, 0, 0 }
191 static u32 __init build_rs(u32 arg)
194 printk(KERN_WARNING "TLB synthesizer field overflow\n");
196 return (arg & RS_MASK) << RS_SH;
199 static u32 __init build_rt(u32 arg)
202 printk(KERN_WARNING "TLB synthesizer field overflow\n");
204 return (arg & RT_MASK) << RT_SH;
207 static u32 __init build_rd(u32 arg)
210 printk(KERN_WARNING "TLB synthesizer field overflow\n");
212 return (arg & RD_MASK) << RD_SH;
215 static u32 __init build_re(u32 arg)
218 printk(KERN_WARNING "TLB synthesizer field overflow\n");
220 return (arg & RE_MASK) << RE_SH;
223 static u32 __init build_simm(s32 arg)
225 if (arg > 0x7fff || arg < -0x8000)
226 printk(KERN_WARNING "TLB synthesizer field overflow\n");
231 static u32 __init build_uimm(u32 arg)
234 printk(KERN_WARNING "TLB synthesizer field overflow\n");
236 return arg & IMM_MASK;
239 static u32 __init build_bimm(s32 arg)
241 if (arg > 0x1ffff || arg < -0x20000)
242 printk(KERN_WARNING "TLB synthesizer field overflow\n");
245 printk(KERN_WARNING "Invalid TLB synthesizer branch target\n");
247 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
250 static u32 __init build_jimm(u32 arg)
252 if (arg & ~((JIMM_MASK) << 2))
253 printk(KERN_WARNING "TLB synthesizer field overflow\n");
255 return (arg >> 2) & JIMM_MASK;
258 static u32 __init build_func(u32 arg)
260 if (arg & ~FUNC_MASK)
261 printk(KERN_WARNING "TLB synthesizer field overflow\n");
263 return arg & FUNC_MASK;
266 static u32 __init build_set(u32 arg)
269 printk(KERN_WARNING "TLB synthesizer field overflow\n");
271 return arg & SET_MASK;
275 * The order of opcode arguments is implicitly left to right,
276 * starting with RS and ending with FUNC or IMM.
278 static void __init build_insn(u32 **buf, enum opcode opc, ...)
280 struct insn *ip = NULL;
285 for (i = 0; insn_table[i].opcode != insn_invalid; i++)
286 if (insn_table[i].opcode == opc) {
291 if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
292 panic("Unsupported TLB synthesizer instruction %d", opc);
296 if (ip->fields & RS) op |= build_rs(va_arg(ap, u32));
297 if (ip->fields & RT) op |= build_rt(va_arg(ap, u32));
298 if (ip->fields & RD) op |= build_rd(va_arg(ap, u32));
299 if (ip->fields & RE) op |= build_re(va_arg(ap, u32));
300 if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32));
301 if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32));
302 if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32));
303 if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32));
304 if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32));
305 if (ip->fields & SET) op |= build_set(va_arg(ap, u32));
312 #define I_u1u2u3(op) \
313 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
314 unsigned int b, unsigned int c) \
316 build_insn(buf, insn##op, a, b, c); \
319 #define I_u2u1u3(op) \
320 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
321 unsigned int b, unsigned int c) \
323 build_insn(buf, insn##op, b, a, c); \
326 #define I_u3u1u2(op) \
327 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
328 unsigned int b, unsigned int c) \
330 build_insn(buf, insn##op, b, c, a); \
333 #define I_u1u2s3(op) \
334 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
335 unsigned int b, signed int c) \
337 build_insn(buf, insn##op, a, b, c); \
340 #define I_u2s3u1(op) \
341 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
342 signed int b, unsigned int c) \
344 build_insn(buf, insn##op, c, a, b); \
347 #define I_u2u1s3(op) \
348 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
349 unsigned int b, signed int c) \
351 build_insn(buf, insn##op, b, a, c); \
355 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
358 build_insn(buf, insn##op, a, b); \
362 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
365 build_insn(buf, insn##op, a, b); \
369 static void __init __maybe_unused i##op(u32 **buf, unsigned int a) \
371 build_insn(buf, insn##op, a); \
375 static void __init __maybe_unused i##op(u32 **buf) \
377 build_insn(buf, insn##op); \
446 label_smp_pgtable_change,
447 label_r3000_write_probe_fail,
455 static void __init build_label(struct label **lab, u32 *addr,
464 static inline void __init l##lb(struct label **lab, u32 *addr) \
466 build_label(lab, addr, label##lb); \
481 L_LA(_smp_pgtable_change)
482 L_LA(_r3000_write_probe_fail)
484 /* convenience macros for instructions */
486 # define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off)
487 # define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off)
488 # define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh)
489 # define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh)
490 # define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh)
491 # define i_MFC0(buf, rt, rd...) i_dmfc0(buf, rt, rd)
492 # define i_MTC0(buf, rt, rd...) i_dmtc0(buf, rt, rd)
493 # define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val)
494 # define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd)
495 # define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd)
496 # define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off)
497 # define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off)
499 # define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off)
500 # define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off)
501 # define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh)
502 # define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh)
503 # define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh)
504 # define i_MFC0(buf, rt, rd...) i_mfc0(buf, rt, rd)
505 # define i_MTC0(buf, rt, rd...) i_mtc0(buf, rt, rd)
506 # define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val)
507 # define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd)
508 # define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd)
509 # define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off)
510 # define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off)
513 #define i_b(buf, off) i_beq(buf, 0, 0, off)
514 #define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off)
515 #define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off)
516 #define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off)
517 #define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off)
518 #define i_move(buf, a, b) i_ADDU(buf, a, 0, b)
519 #define i_nop(buf) i_sll(buf, 0, 0, 0)
520 #define i_ssnop(buf) i_sll(buf, 0, 0, 1)
521 #define i_ehb(buf) i_sll(buf, 0, 0, 3)
523 static int __init __maybe_unused in_compat_space_p(long addr)
525 /* Is this address in 32bit compat space? */
527 return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
533 static int __init __maybe_unused rel_highest(long val)
536 return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
542 static int __init __maybe_unused rel_higher(long val)
545 return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
551 static int __init rel_hi(long val)
553 return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
556 static int __init rel_lo(long val)
558 return ((val & 0xffff) ^ 0x8000) - 0x8000;
561 static void __init i_LA_mostly(u32 **buf, unsigned int rs, long addr)
563 if (!in_compat_space_p(addr)) {
564 i_lui(buf, rs, rel_highest(addr));
565 if (rel_higher(addr))
566 i_daddiu(buf, rs, rs, rel_higher(addr));
568 i_dsll(buf, rs, rs, 16);
569 i_daddiu(buf, rs, rs, rel_hi(addr));
570 i_dsll(buf, rs, rs, 16);
572 i_dsll32(buf, rs, rs, 0);
574 i_lui(buf, rs, rel_hi(addr));
577 static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr)
579 i_LA_mostly(buf, rs, addr);
581 if (!in_compat_space_p(addr))
582 i_daddiu(buf, rs, rs, rel_lo(addr));
584 i_addiu(buf, rs, rs, rel_lo(addr));
598 static void __init r_mips_pc16(struct reloc **rel, u32 *addr,
602 (*rel)->type = R_MIPS_PC16;
607 static inline void __resolve_relocs(struct reloc *rel, struct label *lab)
609 long laddr = (long)lab->addr;
610 long raddr = (long)rel->addr;
614 *rel->addr |= build_bimm(laddr - (raddr + 4));
618 panic("Unsupported TLB synthesizer relocation %d",
623 static void __init resolve_relocs(struct reloc *rel, struct label *lab)
627 for (; rel->lab != label_invalid; rel++)
628 for (l = lab; l->lab != label_invalid; l++)
629 if (rel->lab == l->lab)
630 __resolve_relocs(rel, l);
633 static void __init move_relocs(struct reloc *rel, u32 *first, u32 *end,
636 for (; rel->lab != label_invalid; rel++)
637 if (rel->addr >= first && rel->addr < end)
641 static void __init move_labels(struct label *lab, u32 *first, u32 *end,
644 for (; lab->lab != label_invalid; lab++)
645 if (lab->addr >= first && lab->addr < end)
649 static void __init copy_handler(struct reloc *rel, struct label *lab,
650 u32 *first, u32 *end, u32 *target)
652 long off = (long)(target - first);
654 memcpy(target, first, (end - first) * sizeof(u32));
656 move_relocs(rel, first, end, off);
657 move_labels(lab, first, end, off);
660 static int __init __maybe_unused insn_has_bdelay(struct reloc *rel,
663 for (; rel->lab != label_invalid; rel++) {
664 if (rel->addr == addr
665 && (rel->type == R_MIPS_PC16
666 || rel->type == R_MIPS_26))
673 /* convenience functions for labeled branches */
674 static void __init __maybe_unused
675 il_bltz(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
677 r_mips_pc16(r, *p, l);
681 static void __init __maybe_unused il_b(u32 **p, struct reloc **r,
684 r_mips_pc16(r, *p, l);
688 static void __init il_beqz(u32 **p, struct reloc **r, unsigned int reg,
691 r_mips_pc16(r, *p, l);
695 static void __init __maybe_unused
696 il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
698 r_mips_pc16(r, *p, l);
702 static void __init il_bnez(u32 **p, struct reloc **r, unsigned int reg,
705 r_mips_pc16(r, *p, l);
709 static void __init il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
712 r_mips_pc16(r, *p, l);
716 static void __init __maybe_unused
717 il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
719 r_mips_pc16(r, *p, l);
724 * For debug purposes.
726 static inline void dump_handler(const u32 *handler, int count)
730 pr_debug("\t.set push\n");
731 pr_debug("\t.set noreorder\n");
733 for (i = 0; i < count; i++)
734 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
736 pr_debug("\t.set pop\n");
739 /* The only general purpose registers allowed in TLB handlers. */
743 /* Some CP0 registers */
744 #define C0_INDEX 0, 0
745 #define C0_ENTRYLO0 2, 0
746 #define C0_TCBIND 2, 2
747 #define C0_ENTRYLO1 3, 0
748 #define C0_CONTEXT 4, 0
749 #define C0_BADVADDR 8, 0
750 #define C0_ENTRYHI 10, 0
752 #define C0_XCONTEXT 20, 0
755 # define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
757 # define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
760 /* The worst case length of the handler is around 18 instructions for
761 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
762 * Maximum space available is 32 instructions for R3000 and 64
763 * instructions for R4000.
765 * We deliberately chose a buffer size of 128, so we won't scribble
766 * over anything important on overflow before we panic.
768 static u32 tlb_handler[128] __initdata;
770 /* simply assume worst case size for labels and relocs */
771 static struct label labels[128] __initdata;
772 static struct reloc relocs[128] __initdata;
775 * The R3000 TLB handler is simple.
777 static void __init build_r3000_tlb_refill_handler(void)
779 long pgdc = (long)pgd_current;
782 memset(tlb_handler, 0, sizeof(tlb_handler));
785 i_mfc0(&p, K0, C0_BADVADDR);
786 i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
787 i_lw(&p, K1, rel_lo(pgdc), K1);
788 i_srl(&p, K0, K0, 22); /* load delay */
789 i_sll(&p, K0, K0, 2);
790 i_addu(&p, K1, K1, K0);
791 i_mfc0(&p, K0, C0_CONTEXT);
792 i_lw(&p, K1, 0, K1); /* cp0 delay */
793 i_andi(&p, K0, K0, 0xffc); /* load delay */
794 i_addu(&p, K1, K1, K0);
796 i_nop(&p); /* load delay */
797 i_mtc0(&p, K0, C0_ENTRYLO0);
798 i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
799 i_tlbwr(&p); /* cp0 delay */
801 i_rfe(&p); /* branch delay */
803 if (p > tlb_handler + 32)
804 panic("TLB refill handler space exceeded");
806 pr_info("Synthesized TLB refill handler (%u instructions).\n",
807 (unsigned int)(p - tlb_handler));
809 memcpy((void *)ebase, tlb_handler, 0x80);
811 dump_handler((u32 *)ebase, 32);
815 * The R4000 TLB handler is much more complicated. We have two
816 * consecutive handler areas with 32 instructions space each.
817 * Since they aren't used at the same time, we can overflow in the
818 * other one.To keep things simple, we first assume linear space,
819 * then we relocate it to the final handler layout as needed.
821 static u32 final_handler[64] __initdata;
826 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
827 * 2. A timing hazard exists for the TLBP instruction.
829 * stalling_instruction
832 * The JTLB is being read for the TLBP throughout the stall generated by the
833 * previous instruction. This is not really correct as the stalling instruction
834 * can modify the address used to access the JTLB. The failure symptom is that
835 * the TLBP instruction will use an address created for the stalling instruction
836 * and not the address held in C0_ENHI and thus report the wrong results.
838 * The software work-around is to not allow the instruction preceding the TLBP
839 * to stall - make it an NOP or some other instruction guaranteed not to stall.
841 * Errata 2 will not be fixed. This errata is also on the R5000.
843 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
845 static void __init __maybe_unused build_tlb_probe_entry(u32 **p)
847 switch (current_cpu_type()) {
848 /* Found by experiment: R4600 v2.0 needs this, too. */
864 * Write random or indexed TLB entry, and care about the hazards from
865 * the preceeding mtc0 and for the following eret.
867 enum tlb_write_entry { tlb_random, tlb_indexed };
869 static void __init build_tlb_write_entry(u32 **p, struct label **l,
871 enum tlb_write_entry wmode)
873 void(*tlbw)(u32 **) = NULL;
876 case tlb_random: tlbw = i_tlbwr; break;
877 case tlb_indexed: tlbw = i_tlbwi; break;
880 if (cpu_has_mips_r2) {
886 switch (current_cpu_type()) {
894 * This branch uses up a mtc0 hazard nop slot and saves
895 * two nops after the tlbw instruction.
897 il_bgezl(p, r, 0, label_tlbw_hazard);
899 l_tlbw_hazard(l, *p);
945 i_nop(p); /* QED specifies 2 nops hazard */
947 * This branch uses up a mtc0 hazard nop slot and saves
948 * a nop after the tlbw instruction.
950 il_bgezl(p, r, 0, label_tlbw_hazard);
952 l_tlbw_hazard(l, *p);
965 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
966 * use of the JTLB for instructions should not occur for 4
967 * cpu cycles and use for data translations should not occur
1002 panic("No TLB refill handler yet (CPU type: %d)",
1003 current_cpu_data.cputype);
1010 * TMP and PTR are scratch.
1011 * TMP will be clobbered, PTR will hold the pmd entry.
1014 build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
1015 unsigned int tmp, unsigned int ptr)
1017 long pgdc = (long)pgd_current;
1020 * The vmalloc handling is not in the hotpath.
1022 i_dmfc0(p, tmp, C0_BADVADDR);
1024 il_bltz(p, r, tmp, label_module_alloc);
1026 il_bltz(p, r, tmp, label_vmalloc);
1028 /* No i_nop needed here, since the next insn doesn't touch TMP. */
1031 # ifdef CONFIG_MIPS_MT_SMTC
1033 * SMTC uses TCBind value as "CPU" index
1035 i_mfc0(p, ptr, C0_TCBIND);
1036 i_dsrl(p, ptr, ptr, 19);
1039 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
1040 * stored in CONTEXT.
1042 i_dmfc0(p, ptr, C0_CONTEXT);
1043 i_dsrl(p, ptr, ptr, 23);
1045 i_LA_mostly(p, tmp, pgdc);
1046 i_daddu(p, ptr, ptr, tmp);
1047 i_dmfc0(p, tmp, C0_BADVADDR);
1048 i_ld(p, ptr, rel_lo(pgdc), ptr);
1050 i_LA_mostly(p, ptr, pgdc);
1051 i_ld(p, ptr, rel_lo(pgdc), ptr);
1054 l_vmalloc_done(l, *p);
1056 if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */
1057 i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
1059 i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
1061 i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
1062 i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
1063 i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
1064 i_ld(p, ptr, 0, ptr); /* get pmd pointer */
1065 i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
1066 i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
1067 i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
1071 * BVADDR is the faulting address, PTR is scratch.
1072 * PTR will hold the pgd for vmalloc.
1075 build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
1076 unsigned int bvaddr, unsigned int ptr)
1078 long swpd = (long)swapper_pg_dir;
1081 long modd = (long)module_pg_dir;
1083 l_module_alloc(l, *p);
1086 * VMALLOC_START >= 0xc000000000000000UL
1087 * MODULE_START >= 0xe000000000000000UL
1089 i_SLL(p, ptr, bvaddr, 2);
1090 il_bgez(p, r, ptr, label_vmalloc);
1092 if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START)) {
1093 i_lui(p, ptr, rel_hi(MODULE_START)); /* delay slot */
1095 /* unlikely configuration */
1096 i_nop(p); /* delay slot */
1097 i_LA(p, ptr, MODULE_START);
1099 i_dsubu(p, bvaddr, bvaddr, ptr);
1101 if (in_compat_space_p(modd) && !rel_lo(modd)) {
1102 il_b(p, r, label_vmalloc_done);
1103 i_lui(p, ptr, rel_hi(modd));
1105 i_LA_mostly(p, ptr, modd);
1106 il_b(p, r, label_vmalloc_done);
1107 if (in_compat_space_p(modd))
1108 i_addiu(p, ptr, ptr, rel_lo(modd));
1110 i_daddiu(p, ptr, ptr, rel_lo(modd));
1114 if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START) &&
1115 MODULE_START << 32 == VMALLOC_START)
1116 i_dsll32(p, ptr, ptr, 0); /* typical case */
1118 i_LA(p, ptr, VMALLOC_START);
1121 i_LA(p, ptr, VMALLOC_START);
1123 i_dsubu(p, bvaddr, bvaddr, ptr);
1125 if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
1126 il_b(p, r, label_vmalloc_done);
1127 i_lui(p, ptr, rel_hi(swpd));
1129 i_LA_mostly(p, ptr, swpd);
1130 il_b(p, r, label_vmalloc_done);
1131 if (in_compat_space_p(swpd))
1132 i_addiu(p, ptr, ptr, rel_lo(swpd));
1134 i_daddiu(p, ptr, ptr, rel_lo(swpd));
1138 #else /* !CONFIG_64BIT */
1141 * TMP and PTR are scratch.
1142 * TMP will be clobbered, PTR will hold the pgd entry.
1144 static void __init __maybe_unused
1145 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
1147 long pgdc = (long)pgd_current;
1149 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
1151 #ifdef CONFIG_MIPS_MT_SMTC
1153 * SMTC uses TCBind value as "CPU" index
1155 i_mfc0(p, ptr, C0_TCBIND);
1156 i_LA_mostly(p, tmp, pgdc);
1157 i_srl(p, ptr, ptr, 19);
1160 * smp_processor_id() << 3 is stored in CONTEXT.
1162 i_mfc0(p, ptr, C0_CONTEXT);
1163 i_LA_mostly(p, tmp, pgdc);
1164 i_srl(p, ptr, ptr, 23);
1166 i_addu(p, ptr, tmp, ptr);
1168 i_LA_mostly(p, ptr, pgdc);
1170 i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
1171 i_lw(p, ptr, rel_lo(pgdc), ptr);
1172 i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
1173 i_sll(p, tmp, tmp, PGD_T_LOG2);
1174 i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
1177 #endif /* !CONFIG_64BIT */
1179 static void __init build_adjust_context(u32 **p, unsigned int ctx)
1181 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
1182 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
1184 switch (current_cpu_type()) {
1201 i_SRL(p, ctx, ctx, shift);
1202 i_andi(p, ctx, ctx, mask);
1205 static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
1208 * Bug workaround for the Nevada. It seems as if under certain
1209 * circumstances the move from cp0_context might produce a
1210 * bogus result when the mfc0 instruction and its consumer are
1211 * in a different cacheline or a load instruction, probably any
1212 * memory reference, is between them.
1214 switch (current_cpu_type()) {
1216 i_LW(p, ptr, 0, ptr);
1217 GET_CONTEXT(p, tmp); /* get context reg */
1221 GET_CONTEXT(p, tmp); /* get context reg */
1222 i_LW(p, ptr, 0, ptr);
1226 build_adjust_context(p, tmp);
1227 i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1230 static void __init build_update_entries(u32 **p, unsigned int tmp,
1234 * 64bit address support (36bit on a 32bit CPU) in a 32bit
1235 * Kernel is a special case. Only a few CPUs use it.
1237 #ifdef CONFIG_64BIT_PHYS_ADDR
1238 if (cpu_has_64bits) {
1239 i_ld(p, tmp, 0, ptep); /* get even pte */
1240 i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1241 i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
1242 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1243 i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
1244 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1246 int pte_off_even = sizeof(pte_t) / 2;
1247 int pte_off_odd = pte_off_even + sizeof(pte_t);
1249 /* The pte entries are pre-shifted */
1250 i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
1251 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1252 i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
1253 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1256 i_LW(p, tmp, 0, ptep); /* get even pte */
1257 i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1258 if (r45k_bvahwbug())
1259 build_tlb_probe_entry(p);
1260 i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
1261 if (r4k_250MHZhwbug())
1262 i_mtc0(p, 0, C0_ENTRYLO0);
1263 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1264 i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
1265 if (r45k_bvahwbug())
1266 i_mfc0(p, tmp, C0_INDEX);
1267 if (r4k_250MHZhwbug())
1268 i_mtc0(p, 0, C0_ENTRYLO1);
1269 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1273 static void __init build_r4000_tlb_refill_handler(void)
1275 u32 *p = tlb_handler;
1276 struct label *l = labels;
1277 struct reloc *r = relocs;
1279 unsigned int final_len;
1281 memset(tlb_handler, 0, sizeof(tlb_handler));
1282 memset(labels, 0, sizeof(labels));
1283 memset(relocs, 0, sizeof(relocs));
1284 memset(final_handler, 0, sizeof(final_handler));
1287 * create the plain linear handler
1289 if (bcm1250_m3_war()) {
1290 i_MFC0(&p, K0, C0_BADVADDR);
1291 i_MFC0(&p, K1, C0_ENTRYHI);
1292 i_xor(&p, K0, K0, K1);
1293 i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1294 il_bnez(&p, &r, K0, label_leave);
1295 /* No need for i_nop */
1299 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1301 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1304 build_get_ptep(&p, K0, K1);
1305 build_update_entries(&p, K0, K1);
1306 build_tlb_write_entry(&p, &l, &r, tlb_random);
1308 i_eret(&p); /* return from trap */
1311 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
1315 * Overflow check: For the 64bit handler, we need at least one
1316 * free instruction slot for the wrap-around branch. In worst
1317 * case, if the intended insertion point is a delay slot, we
1318 * need three, with the second nop'ed and the third being
1321 /* Loongson2 ebase is different than r4k, we have more space */
1322 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
1323 if ((p - tlb_handler) > 64)
1324 panic("TLB refill handler space exceeded");
1326 if (((p - tlb_handler) > 63)
1327 || (((p - tlb_handler) > 61)
1328 && insn_has_bdelay(relocs, tlb_handler + 29)))
1329 panic("TLB refill handler space exceeded");
1333 * Now fold the handler in the TLB refill handler space.
1335 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
1337 /* Simplest case, just copy the handler. */
1338 copy_handler(relocs, labels, tlb_handler, p, f);
1339 final_len = p - tlb_handler;
1340 #else /* CONFIG_64BIT */
1341 f = final_handler + 32;
1342 if ((p - tlb_handler) <= 32) {
1343 /* Just copy the handler. */
1344 copy_handler(relocs, labels, tlb_handler, p, f);
1345 final_len = p - tlb_handler;
1347 u32 *split = tlb_handler + 30;
1350 * Find the split point.
1352 if (insn_has_bdelay(relocs, split - 1))
1355 /* Copy first part of the handler. */
1356 copy_handler(relocs, labels, tlb_handler, split, f);
1357 f += split - tlb_handler;
1359 /* Insert branch. */
1360 l_split(&l, final_handler);
1361 il_b(&f, &r, label_split);
1362 if (insn_has_bdelay(relocs, split))
1365 copy_handler(relocs, labels, split, split + 1, f);
1366 move_labels(labels, f, f + 1, -1);
1371 /* Copy the rest of the handler. */
1372 copy_handler(relocs, labels, split, p, final_handler);
1373 final_len = (f - (final_handler + 32)) + (p - split);
1375 #endif /* CONFIG_64BIT */
1377 resolve_relocs(relocs, labels);
1378 pr_info("Synthesized TLB refill handler (%u instructions).\n",
1381 memcpy((void *)ebase, final_handler, 0x100);
1383 dump_handler((u32 *)ebase, 64);
1387 * TLB load/store/modify handlers.
1389 * Only the fastpath gets synthesized at runtime, the slowpath for
1390 * do_page_fault remains normal asm.
1392 extern void tlb_do_page_fault_0(void);
1393 extern void tlb_do_page_fault_1(void);
1396 * 128 instructions for the fastpath handler is generous and should
1397 * never be exceeded.
1399 #define FASTPATH_SIZE 128
1401 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
1402 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
1403 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
1406 iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
1409 # ifdef CONFIG_64BIT_PHYS_ADDR
1411 i_lld(p, pte, 0, ptr);
1414 i_LL(p, pte, 0, ptr);
1416 # ifdef CONFIG_64BIT_PHYS_ADDR
1418 i_ld(p, pte, 0, ptr);
1421 i_LW(p, pte, 0, ptr);
1426 iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr,
1429 #ifdef CONFIG_64BIT_PHYS_ADDR
1430 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
1433 i_ori(p, pte, pte, mode);
1435 # ifdef CONFIG_64BIT_PHYS_ADDR
1437 i_scd(p, pte, 0, ptr);
1440 i_SC(p, pte, 0, ptr);
1442 if (r10000_llsc_war())
1443 il_beqzl(p, r, pte, label_smp_pgtable_change);
1445 il_beqz(p, r, pte, label_smp_pgtable_change);
1447 # ifdef CONFIG_64BIT_PHYS_ADDR
1448 if (!cpu_has_64bits) {
1449 /* no i_nop needed */
1450 i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1451 i_ori(p, pte, pte, hwmode);
1452 i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1453 il_beqz(p, r, pte, label_smp_pgtable_change);
1454 /* no i_nop needed */
1455 i_lw(p, pte, 0, ptr);
1462 # ifdef CONFIG_64BIT_PHYS_ADDR
1464 i_sd(p, pte, 0, ptr);
1467 i_SW(p, pte, 0, ptr);
1469 # ifdef CONFIG_64BIT_PHYS_ADDR
1470 if (!cpu_has_64bits) {
1471 i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1472 i_ori(p, pte, pte, hwmode);
1473 i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1474 i_lw(p, pte, 0, ptr);
1481 * Check if PTE is present, if not then jump to LABEL. PTR points to
1482 * the page table where this PTE is located, PTE will be re-loaded
1483 * with it's original value.
1486 build_pte_present(u32 **p, struct label **l, struct reloc **r,
1487 unsigned int pte, unsigned int ptr, enum label_id lid)
1489 i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1490 i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1491 il_bnez(p, r, pte, lid);
1492 iPTE_LW(p, l, pte, ptr);
1495 /* Make PTE valid, store result in PTR. */
1497 build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
1500 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1502 iPTE_SW(p, r, pte, ptr, mode);
1506 * Check if PTE can be written to, if not branch to LABEL. Regardless
1507 * restore PTE with value from PTR when done.
1510 build_pte_writable(u32 **p, struct label **l, struct reloc **r,
1511 unsigned int pte, unsigned int ptr, enum label_id lid)
1513 i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1514 i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1515 il_bnez(p, r, pte, lid);
1516 iPTE_LW(p, l, pte, ptr);
1519 /* Make PTE writable, update software status bits as well, then store
1523 build_make_write(u32 **p, struct reloc **r, unsigned int pte,
1526 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1529 iPTE_SW(p, r, pte, ptr, mode);
1533 * Check if PTE can be modified, if not branch to LABEL. Regardless
1534 * restore PTE with value from PTR when done.
1537 build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
1538 unsigned int pte, unsigned int ptr, enum label_id lid)
1540 i_andi(p, pte, pte, _PAGE_WRITE);
1541 il_beqz(p, r, pte, lid);
1542 iPTE_LW(p, l, pte, ptr);
1546 * R3000 style TLB load/store/modify handlers.
1550 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1554 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1556 i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1557 i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1560 i_rfe(p); /* branch delay */
1564 * This places the pte into ENTRYLO0 and writes it with tlbwi
1565 * or tlbwr as appropriate. This is because the index register
1566 * may have the probe fail bit set as a result of a trap on a
1567 * kseg2 access, i.e. without refill. Then it returns.
1570 build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r,
1571 unsigned int pte, unsigned int tmp)
1573 i_mfc0(p, tmp, C0_INDEX);
1574 i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1575 il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1576 i_mfc0(p, tmp, C0_EPC); /* branch delay */
1577 i_tlbwi(p); /* cp0 delay */
1579 i_rfe(p); /* branch delay */
1580 l_r3000_write_probe_fail(l, *p);
1581 i_tlbwr(p); /* cp0 delay */
1583 i_rfe(p); /* branch delay */
1587 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1590 long pgdc = (long)pgd_current;
1592 i_mfc0(p, pte, C0_BADVADDR);
1593 i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
1594 i_lw(p, ptr, rel_lo(pgdc), ptr);
1595 i_srl(p, pte, pte, 22); /* load delay */
1596 i_sll(p, pte, pte, 2);
1597 i_addu(p, ptr, ptr, pte);
1598 i_mfc0(p, pte, C0_CONTEXT);
1599 i_lw(p, ptr, 0, ptr); /* cp0 delay */
1600 i_andi(p, pte, pte, 0xffc); /* load delay */
1601 i_addu(p, ptr, ptr, pte);
1602 i_lw(p, pte, 0, ptr);
1603 i_tlbp(p); /* load delay */
1606 static void __init build_r3000_tlb_load_handler(void)
1608 u32 *p = handle_tlbl;
1609 struct label *l = labels;
1610 struct reloc *r = relocs;
1612 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1613 memset(labels, 0, sizeof(labels));
1614 memset(relocs, 0, sizeof(relocs));
1616 build_r3000_tlbchange_handler_head(&p, K0, K1);
1617 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1618 i_nop(&p); /* load delay */
1619 build_make_valid(&p, &r, K0, K1);
1620 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1622 l_nopage_tlbl(&l, p);
1623 i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1626 if ((p - handle_tlbl) > FASTPATH_SIZE)
1627 panic("TLB load handler fastpath space exceeded");
1629 resolve_relocs(relocs, labels);
1630 pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
1631 (unsigned int)(p - handle_tlbl));
1633 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1636 static void __init build_r3000_tlb_store_handler(void)
1638 u32 *p = handle_tlbs;
1639 struct label *l = labels;
1640 struct reloc *r = relocs;
1642 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1643 memset(labels, 0, sizeof(labels));
1644 memset(relocs, 0, sizeof(relocs));
1646 build_r3000_tlbchange_handler_head(&p, K0, K1);
1647 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1648 i_nop(&p); /* load delay */
1649 build_make_write(&p, &r, K0, K1);
1650 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1652 l_nopage_tlbs(&l, p);
1653 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1656 if ((p - handle_tlbs) > FASTPATH_SIZE)
1657 panic("TLB store handler fastpath space exceeded");
1659 resolve_relocs(relocs, labels);
1660 pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
1661 (unsigned int)(p - handle_tlbs));
1663 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1666 static void __init build_r3000_tlb_modify_handler(void)
1668 u32 *p = handle_tlbm;
1669 struct label *l = labels;
1670 struct reloc *r = relocs;
1672 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1673 memset(labels, 0, sizeof(labels));
1674 memset(relocs, 0, sizeof(relocs));
1676 build_r3000_tlbchange_handler_head(&p, K0, K1);
1677 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1678 i_nop(&p); /* load delay */
1679 build_make_write(&p, &r, K0, K1);
1680 build_r3000_pte_reload_tlbwi(&p, K0, K1);
1682 l_nopage_tlbm(&l, p);
1683 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1686 if ((p - handle_tlbm) > FASTPATH_SIZE)
1687 panic("TLB modify handler fastpath space exceeded");
1689 resolve_relocs(relocs, labels);
1690 pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
1691 (unsigned int)(p - handle_tlbm));
1693 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1697 * R4000 style TLB load/store/modify handlers.
1700 build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
1701 struct reloc **r, unsigned int pte,
1705 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
1707 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
1710 i_MFC0(p, pte, C0_BADVADDR);
1711 i_LW(p, ptr, 0, ptr);
1712 i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1713 i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1714 i_ADDU(p, ptr, ptr, pte);
1717 l_smp_pgtable_change(l, *p);
1719 iPTE_LW(p, l, pte, ptr); /* get even pte */
1720 if (!m4kc_tlbp_war())
1721 build_tlb_probe_entry(p);
1725 build_r4000_tlbchange_handler_tail(u32 **p, struct label **l,
1726 struct reloc **r, unsigned int tmp,
1729 i_ori(p, ptr, ptr, sizeof(pte_t));
1730 i_xori(p, ptr, ptr, sizeof(pte_t));
1731 build_update_entries(p, tmp, ptr);
1732 build_tlb_write_entry(p, l, r, tlb_indexed);
1734 i_eret(p); /* return from trap */
1737 build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
1741 static void __init build_r4000_tlb_load_handler(void)
1743 u32 *p = handle_tlbl;
1744 struct label *l = labels;
1745 struct reloc *r = relocs;
1747 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1748 memset(labels, 0, sizeof(labels));
1749 memset(relocs, 0, sizeof(relocs));
1751 if (bcm1250_m3_war()) {
1752 i_MFC0(&p, K0, C0_BADVADDR);
1753 i_MFC0(&p, K1, C0_ENTRYHI);
1754 i_xor(&p, K0, K0, K1);
1755 i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1756 il_bnez(&p, &r, K0, label_leave);
1757 /* No need for i_nop */
1760 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1761 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1762 if (m4kc_tlbp_war())
1763 build_tlb_probe_entry(&p);
1764 build_make_valid(&p, &r, K0, K1);
1765 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1767 l_nopage_tlbl(&l, p);
1768 i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1771 if ((p - handle_tlbl) > FASTPATH_SIZE)
1772 panic("TLB load handler fastpath space exceeded");
1774 resolve_relocs(relocs, labels);
1775 pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
1776 (unsigned int)(p - handle_tlbl));
1778 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1781 static void __init build_r4000_tlb_store_handler(void)
1783 u32 *p = handle_tlbs;
1784 struct label *l = labels;
1785 struct reloc *r = relocs;
1787 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1788 memset(labels, 0, sizeof(labels));
1789 memset(relocs, 0, sizeof(relocs));
1791 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1792 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1793 if (m4kc_tlbp_war())
1794 build_tlb_probe_entry(&p);
1795 build_make_write(&p, &r, K0, K1);
1796 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1798 l_nopage_tlbs(&l, p);
1799 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1802 if ((p - handle_tlbs) > FASTPATH_SIZE)
1803 panic("TLB store handler fastpath space exceeded");
1805 resolve_relocs(relocs, labels);
1806 pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
1807 (unsigned int)(p - handle_tlbs));
1809 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1812 static void __init build_r4000_tlb_modify_handler(void)
1814 u32 *p = handle_tlbm;
1815 struct label *l = labels;
1816 struct reloc *r = relocs;
1818 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1819 memset(labels, 0, sizeof(labels));
1820 memset(relocs, 0, sizeof(relocs));
1822 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1823 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1824 if (m4kc_tlbp_war())
1825 build_tlb_probe_entry(&p);
1826 /* Present and writable bits set, set accessed and dirty bits. */
1827 build_make_write(&p, &r, K0, K1);
1828 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1830 l_nopage_tlbm(&l, p);
1831 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1834 if ((p - handle_tlbm) > FASTPATH_SIZE)
1835 panic("TLB modify handler fastpath space exceeded");
1837 resolve_relocs(relocs, labels);
1838 pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
1839 (unsigned int)(p - handle_tlbm));
1841 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1844 void __init build_tlb_refill_handler(void)
1847 * The refill handler is generated per-CPU, multi-node systems
1848 * may have local storage for it. The other handlers are only
1851 static int run_once = 0;
1853 switch (current_cpu_type()) {
1861 build_r3000_tlb_refill_handler();
1863 build_r3000_tlb_load_handler();
1864 build_r3000_tlb_store_handler();
1865 build_r3000_tlb_modify_handler();
1872 panic("No R6000 TLB refill handler yet");
1876 panic("No R8000 TLB refill handler yet");
1880 build_r4000_tlb_refill_handler();
1882 build_r4000_tlb_load_handler();
1883 build_r4000_tlb_store_handler();
1884 build_r4000_tlb_modify_handler();
1890 void __init flush_tlb_handlers(void)
1892 flush_icache_range((unsigned long)handle_tlbl,
1893 (unsigned long)handle_tlbl + sizeof(handle_tlbl));
1894 flush_icache_range((unsigned long)handle_tlbs,
1895 (unsigned long)handle_tlbs + sizeof(handle_tlbs));
1896 flush_icache_range((unsigned long)handle_tlbm,
1897 (unsigned long)handle_tlbm + sizeof(handle_tlbm));