X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=arch%2Fpowerpc%2Fkernel%2Falign.c;h=e06f75daeba368ca231ec3d8af43db2285e68aba;hb=a58d52443fca9cc28ed580e3ccfcae3c3d49df33;hp=faaec9c6f78f1aaf5f7f572e5cbd8035c4c063e9;hpb=97f2aab6698f3ab2552c41c1024a65ffd0763a6d;p=linux-2.6 diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c index faaec9c6f7..e06f75daeb 100644 --- a/arch/powerpc/kernel/align.c +++ b/arch/powerpc/kernel/align.c @@ -35,17 +35,21 @@ struct aligninfo { #define INVALID { 0, 0 } -#define LD 1 /* load */ -#define ST 2 /* store */ -#define SE 4 /* sign-extend value */ -#define F 8 /* to/from fp regs */ -#define U 0x10 /* update index register */ -#define M 0x20 /* multiple load/store */ -#define SW 0x40 /* byte swap int or ... */ -#define S 0x40 /* ... single-precision fp */ -#define SX 0x40 /* byte count in XER */ +/* Bits in the flags field */ +#define LD 0 /* load */ +#define ST 1 /* store */ +#define SE 2 /* sign-extend value, or FP ld/st as word */ +#define F 4 /* to/from fp regs */ +#define U 8 /* update index register */ +#define M 0x10 /* multiple load/store */ +#define SW 0x20 /* byte swap */ +#define S 0x40 /* single-precision fp or... */ +#define SX 0x40 /* ... byte count in XER */ #define HARD 0x80 /* string, stwcx. */ +#define E4 0x40 /* SPE endianness is word */ +#define E8 0x80 /* SPE endianness is double word */ +/* DSISR bits reported for a DCBZ instruction: */ #define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */ #define SWAP(a, b) (t = (a), (a) = (b), (b) = t) @@ -85,9 +89,9 @@ static struct aligninfo aligninfo[128] = { { 8, LD+F+U }, /* 00 1 1001: lfdu */ { 4, ST+F+S+U }, /* 00 1 1010: stfsu */ { 8, ST+F+U }, /* 00 1 1011: stfdu */ - INVALID, /* 00 1 1100 */ + { 16, LD+F }, /* 00 1 1100: lfdp */ INVALID, /* 00 1 1101 */ - INVALID, /* 00 1 1110 */ + { 16, ST+F }, /* 00 1 1110: stfdp */ INVALID, /* 00 1 1111 */ { 8, LD }, /* 01 0 0000: ldx */ INVALID, /* 01 0 0001 */ @@ -165,10 +169,10 @@ static struct aligninfo aligninfo[128] = { { 8, LD+F }, /* 11 0 1001: lfdx */ { 4, ST+F+S }, /* 11 0 1010: stfsx */ { 8, ST+F }, /* 11 0 1011: stfdx */ - INVALID, /* 11 0 1100 */ - { 8, LD+M }, /* 11 0 1101: lmd */ - INVALID, /* 11 0 1110 */ - { 8, ST+M }, /* 11 0 1111: stmd */ + { 16, LD+F }, /* 11 0 1100: lfdpx */ + { 4, LD+F+SE }, /* 11 0 1101: lfiwax */ + { 16, ST+F }, /* 11 0 1110: stfdpx */ + { 4, ST+F }, /* 11 0 1111: stfiwx */ { 4, LD+U }, /* 11 1 0000: lwzux */ INVALID, /* 11 1 0001 */ { 4, ST+U }, /* 11 1 0010: stwux */ @@ -239,7 +243,7 @@ static int emulate_dcbz(struct pt_regs *regs, unsigned char __user *addr) if (user_mode(regs) && !access_ok(VERIFY_WRITE, p, size)) return -EFAULT; for (i = 0; i < size / sizeof(long); ++i) - if (__put_user(0, p+i)) + if (__put_user_inatomic(0, p+i)) return -EFAULT; return 1; } @@ -256,12 +260,16 @@ static int emulate_dcbz(struct pt_regs *regs, unsigned char __user *addr) #define REG_BYTE(rp, i) *((u8 *)(rp) + (i)) #endif +#define SWIZ_PTR(p) ((unsigned char __user *)((p) ^ swiz)) + static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, unsigned int reg, unsigned int nb, - unsigned int flags, unsigned int instr) + unsigned int flags, unsigned int instr, + unsigned long swiz) { unsigned long *rptr; - unsigned int nb0, i; + unsigned int nb0, i, bswiz; + unsigned long p; /* * We do not try to emulate 8 bytes multiple as they aren't really @@ -280,9 +288,13 @@ static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, if (nb == 0) return 1; } else { - if (__get_user(instr, - (unsigned int __user *)regs->nip)) + unsigned long pc = regs->nip ^ (swiz & 4); + + if (__get_user_inatomic(instr, + (unsigned int __user *)pc)) return -EFAULT; + if (swiz == 0 && (flags & SW)) + instr = cpu_to_le32(instr); nb = (instr >> 11) & 0x1f; if (nb == 0) nb = 32; @@ -300,7 +312,10 @@ static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, return -EFAULT; /* bad address */ rptr = ®s->gpr[reg]; - if (flags & LD) { + p = (unsigned long) addr; + bswiz = (flags & SW)? 3: 0; + + if (!(flags & ST)) { /* * This zeroes the top 4 bytes of the affected registers * in 64-bit mode, and also zeroes out any remaining @@ -311,32 +326,316 @@ static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr, memset(®s->gpr[0], 0, ((nb0 + 3) / 4) * sizeof(unsigned long)); - for (i = 0; i < nb; ++i) - if (__get_user(REG_BYTE(rptr, i), addr + i)) + for (i = 0; i < nb; ++i, ++p) + if (__get_user_inatomic(REG_BYTE(rptr, i ^ bswiz), + SWIZ_PTR(p))) return -EFAULT; if (nb0 > 0) { rptr = ®s->gpr[0]; addr += nb; - for (i = 0; i < nb0; ++i) - if (__get_user(REG_BYTE(rptr, i), addr + i)) + for (i = 0; i < nb0; ++i, ++p) + if (__get_user_inatomic(REG_BYTE(rptr, + i ^ bswiz), + SWIZ_PTR(p))) return -EFAULT; } } else { - for (i = 0; i < nb; ++i) - if (__put_user(REG_BYTE(rptr, i), addr + i)) + for (i = 0; i < nb; ++i, ++p) + if (__put_user_inatomic(REG_BYTE(rptr, i ^ bswiz), + SWIZ_PTR(p))) return -EFAULT; if (nb0 > 0) { rptr = ®s->gpr[0]; addr += nb; - for (i = 0; i < nb0; ++i) - if (__put_user(REG_BYTE(rptr, i), addr + i)) + for (i = 0; i < nb0; ++i, ++p) + if (__put_user_inatomic(REG_BYTE(rptr, + i ^ bswiz), + SWIZ_PTR(p))) return -EFAULT; } } return 1; } +/* + * Emulate floating-point pair loads and stores. + * Only POWER6 has these instructions, and it does true little-endian, + * so we don't need the address swizzling. + */ +static int emulate_fp_pair(struct pt_regs *regs, unsigned char __user *addr, + unsigned int reg, unsigned int flags) +{ + char *ptr = (char *) ¤t->thread.fpr[reg]; + int i, ret; + + if (!(flags & F)) + return 0; + if (reg & 1) + return 0; /* invalid form: FRS/FRT must be even */ + if (!(flags & SW)) { + /* not byte-swapped - easy */ + if (!(flags & ST)) + ret = __copy_from_user(ptr, addr, 16); + else + ret = __copy_to_user(addr, ptr, 16); + } else { + /* each FPR value is byte-swapped separately */ + ret = 0; + for (i = 0; i < 16; ++i) { + if (!(flags & ST)) + ret |= __get_user(ptr[i^7], addr + i); + else + ret |= __put_user(ptr[i^7], addr + i); + } + } + if (ret) + return -EFAULT; + return 1; /* exception handled and fixed up */ +} + +#ifdef CONFIG_SPE + +static struct aligninfo spe_aligninfo[32] = { + { 8, LD+E8 }, /* 0 00 00: evldd[x] */ + { 8, LD+E4 }, /* 0 00 01: evldw[x] */ + { 8, LD }, /* 0 00 10: evldh[x] */ + INVALID, /* 0 00 11 */ + { 2, LD }, /* 0 01 00: evlhhesplat[x] */ + INVALID, /* 0 01 01 */ + { 2, LD }, /* 0 01 10: evlhhousplat[x] */ + { 2, LD+SE }, /* 0 01 11: evlhhossplat[x] */ + { 4, LD }, /* 0 10 00: evlwhe[x] */ + INVALID, /* 0 10 01 */ + { 4, LD }, /* 0 10 10: evlwhou[x] */ + { 4, LD+SE }, /* 0 10 11: evlwhos[x] */ + { 4, LD+E4 }, /* 0 11 00: evlwwsplat[x] */ + INVALID, /* 0 11 01 */ + { 4, LD }, /* 0 11 10: evlwhsplat[x] */ + INVALID, /* 0 11 11 */ + + { 8, ST+E8 }, /* 1 00 00: evstdd[x] */ + { 8, ST+E4 }, /* 1 00 01: evstdw[x] */ + { 8, ST }, /* 1 00 10: evstdh[x] */ + INVALID, /* 1 00 11 */ + INVALID, /* 1 01 00 */ + INVALID, /* 1 01 01 */ + INVALID, /* 1 01 10 */ + INVALID, /* 1 01 11 */ + { 4, ST }, /* 1 10 00: evstwhe[x] */ + INVALID, /* 1 10 01 */ + { 4, ST }, /* 1 10 10: evstwho[x] */ + INVALID, /* 1 10 11 */ + { 4, ST+E4 }, /* 1 11 00: evstwwe[x] */ + INVALID, /* 1 11 01 */ + { 4, ST+E4 }, /* 1 11 10: evstwwo[x] */ + INVALID, /* 1 11 11 */ +}; + +#define EVLDD 0x00 +#define EVLDW 0x01 +#define EVLDH 0x02 +#define EVLHHESPLAT 0x04 +#define EVLHHOUSPLAT 0x06 +#define EVLHHOSSPLAT 0x07 +#define EVLWHE 0x08 +#define EVLWHOU 0x0A +#define EVLWHOS 0x0B +#define EVLWWSPLAT 0x0C +#define EVLWHSPLAT 0x0E +#define EVSTDD 0x10 +#define EVSTDW 0x11 +#define EVSTDH 0x12 +#define EVSTWHE 0x18 +#define EVSTWHO 0x1A +#define EVSTWWE 0x1C +#define EVSTWWO 0x1E + +/* + * Emulate SPE loads and stores. + * Only Book-E has these instructions, and it does true little-endian, + * so we don't need the address swizzling. + */ +static int emulate_spe(struct pt_regs *regs, unsigned int reg, + unsigned int instr) +{ + int t, ret; + union { + u64 ll; + u32 w[2]; + u16 h[4]; + u8 v[8]; + } data, temp; + unsigned char __user *p, *addr; + unsigned long *evr = ¤t->thread.evr[reg]; + unsigned int nb, flags; + + instr = (instr >> 1) & 0x1f; + + /* DAR has the operand effective address */ + addr = (unsigned char __user *)regs->dar; + + nb = spe_aligninfo[instr].len; + flags = spe_aligninfo[instr].flags; + + /* Verify the address of the operand */ + if (unlikely(user_mode(regs) && + !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ), + addr, nb))) + return -EFAULT; + + /* userland only */ + if (unlikely(!user_mode(regs))) + return 0; + + flush_spe_to_thread(current); + + /* If we are loading, get the data from user space, else + * get it from register values + */ + if (flags & ST) { + data.ll = 0; + switch (instr) { + case EVSTDD: + case EVSTDW: + case EVSTDH: + data.w[0] = *evr; + data.w[1] = regs->gpr[reg]; + break; + case EVSTWHE: + data.h[2] = *evr >> 16; + data.h[3] = regs->gpr[reg] >> 16; + break; + case EVSTWHO: + data.h[2] = *evr & 0xffff; + data.h[3] = regs->gpr[reg] & 0xffff; + break; + case EVSTWWE: + data.w[1] = *evr; + break; + case EVSTWWO: + data.w[1] = regs->gpr[reg]; + break; + default: + return -EINVAL; + } + } else { + temp.ll = data.ll = 0; + ret = 0; + p = addr; + + switch (nb) { + case 8: + ret |= __get_user_inatomic(temp.v[0], p++); + ret |= __get_user_inatomic(temp.v[1], p++); + ret |= __get_user_inatomic(temp.v[2], p++); + ret |= __get_user_inatomic(temp.v[3], p++); + case 4: + ret |= __get_user_inatomic(temp.v[4], p++); + ret |= __get_user_inatomic(temp.v[5], p++); + case 2: + ret |= __get_user_inatomic(temp.v[6], p++); + ret |= __get_user_inatomic(temp.v[7], p++); + if (unlikely(ret)) + return -EFAULT; + } + + switch (instr) { + case EVLDD: + case EVLDW: + case EVLDH: + data.ll = temp.ll; + break; + case EVLHHESPLAT: + data.h[0] = temp.h[3]; + data.h[2] = temp.h[3]; + break; + case EVLHHOUSPLAT: + case EVLHHOSSPLAT: + data.h[1] = temp.h[3]; + data.h[3] = temp.h[3]; + break; + case EVLWHE: + data.h[0] = temp.h[2]; + data.h[2] = temp.h[3]; + break; + case EVLWHOU: + case EVLWHOS: + data.h[1] = temp.h[2]; + data.h[3] = temp.h[3]; + break; + case EVLWWSPLAT: + data.w[0] = temp.w[1]; + data.w[1] = temp.w[1]; + break; + case EVLWHSPLAT: + data.h[0] = temp.h[2]; + data.h[1] = temp.h[2]; + data.h[2] = temp.h[3]; + data.h[3] = temp.h[3]; + break; + default: + return -EINVAL; + } + } + + if (flags & SW) { + switch (flags & 0xf0) { + case E8: + SWAP(data.v[0], data.v[7]); + SWAP(data.v[1], data.v[6]); + SWAP(data.v[2], data.v[5]); + SWAP(data.v[3], data.v[4]); + break; + case E4: + + SWAP(data.v[0], data.v[3]); + SWAP(data.v[1], data.v[2]); + SWAP(data.v[4], data.v[7]); + SWAP(data.v[5], data.v[6]); + break; + /* Its half word endian */ + default: + SWAP(data.v[0], data.v[1]); + SWAP(data.v[2], data.v[3]); + SWAP(data.v[4], data.v[5]); + SWAP(data.v[6], data.v[7]); + break; + } + } + + if (flags & SE) { + data.w[0] = (s16)data.h[1]; + data.w[1] = (s16)data.h[3]; + } + + /* Store result to memory or update registers */ + if (flags & ST) { + ret = 0; + p = addr; + switch (nb) { + case 8: + ret |= __put_user_inatomic(data.v[0], p++); + ret |= __put_user_inatomic(data.v[1], p++); + ret |= __put_user_inatomic(data.v[2], p++); + ret |= __put_user_inatomic(data.v[3], p++); + case 4: + ret |= __put_user_inatomic(data.v[4], p++); + ret |= __put_user_inatomic(data.v[5], p++); + case 2: + ret |= __put_user_inatomic(data.v[6], p++); + ret |= __put_user_inatomic(data.v[7], p++); + } + if (unlikely(ret)) + return -EFAULT; + } else { + *evr = data.w[0]; + regs->gpr[reg] = data.w[1]; + } + + return 1; +} +#endif /* CONFIG_SPE */ /* * Called on alignment exception. Attempts to fixup @@ -352,7 +651,7 @@ int fix_alignment(struct pt_regs *regs) unsigned int reg, areg; unsigned int dsisr; unsigned char __user *addr; - unsigned char __user *p; + unsigned long p, swiz; int ret, t; union { u64 ll; @@ -380,16 +679,27 @@ int fix_alignment(struct pt_regs *regs) * let's make one up from the instruction */ if (cpu_has_feature(CPU_FTR_NODSISRALIGN)) { - unsigned int real_instr; - if (unlikely(__get_user(real_instr, - (unsigned int __user *)regs->nip))) + unsigned long pc = regs->nip; + + if (cpu_has_feature(CPU_FTR_PPC_LE) && (regs->msr & MSR_LE)) + pc ^= 4; + if (unlikely(__get_user_inatomic(instr, + (unsigned int __user *)pc))) return -EFAULT; - dsisr = make_dsisr(real_instr); + if (cpu_has_feature(CPU_FTR_REAL_LE) && (regs->msr & MSR_LE)) + instr = cpu_to_le32(instr); + dsisr = make_dsisr(instr); } /* extract the operation and registers from the dsisr */ reg = (dsisr >> 5) & 0x1f; /* source/dest register */ areg = dsisr & 0x1f; /* register to update */ + +#ifdef CONFIG_SPE + if ((instr >> 26) == 0x4) + return emulate_spe(regs, reg, instr); +#endif + instr = (dsisr >> 10) & 0x7f; instr |= (dsisr >> 13) & 0x60; @@ -397,6 +707,24 @@ int fix_alignment(struct pt_regs *regs) nb = aligninfo[instr].len; flags = aligninfo[instr].flags; + /* Byteswap little endian loads and stores */ + swiz = 0; + if (regs->msr & MSR_LE) { + flags ^= SW; + /* + * So-called "PowerPC little endian" mode works by + * swizzling addresses rather than by actually doing + * any byte-swapping. To emulate this, we XOR each + * byte address with 7. We also byte-swap, because + * the processor's address swizzling depends on the + * operand size (it xors the address with 7 for bytes, + * 6 for halfwords, 4 for words, 0 for doublewords) but + * we will xor with 7 and load/store each byte separately. + */ + if (cpu_has_feature(CPU_FTR_PPC_LE)) + swiz = 7; + } + /* DAR has the operand effective address */ addr = (unsigned char __user *)regs->dar; @@ -412,7 +740,8 @@ int fix_alignment(struct pt_regs *regs) * function */ if (flags & M) - return emulate_multiple(regs, addr, reg, nb, flags, instr); + return emulate_multiple(regs, addr, reg, nb, + flags, instr, swiz); /* Verify the address of the operand */ if (unlikely(user_mode(regs) && @@ -428,54 +757,79 @@ int fix_alignment(struct pt_regs *regs) flush_fp_to_thread(current); } + /* Special case for 16-byte FP loads and stores */ + if (nb == 16) + return emulate_fp_pair(regs, addr, reg, flags); + /* If we are loading, get the data from user space, else * get it from register values */ - if (flags & LD) { + if (!(flags & ST)) { data.ll = 0; ret = 0; - p = addr; + p = (unsigned long) addr; switch (nb) { case 8: - ret |= __get_user(data.v[0], p++); - ret |= __get_user(data.v[1], p++); - ret |= __get_user(data.v[2], p++); - ret |= __get_user(data.v[3], p++); + ret |= __get_user_inatomic(data.v[0], SWIZ_PTR(p++)); + ret |= __get_user_inatomic(data.v[1], SWIZ_PTR(p++)); + ret |= __get_user_inatomic(data.v[2], SWIZ_PTR(p++)); + ret |= __get_user_inatomic(data.v[3], SWIZ_PTR(p++)); case 4: - ret |= __get_user(data.v[4], p++); - ret |= __get_user(data.v[5], p++); + ret |= __get_user_inatomic(data.v[4], SWIZ_PTR(p++)); + ret |= __get_user_inatomic(data.v[5], SWIZ_PTR(p++)); case 2: - ret |= __get_user(data.v[6], p++); - ret |= __get_user(data.v[7], p++); + ret |= __get_user_inatomic(data.v[6], SWIZ_PTR(p++)); + ret |= __get_user_inatomic(data.v[7], SWIZ_PTR(p++)); if (unlikely(ret)) return -EFAULT; } - } else if (flags & F) + } else if (flags & F) { data.dd = current->thread.fpr[reg]; - else + if (flags & S) { + /* Single-precision FP store requires conversion... */ +#ifdef CONFIG_PPC_FPU + preempt_disable(); + enable_kernel_fp(); + cvt_df(&data.dd, (float *)&data.v[4], ¤t->thread); + preempt_enable(); +#else + return 0; +#endif + } + } else data.ll = regs->gpr[reg]; - /* Perform other misc operations like sign extension, byteswap, + if (flags & SW) { + switch (nb) { + case 8: + SWAP(data.v[0], data.v[7]); + SWAP(data.v[1], data.v[6]); + SWAP(data.v[2], data.v[5]); + SWAP(data.v[3], data.v[4]); + break; + case 4: + SWAP(data.v[4], data.v[7]); + SWAP(data.v[5], data.v[6]); + break; + case 2: + SWAP(data.v[6], data.v[7]); + break; + } + } + + /* Perform other misc operations like sign extension * or floating point single precision conversion */ - switch (flags & ~U) { - case LD+SE: /* sign extend */ + switch (flags & ~(U|SW)) { + case LD+SE: /* sign extending integer loads */ + case LD+F+SE: /* sign extend for lfiwax */ if ( nb == 2 ) data.ll = data.x16.low16; else /* nb must be 4 */ data.ll = data.x32.low32; break; - case LD+S: /* byte-swap */ - case ST+S: - if (nb == 2) { - SWAP(data.v[6], data.v[7]); - } else { - SWAP(data.v[4], data.v[7]); - SWAP(data.v[5], data.v[6]); - } - break; - /* Single-precision FP load and store require conversions... */ + /* Single-precision FP load requires conversion... */ case LD+F+S: #ifdef CONFIG_PPC_FPU preempt_disable(); @@ -484,16 +838,6 @@ int fix_alignment(struct pt_regs *regs) preempt_enable(); #else return 0; -#endif - break; - case ST+F+S: -#ifdef CONFIG_PPC_FPU - preempt_disable(); - enable_kernel_fp(); - cvt_df(&data.dd, (float *)&data.v[4], ¤t->thread); - preempt_enable(); -#else - return 0; #endif break; } @@ -501,19 +845,19 @@ int fix_alignment(struct pt_regs *regs) /* Store result to memory or update registers */ if (flags & ST) { ret = 0; - p = addr; + p = (unsigned long) addr; switch (nb) { case 8: - ret |= __put_user(data.v[0], p++); - ret |= __put_user(data.v[1], p++); - ret |= __put_user(data.v[2], p++); - ret |= __put_user(data.v[3], p++); + ret |= __put_user_inatomic(data.v[0], SWIZ_PTR(p++)); + ret |= __put_user_inatomic(data.v[1], SWIZ_PTR(p++)); + ret |= __put_user_inatomic(data.v[2], SWIZ_PTR(p++)); + ret |= __put_user_inatomic(data.v[3], SWIZ_PTR(p++)); case 4: - ret |= __put_user(data.v[4], p++); - ret |= __put_user(data.v[5], p++); + ret |= __put_user_inatomic(data.v[4], SWIZ_PTR(p++)); + ret |= __put_user_inatomic(data.v[5], SWIZ_PTR(p++)); case 2: - ret |= __put_user(data.v[6], p++); - ret |= __put_user(data.v[7], p++); + ret |= __put_user_inatomic(data.v[6], SWIZ_PTR(p++)); + ret |= __put_user_inatomic(data.v[7], SWIZ_PTR(p++)); } if (unlikely(ret)) return -EFAULT;